vidarchive

vidarchive/internal/service/library.go
library.go
⎇
Raw
1package service
2
3import (
4 "bytes"
5 "encoding/json"
6 "fmt"
7 "log/slog"
8 "os"
9 "path/filepath"
10 "sort"
11 "strconv"
12 "strings"
13 "sync"
14 "time"
15
16 "github.com/BurntSushi/toml"
17
18 "vidarchive/internal/models"
19)
20
21const (
22 itemMarkerName = ".vidarchive-item.toml"
23 subtitlesDirName = "subtitles"
24)
25
26var mediaExts = map[string]struct{}{
27 ".mp4": {}, ".webm": {}, ".mkv": {}, ".avi": {}, ".mov": {},
28 ".mp3": {}, ".wav": {}, ".flac": {}, ".aac": {}, ".opus": {}, ".m4a": {},
29}
30
31var audioExts = map[string]struct{}{
32 ".mp3": {}, ".wav": {}, ".flac": {}, ".aac": {}, ".opus": {}, ".m4a": {},
33}
34
35type LibraryService struct {
36 libraryDir string
37 // ffmpegPath/ffprobePath are the binaries used for thumbnail extraction,
38 // subtitle conversion, and media probing. Configurable so non-PATH installs
39 // (e.g. a pinned build) can be pointed at directly.
40 ffmpegPath string
41 ffprobePath string
42 // thumbLocks holds a per-media-file mutex serializing extraction so two
43 // callers never write the same temp file at once. Entries are reference
44 // counted and removed once nobody holds them. Evicting a lock that is still
45 // held would let a second caller take a fresh one and reintroduce the race.
46 thumbMu sync.Mutex
47 thumbLocks map[string]*refLock
48 thumbSem chan struct{}
49
50 // thumbFailed records media filepaths whose extraction already failed this
51 // run, so we trust ffmpeg's verdict and don't re-run it on every request.
52 // Bounded: an evicted path just means one more ffmpeg attempt.
53 thumbFailed *lru[bool]
54
55 // scanCache memoizes scanned items for a short TTL so the listing page (which
56 // fans out one thumbnail request per media file) and quick auto-refreshes
57 // don't re-parse each item's marker + info.json on every request. Only item
58 // scans are cached — the directory listing itself is always read fresh, so
59 // newly added/removed items and subfolders appear immediately.
60 scanCache *lru[scanCacheEntry]
61 scanTTL time.Duration
62}
63
64type scanCacheEntry struct {
65 item *models.LibraryItem
66 at time.Time
67}
68
69// scanCacheTTL is how long a scanned item is reused before being re-read.
70const scanCacheTTL = 10 * time.Second
71
72// maxCachedPaths bounds each per-path cache. Without it both grow with the
73// number of distinct files touched over the process lifetime, which is fine for
74// a personal archive and not for a large one.
75const maxCachedPaths = 1024
76
77func NewLibraryService(libraryDir, ffmpegPath, ffprobePath string) *LibraryService {
78 return &LibraryService{
79 libraryDir: libraryDir,
80 ffmpegPath: ffmpegPath,
81 ffprobePath: ffprobePath,
82 thumbLocks: make(map[string]*refLock),
83 thumbSem: make(chan struct{}, maxConcurrentThumbnails),
84 thumbFailed: newLRU[bool](maxCachedPaths),
85 scanCache: newLRU[scanCacheEntry](maxCachedPaths),
86 scanTTL: scanCacheTTL,
87 }
88}
89
90func (s *LibraryService) getCachedScan(relPath string) (*models.LibraryItem, bool) {
91 if s.scanTTL <= 0 {
92 return nil, false
93 }
94 e, ok := s.scanCache.Get(relPath)
95 if !ok || time.Since(e.at) > s.scanTTL {
96 return nil, false
97 }
98 return e.item, true
99}
100
101func (s *LibraryService) putCachedScan(relPath string, item *models.LibraryItem) {
102 if s.scanTTL <= 0 {
103 return
104 }
105 s.scanCache.Put(relPath, scanCacheEntry{item: item, at: time.Now()})
106}
107
108func (s *LibraryService) evictCachedScan(relPath string) {
109 s.scanCache.Delete(relPath)
110}
111
112// refLock is a mutex plus the number of callers holding or waiting for it.
113type refLock struct {
114 mu sync.Mutex
115 refs int
116}
117
118// lockThumbFile locks the mutex guarding path and returns its release func.
119func (s *LibraryService) lockThumbFile(path string) func() {
120 s.thumbMu.Lock()
121 l, ok := s.thumbLocks[path]
122 if !ok {
123 l = &refLock{}
124 s.thumbLocks[path] = l
125 }
126 l.refs++
127 s.thumbMu.Unlock()
128
129 l.mu.Lock()
130
131 return func() {
132 l.mu.Unlock()
133 s.thumbMu.Lock()
134 l.refs--
135 if l.refs == 0 {
136 delete(s.thumbLocks, path)
137 }
138 s.thumbMu.Unlock()
139 }
140}
141
142// scannedItem returns a cached scan if fresh, otherwise scans and caches it.
143func (s *LibraryService) scannedItem(itemDir, relPath string) (*models.LibraryItem, error) {
144 if item, ok := s.getCachedScan(relPath); ok {
145 return item, nil
146 }
147 item, err := s.scanItem(itemDir, relPath)
148 if err != nil {
149 return nil, err
150 }
151 s.putCachedScan(relPath, item)
152 return item, nil
153}
154
155// ResolveWithinLibrary resolves a caller-supplied relative directory against the
156// library root and rejects anything that escapes it. The directory need not
157// exist yet, so it is safe to use when choosing a download's output location.
158func (s *LibraryService) ResolveWithinLibrary(relPath string) (string, error) {
159 return s.resolveItemDir(filepath.Clean(relPath))
160}
161
162// libraryRoot returns the symlink-resolved library root.
163func (s *LibraryService) libraryRoot() string {
164 if base, err := filepath.EvalSymlinks(s.libraryDir); err == nil {
165 return base
166 }
167 return filepath.Clean(s.libraryDir)
168}
169
170func (s *LibraryService) resolveItemDir(relPath string) (string, error) {
171 relPath = strings.Trim(relPath, string(filepath.Separator))
172 base := s.libraryRoot()
173 if relPath == "" || relPath == "." {
174 return base, nil
175 }
176 itemDir := filepath.Join(base, relPath)
177 cleanDir, err := filepath.EvalSymlinks(itemDir)
178 if err != nil {
179 cleanDir = filepath.Clean(itemDir)
180 }
181 if !strings.HasPrefix(cleanDir, base+string(filepath.Separator)) && cleanDir != base {
182 return "", fmt.Errorf("invalid path")
183 }
184 // Return the cleaned/symlink-resolved path we just validated, so callers do
185 // I/O on exactly the path that passed the boundary check.
186 return cleanDir, nil
187}
188
189func (s *LibraryService) GetAll(path, sortBy, filter string) ([]*models.LibraryItem, []string, error) {
190 if path != "" && !strings.HasSuffix(path, "/") {
191 path += "/"
192 }
193
194 dir, err := s.resolveItemDir(path)
195 if err != nil {
196 slog.Warn("library listing: invalid path", "path", path, "err", err)
197 return nil, nil, nil
198 }
199 entries, err := os.ReadDir(dir)
200 if err != nil {
201 if os.IsNotExist(err) {
202 return nil, nil, nil
203 }
204 return nil, nil, err
205 }
206
207 var items []*models.LibraryItem
208 var folders []string
209
210 for _, entry := range entries {
211 if !entry.IsDir() {
212 continue
213 }
214 name := entry.Name()
215 if name == subtitlesDirName {
216 continue
217 }
218 itemDir := filepath.Join(dir, name)
219 relPath := filepath.ToSlash(filepath.Join(path, name))
220
221 markerPath := filepath.Join(itemDir, itemMarkerName)
222 if _, err := os.Stat(markerPath); err == nil {
223 item, err := s.scannedItem(itemDir, relPath)
224 if err != nil {
225 slog.Warn("failed to scan item", "path", relPath, "err", err)
226 continue
227 }
228 if filter != "" && !strings.Contains(strings.ToLower(item.Name), strings.ToLower(filter)) && !strings.Contains(strings.ToLower(item.RelPath), strings.ToLower(filter)) {
229 continue
230 }
231 items = append(items, item)
232 } else {
233 folders = append(folders, name)
234 }
235 }
236
237 switch sortBy {
238 case "title":
239 sort.Slice(items, func(i, j int) bool { return strings.ToLower(items[i].Name) < strings.ToLower(items[j].Name) })
240 case "duration":
241 sort.Slice(items, func(i, j int) bool { return items[i].Duration > items[j].Duration })
242 case "date":
243 fallthrough
244 default:
245 // Stat each item once up front rather than twice per comparison.
246 modTime := make(map[string]int64, len(items))
247 for _, it := range items {
248 if info, err := os.Stat(it.DirPath); err == nil {
249 modTime[it.RelPath] = info.ModTime().UnixNano()
250 }
251 }
252 sort.Slice(items, func(i, j int) bool {
253 return modTime[items[i].RelPath] > modTime[items[j].RelPath]
254 })
255 }
256
257 sort.Strings(folders)
258
259 return items, folders, nil
260}
261
262// GetByRelPath returns the item at relPath, reusing a recent cached scan when
263// available (see scanCache).
264func (s *LibraryService) GetByRelPath(relPath string) (*models.LibraryItem, error) {
265 relPath = strings.Trim(relPath, "/")
266
267 if item, ok := s.getCachedScan(relPath); ok {
268 return item, nil
269 }
270
271 itemDir, err := s.resolveItemDir(relPath)
272 if err != nil {
273 return nil, fmt.Errorf("item not found")
274 }
275 if _, err := os.Stat(filepath.Join(itemDir, itemMarkerName)); err != nil {
276 return nil, fmt.Errorf("item not found")
277 }
278 return s.scannedItem(itemDir, relPath)
279}
280
281func (s *LibraryService) scanItem(itemDir, relPath string) (*models.LibraryItem, error) {
282 metadata, err := s.readMetadata(itemDir)
283 if err != nil {
284 return nil, err
285 }
286
287 mediaFiles, infoJSONPath, err := s.listItemFiles(itemDir)
288 if err != nil {
289 return nil, err
290 }
291
292 var info map[string]interface{}
293 if infoJSONPath != "" {
294 data, err := os.ReadFile(infoJSONPath)
295 if err == nil {
296 if err := json.Unmarshal(data, &info); err != nil {
297 slog.Warn("ignoring malformed info.json", "path", infoJSONPath, "err", err)
298 }
299 }
300 }
301
302 // dirty tracks whether we derived any new metadata worth persisting, so a
303 // plain listing or detail view doesn't rewrite the marker file on every read.
304 dirty := false
305
306 if metadata.Name == "" {
307 if title, ok := infoString(info, "title"); ok && title != "" {
308 metadata.Name = title
309 } else if len(mediaFiles) > 0 {
310 metadata.Name = mediaFileStem(mediaFiles[0])
311 } else {
312 metadata.Name = filepath.Base(itemDir)
313 }
314 dirty = true
315 }
316
317 if metadata.SourceURL == "" {
318 dirty = backfillString(&metadata.SourceURL, info, "webpage_url", "url") || dirty
319 }
320
321 if metadata.Description == "" {
322 dirty = backfillString(&metadata.Description, info, "description") || dirty
323 }
324
325 // Backfill the stable identity (yt-dlp's video id) from info.json so
326 // pre-existing items gain an identity on their next scan. Subscriptions match
327 // and prune items by this id (see FindByVideoID / PruneToIDSet).
328 if metadata.VideoID == "" {
329 dirty = backfillString(&metadata.VideoID, info, "id") || dirty
330 }
331
332 if metadata.FileDurations == nil {
333 metadata.FileDurations = make(map[string]int)
334 }
335
336 // Per-file durations come from the marker's file_durations map (populated at
337 // import time). For a single-file item we also seed it from info.json's
338 // duration, which covers the common case without a probe. We deliberately do
339 // not run ffprobe here — keeping it off the scan/listing path is the point of
340 // the marker cache. Files without a known duration simply show no badge.
341 for i := range mediaFiles {
342 mf := &mediaFiles[i]
343 if d, ok := metadata.FileDurations[mf.Filename]; ok {
344 mf.Duration = d
345 continue
346 }
347 if len(mediaFiles) == 1 {
348 if d, ok := infoDuration(info); ok && d > 0 {
349 mf.Duration = d
350 metadata.FileDurations[mf.Filename] = d
351 dirty = true
352 }
353 }
354 }
355
356 // The item-level duration is the sum of known per-file durations, used only
357 // for the "duration" sort — there is no single "overall" duration shown.
358 total := 0
359 for _, mf := range mediaFiles {
360 if mf.Duration > 0 {
361 total += mf.Duration
362 }
363 }
364
365 item := &models.LibraryItem{
366 Name: metadata.Name,
367 RelPath: relPath,
368 DirPath: itemDir,
369 SourceURL: metadata.SourceURL,
370 Duration: total,
371 Description: metadata.Description,
372 YtdlpFlags: metadata.YtdlpFlags,
373 MediaFiles: mediaFiles,
374 }
375
376 if dirty {
377 if err := s.writeMetadata(itemDir, metadata); err != nil {
378 slog.Warn("failed to persist derived metadata", "dir", itemDir, "err", err)
379 }
380 }
381
382 return item, nil
383}
384
385// backfillString sets *field from the first non-empty string value among
386// info's keys, reporting whether it changed anything. An empty value in
387// info.json must not count as a change: the marker would be rewritten on every
388// scan forever without ever gaining a value.
389func backfillString(field *string, info map[string]interface{}, keys ...string) bool {
390 for _, k := range keys {
391 if v, ok := infoString(info, k); ok && v != "" {
392 *field = v
393 return true
394 }
395 }
396 return false
397}
398
399func (s *LibraryService) readMetadata(itemDir string) (models.ItemMetadata, error) {
400 markerPath := filepath.Join(itemDir, itemMarkerName)
401 var metadata models.ItemMetadata
402 data, err := os.ReadFile(markerPath)
403 if err == nil {
404 if _, err := toml.Decode(string(data), &metadata); err != nil {
405 slog.Warn("failed to parse item marker", "path", markerPath, "err", err)
406 }
407 }
408 return metadata, nil
409}
410
411func (s *LibraryService) writeMetadata(itemDir string, metadata models.ItemMetadata) error {
412 var buf bytes.Buffer
413 if err := toml.NewEncoder(&buf).Encode(metadata); err != nil {
414 return err
415 }
416 return atomicWrite(filepath.Join(itemDir, itemMarkerName), buf.Bytes(), markerFileMode)
417}
418
419// markerFileMode matches the info.json sidecar: both sit in the library next to
420// the media and are meant to be readable (and hand-editable) by the operator.
421const markerFileMode = 0o644
422
423// atomicWrite writes data to path via a uniquely-named temp file in the same
424// directory and a rename, so a crash mid-write can't leave a truncated file and
425// two concurrent writers never collide on a shared temp path.
426func atomicWrite(path string, data []byte, mode os.FileMode) error {
427 f, err := os.CreateTemp(filepath.Dir(path), ".vidarchive-*.tmp")
428 if err != nil {
429 return err
430 }
431 tmpPath := f.Name()
432 // A no-op once the rename below has succeeded.
433 defer os.Remove(tmpPath)
434
435 if err := f.Chmod(mode); err != nil {
436 f.Close()
437 return err
438 }
439 if _, err := f.Write(data); err != nil {
440 f.Close()
441 return err
442 }
443 // Close explicitly: a deferred close would hide a flush error on the write.
444 if err := f.Close(); err != nil {
445 return err
446 }
447 return os.Rename(tmpPath, path)
448}
449
450func (s *LibraryService) listItemFiles(itemDir string) ([]models.MediaFile, string, error) {
451 entries, err := os.ReadDir(itemDir)
452 if err != nil {
453 return nil, "", err
454 }
455
456 var mediaFiles []models.MediaFile
457 var infoJSONFiles []string
458
459 for _, entry := range entries {
460 if entry.IsDir() {
461 continue
462 }
463 name := entry.Name()
464 path := filepath.Join(itemDir, name)
465 ext := strings.ToLower(filepath.Ext(name))
466
467 if name == itemMarkerName {
468 continue
469 }
470 if name == "info.json" || strings.HasSuffix(name, ".info.json") {
471 infoJSONFiles = append(infoJSONFiles, path)
472 continue
473 }
474 if _, ok := mediaExts[ext]; !ok {
475 continue
476 }
477
478 _, isAudio := audioExts[ext]
479
480 mediaFiles = append(mediaFiles, models.MediaFile{
481 Filename: name,
482 Filepath: path,
483 IsAudio: isAudio,
484 Duration: -1,
485 })
486 }
487
488 sort.Slice(mediaFiles, func(i, j int) bool {
489 return mediaFiles[i].Filepath < mediaFiles[j].Filepath
490 })
491
492 var infoJSONPath string
493 if len(infoJSONFiles) > 0 {
494 sort.Strings(infoJSONFiles)
495 infoJSONPath = infoJSONFiles[0]
496 if len(infoJSONFiles) > 1 {
497 slog.Warn("multiple info.json files", "dir", itemDir, "using", infoJSONPath)
498 }
499 }
500
501 return mediaFiles, infoJSONPath, nil
502}
503
504func infoString(info map[string]interface{}, key string) (string, bool) {
505 if info == nil {
506 return "", false
507 }
508 // Only accept genuine strings: title/url/description are always strings in
509 // yt-dlp output, and stringifying an arbitrary JSON value (map, slice) would
510 // store junk like "map[...]" into the field.
511 if s, ok := info[key].(string); ok {
512 return s, true
513 }
514 return "", false
515}
516
517func infoDuration(info map[string]interface{}) (int, bool) {
518 if info == nil {
519 return 0, false
520 }
521 v, ok := info["duration"]
522 if !ok {
523 return 0, false
524 }
525 switch n := v.(type) {
526 case float64:
527 return int(n + 0.5), true
528 case string:
529 if f, err := strconv.ParseFloat(n, 64); err == nil {
530 return int(f + 0.5), true
531 }
532 }
533 return 0, false
534}
535
536func mediaFileStem(mf models.MediaFile) string {
537 return strings.TrimSuffix(filepath.Base(mf.Filename), filepath.Ext(mf.Filename))
538}
539
540// primaryMediaFile picks the representative file for an item: the largest video
541// file, or — if there are none — the largest file overall. Returns nil for an
542// item with no media files. Used for the item-level thumbnail and as the default
543// target for metadata, keeping those two consistent.
544func primaryMediaFile(item *models.LibraryItem) *models.MediaFile {
545 size := func(mf *models.MediaFile) int64 {
546 if info, err := os.Stat(mf.Filepath); err == nil {
547 return info.Size()
548 }
549 return 0
550 }
551 var best *models.MediaFile
552 for i := range item.MediaFiles {
553 mf := &item.MediaFiles[i]
554 switch {
555 case best == nil:
556 best = mf
557 case best.IsAudio && !mf.IsAudio:
558 // Prefer any video over audio.
559 best = mf
560 case best.IsAudio == mf.IsAudio && size(mf) > size(best):
561 best = mf
562 }
563 }
564 return best
565}
566
567func (s *LibraryService) GetMediaFile(relPath, filename string) (string, error) {
568 item, err := s.GetByRelPath(relPath)
569 if err != nil {
570 return "", err
571 }
572 for _, mf := range item.MediaFiles {
573 if mf.Filename == filename {
574 return mf.Filepath, nil
575 }
576 }
577 return "", fmt.Errorf("media file not found")
578}
579
580func (s *LibraryService) Delete(relPath string) error {
581 itemDir, err := s.resolveItemDir(relPath)
582 if err != nil {
583 return err
584 }
585 // resolveItemDir maps ""/"." to the library root and resolves symlinks, so a
586 // result equal to the root (reachable via a URL-encoded slash, "sub/..", or a
587 // symlink pointing back at the root) must be refused — deleting it would
588 // wipe the entire library.
589 if itemDir == s.libraryRoot() {
590 return fmt.Errorf("refusing to delete library root")
591 }
592 // Evict the cached scan so the deletion is reflected immediately rather than
593 // lingering until the TTL expires.
594 s.evictCachedScan(strings.Trim(relPath, "/"))
595 return os.RemoveAll(itemDir)
596}
597
598// FindByVideoID returns the absolute directory of the item under baseDir whose
599// marker matches the given yt-dlp video id, scanning only that directory (the
600// subscription's owned folder). Matching on the id alone is safe here because
601// each subscription owns a single source, so ids don't collide across
602// extractors within the folder. ok is false when no match is found or id is
603// empty.
604func (s *LibraryService) FindByVideoID(baseDir, id string) (string, bool) {
605 if id == "" {
606 return "", false
607 }
608
609 var match string
610 // An unreadable base dir simply means no match here.
611 _ = s.eachItemDir(baseDir, "FindByVideoID", func(itemDir string, meta models.ItemMetadata) bool {
612 if meta.VideoID != id {
613 return true
614 }
615 match = itemDir
616 return false
617 })
618
619 return match, match != ""
620}
621
622// eachItemDir walks the marked library items directly under baseDir, reading
623// each one's metadata, and calls fn until it returns false. Entries that aren't
624// items, or whose metadata can't be read, are skipped — a single bad item must
625// not abort a scan. logLabel names the caller in those skip messages.
626func (s *LibraryService) eachItemDir(baseDir, logLabel string, fn func(itemDir string, meta models.ItemMetadata) bool) error {
627 entries, err := os.ReadDir(baseDir)
628 if err != nil {
629 return err
630 }
631
632 for _, entry := range entries {
633 if !entry.IsDir() || entry.Name() == subtitlesDirName {
634 continue
635 }
636 itemDir := filepath.Join(baseDir, entry.Name())
637 if _, err := os.Stat(filepath.Join(itemDir, itemMarkerName)); err != nil {
638 continue
639 }
640 meta, err := s.readMetadata(itemDir)
641 if err != nil {
642 slog.Warn("skipping item", "op", logLabel, "dir", itemDir, "err", err)
643 continue
644 }
645 if !fn(itemDir, meta) {
646 return nil
647 }
648 }
649
650 return nil
651}
652
653// PruneToIDSet deletes items directly under baseDir whose identity key is not in
654// keep. It is used to mirror a subscription's source: entries removed upstream
655// are removed locally. Items without a known identity key are left untouched (we
656// never delete something we can't positively identify). Returns the number
657// removed.
658func (s *LibraryService) PruneToIDSet(baseDir string, keep map[string]bool) (int, error) {
659 removed := 0
660 err := s.eachItemDir(baseDir, "PruneToIDSet", func(itemDir string, meta models.ItemMetadata) bool {
661 if meta.VideoID == "" || keep[meta.VideoID] {
662 return true
663 }
664 if rel, err := filepath.Rel(s.libraryDir, itemDir); err == nil {
665 s.evictCachedScan(filepath.ToSlash(rel))
666 }
667 if err := os.RemoveAll(itemDir); err != nil {
668 slog.Warn("prune failed to remove item", "dir", itemDir, "err", err)
669 return true
670 }
671 removed++
672 return true
673 })
674
675 return removed, err
676}
677