vidarchive

vidarchive/internal/service/library.go
library.go
⎇
Raw
1package service
2
3import (
4 "bytes"
5 "encoding/json"
6 "fmt"
7 "log/slog"
8 "os"
9 "path/filepath"
10 "sort"
11 "strconv"
12 "strings"
13 "sync"
14 "time"
15
16 "github.com/BurntSushi/toml"
17
18 "vidarchive/internal/models"
19)
20
21const (
22 itemMarkerName = ".vidarchive-item.toml"
23 subtitlesDirName = "subtitles"
24)
25
26var mediaExts = map[string]struct{}{
27 ".mp4": {}, ".webm": {}, ".mkv": {}, ".avi": {}, ".mov": {},
28 ".mp3": {}, ".wav": {}, ".flac": {}, ".aac": {}, ".opus": {}, ".m4a": {},
29}
30
31var audioExts = map[string]struct{}{
32 ".mp3": {}, ".wav": {}, ".flac": {}, ".aac": {}, ".opus": {}, ".m4a": {},
33}
34
35type LibraryService struct {
36 libraryDir string
37 // ffmpegPath/ffprobePath are the binaries used for thumbnail extraction,
38 // subtitle conversion, and media probing. Configurable so non-PATH installs
39 // (e.g. a pinned build) can be pointed at directly.
40 ffmpegPath string
41 ffprobePath string
42 // thumbLocks holds a per-media-file mutex serializing extraction so two
43 // callers never write the same temp file at once. Entries are reference
44 // counted and removed once nobody holds them. Evicting a lock that is still
45 // held would let a second caller take a fresh one and reintroduce the race.
46 thumbMu sync.Mutex
47 thumbLocks map[string]*refLock
48 thumbSem chan struct{}
49
50 // thumbFailed records media filepaths whose extraction already failed this
51 // run, so we trust ffmpeg's verdict and don't re-run it on every request.
52 // Bounded: an evicted path just means one more ffmpeg attempt.
53 thumbFailed *lru[bool]
54
55 // scanCache memoizes scanned items for a short TTL so the listing page (which
56 // fans out one thumbnail request per media file) and quick auto-refreshes
57 // don't re-parse each item's marker + info.json on every request. Only item
58 // scans are cached — the directory listing itself is always read fresh, so
59 // newly added/removed items and subfolders appear immediately.
60 scanCache *lru[scanCacheEntry]
61 scanTTL time.Duration
62}
63
64type scanCacheEntry struct {
65 item *models.LibraryItem
66 at time.Time
67}
68
69// scanCacheTTL is how long a scanned item is reused before being re-read.
70const scanCacheTTL = 10 * time.Second
71
72// maxCachedPaths bounds each per-path cache. Without it both grow with the
73// number of distinct files touched over the process lifetime, which is fine for
74// a personal archive and not for a large one.
75const maxCachedPaths = 1024
76
77func NewLibraryService(libraryDir, ffmpegPath, ffprobePath string) *LibraryService {
78 return &LibraryService{
79 libraryDir: libraryDir,
80 ffmpegPath: ffmpegPath,
81 ffprobePath: ffprobePath,
82 thumbLocks: make(map[string]*refLock),
83 thumbSem: make(chan struct{}, maxConcurrentThumbnails),
84 thumbFailed: newLRU[bool](maxCachedPaths),
85 scanCache: newLRU[scanCacheEntry](maxCachedPaths),
86 scanTTL: scanCacheTTL,
87 }
88}
89
90func (s *LibraryService) getCachedScan(relPath string) (*models.LibraryItem, bool) {
91 if s.scanTTL <= 0 {
92 return nil, false
93 }
94 e, ok := s.scanCache.Get(relPath)
95 if !ok || time.Since(e.at) > s.scanTTL {
96 return nil, false
97 }
98 return e.item, true
99}
100
101func (s *LibraryService) putCachedScan(relPath string, item *models.LibraryItem) {
102 if s.scanTTL <= 0 {
103 return
104 }
105 s.scanCache.Put(relPath, scanCacheEntry{item: item, at: time.Now()})
106}
107
108func (s *LibraryService) evictCachedScan(relPath string) {
109 s.scanCache.Delete(relPath)
110}
111
112// evictCachedDir drops the cached scan for an absolute item directory.
113//
114// The key must be relative to the symlink-resolved root, because that is what
115// resolveItemDir stored it under. Computing it against the raw libraryDir misses
116// whenever that is a symlink, leaving a deleted or replaced item visible until
117// the TTL expires.
118func (s *LibraryService) evictCachedDir(itemDir string) {
119 rel, err := filepath.Rel(s.libraryRoot(), itemDir)
120 if err != nil {
121 return
122 }
123 s.evictCachedScan(filepath.ToSlash(rel))
124}
125
126// refLock is a mutex plus the number of callers holding or waiting for it.
127type refLock struct {
128 mu sync.Mutex
129 refs int
130}
131
132// lockThumbFile locks the mutex guarding path and returns its release func.
133func (s *LibraryService) lockThumbFile(path string) func() {
134 s.thumbMu.Lock()
135 l, ok := s.thumbLocks[path]
136 if !ok {
137 l = &refLock{}
138 s.thumbLocks[path] = l
139 }
140 l.refs++
141 s.thumbMu.Unlock()
142
143 l.mu.Lock()
144
145 return func() {
146 l.mu.Unlock()
147 s.thumbMu.Lock()
148 l.refs--
149 if l.refs == 0 {
150 delete(s.thumbLocks, path)
151 }
152 s.thumbMu.Unlock()
153 }
154}
155
156// scannedItem returns a cached scan if fresh, otherwise scans and caches it.
157func (s *LibraryService) scannedItem(itemDir, relPath string) (*models.LibraryItem, error) {
158 if item, ok := s.getCachedScan(relPath); ok {
159 return item, nil
160 }
161 item, err := s.scanItem(itemDir, relPath)
162 if err != nil {
163 return nil, err
164 }
165 s.putCachedScan(relPath, item)
166 return item, nil
167}
168
169// ResolveWithinLibrary resolves a caller-supplied relative directory against the
170// library root and rejects anything that escapes it. The directory need not
171// exist yet, so it is safe to use when choosing a download's output location.
172func (s *LibraryService) ResolveWithinLibrary(relPath string) (string, error) {
173 return s.resolveItemDir(filepath.Clean(relPath))
174}
175
176// libraryRoot returns the symlink-resolved library root.
177func (s *LibraryService) libraryRoot() string {
178 if base, err := filepath.EvalSymlinks(s.libraryDir); err == nil {
179 return base
180 }
181 return filepath.Clean(s.libraryDir)
182}
183
184func (s *LibraryService) resolveItemDir(relPath string) (string, error) {
185 relPath = strings.Trim(relPath, string(filepath.Separator))
186 base := s.libraryRoot()
187 if relPath == "" || relPath == "." {
188 return base, nil
189 }
190 itemDir := filepath.Join(base, relPath)
191 cleanDir, err := filepath.EvalSymlinks(itemDir)
192 if err != nil {
193 cleanDir = filepath.Clean(itemDir)
194 }
195 if !strings.HasPrefix(cleanDir, base+string(filepath.Separator)) && cleanDir != base {
196 return "", fmt.Errorf("invalid path")
197 }
198 // Return the cleaned/symlink-resolved path we just validated, so callers do
199 // I/O on exactly the path that passed the boundary check.
200 return cleanDir, nil
201}
202
203func (s *LibraryService) GetAll(path, sortBy, filter string) ([]*models.LibraryItem, []string, error) {
204 if path != "" && !strings.HasSuffix(path, "/") {
205 path += "/"
206 }
207
208 dir, err := s.resolveItemDir(path)
209 if err != nil {
210 slog.Warn("library listing: invalid path", "path", path, "err", err)
211 return nil, nil, nil
212 }
213 entries, err := os.ReadDir(dir)
214 if err != nil {
215 if os.IsNotExist(err) {
216 return nil, nil, nil
217 }
218 return nil, nil, err
219 }
220
221 var items []*models.LibraryItem
222 var folders []string
223
224 for _, entry := range entries {
225 if !entry.IsDir() {
226 continue
227 }
228 name := entry.Name()
229 if name == subtitlesDirName {
230 continue
231 }
232 itemDir := filepath.Join(dir, name)
233 relPath := filepath.ToSlash(filepath.Join(path, name))
234
235 markerPath := filepath.Join(itemDir, itemMarkerName)
236 if _, err := os.Stat(markerPath); err == nil {
237 item, err := s.scannedItem(itemDir, relPath)
238 if err != nil {
239 slog.Warn("failed to scan item", "path", relPath, "err", err)
240 continue
241 }
242 if filter != "" && !strings.Contains(strings.ToLower(item.Name), strings.ToLower(filter)) && !strings.Contains(strings.ToLower(item.RelPath), strings.ToLower(filter)) {
243 continue
244 }
245 items = append(items, item)
246 } else {
247 folders = append(folders, name)
248 }
249 }
250
251 switch sortBy {
252 case "title":
253 sort.Slice(items, func(i, j int) bool { return strings.ToLower(items[i].Name) < strings.ToLower(items[j].Name) })
254 case "duration":
255 sort.Slice(items, func(i, j int) bool { return items[i].Duration > items[j].Duration })
256 case "date":
257 fallthrough
258 default:
259 // Stat each item once up front rather than twice per comparison.
260 modTime := make(map[string]int64, len(items))
261 for _, it := range items {
262 if info, err := os.Stat(it.DirPath); err == nil {
263 modTime[it.RelPath] = info.ModTime().UnixNano()
264 }
265 }
266 sort.Slice(items, func(i, j int) bool {
267 return modTime[items[i].RelPath] > modTime[items[j].RelPath]
268 })
269 }
270
271 sort.Strings(folders)
272
273 return items, folders, nil
274}
275
276// GetByRelPath returns the item at relPath, reusing a recent cached scan when
277// available (see scanCache).
278func (s *LibraryService) GetByRelPath(relPath string) (*models.LibraryItem, error) {
279 relPath = strings.Trim(relPath, "/")
280
281 if item, ok := s.getCachedScan(relPath); ok {
282 return item, nil
283 }
284
285 itemDir, err := s.resolveItemDir(relPath)
286 if err != nil {
287 return nil, fmt.Errorf("item not found")
288 }
289 if _, err := os.Stat(filepath.Join(itemDir, itemMarkerName)); err != nil {
290 return nil, fmt.Errorf("item not found")
291 }
292 return s.scannedItem(itemDir, relPath)
293}
294
295func (s *LibraryService) scanItem(itemDir, relPath string) (*models.LibraryItem, error) {
296 metadata, err := s.readMetadata(itemDir)
297 if err != nil {
298 return nil, err
299 }
300
301 mediaFiles, infoJSONPath, err := s.listItemFiles(itemDir)
302 if err != nil {
303 return nil, err
304 }
305
306 var info map[string]interface{}
307 if infoJSONPath != "" {
308 data, err := os.ReadFile(infoJSONPath)
309 if err == nil {
310 if err := json.Unmarshal(data, &info); err != nil {
311 slog.Warn("ignoring malformed info.json", "path", infoJSONPath, "err", err)
312 }
313 }
314 }
315
316 // dirty tracks whether we derived any new metadata worth persisting, so a
317 // plain listing or detail view doesn't rewrite the marker file on every read.
318 dirty := false
319
320 if metadata.Name == "" {
321 if title, ok := infoString(info, "title"); ok && title != "" {
322 metadata.Name = title
323 } else if len(mediaFiles) > 0 {
324 metadata.Name = mediaFileStem(mediaFiles[0])
325 } else {
326 metadata.Name = filepath.Base(itemDir)
327 }
328 dirty = true
329 }
330
331 if metadata.SourceURL == "" {
332 dirty = backfillString(&metadata.SourceURL, info, "webpage_url", "url") || dirty
333 }
334
335 if metadata.Description == "" {
336 dirty = backfillString(&metadata.Description, info, "description") || dirty
337 }
338
339 // Backfill the stable identity (yt-dlp's video id) from info.json so
340 // pre-existing items gain an identity on their next scan. Subscriptions match
341 // and prune items by this id (see FindByVideoID / PruneToIDSet).
342 if metadata.VideoID == "" {
343 dirty = backfillString(&metadata.VideoID, info, "id") || dirty
344 }
345
346 if metadata.FileDurations == nil {
347 metadata.FileDurations = make(map[string]int)
348 }
349
350 // Per-file durations come from the marker's file_durations map (populated at
351 // import time). For a single-file item we also seed it from info.json's
352 // duration, which covers the common case without a probe. We deliberately do
353 // not run ffprobe here — keeping it off the scan/listing path is the point of
354 // the marker cache. Files without a known duration simply show no badge.
355 for i := range mediaFiles {
356 mf := &mediaFiles[i]
357 if d, ok := metadata.FileDurations[mf.Filename]; ok {
358 mf.Duration = d
359 continue
360 }
361 if len(mediaFiles) == 1 {
362 if d, ok := infoDuration(info); ok && d > 0 {
363 mf.Duration = d
364 metadata.FileDurations[mf.Filename] = d
365 dirty = true
366 }
367 }
368 }
369
370 // The item-level duration is the sum of known per-file durations, used only
371 // for the "duration" sort — there is no single "overall" duration shown.
372 total := 0
373 for _, mf := range mediaFiles {
374 if mf.Duration > 0 {
375 total += mf.Duration
376 }
377 }
378
379 item := &models.LibraryItem{
380 Name: metadata.Name,
381 RelPath: relPath,
382 DirPath: itemDir,
383 SourceURL: metadata.SourceURL,
384 Duration: total,
385 Description: metadata.Description,
386 YtdlpFlags: metadata.YtdlpFlags,
387 MediaFiles: mediaFiles,
388 }
389
390 if dirty {
391 if err := s.writeMetadata(itemDir, metadata); err != nil {
392 slog.Warn("failed to persist derived metadata", "dir", itemDir, "err", err)
393 }
394 }
395
396 return item, nil
397}
398
399// backfillString sets *field from the first non-empty string value among
400// info's keys, reporting whether it changed anything. An empty value in
401// info.json must not count as a change: the marker would be rewritten on every
402// scan forever without ever gaining a value.
403func backfillString(field *string, info map[string]interface{}, keys ...string) bool {
404 for _, k := range keys {
405 if v, ok := infoString(info, k); ok && v != "" {
406 *field = v
407 return true
408 }
409 }
410 return false
411}
412
413func (s *LibraryService) readMetadata(itemDir string) (models.ItemMetadata, error) {
414 markerPath := filepath.Join(itemDir, itemMarkerName)
415 var metadata models.ItemMetadata
416 data, err := os.ReadFile(markerPath)
417 if err == nil {
418 if _, err := toml.Decode(string(data), &metadata); err != nil {
419 slog.Warn("failed to parse item marker", "path", markerPath, "err", err)
420 }
421 }
422 return metadata, nil
423}
424
425func (s *LibraryService) writeMetadata(itemDir string, metadata models.ItemMetadata) error {
426 var buf bytes.Buffer
427 if err := toml.NewEncoder(&buf).Encode(metadata); err != nil {
428 return err
429 }
430 return atomicWrite(filepath.Join(itemDir, itemMarkerName), buf.Bytes(), markerFileMode)
431}
432
433// markerFileMode matches the info.json sidecar: both sit in the library next to
434// the media and are meant to be readable (and hand-editable) by the operator.
435const markerFileMode = 0o644
436
437// atomicWrite writes data to path via a uniquely-named temp file in the same
438// directory and a rename, so a crash mid-write can't leave a truncated file and
439// two concurrent writers never collide on a shared temp path.
440func atomicWrite(path string, data []byte, mode os.FileMode) error {
441 f, err := os.CreateTemp(filepath.Dir(path), ".vidarchive-*.tmp")
442 if err != nil {
443 return err
444 }
445 tmpPath := f.Name()
446 // A no-op once the rename below has succeeded.
447 defer os.Remove(tmpPath)
448
449 if err := f.Chmod(mode); err != nil {
450 f.Close()
451 return err
452 }
453 if _, err := f.Write(data); err != nil {
454 f.Close()
455 return err
456 }
457 // Close explicitly: a deferred close would hide a flush error on the write.
458 if err := f.Close(); err != nil {
459 return err
460 }
461 return os.Rename(tmpPath, path)
462}
463
464func (s *LibraryService) listItemFiles(itemDir string) ([]models.MediaFile, string, error) {
465 entries, err := os.ReadDir(itemDir)
466 if err != nil {
467 return nil, "", err
468 }
469
470 var mediaFiles []models.MediaFile
471 var infoJSONFiles []string
472
473 for _, entry := range entries {
474 if entry.IsDir() {
475 continue
476 }
477 name := entry.Name()
478 path := filepath.Join(itemDir, name)
479 ext := strings.ToLower(filepath.Ext(name))
480
481 if name == itemMarkerName {
482 continue
483 }
484 if name == "info.json" || strings.HasSuffix(name, ".info.json") {
485 infoJSONFiles = append(infoJSONFiles, path)
486 continue
487 }
488 if _, ok := mediaExts[ext]; !ok {
489 continue
490 }
491
492 _, isAudio := audioExts[ext]
493
494 mediaFiles = append(mediaFiles, models.MediaFile{
495 Filename: name,
496 Filepath: path,
497 IsAudio: isAudio,
498 Duration: -1,
499 })
500 }
501
502 sort.Slice(mediaFiles, func(i, j int) bool {
503 return mediaFiles[i].Filepath < mediaFiles[j].Filepath
504 })
505
506 var infoJSONPath string
507 if len(infoJSONFiles) > 0 {
508 sort.Strings(infoJSONFiles)
509 infoJSONPath = infoJSONFiles[0]
510 if len(infoJSONFiles) > 1 {
511 slog.Warn("multiple info.json files", "dir", itemDir, "using", infoJSONPath)
512 }
513 }
514
515 return mediaFiles, infoJSONPath, nil
516}
517
518func infoString(info map[string]interface{}, key string) (string, bool) {
519 if info == nil {
520 return "", false
521 }
522 // Only accept genuine strings: title/url/description are always strings in
523 // yt-dlp output, and stringifying an arbitrary JSON value (map, slice) would
524 // store junk like "map[...]" into the field.
525 if s, ok := info[key].(string); ok {
526 return s, true
527 }
528 return "", false
529}
530
531func infoDuration(info map[string]interface{}) (int, bool) {
532 if info == nil {
533 return 0, false
534 }
535 v, ok := info["duration"]
536 if !ok {
537 return 0, false
538 }
539 switch n := v.(type) {
540 case float64:
541 return int(n + 0.5), true
542 case string:
543 if f, err := strconv.ParseFloat(n, 64); err == nil {
544 return int(f + 0.5), true
545 }
546 }
547 return 0, false
548}
549
550func mediaFileStem(mf models.MediaFile) string {
551 return strings.TrimSuffix(filepath.Base(mf.Filename), filepath.Ext(mf.Filename))
552}
553
554// primaryMediaFile picks the representative file for an item: the largest video
555// file, or — if there are none — the largest file overall. Returns nil for an
556// item with no media files. Used for the item-level thumbnail and as the default
557// target for metadata, keeping those two consistent.
558func primaryMediaFile(item *models.LibraryItem) *models.MediaFile {
559 size := func(mf *models.MediaFile) int64 {
560 if info, err := os.Stat(mf.Filepath); err == nil {
561 return info.Size()
562 }
563 return 0
564 }
565 var best *models.MediaFile
566 for i := range item.MediaFiles {
567 mf := &item.MediaFiles[i]
568 switch {
569 case best == nil:
570 best = mf
571 case best.IsAudio && !mf.IsAudio:
572 // Prefer any video over audio.
573 best = mf
574 case best.IsAudio == mf.IsAudio && size(mf) > size(best):
575 best = mf
576 }
577 }
578 return best
579}
580
581func (s *LibraryService) GetMediaFile(relPath, filename string) (string, error) {
582 item, err := s.GetByRelPath(relPath)
583 if err != nil {
584 return "", err
585 }
586 for _, mf := range item.MediaFiles {
587 if mf.Filename == filename {
588 return mf.Filepath, nil
589 }
590 }
591 return "", fmt.Errorf("media file not found")
592}
593
594func (s *LibraryService) Delete(relPath string) error {
595 itemDir, err := s.resolveItemDir(relPath)
596 if err != nil {
597 return err
598 }
599 // resolveItemDir maps ""/"." to the library root and resolves symlinks, so a
600 // result equal to the root (reachable via a URL-encoded slash, "sub/..", or a
601 // symlink pointing back at the root) must be refused — deleting it would
602 // wipe the entire library.
603 if itemDir == s.libraryRoot() {
604 return fmt.Errorf("refusing to delete library root")
605 }
606 // Evict the cached scan so the deletion is reflected immediately rather than
607 // lingering until the TTL expires.
608 s.evictCachedScan(strings.Trim(relPath, "/"))
609 return os.RemoveAll(itemDir)
610}
611
612// FindByVideoID returns the absolute directory of the item under baseDir whose
613// marker matches the given yt-dlp video id, scanning only that directory (the
614// subscription's owned folder). Matching on the id alone is safe here because
615// each subscription owns a single source, so ids don't collide across
616// extractors within the folder. ok is false when no match is found or id is
617// empty.
618func (s *LibraryService) FindByVideoID(baseDir, id string) (string, bool) {
619 if id == "" {
620 return "", false
621 }
622
623 var match string
624 // An unreadable base dir simply means no match here.
625 _ = s.eachItemDir(baseDir, "FindByVideoID", func(itemDir string, meta models.ItemMetadata) bool {
626 if meta.VideoID != id {
627 return true
628 }
629 match = itemDir
630 return false
631 })
632
633 return match, match != ""
634}
635
636// eachItemDir walks the marked library items directly under baseDir, reading
637// each one's metadata, and calls fn until it returns false. Entries that aren't
638// items, or whose metadata can't be read, are skipped — a single bad item must
639// not abort a scan. logLabel names the caller in those skip messages.
640func (s *LibraryService) eachItemDir(baseDir, logLabel string, fn func(itemDir string, meta models.ItemMetadata) bool) error {
641 entries, err := os.ReadDir(baseDir)
642 if err != nil {
643 return err
644 }
645
646 for _, entry := range entries {
647 if !entry.IsDir() || entry.Name() == subtitlesDirName {
648 continue
649 }
650 itemDir := filepath.Join(baseDir, entry.Name())
651 if _, err := os.Stat(filepath.Join(itemDir, itemMarkerName)); err != nil {
652 continue
653 }
654 meta, err := s.readMetadata(itemDir)
655 if err != nil {
656 slog.Warn("skipping item", "op", logLabel, "dir", itemDir, "err", err)
657 continue
658 }
659 if !fn(itemDir, meta) {
660 return nil
661 }
662 }
663
664 return nil
665}
666
667// PruneToIDSet deletes items directly under baseDir whose identity key is not in
668// keep. It is used to mirror a subscription's source: entries removed upstream
669// are removed locally. Items without a known identity key are left untouched (we
670// never delete something we can't positively identify). Returns the number
671// removed.
672func (s *LibraryService) PruneToIDSet(baseDir string, keep map[string]bool) (int, error) {
673 removed := 0
674 err := s.eachItemDir(baseDir, "PruneToIDSet", func(itemDir string, meta models.ItemMetadata) bool {
675 if meta.VideoID == "" || keep[meta.VideoID] {
676 return true
677 }
678 s.evictCachedDir(itemDir)
679 if err := os.RemoveAll(itemDir); err != nil {
680 slog.Warn("prune failed to remove item", "dir", itemDir, "err", err)
681 return true
682 }
683 removed++
684 return true
685 })
686
687 return removed, err
688}
689