vidarchive

vidarchive/internal/service/library.go
library.go
⎇
Raw
1package service
2
3import (
4 "bytes"
5 "encoding/json"
6 "fmt"
7 "log/slog"
8 "os"
9 "path/filepath"
10 "sort"
11 "strconv"
12 "strings"
13 "sync"
14 "time"
15
16 "github.com/BurntSushi/toml"
17
18 "vidarchive/internal/models"
19)
20
21const (
22 itemMarkerName = ".vidarchive-item.toml"
23 subtitlesDirName = "subtitles"
24)
25
26var mediaExts = map[string]struct{}{
27 ".mp4": {}, ".webm": {}, ".mkv": {}, ".avi": {}, ".mov": {},
28 ".mp3": {}, ".wav": {}, ".flac": {}, ".aac": {}, ".opus": {}, ".m4a": {},
29}
30
31var audioExts = map[string]struct{}{
32 ".mp3": {}, ".wav": {}, ".flac": {}, ".aac": {}, ".opus": {}, ".m4a": {},
33}
34
35type LibraryService struct {
36 libraryDir string
37 ffmpegPath string
38 ffprobePath string
39 // thumbLocks serializes extraction per media file, so two callers never write
40 // the same temp file at once. Entries are reference counted: evicting a lock
41 // someone still holds would let the next caller take a fresh one and
42 // reintroduce the race.
43 thumbMu sync.Mutex
44 thumbLocks map[string]*refLock
45 thumbSem chan struct{}
46
47 // thumbFailed remembers paths whose extraction already failed this run, so
48 // ffmpeg is not re-run on every request. An evicted path costs one retry.
49 thumbFailed *lru[bool]
50
51 // scanCache memoizes item scans for a short TTL so the listing page does not
52 // re-parse every marker + info.json per request. Only scans are cached: the
53 // directory listing is always read fresh, so added and removed items appear
54 // immediately.
55 scanCache *lru[scanCacheEntry]
56 scanTTL time.Duration
57}
58
59type scanCacheEntry struct {
60 item *models.LibraryItem
61 at time.Time
62}
63
64const scanCacheTTL = 10 * time.Second
65
66// maxCachedPaths bounds each per-path cache. Unbounded, both grow with every
67// distinct file touched over the process lifetime.
68const maxCachedPaths = 1024
69
70func NewLibraryService(libraryDir, ffmpegPath, ffprobePath string) *LibraryService {
71 return &LibraryService{
72 libraryDir: libraryDir,
73 ffmpegPath: ffmpegPath,
74 ffprobePath: ffprobePath,
75 thumbLocks: make(map[string]*refLock),
76 thumbSem: make(chan struct{}, maxConcurrentThumbnails),
77 thumbFailed: newLRU[bool](maxCachedPaths),
78 scanCache: newLRU[scanCacheEntry](maxCachedPaths),
79 scanTTL: scanCacheTTL,
80 }
81}
82
83func (s *LibraryService) getCachedScan(relPath string) (*models.LibraryItem, bool) {
84 if s.scanTTL <= 0 {
85 return nil, false
86 }
87 e, ok := s.scanCache.Get(relPath)
88 if !ok || time.Since(e.at) > s.scanTTL {
89 return nil, false
90 }
91 return e.item, true
92}
93
94func (s *LibraryService) putCachedScan(relPath string, item *models.LibraryItem) {
95 if s.scanTTL <= 0 {
96 return
97 }
98 s.scanCache.Put(relPath, scanCacheEntry{item: item, at: time.Now()})
99}
100
101func (s *LibraryService) evictCachedScan(relPath string) {
102 s.scanCache.Delete(relPath)
103}
104
105// evictCachedDir keys off the symlink-resolved root, because that is what
106// resolveItemDir stored the entry under. Against the raw libraryDir it misses
107// whenever that is a symlink.
108func (s *LibraryService) evictCachedDir(itemDir string) {
109 rel, err := filepath.Rel(s.libraryRoot(), itemDir)
110 if err != nil {
111 return
112 }
113 s.evictCachedScan(filepath.ToSlash(rel))
114}
115
116// refLock is a mutex plus the number of callers holding or waiting for it.
117type refLock struct {
118 mu sync.Mutex
119 refs int
120}
121
122func (s *LibraryService) lockThumbFile(path string) func() {
123 s.thumbMu.Lock()
124 l, ok := s.thumbLocks[path]
125 if !ok {
126 l = &refLock{}
127 s.thumbLocks[path] = l
128 }
129 l.refs++
130 s.thumbMu.Unlock()
131
132 l.mu.Lock()
133
134 return func() {
135 l.mu.Unlock()
136 s.thumbMu.Lock()
137 l.refs--
138 if l.refs == 0 {
139 delete(s.thumbLocks, path)
140 }
141 s.thumbMu.Unlock()
142 }
143}
144
145func (s *LibraryService) scannedItem(itemDir, relPath string) (*models.LibraryItem, error) {
146 if item, ok := s.getCachedScan(relPath); ok {
147 return item, nil
148 }
149 item, err := s.scanItem(itemDir, relPath)
150 if err != nil {
151 return nil, err
152 }
153 s.putCachedScan(relPath, item)
154 return item, nil
155}
156
157// ResolveWithinLibrary rejects anything that escapes the library root. The
158// directory need not exist yet, so it also works for a download's output dir.
159func (s *LibraryService) ResolveWithinLibrary(relPath string) (string, error) {
160 return s.resolveItemDir(filepath.Clean(relPath))
161}
162
163func (s *LibraryService) libraryRoot() string {
164 if base, err := filepath.EvalSymlinks(s.libraryDir); err == nil {
165 return base
166 }
167 return filepath.Clean(s.libraryDir)
168}
169
170func (s *LibraryService) resolveItemDir(relPath string) (string, error) {
171 relPath = strings.Trim(relPath, string(filepath.Separator))
172 base := s.libraryRoot()
173 if relPath == "" || relPath == "." {
174 return base, nil
175 }
176 itemDir := filepath.Join(base, relPath)
177 cleanDir, err := filepath.EvalSymlinks(itemDir)
178 if err != nil {
179 cleanDir = filepath.Clean(itemDir)
180 }
181 if !strings.HasPrefix(cleanDir, base+string(filepath.Separator)) && cleanDir != base {
182 return "", fmt.Errorf("invalid path")
183 }
184 // Return the path that passed the boundary check, so callers do I/O on
185 // exactly that one.
186 return cleanDir, nil
187}
188
189func (s *LibraryService) GetAll(path, sortBy, filter string) ([]*models.LibraryItem, []string, error) {
190 if path != "" && !strings.HasSuffix(path, "/") {
191 path += "/"
192 }
193
194 dir, err := s.resolveItemDir(path)
195 if err != nil {
196 slog.Warn("library listing: invalid path", "path", path, "err", err)
197 return nil, nil, nil
198 }
199 entries, err := os.ReadDir(dir)
200 if err != nil {
201 if os.IsNotExist(err) {
202 return nil, nil, nil
203 }
204 return nil, nil, err
205 }
206
207 var items []*models.LibraryItem
208 var folders []string
209
210 for _, entry := range entries {
211 if !entry.IsDir() {
212 continue
213 }
214 name := entry.Name()
215 if name == subtitlesDirName {
216 continue
217 }
218 itemDir := filepath.Join(dir, name)
219 relPath := filepath.ToSlash(filepath.Join(path, name))
220
221 markerPath := filepath.Join(itemDir, itemMarkerName)
222 if _, err := os.Stat(markerPath); err == nil {
223 item, err := s.scannedItem(itemDir, relPath)
224 if err != nil {
225 slog.Warn("failed to scan item", "path", relPath, "err", err)
226 continue
227 }
228 if filter != "" && !strings.Contains(strings.ToLower(item.Name), strings.ToLower(filter)) && !strings.Contains(strings.ToLower(item.RelPath), strings.ToLower(filter)) {
229 continue
230 }
231 items = append(items, item)
232 } else {
233 folders = append(folders, name)
234 }
235 }
236
237 switch sortBy {
238 case "title":
239 sort.Slice(items, func(i, j int) bool { return strings.ToLower(items[i].Name) < strings.ToLower(items[j].Name) })
240 case "duration":
241 sort.Slice(items, func(i, j int) bool { return items[i].Duration > items[j].Duration })
242 case "date":
243 fallthrough
244 default:
245 // Stat each item once up front rather than twice per comparison.
246 modTime := make(map[string]int64, len(items))
247 for _, it := range items {
248 if info, err := os.Stat(it.DirPath); err == nil {
249 modTime[it.RelPath] = info.ModTime().UnixNano()
250 }
251 }
252 sort.Slice(items, func(i, j int) bool {
253 return modTime[items[i].RelPath] > modTime[items[j].RelPath]
254 })
255 }
256
257 sort.Strings(folders)
258
259 return items, folders, nil
260}
261
262func (s *LibraryService) GetByRelPath(relPath string) (*models.LibraryItem, error) {
263 relPath = strings.Trim(relPath, "/")
264
265 if item, ok := s.getCachedScan(relPath); ok {
266 return item, nil
267 }
268
269 itemDir, err := s.resolveItemDir(relPath)
270 if err != nil {
271 return nil, fmt.Errorf("item not found")
272 }
273 if _, err := os.Stat(filepath.Join(itemDir, itemMarkerName)); err != nil {
274 return nil, fmt.Errorf("item not found")
275 }
276 return s.scannedItem(itemDir, relPath)
277}
278
279func (s *LibraryService) scanItem(itemDir, relPath string) (*models.LibraryItem, error) {
280 metadata, err := s.readMetadata(itemDir)
281 if err != nil {
282 return nil, err
283 }
284
285 mediaFiles, infoJSONPath, err := s.listItemFiles(itemDir)
286 if err != nil {
287 return nil, err
288 }
289
290 var info map[string]interface{}
291 if infoJSONPath != "" {
292 data, err := os.ReadFile(infoJSONPath)
293 if err == nil {
294 if err := json.Unmarshal(data, &info); err != nil {
295 slog.Warn("ignoring malformed info.json", "path", infoJSONPath, "err", err)
296 }
297 }
298 }
299
300 // Only derived metadata makes this dirty, so a plain listing does not rewrite
301 // the marker on every read.
302 dirty := false
303
304 if metadata.Name == "" {
305 if title, ok := infoString(info, "title"); ok && title != "" {
306 metadata.Name = title
307 } else if len(mediaFiles) > 0 {
308 metadata.Name = mediaFileStem(mediaFiles[0])
309 } else {
310 metadata.Name = filepath.Base(itemDir)
311 }
312 dirty = true
313 }
314
315 if metadata.SourceURL == "" {
316 dirty = backfillString(&metadata.SourceURL, info, "webpage_url", "url") || dirty
317 }
318
319 if metadata.Description == "" {
320 dirty = backfillString(&metadata.Description, info, "description") || dirty
321 }
322
323 // Backfilling the video id gives pre-existing items an identity on their next
324 // scan; subscriptions match and prune by it.
325 if metadata.VideoID == "" {
326 dirty = backfillString(&metadata.VideoID, info, "id") || dirty
327 }
328
329 if metadata.FileDurations == nil {
330 metadata.FileDurations = make(map[string]int)
331 }
332
333 // Durations come from the marker (written at import), plus info.json for a
334 // single-file item. No ffprobe here: keeping it off the scan path is the point
335 // of the marker cache. A file without a known duration shows no badge.
336 for i := range mediaFiles {
337 mf := &mediaFiles[i]
338 if d, ok := metadata.FileDurations[mf.Filename]; ok {
339 mf.Duration = d
340 continue
341 }
342 if len(mediaFiles) == 1 {
343 if d, ok := infoDuration(info); ok && d > 0 {
344 mf.Duration = d
345 metadata.FileDurations[mf.Filename] = d
346 dirty = true
347 }
348 }
349 }
350
351 // The item-level duration only feeds the "duration" sort; nothing displays it.
352 total := 0
353 for _, mf := range mediaFiles {
354 if mf.Duration > 0 {
355 total += mf.Duration
356 }
357 }
358
359 item := &models.LibraryItem{
360 Name: metadata.Name,
361 RelPath: relPath,
362 DirPath: itemDir,
363 SourceURL: metadata.SourceURL,
364 Duration: total,
365 Description: metadata.Description,
366 YtdlpFlags: metadata.YtdlpFlags,
367 MediaFiles: mediaFiles,
368 }
369
370 if dirty {
371 if err := s.writeMetadata(itemDir, metadata); err != nil {
372 slog.Warn("failed to persist derived metadata", "dir", itemDir, "err", err)
373 }
374 }
375
376 return item, nil
377}
378
379// backfillString takes the first non-empty value among keys and reports whether
380// it changed anything. An empty value must not count as a change: the marker
381// would be rewritten on every scan forever without ever gaining a value.
382func backfillString(field *string, info map[string]interface{}, keys ...string) bool {
383 for _, k := range keys {
384 if v, ok := infoString(info, k); ok && v != "" {
385 *field = v
386 return true
387 }
388 }
389 return false
390}
391
392func (s *LibraryService) readMetadata(itemDir string) (models.ItemMetadata, error) {
393 markerPath := filepath.Join(itemDir, itemMarkerName)
394 var metadata models.ItemMetadata
395 data, err := os.ReadFile(markerPath)
396 if err == nil {
397 if _, err := toml.Decode(string(data), &metadata); err != nil {
398 slog.Warn("failed to parse item marker", "path", markerPath, "err", err)
399 }
400 }
401 return metadata, nil
402}
403
404func (s *LibraryService) writeMetadata(itemDir string, metadata models.ItemMetadata) error {
405 var buf bytes.Buffer
406 if err := toml.NewEncoder(&buf).Encode(metadata); err != nil {
407 return err
408 }
409 return atomicWrite(filepath.Join(itemDir, itemMarkerName), buf.Bytes(), markerFileMode)
410}
411
412// markerFileMode matches the info.json sidecar: hand-editable by the operator.
413const markerFileMode = 0o644
414
415// atomicWrite writes via a uniquely-named temp file in the same directory, so a
416// crash can't leave a truncated file and two writers can't collide.
417func atomicWrite(path string, data []byte, mode os.FileMode) error {
418 f, err := os.CreateTemp(filepath.Dir(path), ".vidarchive-*.tmp")
419 if err != nil {
420 return err
421 }
422 tmpPath := f.Name()
423 // A no-op once the rename below has succeeded.
424 defer os.Remove(tmpPath)
425
426 if err := f.Chmod(mode); err != nil {
427 f.Close()
428 return err
429 }
430 if _, err := f.Write(data); err != nil {
431 f.Close()
432 return err
433 }
434 // Close explicitly: a deferred close would hide a flush error on the write.
435 if err := f.Close(); err != nil {
436 return err
437 }
438 return os.Rename(tmpPath, path)
439}
440
441func (s *LibraryService) listItemFiles(itemDir string) ([]models.MediaFile, string, error) {
442 entries, err := os.ReadDir(itemDir)
443 if err != nil {
444 return nil, "", err
445 }
446
447 var mediaFiles []models.MediaFile
448 var infoJSONFiles []string
449
450 for _, entry := range entries {
451 if entry.IsDir() {
452 continue
453 }
454 name := entry.Name()
455 path := filepath.Join(itemDir, name)
456 ext := strings.ToLower(filepath.Ext(name))
457
458 if name == itemMarkerName {
459 continue
460 }
461 if name == "info.json" || strings.HasSuffix(name, ".info.json") {
462 infoJSONFiles = append(infoJSONFiles, path)
463 continue
464 }
465 if _, ok := mediaExts[ext]; !ok {
466 continue
467 }
468
469 _, isAudio := audioExts[ext]
470
471 mediaFiles = append(mediaFiles, models.MediaFile{
472 Filename: name,
473 Filepath: path,
474 IsAudio: isAudio,
475 Duration: -1,
476 })
477 }
478
479 sort.Slice(mediaFiles, func(i, j int) bool {
480 return mediaFiles[i].Filepath < mediaFiles[j].Filepath
481 })
482
483 var infoJSONPath string
484 if len(infoJSONFiles) > 0 {
485 sort.Strings(infoJSONFiles)
486 infoJSONPath = infoJSONFiles[0]
487 if len(infoJSONFiles) > 1 {
488 slog.Warn("multiple info.json files", "dir", itemDir, "using", infoJSONPath)
489 }
490 }
491
492 return mediaFiles, infoJSONPath, nil
493}
494
495func infoString(info map[string]interface{}, key string) (string, bool) {
496 if info == nil {
497 return "", false
498 }
499 // Only genuine strings: stringifying a map or slice would store junk like
500 // "map[...]" into the field.
501 if s, ok := info[key].(string); ok {
502 return s, true
503 }
504 return "", false
505}
506
507func infoDuration(info map[string]interface{}) (int, bool) {
508 if info == nil {
509 return 0, false
510 }
511 v, ok := info["duration"]
512 if !ok {
513 return 0, false
514 }
515 switch n := v.(type) {
516 case float64:
517 return int(n + 0.5), true
518 case string:
519 if f, err := strconv.ParseFloat(n, 64); err == nil {
520 return int(f + 0.5), true
521 }
522 }
523 return 0, false
524}
525
526func mediaFileStem(mf models.MediaFile) string {
527 return strings.TrimSuffix(filepath.Base(mf.Filename), filepath.Ext(mf.Filename))
528}
529
530// primaryMediaFile picks the largest video file, or the largest file overall if
531// the item has no video. It backs both the item thumbnail and the default
532// metadata target, which keeps those two consistent.
533func primaryMediaFile(item *models.LibraryItem) *models.MediaFile {
534 size := func(mf *models.MediaFile) int64 {
535 if info, err := os.Stat(mf.Filepath); err == nil {
536 return info.Size()
537 }
538 return 0
539 }
540 var best *models.MediaFile
541 for i := range item.MediaFiles {
542 mf := &item.MediaFiles[i]
543 switch {
544 case best == nil:
545 best = mf
546 case best.IsAudio && !mf.IsAudio:
547 best = mf
548 case best.IsAudio == mf.IsAudio && size(mf) > size(best):
549 best = mf
550 }
551 }
552 return best
553}
554
555func (s *LibraryService) GetMediaFile(relPath, filename string) (string, error) {
556 item, err := s.GetByRelPath(relPath)
557 if err != nil {
558 return "", err
559 }
560 for _, mf := range item.MediaFiles {
561 if mf.Filename == filename {
562 return mf.Filepath, nil
563 }
564 }
565 return "", fmt.Errorf("media file not found")
566}
567
568// Rename changes only the display name in the marker. The directory keeps its
569// name, so existing URLs and subscription matching are unaffected.
570func (s *LibraryService) Rename(relPath, name string) error {
571 name = strings.TrimSpace(name)
572 if name == "" {
573 return fmt.Errorf("name must not be empty")
574 }
575 itemDir, err := s.resolveItemDir(relPath)
576 if err != nil {
577 return err
578 }
579 if _, err := os.Stat(filepath.Join(itemDir, itemMarkerName)); err != nil {
580 return fmt.Errorf("item not found")
581 }
582 metadata, err := s.readMetadata(itemDir)
583 if err != nil {
584 return err
585 }
586 metadata.Name = name
587 if err := s.writeMetadata(itemDir, metadata); err != nil {
588 return err
589 }
590 // Evict so the new name shows immediately instead of at TTL expiry.
591 s.evictCachedScan(strings.Trim(relPath, "/"))
592 return nil
593}
594
595func (s *LibraryService) Delete(relPath string) error {
596 itemDir, err := s.resolveItemDir(relPath)
597 if err != nil {
598 return err
599 }
600 // resolveItemDir maps ""/"." to the root and resolves symlinks, so the root is
601 // reachable via a URL-encoded slash, "sub/..", or a symlink pointing back at
602 // it. Deleting that would wipe the entire library.
603 if itemDir == s.libraryRoot() {
604 return fmt.Errorf("refusing to delete library root")
605 }
606 // Evict so the deletion shows immediately instead of at TTL expiry.
607 s.evictCachedScan(strings.Trim(relPath, "/"))
608 return os.RemoveAll(itemDir)
609}
610
611// FindByVideoID scans only baseDir, the folder one subscription owns. Matching
612// on the yt-dlp video id alone is safe there: a single source can't collide with
613// another extractor's ids.
614func (s *LibraryService) FindByVideoID(baseDir, id string) (string, bool) {
615 if id == "" {
616 return "", false
617 }
618
619 var match string
620 // An unreadable base dir simply means no match here.
621 _ = s.eachItemDir(baseDir, "FindByVideoID", func(itemDir string, meta models.ItemMetadata) bool {
622 if meta.VideoID != id {
623 return true
624 }
625 match = itemDir
626 return false
627 })
628
629 return match, match != ""
630}
631
632// eachItemDir calls fn for each marked item directly under baseDir until fn
633// returns false. Unreadable items are skipped: one bad item must not abort a
634// scan. logLabel names the caller in those skip messages.
635func (s *LibraryService) eachItemDir(baseDir, logLabel string, fn func(itemDir string, meta models.ItemMetadata) bool) error {
636 entries, err := os.ReadDir(baseDir)
637 if err != nil {
638 return err
639 }
640
641 for _, entry := range entries {
642 if !entry.IsDir() || entry.Name() == subtitlesDirName {
643 continue
644 }
645 itemDir := filepath.Join(baseDir, entry.Name())
646 if _, err := os.Stat(filepath.Join(itemDir, itemMarkerName)); err != nil {
647 continue
648 }
649 meta, err := s.readMetadata(itemDir)
650 if err != nil {
651 slog.Warn("skipping item", "op", logLabel, "dir", itemDir, "err", err)
652 continue
653 }
654 if !fn(itemDir, meta) {
655 return nil
656 }
657 }
658
659 return nil
660}
661
662// PruneToIDSet deletes items under baseDir whose identity key is not in keep,
663// mirroring a subscription's source. An item without an identity key is left
664// alone: never delete what can't be positively identified.
665func (s *LibraryService) PruneToIDSet(baseDir string, keep map[string]bool) (int, error) {
666 removed := 0
667 err := s.eachItemDir(baseDir, "PruneToIDSet", func(itemDir string, meta models.ItemMetadata) bool {
668 if meta.VideoID == "" || keep[meta.VideoID] {
669 return true
670 }
671 s.evictCachedDir(itemDir)
672 if err := os.RemoveAll(itemDir); err != nil {
673 slog.Warn("prune failed to remove item", "dir", itemDir, "err", err)
674 return true
675 }
676 removed++
677 return true
678 })
679
680 return removed, err
681}
682