vidarchive

vidarchive/internal/service/library.go
library.go
⎇
Raw
1package service
2
3import (
4 "encoding/json"
5 "fmt"
6 "log"
7 "os"
8 "path/filepath"
9 "sort"
10 "strconv"
11 "strings"
12 "sync"
13 "time"
14
15 "github.com/BurntSushi/toml"
16
17 "vidarchive/internal/models"
18)
19
20const itemMarkerName = ".vidarchive-item.toml"
21const subtitlesDirName = "subtitles"
22
23var mediaExts = map[string]struct{}{
24 ".mp4": {}, ".webm": {}, ".mkv": {}, ".avi": {}, ".mov": {},
25 ".mp3": {}, ".wav": {}, ".flac": {}, ".aac": {}, ".opus": {},
26}
27
28var audioExts = map[string]struct{}{
29 ".mp3": {}, ".wav": {}, ".flac": {}, ".aac": {}, ".opus": {},
30}
31
32var imageExts = map[string]struct{}{
33 ".webp": {}, ".jpg": {}, ".jpeg": {}, ".png": {}, ".gif": {}, ".bmp": {},
34}
35
36type LibraryService struct {
37 libraryDir string
38 // ffmpegPath/ffprobePath are the binaries used for thumbnail extraction,
39 // subtitle conversion, and media probing. Configurable so non-PATH installs
40 // (e.g. a pinned build) can be pointed at directly.
41 ffmpegPath string
42 ffprobePath string
43 // thumbLocks holds a per-media-file mutex serializing extraction so two
44 // callers never write the same temp file at once. Entries are deliberately
45 // never pruned: dropping one would reintroduce the race it prevents.
46 thumbLocks sync.Map
47 thumbSem chan struct{}
48
49 // thumbFailed records media filepaths whose extraction already failed this
50 // run, so we trust ffmpeg's verdict and don't re-run it on every request.
51 thumbFailed sync.Map
52
53 // extraction counters, for observability/tests.
54 extractInFlight int32
55 extractMaxConcurrent int32
56 extractAttempts int32
57
58 // scanCache memoizes scanned items for a short TTL so the listing page (which
59 // fans out one thumbnail request per media file) and quick auto-refreshes
60 // don't re-parse each item's marker + info.json on every request. Only item
61 // scans are cached — the directory listing itself is always read fresh, so
62 // newly added/removed items and subfolders appear immediately.
63 scanMu sync.Mutex
64 scanCache map[string]scanCacheEntry
65 scanTTL time.Duration
66}
67
68type scanCacheEntry struct {
69 item *models.LibraryItem
70 at time.Time
71}
72
73// scanCacheTTL is how long a scanned item is reused before being re-read.
74const scanCacheTTL = 10 * time.Second
75
76func NewLibraryService(libraryDir, ffmpegPath, ffprobePath string) *LibraryService {
77 return &LibraryService{
78 libraryDir: libraryDir,
79 ffmpegPath: ffmpegPath,
80 ffprobePath: ffprobePath,
81 thumbSem: make(chan struct{}, maxConcurrentThumbnails),
82 scanCache: make(map[string]scanCacheEntry),
83 scanTTL: scanCacheTTL,
84 }
85}
86
87func (s *LibraryService) getCachedScan(relPath string) (*models.LibraryItem, bool) {
88 if s.scanTTL <= 0 {
89 return nil, false
90 }
91 s.scanMu.Lock()
92 defer s.scanMu.Unlock()
93 e, ok := s.scanCache[relPath]
94 if !ok || time.Since(e.at) > s.scanTTL {
95 return nil, false
96 }
97 return e.item, true
98}
99
100func (s *LibraryService) putCachedScan(relPath string, item *models.LibraryItem) {
101 if s.scanTTL <= 0 {
102 return
103 }
104 s.scanMu.Lock()
105 s.scanCache[relPath] = scanCacheEntry{item: item, at: time.Now()}
106 s.scanMu.Unlock()
107}
108
109func (s *LibraryService) evictCachedScan(relPath string) {
110 s.scanMu.Lock()
111 delete(s.scanCache, relPath)
112 s.scanMu.Unlock()
113}
114
115// scannedItem returns a cached scan if fresh, otherwise scans and caches it.
116func (s *LibraryService) scannedItem(itemDir, relPath string) (*models.LibraryItem, error) {
117 if item, ok := s.getCachedScan(relPath); ok {
118 return item, nil
119 }
120 item, err := s.scanItem(itemDir, relPath)
121 if err != nil {
122 return nil, err
123 }
124 s.putCachedScan(relPath, item)
125 return item, nil
126}
127
128// ResolveWithinLibrary resolves a caller-supplied relative directory against the
129// library root and rejects anything that escapes it. The directory need not
130// exist yet, so it is safe to use when choosing a download's output location.
131func (s *LibraryService) ResolveWithinLibrary(relPath string) (string, error) {
132 return s.resolveItemDir(filepath.Clean(relPath))
133}
134
135// libraryRoot returns the symlink-resolved library root.
136func (s *LibraryService) libraryRoot() string {
137 if base, err := filepath.EvalSymlinks(s.libraryDir); err == nil {
138 return base
139 }
140 return filepath.Clean(s.libraryDir)
141}
142
143func (s *LibraryService) resolveItemDir(relPath string) (string, error) {
144 relPath = strings.Trim(relPath, string(filepath.Separator))
145 base := s.libraryRoot()
146 if relPath == "" || relPath == "." {
147 return base, nil
148 }
149 itemDir := filepath.Join(base, relPath)
150 cleanDir, err := filepath.EvalSymlinks(itemDir)
151 if err != nil {
152 cleanDir = filepath.Clean(itemDir)
153 }
154 if !strings.HasPrefix(cleanDir, base+string(filepath.Separator)) && cleanDir != base {
155 return "", fmt.Errorf("invalid path")
156 }
157 // Return the cleaned/symlink-resolved path we just validated, so callers do
158 // I/O on exactly the path that passed the boundary check.
159 return cleanDir, nil
160}
161
162func (s *LibraryService) GetAll(path, sortBy, filter string) ([]*models.LibraryItem, []string, error) {
163 if path != "" && !strings.HasSuffix(path, "/") {
164 path += "/"
165 }
166
167 dir, err := s.resolveItemDir(path)
168 if err != nil {
169 log.Printf("GetAll: invalid path %q: %v", path, err)
170 return nil, nil, nil
171 }
172 entries, err := os.ReadDir(dir)
173 if err != nil {
174 if os.IsNotExist(err) {
175 return nil, nil, nil
176 }
177 return nil, nil, err
178 }
179
180 var items []*models.LibraryItem
181 var folders []string
182
183 for _, entry := range entries {
184 if !entry.IsDir() {
185 continue
186 }
187 name := entry.Name()
188 if name == subtitlesDirName {
189 continue
190 }
191 itemDir := filepath.Join(dir, name)
192 relPath := filepath.ToSlash(filepath.Join(path, name))
193
194 markerPath := filepath.Join(itemDir, itemMarkerName)
195 if _, err := os.Stat(markerPath); err == nil {
196 item, err := s.scannedItem(itemDir, relPath)
197 if err != nil {
198 log.Printf("warning: failed to scan item %s: %v", relPath, err)
199 continue
200 }
201 if filter != "" && !strings.Contains(strings.ToLower(item.Name), strings.ToLower(filter)) && !strings.Contains(strings.ToLower(item.RelPath), strings.ToLower(filter)) {
202 continue
203 }
204 items = append(items, item)
205 } else {
206 folders = append(folders, name)
207 }
208 }
209
210 switch sortBy {
211 case "title":
212 sort.Slice(items, func(i, j int) bool { return strings.ToLower(items[i].Name) < strings.ToLower(items[j].Name) })
213 case "duration":
214 sort.Slice(items, func(i, j int) bool { return items[i].Duration > items[j].Duration })
215 case "date":
216 fallthrough
217 default:
218 // Stat each item once up front rather than twice per comparison.
219 modTime := make(map[string]int64, len(items))
220 for _, it := range items {
221 if info, err := os.Stat(it.DirPath); err == nil {
222 modTime[it.RelPath] = info.ModTime().UnixNano()
223 }
224 }
225 sort.Slice(items, func(i, j int) bool {
226 return modTime[items[i].RelPath] > modTime[items[j].RelPath]
227 })
228 }
229
230 sort.Strings(folders)
231
232 return items, folders, nil
233}
234
235// GetByRelPath returns the item at relPath, reusing a recent cached scan when
236// available (see scanCache).
237func (s *LibraryService) GetByRelPath(relPath string) (*models.LibraryItem, error) {
238 relPath = strings.Trim(relPath, "/")
239
240 if item, ok := s.getCachedScan(relPath); ok {
241 return item, nil
242 }
243
244 itemDir, err := s.resolveItemDir(relPath)
245 if err != nil {
246 return nil, fmt.Errorf("item not found")
247 }
248 markerPath := filepath.Join(itemDir, itemMarkerName)
249 if _, err := os.Stat(markerPath); err != nil {
250 return nil, fmt.Errorf("item not found")
251 }
252 item, err := s.scanItem(itemDir, relPath)
253 if err != nil {
254 return nil, err
255 }
256 s.putCachedScan(relPath, item)
257 return item, nil
258}
259
260func (s *LibraryService) scanItem(itemDir, relPath string) (*models.LibraryItem, error) {
261 metadata, err := s.readMetadata(itemDir)
262 if err != nil {
263 return nil, err
264 }
265
266 mediaFiles, infoJSONPath, err := s.listItemFiles(itemDir)
267 if err != nil {
268 return nil, err
269 }
270
271 var info map[string]interface{}
272 if infoJSONPath != "" {
273 data, err := os.ReadFile(infoJSONPath)
274 if err == nil {
275 if err := json.Unmarshal(data, &info); err != nil {
276 log.Printf("scanItem: ignoring malformed %s: %v", infoJSONPath, err)
277 }
278 }
279 }
280
281 // dirty tracks whether we derived any new metadata worth persisting, so a
282 // plain listing or detail view doesn't rewrite the marker file on every read.
283 dirty := false
284
285 if metadata.Name == "" {
286 if title, ok := infoString(info, "title"); ok && title != "" {
287 metadata.Name = title
288 } else if len(mediaFiles) > 0 {
289 metadata.Name = mediaFileStem(mediaFiles[0])
290 } else {
291 metadata.Name = filepath.Base(itemDir)
292 }
293 dirty = true
294 }
295
296 if metadata.SourceURL == "" {
297 dirty = backfillString(&metadata.SourceURL, info, "webpage_url", "url") || dirty
298 }
299
300 if metadata.Description == "" {
301 dirty = backfillString(&metadata.Description, info, "description") || dirty
302 }
303
304 // Backfill the stable identity (yt-dlp's video id) from info.json so
305 // pre-existing items gain an identity on their next scan. Subscriptions match
306 // and prune items by this id (see FindByVideoID / PruneToIDSet).
307 if metadata.VideoID == "" {
308 dirty = backfillString(&metadata.VideoID, info, "id") || dirty
309 }
310
311 if metadata.FileDurations == nil {
312 metadata.FileDurations = make(map[string]int)
313 }
314
315 // Per-file durations come from the marker's file_durations map (populated at
316 // import time). For a single-file item we also seed it from info.json's
317 // duration, which covers the common case without a probe. We deliberately do
318 // not run ffprobe here — keeping it off the scan/listing path is the point of
319 // the marker cache. Files without a known duration simply show no badge.
320 for i := range mediaFiles {
321 mf := &mediaFiles[i]
322 if d, ok := metadata.FileDurations[mf.Filename]; ok {
323 mf.Duration = d
324 continue
325 }
326 if len(mediaFiles) == 1 {
327 if d, ok := infoDuration(info); ok && d > 0 {
328 mf.Duration = d
329 metadata.FileDurations[mf.Filename] = d
330 dirty = true
331 }
332 }
333 }
334
335 // The item-level duration is the sum of known per-file durations, used only
336 // for the "duration" sort — there is no single "overall" duration shown.
337 total := 0
338 for _, mf := range mediaFiles {
339 if mf.Duration > 0 {
340 total += mf.Duration
341 }
342 }
343
344 item := &models.LibraryItem{
345 Name: metadata.Name,
346 RelPath: relPath,
347 DirPath: itemDir,
348 SourceURL: metadata.SourceURL,
349 Duration: total,
350 Description: metadata.Description,
351 YtdlpFlags: metadata.YtdlpFlags,
352 MediaFiles: mediaFiles,
353 }
354
355 if dirty {
356 if err := s.writeMetadata(itemDir, metadata); err != nil {
357 log.Printf("scanItem: failed to persist derived metadata for %s: %v", itemDir, err)
358 }
359 }
360
361 return item, nil
362}
363
364// backfillString sets *field from the first non-empty string value among
365// info's keys, reporting whether it changed anything. An empty value in
366// info.json must not count as a change: the marker would be rewritten on every
367// scan forever without ever gaining a value.
368func backfillString(field *string, info map[string]interface{}, keys ...string) bool {
369 for _, k := range keys {
370 if v, ok := infoString(info, k); ok && v != "" {
371 *field = v
372 return true
373 }
374 }
375 return false
376}
377
378func (s *LibraryService) readMetadata(itemDir string) (models.ItemMetadata, error) {
379 markerPath := filepath.Join(itemDir, itemMarkerName)
380 var metadata models.ItemMetadata
381 data, err := os.ReadFile(markerPath)
382 if err == nil {
383 if _, err := toml.Decode(string(data), &metadata); err != nil {
384 log.Printf("warning: failed to parse %s: %v", markerPath, err)
385 }
386 }
387 return metadata, nil
388}
389
390func (s *LibraryService) writeMetadata(itemDir string, metadata models.ItemMetadata) error {
391 markerPath := filepath.Join(itemDir, itemMarkerName)
392 // Write to a uniquely-named temp file and rename so a crash mid-encode can't
393 // leave a truncated marker, and two concurrent writers never collide on a
394 // shared temp path.
395 f, err := os.CreateTemp(itemDir, ".vidarchive-item-*.tmp")
396 if err != nil {
397 return err
398 }
399 tmpPath := f.Name()
400 if err := toml.NewEncoder(f).Encode(metadata); err != nil {
401 f.Close()
402 os.Remove(tmpPath)
403 return err
404 }
405 if err := f.Close(); err != nil {
406 os.Remove(tmpPath)
407 return err
408 }
409 return os.Rename(tmpPath, markerPath)
410}
411
412func (s *LibraryService) listItemFiles(itemDir string) ([]models.MediaFile, string, error) {
413 entries, err := os.ReadDir(itemDir)
414 if err != nil {
415 return nil, "", err
416 }
417
418 var mediaFiles []models.MediaFile
419 var infoJSONFiles []string
420
421 for _, entry := range entries {
422 if entry.IsDir() {
423 continue
424 }
425 name := entry.Name()
426 path := filepath.Join(itemDir, name)
427 ext := strings.ToLower(filepath.Ext(name))
428
429 if name == itemMarkerName {
430 continue
431 }
432 if name == "info.json" || strings.HasSuffix(name, ".info.json") {
433 infoJSONFiles = append(infoJSONFiles, path)
434 continue
435 }
436 if _, ok := imageExts[ext]; ok {
437 continue
438 }
439 if _, ok := mediaExts[ext]; !ok {
440 continue
441 }
442
443 _, isAudio := audioExts[ext]
444
445 mediaFiles = append(mediaFiles, models.MediaFile{
446 Filename: name,
447 Filepath: path,
448 IsAudio: isAudio,
449 Duration: -1,
450 })
451 }
452
453 sort.Slice(mediaFiles, func(i, j int) bool {
454 return mediaFiles[i].Filepath < mediaFiles[j].Filepath
455 })
456
457 var infoJSONPath string
458 if len(infoJSONFiles) > 0 {
459 sort.Strings(infoJSONFiles)
460 infoJSONPath = infoJSONFiles[0]
461 if len(infoJSONFiles) > 1 {
462 log.Printf("warning: multiple info.json files in %s, using %s", itemDir, infoJSONPath)
463 }
464 }
465
466 return mediaFiles, infoJSONPath, nil
467}
468
469func infoString(info map[string]interface{}, key string) (string, bool) {
470 if info == nil {
471 return "", false
472 }
473 // Only accept genuine strings: title/url/description are always strings in
474 // yt-dlp output, and stringifying an arbitrary JSON value (map, slice) would
475 // store junk like "map[...]" into the field.
476 if s, ok := info[key].(string); ok {
477 return s, true
478 }
479 return "", false
480}
481
482func infoDuration(info map[string]interface{}) (int, bool) {
483 if info == nil {
484 return 0, false
485 }
486 v, ok := info["duration"]
487 if !ok {
488 return 0, false
489 }
490 switch n := v.(type) {
491 case float64:
492 return int(n + 0.5), true
493 case string:
494 if f, err := strconv.ParseFloat(n, 64); err == nil {
495 return int(f + 0.5), true
496 }
497 }
498 return 0, false
499}
500
501func mediaFileStem(mf models.MediaFile) string {
502 return strings.TrimSuffix(filepath.Base(mf.Filename), filepath.Ext(mf.Filename))
503}
504
505// primaryMediaFile picks the representative file for an item: the largest video
506// file, or — if there are none — the largest file overall. Returns nil for an
507// item with no media files. Used for the item-level thumbnail and as the default
508// target for metadata, keeping those two consistent.
509func primaryMediaFile(item *models.LibraryItem) *models.MediaFile {
510 size := func(mf *models.MediaFile) int64 {
511 if info, err := os.Stat(mf.Filepath); err == nil {
512 return info.Size()
513 }
514 return 0
515 }
516 var best *models.MediaFile
517 for i := range item.MediaFiles {
518 mf := &item.MediaFiles[i]
519 switch {
520 case best == nil:
521 best = mf
522 case best.IsAudio && !mf.IsAudio:
523 // Prefer any video over audio.
524 best = mf
525 case best.IsAudio == mf.IsAudio && size(mf) > size(best):
526 best = mf
527 }
528 }
529 return best
530}
531
532func (s *LibraryService) GetMediaFile(relPath, filename string) (string, error) {
533 item, err := s.GetByRelPath(relPath)
534 if err != nil {
535 return "", err
536 }
537 for _, mf := range item.MediaFiles {
538 if mf.Filename == filename {
539 return mf.Filepath, nil
540 }
541 }
542 return "", fmt.Errorf("media file not found")
543}
544
545func (s *LibraryService) Delete(relPath string) error {
546 itemDir, err := s.resolveItemDir(relPath)
547 if err != nil {
548 return err
549 }
550 // resolveItemDir maps ""/"." to the library root and resolves symlinks, so a
551 // result equal to the root (reachable via a URL-encoded slash, "sub/..", or a
552 // symlink pointing back at the root) must be refused — deleting it would
553 // wipe the entire library.
554 if itemDir == s.libraryRoot() {
555 return fmt.Errorf("refusing to delete library root")
556 }
557 // Evict the cached scan so the deletion is reflected immediately rather than
558 // lingering until the TTL expires.
559 s.evictCachedScan(strings.Trim(relPath, "/"))
560 return os.RemoveAll(itemDir)
561}
562
563// FindByVideoID returns the absolute directory of the item under baseDir whose
564// marker matches the given yt-dlp video id, scanning only that directory (the
565// subscription's owned folder). Matching on the id alone is safe here because
566// each subscription owns a single source, so ids don't collide across
567// extractors within the folder. ok is false when no match is found or id is
568// empty.
569func (s *LibraryService) FindByVideoID(baseDir, id string) (string, bool) {
570 if id == "" {
571 return "", false
572 }
573
574 var match string
575 // An unreadable base dir simply means no match here.
576 _ = s.eachItemDir(baseDir, "FindByVideoID", func(itemDir string, meta models.ItemMetadata) bool {
577 if meta.VideoID != id {
578 return true
579 }
580 match = itemDir
581 return false
582 })
583
584 return match, match != ""
585}
586
587// eachItemDir walks the marked library items directly under baseDir, reading
588// each one's metadata, and calls fn until it returns false. Entries that aren't
589// items, or whose metadata can't be read, are skipped — a single bad item must
590// not abort a scan. logLabel names the caller in those skip messages.
591func (s *LibraryService) eachItemDir(baseDir, logLabel string, fn func(itemDir string, meta models.ItemMetadata) bool) error {
592 entries, err := os.ReadDir(baseDir)
593 if err != nil {
594 return err
595 }
596
597 for _, entry := range entries {
598 if !entry.IsDir() || entry.Name() == subtitlesDirName {
599 continue
600 }
601 itemDir := filepath.Join(baseDir, entry.Name())
602 if _, err := os.Stat(filepath.Join(itemDir, itemMarkerName)); err != nil {
603 continue
604 }
605 meta, err := s.readMetadata(itemDir)
606 if err != nil {
607 log.Printf("%s: skipping %s: %v", logLabel, itemDir, err)
608 continue
609 }
610 if !fn(itemDir, meta) {
611 return nil
612 }
613 }
614
615 return nil
616}
617
618// PruneToIDSet deletes items directly under baseDir whose identity key is not in
619// keep. It is used to mirror a subscription's source: entries removed upstream
620// are removed locally. Items without a known identity key are left untouched (we
621// never delete something we can't positively identify). Returns the number
622// removed.
623func (s *LibraryService) PruneToIDSet(baseDir string, keep map[string]bool) (int, error) {
624 removed := 0
625 err := s.eachItemDir(baseDir, "PruneToIDSet", func(itemDir string, meta models.ItemMetadata) bool {
626 if meta.VideoID == "" || keep[meta.VideoID] {
627 return true
628 }
629 if rel, err := filepath.Rel(s.libraryDir, itemDir); err == nil {
630 s.evictCachedScan(filepath.ToSlash(rel))
631 }
632 if err := os.RemoveAll(itemDir); err != nil {
633 log.Printf("warning: prune failed to remove %s: %v", itemDir, err)
634 return true
635 }
636 removed++
637 return true
638 })
639
640 return removed, err
641}
642