vidarchive

vidarchive/internal/service/library.go
library.go
⎇
Raw
1package service
2
3import (
4 "encoding/json"
5 "fmt"
6 "log"
7 "os"
8 "path/filepath"
9 "sort"
10 "strconv"
11 "strings"
12 "sync"
13 "time"
14
15 "github.com/BurntSushi/toml"
16
17 "vidarchive/internal/models"
18)
19
20const (
21 itemMarkerName = ".vidarchive-item.toml"
22 subtitlesDirName = "subtitles"
23)
24
25var mediaExts = map[string]struct{}{
26 ".mp4": {}, ".webm": {}, ".mkv": {}, ".avi": {}, ".mov": {},
27 ".mp3": {}, ".wav": {}, ".flac": {}, ".aac": {}, ".opus": {}, ".m4a": {},
28}
29
30var audioExts = map[string]struct{}{
31 ".mp3": {}, ".wav": {}, ".flac": {}, ".aac": {}, ".opus": {}, ".m4a": {},
32}
33
34var imageExts = map[string]struct{}{
35 ".webp": {}, ".jpg": {}, ".jpeg": {}, ".png": {}, ".gif": {}, ".bmp": {},
36}
37
38type LibraryService struct {
39 libraryDir string
40 // ffmpegPath/ffprobePath are the binaries used for thumbnail extraction,
41 // subtitle conversion, and media probing. Configurable so non-PATH installs
42 // (e.g. a pinned build) can be pointed at directly.
43 ffmpegPath string
44 ffprobePath string
45 // thumbLocks holds a per-media-file mutex serializing extraction so two
46 // callers never write the same temp file at once. Entries are deliberately
47 // never pruned: dropping one would reintroduce the race it prevents.
48 thumbLocks sync.Map
49 thumbSem chan struct{}
50
51 // thumbFailed records media filepaths whose extraction already failed this
52 // run, so we trust ffmpeg's verdict and don't re-run it on every request.
53 thumbFailed sync.Map
54
55 // extractAttempts counts ffmpeg extraction runs, so the negative cache can
56 // be verified to actually prevent a re-run.
57 extractAttempts int32
58
59 // scanCache memoizes scanned items for a short TTL so the listing page (which
60 // fans out one thumbnail request per media file) and quick auto-refreshes
61 // don't re-parse each item's marker + info.json on every request. Only item
62 // scans are cached — the directory listing itself is always read fresh, so
63 // newly added/removed items and subfolders appear immediately.
64 scanMu sync.Mutex
65 scanCache map[string]scanCacheEntry
66 scanTTL time.Duration
67}
68
69type scanCacheEntry struct {
70 item *models.LibraryItem
71 at time.Time
72}
73
74// scanCacheTTL is how long a scanned item is reused before being re-read.
75const scanCacheTTL = 10 * time.Second
76
77func NewLibraryService(libraryDir, ffmpegPath, ffprobePath string) *LibraryService {
78 return &LibraryService{
79 libraryDir: libraryDir,
80 ffmpegPath: ffmpegPath,
81 ffprobePath: ffprobePath,
82 thumbSem: make(chan struct{}, maxConcurrentThumbnails),
83 scanCache: make(map[string]scanCacheEntry),
84 scanTTL: scanCacheTTL,
85 }
86}
87
88func (s *LibraryService) getCachedScan(relPath string) (*models.LibraryItem, bool) {
89 if s.scanTTL <= 0 {
90 return nil, false
91 }
92 s.scanMu.Lock()
93 defer s.scanMu.Unlock()
94 e, ok := s.scanCache[relPath]
95 if !ok || time.Since(e.at) > s.scanTTL {
96 return nil, false
97 }
98 return e.item, true
99}
100
101func (s *LibraryService) putCachedScan(relPath string, item *models.LibraryItem) {
102 if s.scanTTL <= 0 {
103 return
104 }
105 s.scanMu.Lock()
106 s.scanCache[relPath] = scanCacheEntry{item: item, at: time.Now()}
107 s.scanMu.Unlock()
108}
109
110func (s *LibraryService) evictCachedScan(relPath string) {
111 s.scanMu.Lock()
112 delete(s.scanCache, relPath)
113 s.scanMu.Unlock()
114}
115
116// scannedItem returns a cached scan if fresh, otherwise scans and caches it.
117func (s *LibraryService) scannedItem(itemDir, relPath string) (*models.LibraryItem, error) {
118 if item, ok := s.getCachedScan(relPath); ok {
119 return item, nil
120 }
121 item, err := s.scanItem(itemDir, relPath)
122 if err != nil {
123 return nil, err
124 }
125 s.putCachedScan(relPath, item)
126 return item, nil
127}
128
129// ResolveWithinLibrary resolves a caller-supplied relative directory against the
130// library root and rejects anything that escapes it. The directory need not
131// exist yet, so it is safe to use when choosing a download's output location.
132func (s *LibraryService) ResolveWithinLibrary(relPath string) (string, error) {
133 return s.resolveItemDir(filepath.Clean(relPath))
134}
135
136// libraryRoot returns the symlink-resolved library root.
137func (s *LibraryService) libraryRoot() string {
138 if base, err := filepath.EvalSymlinks(s.libraryDir); err == nil {
139 return base
140 }
141 return filepath.Clean(s.libraryDir)
142}
143
144func (s *LibraryService) resolveItemDir(relPath string) (string, error) {
145 relPath = strings.Trim(relPath, string(filepath.Separator))
146 base := s.libraryRoot()
147 if relPath == "" || relPath == "." {
148 return base, nil
149 }
150 itemDir := filepath.Join(base, relPath)
151 cleanDir, err := filepath.EvalSymlinks(itemDir)
152 if err != nil {
153 cleanDir = filepath.Clean(itemDir)
154 }
155 if !strings.HasPrefix(cleanDir, base+string(filepath.Separator)) && cleanDir != base {
156 return "", fmt.Errorf("invalid path")
157 }
158 // Return the cleaned/symlink-resolved path we just validated, so callers do
159 // I/O on exactly the path that passed the boundary check.
160 return cleanDir, nil
161}
162
163func (s *LibraryService) GetAll(path, sortBy, filter string) ([]*models.LibraryItem, []string, error) {
164 if path != "" && !strings.HasSuffix(path, "/") {
165 path += "/"
166 }
167
168 dir, err := s.resolveItemDir(path)
169 if err != nil {
170 log.Printf("GetAll: invalid path %q: %v", path, err)
171 return nil, nil, nil
172 }
173 entries, err := os.ReadDir(dir)
174 if err != nil {
175 if os.IsNotExist(err) {
176 return nil, nil, nil
177 }
178 return nil, nil, err
179 }
180
181 var items []*models.LibraryItem
182 var folders []string
183
184 for _, entry := range entries {
185 if !entry.IsDir() {
186 continue
187 }
188 name := entry.Name()
189 if name == subtitlesDirName {
190 continue
191 }
192 itemDir := filepath.Join(dir, name)
193 relPath := filepath.ToSlash(filepath.Join(path, name))
194
195 markerPath := filepath.Join(itemDir, itemMarkerName)
196 if _, err := os.Stat(markerPath); err == nil {
197 item, err := s.scannedItem(itemDir, relPath)
198 if err != nil {
199 log.Printf("warning: failed to scan item %s: %v", relPath, err)
200 continue
201 }
202 if filter != "" && !strings.Contains(strings.ToLower(item.Name), strings.ToLower(filter)) && !strings.Contains(strings.ToLower(item.RelPath), strings.ToLower(filter)) {
203 continue
204 }
205 items = append(items, item)
206 } else {
207 folders = append(folders, name)
208 }
209 }
210
211 switch sortBy {
212 case "title":
213 sort.Slice(items, func(i, j int) bool { return strings.ToLower(items[i].Name) < strings.ToLower(items[j].Name) })
214 case "duration":
215 sort.Slice(items, func(i, j int) bool { return items[i].Duration > items[j].Duration })
216 case "date":
217 fallthrough
218 default:
219 // Stat each item once up front rather than twice per comparison.
220 modTime := make(map[string]int64, len(items))
221 for _, it := range items {
222 if info, err := os.Stat(it.DirPath); err == nil {
223 modTime[it.RelPath] = info.ModTime().UnixNano()
224 }
225 }
226 sort.Slice(items, func(i, j int) bool {
227 return modTime[items[i].RelPath] > modTime[items[j].RelPath]
228 })
229 }
230
231 sort.Strings(folders)
232
233 return items, folders, nil
234}
235
236// GetByRelPath returns the item at relPath, reusing a recent cached scan when
237// available (see scanCache).
238func (s *LibraryService) GetByRelPath(relPath string) (*models.LibraryItem, error) {
239 relPath = strings.Trim(relPath, "/")
240
241 if item, ok := s.getCachedScan(relPath); ok {
242 return item, nil
243 }
244
245 itemDir, err := s.resolveItemDir(relPath)
246 if err != nil {
247 return nil, fmt.Errorf("item not found")
248 }
249 markerPath := filepath.Join(itemDir, itemMarkerName)
250 if _, err := os.Stat(markerPath); err != nil {
251 return nil, fmt.Errorf("item not found")
252 }
253 item, err := s.scanItem(itemDir, relPath)
254 if err != nil {
255 return nil, err
256 }
257 s.putCachedScan(relPath, item)
258 return item, nil
259}
260
261func (s *LibraryService) scanItem(itemDir, relPath string) (*models.LibraryItem, error) {
262 metadata, err := s.readMetadata(itemDir)
263 if err != nil {
264 return nil, err
265 }
266
267 mediaFiles, infoJSONPath, err := s.listItemFiles(itemDir)
268 if err != nil {
269 return nil, err
270 }
271
272 var info map[string]interface{}
273 if infoJSONPath != "" {
274 data, err := os.ReadFile(infoJSONPath)
275 if err == nil {
276 if err := json.Unmarshal(data, &info); err != nil {
277 log.Printf("scanItem: ignoring malformed %s: %v", infoJSONPath, err)
278 }
279 }
280 }
281
282 // dirty tracks whether we derived any new metadata worth persisting, so a
283 // plain listing or detail view doesn't rewrite the marker file on every read.
284 dirty := false
285
286 if metadata.Name == "" {
287 if title, ok := infoString(info, "title"); ok && title != "" {
288 metadata.Name = title
289 } else if len(mediaFiles) > 0 {
290 metadata.Name = mediaFileStem(mediaFiles[0])
291 } else {
292 metadata.Name = filepath.Base(itemDir)
293 }
294 dirty = true
295 }
296
297 if metadata.SourceURL == "" {
298 dirty = backfillString(&metadata.SourceURL, info, "webpage_url", "url") || dirty
299 }
300
301 if metadata.Description == "" {
302 dirty = backfillString(&metadata.Description, info, "description") || dirty
303 }
304
305 // Backfill the stable identity (yt-dlp's video id) from info.json so
306 // pre-existing items gain an identity on their next scan. Subscriptions match
307 // and prune items by this id (see FindByVideoID / PruneToIDSet).
308 if metadata.VideoID == "" {
309 dirty = backfillString(&metadata.VideoID, info, "id") || dirty
310 }
311
312 if metadata.FileDurations == nil {
313 metadata.FileDurations = make(map[string]int)
314 }
315
316 // Per-file durations come from the marker's file_durations map (populated at
317 // import time). For a single-file item we also seed it from info.json's
318 // duration, which covers the common case without a probe. We deliberately do
319 // not run ffprobe here — keeping it off the scan/listing path is the point of
320 // the marker cache. Files without a known duration simply show no badge.
321 for i := range mediaFiles {
322 mf := &mediaFiles[i]
323 if d, ok := metadata.FileDurations[mf.Filename]; ok {
324 mf.Duration = d
325 continue
326 }
327 if len(mediaFiles) == 1 {
328 if d, ok := infoDuration(info); ok && d > 0 {
329 mf.Duration = d
330 metadata.FileDurations[mf.Filename] = d
331 dirty = true
332 }
333 }
334 }
335
336 // The item-level duration is the sum of known per-file durations, used only
337 // for the "duration" sort — there is no single "overall" duration shown.
338 total := 0
339 for _, mf := range mediaFiles {
340 if mf.Duration > 0 {
341 total += mf.Duration
342 }
343 }
344
345 item := &models.LibraryItem{
346 Name: metadata.Name,
347 RelPath: relPath,
348 DirPath: itemDir,
349 SourceURL: metadata.SourceURL,
350 Duration: total,
351 Description: metadata.Description,
352 YtdlpFlags: metadata.YtdlpFlags,
353 MediaFiles: mediaFiles,
354 }
355
356 if dirty {
357 if err := s.writeMetadata(itemDir, metadata); err != nil {
358 log.Printf("scanItem: failed to persist derived metadata for %s: %v", itemDir, err)
359 }
360 }
361
362 return item, nil
363}
364
365// backfillString sets *field from the first non-empty string value among
366// info's keys, reporting whether it changed anything. An empty value in
367// info.json must not count as a change: the marker would be rewritten on every
368// scan forever without ever gaining a value.
369func backfillString(field *string, info map[string]interface{}, keys ...string) bool {
370 for _, k := range keys {
371 if v, ok := infoString(info, k); ok && v != "" {
372 *field = v
373 return true
374 }
375 }
376 return false
377}
378
379func (s *LibraryService) readMetadata(itemDir string) (models.ItemMetadata, error) {
380 markerPath := filepath.Join(itemDir, itemMarkerName)
381 var metadata models.ItemMetadata
382 data, err := os.ReadFile(markerPath)
383 if err == nil {
384 if _, err := toml.Decode(string(data), &metadata); err != nil {
385 log.Printf("warning: failed to parse %s: %v", markerPath, err)
386 }
387 }
388 return metadata, nil
389}
390
391func (s *LibraryService) writeMetadata(itemDir string, metadata models.ItemMetadata) error {
392 markerPath := filepath.Join(itemDir, itemMarkerName)
393 // Write to a uniquely-named temp file and rename so a crash mid-encode can't
394 // leave a truncated marker, and two concurrent writers never collide on a
395 // shared temp path.
396 f, err := os.CreateTemp(itemDir, ".vidarchive-item-*.tmp")
397 if err != nil {
398 return err
399 }
400 tmpPath := f.Name()
401 if err := toml.NewEncoder(f).Encode(metadata); err != nil {
402 f.Close()
403 os.Remove(tmpPath)
404 return err
405 }
406 if err := f.Close(); err != nil {
407 os.Remove(tmpPath)
408 return err
409 }
410 return os.Rename(tmpPath, markerPath)
411}
412
413func (s *LibraryService) listItemFiles(itemDir string) ([]models.MediaFile, string, error) {
414 entries, err := os.ReadDir(itemDir)
415 if err != nil {
416 return nil, "", err
417 }
418
419 var mediaFiles []models.MediaFile
420 var infoJSONFiles []string
421
422 for _, entry := range entries {
423 if entry.IsDir() {
424 continue
425 }
426 name := entry.Name()
427 path := filepath.Join(itemDir, name)
428 ext := strings.ToLower(filepath.Ext(name))
429
430 if name == itemMarkerName {
431 continue
432 }
433 if name == "info.json" || strings.HasSuffix(name, ".info.json") {
434 infoJSONFiles = append(infoJSONFiles, path)
435 continue
436 }
437 if _, ok := imageExts[ext]; ok {
438 continue
439 }
440 if _, ok := mediaExts[ext]; !ok {
441 continue
442 }
443
444 _, isAudio := audioExts[ext]
445
446 mediaFiles = append(mediaFiles, models.MediaFile{
447 Filename: name,
448 Filepath: path,
449 IsAudio: isAudio,
450 Duration: -1,
451 })
452 }
453
454 sort.Slice(mediaFiles, func(i, j int) bool {
455 return mediaFiles[i].Filepath < mediaFiles[j].Filepath
456 })
457
458 var infoJSONPath string
459 if len(infoJSONFiles) > 0 {
460 sort.Strings(infoJSONFiles)
461 infoJSONPath = infoJSONFiles[0]
462 if len(infoJSONFiles) > 1 {
463 log.Printf("warning: multiple info.json files in %s, using %s", itemDir, infoJSONPath)
464 }
465 }
466
467 return mediaFiles, infoJSONPath, nil
468}
469
470func infoString(info map[string]interface{}, key string) (string, bool) {
471 if info == nil {
472 return "", false
473 }
474 // Only accept genuine strings: title/url/description are always strings in
475 // yt-dlp output, and stringifying an arbitrary JSON value (map, slice) would
476 // store junk like "map[...]" into the field.
477 if s, ok := info[key].(string); ok {
478 return s, true
479 }
480 return "", false
481}
482
483func infoDuration(info map[string]interface{}) (int, bool) {
484 if info == nil {
485 return 0, false
486 }
487 v, ok := info["duration"]
488 if !ok {
489 return 0, false
490 }
491 switch n := v.(type) {
492 case float64:
493 return int(n + 0.5), true
494 case string:
495 if f, err := strconv.ParseFloat(n, 64); err == nil {
496 return int(f + 0.5), true
497 }
498 }
499 return 0, false
500}
501
502func mediaFileStem(mf models.MediaFile) string {
503 return strings.TrimSuffix(filepath.Base(mf.Filename), filepath.Ext(mf.Filename))
504}
505
506// primaryMediaFile picks the representative file for an item: the largest video
507// file, or — if there are none — the largest file overall. Returns nil for an
508// item with no media files. Used for the item-level thumbnail and as the default
509// target for metadata, keeping those two consistent.
510func primaryMediaFile(item *models.LibraryItem) *models.MediaFile {
511 size := func(mf *models.MediaFile) int64 {
512 if info, err := os.Stat(mf.Filepath); err == nil {
513 return info.Size()
514 }
515 return 0
516 }
517 var best *models.MediaFile
518 for i := range item.MediaFiles {
519 mf := &item.MediaFiles[i]
520 switch {
521 case best == nil:
522 best = mf
523 case best.IsAudio && !mf.IsAudio:
524 // Prefer any video over audio.
525 best = mf
526 case best.IsAudio == mf.IsAudio && size(mf) > size(best):
527 best = mf
528 }
529 }
530 return best
531}
532
533func (s *LibraryService) GetMediaFile(relPath, filename string) (string, error) {
534 item, err := s.GetByRelPath(relPath)
535 if err != nil {
536 return "", err
537 }
538 for _, mf := range item.MediaFiles {
539 if mf.Filename == filename {
540 return mf.Filepath, nil
541 }
542 }
543 return "", fmt.Errorf("media file not found")
544}
545
546func (s *LibraryService) Delete(relPath string) error {
547 itemDir, err := s.resolveItemDir(relPath)
548 if err != nil {
549 return err
550 }
551 // resolveItemDir maps ""/"." to the library root and resolves symlinks, so a
552 // result equal to the root (reachable via a URL-encoded slash, "sub/..", or a
553 // symlink pointing back at the root) must be refused — deleting it would
554 // wipe the entire library.
555 if itemDir == s.libraryRoot() {
556 return fmt.Errorf("refusing to delete library root")
557 }
558 // Evict the cached scan so the deletion is reflected immediately rather than
559 // lingering until the TTL expires.
560 s.evictCachedScan(strings.Trim(relPath, "/"))
561 return os.RemoveAll(itemDir)
562}
563
564// FindByVideoID returns the absolute directory of the item under baseDir whose
565// marker matches the given yt-dlp video id, scanning only that directory (the
566// subscription's owned folder). Matching on the id alone is safe here because
567// each subscription owns a single source, so ids don't collide across
568// extractors within the folder. ok is false when no match is found or id is
569// empty.
570func (s *LibraryService) FindByVideoID(baseDir, id string) (string, bool) {
571 if id == "" {
572 return "", false
573 }
574
575 var match string
576 // An unreadable base dir simply means no match here.
577 _ = s.eachItemDir(baseDir, "FindByVideoID", func(itemDir string, meta models.ItemMetadata) bool {
578 if meta.VideoID != id {
579 return true
580 }
581 match = itemDir
582 return false
583 })
584
585 return match, match != ""
586}
587
588// eachItemDir walks the marked library items directly under baseDir, reading
589// each one's metadata, and calls fn until it returns false. Entries that aren't
590// items, or whose metadata can't be read, are skipped — a single bad item must
591// not abort a scan. logLabel names the caller in those skip messages.
592func (s *LibraryService) eachItemDir(baseDir, logLabel string, fn func(itemDir string, meta models.ItemMetadata) bool) error {
593 entries, err := os.ReadDir(baseDir)
594 if err != nil {
595 return err
596 }
597
598 for _, entry := range entries {
599 if !entry.IsDir() || entry.Name() == subtitlesDirName {
600 continue
601 }
602 itemDir := filepath.Join(baseDir, entry.Name())
603 if _, err := os.Stat(filepath.Join(itemDir, itemMarkerName)); err != nil {
604 continue
605 }
606 meta, err := s.readMetadata(itemDir)
607 if err != nil {
608 log.Printf("%s: skipping %s: %v", logLabel, itemDir, err)
609 continue
610 }
611 if !fn(itemDir, meta) {
612 return nil
613 }
614 }
615
616 return nil
617}
618
619// PruneToIDSet deletes items directly under baseDir whose identity key is not in
620// keep. It is used to mirror a subscription's source: entries removed upstream
621// are removed locally. Items without a known identity key are left untouched (we
622// never delete something we can't positively identify). Returns the number
623// removed.
624func (s *LibraryService) PruneToIDSet(baseDir string, keep map[string]bool) (int, error) {
625 removed := 0
626 err := s.eachItemDir(baseDir, "PruneToIDSet", func(itemDir string, meta models.ItemMetadata) bool {
627 if meta.VideoID == "" || keep[meta.VideoID] {
628 return true
629 }
630 if rel, err := filepath.Rel(s.libraryDir, itemDir); err == nil {
631 s.evictCachedScan(filepath.ToSlash(rel))
632 }
633 if err := os.RemoveAll(itemDir); err != nil {
634 log.Printf("warning: prune failed to remove %s: %v", itemDir, err)
635 return true
636 }
637 removed++
638 return true
639 })
640
641 return removed, err
642}
643