vidarchive

vidarchive/internal/service/library.go
library.go
⎇
Raw
1package service
2
3import (
4 "encoding/json"
5 "fmt"
6 "log"
7 "os"
8 "path/filepath"
9 "sort"
10 "strconv"
11 "strings"
12 "sync"
13 "time"
14
15 "github.com/BurntSushi/toml"
16
17 "vidarchive/internal/models"
18)
19
20const itemMarkerName = ".vidarchive-item.toml"
21const subtitlesDirName = "subtitles"
22
23var mediaExts = map[string]struct{}{
24 ".mp4": {}, ".webm": {}, ".mkv": {}, ".avi": {}, ".mov": {},
25 ".mp3": {}, ".wav": {}, ".flac": {}, ".aac": {}, ".opus": {}, ".m4a": {},
26}
27
28var audioExts = map[string]struct{}{
29 ".mp3": {}, ".wav": {}, ".flac": {}, ".aac": {}, ".opus": {}, ".m4a": {},
30}
31
32var imageExts = map[string]struct{}{
33 ".webp": {}, ".jpg": {}, ".jpeg": {}, ".png": {}, ".gif": {}, ".bmp": {},
34}
35
36type LibraryService struct {
37 libraryDir string
38 // ffmpegPath/ffprobePath are the binaries used for thumbnail extraction,
39 // subtitle conversion, and media probing. Configurable so non-PATH installs
40 // (e.g. a pinned build) can be pointed at directly.
41 ffmpegPath string
42 ffprobePath string
43 // thumbLocks holds a per-media-file mutex serializing extraction so two
44 // callers never write the same temp file at once. Entries are deliberately
45 // never pruned: dropping one would reintroduce the race it prevents.
46 thumbLocks sync.Map
47 thumbSem chan struct{}
48
49 // thumbFailed records media filepaths whose extraction already failed this
50 // run, so we trust ffmpeg's verdict and don't re-run it on every request.
51 thumbFailed sync.Map
52
53 // extractAttempts counts ffmpeg extraction runs, so the negative cache can
54 // be verified to actually prevent a re-run.
55 extractAttempts int32
56
57 // scanCache memoizes scanned items for a short TTL so the listing page (which
58 // fans out one thumbnail request per media file) and quick auto-refreshes
59 // don't re-parse each item's marker + info.json on every request. Only item
60 // scans are cached — the directory listing itself is always read fresh, so
61 // newly added/removed items and subfolders appear immediately.
62 scanMu sync.Mutex
63 scanCache map[string]scanCacheEntry
64 scanTTL time.Duration
65}
66
67type scanCacheEntry struct {
68 item *models.LibraryItem
69 at time.Time
70}
71
72// scanCacheTTL is how long a scanned item is reused before being re-read.
73const scanCacheTTL = 10 * time.Second
74
75func NewLibraryService(libraryDir, ffmpegPath, ffprobePath string) *LibraryService {
76 return &LibraryService{
77 libraryDir: libraryDir,
78 ffmpegPath: ffmpegPath,
79 ffprobePath: ffprobePath,
80 thumbSem: make(chan struct{}, maxConcurrentThumbnails),
81 scanCache: make(map[string]scanCacheEntry),
82 scanTTL: scanCacheTTL,
83 }
84}
85
86func (s *LibraryService) getCachedScan(relPath string) (*models.LibraryItem, bool) {
87 if s.scanTTL <= 0 {
88 return nil, false
89 }
90 s.scanMu.Lock()
91 defer s.scanMu.Unlock()
92 e, ok := s.scanCache[relPath]
93 if !ok || time.Since(e.at) > s.scanTTL {
94 return nil, false
95 }
96 return e.item, true
97}
98
99func (s *LibraryService) putCachedScan(relPath string, item *models.LibraryItem) {
100 if s.scanTTL <= 0 {
101 return
102 }
103 s.scanMu.Lock()
104 s.scanCache[relPath] = scanCacheEntry{item: item, at: time.Now()}
105 s.scanMu.Unlock()
106}
107
108func (s *LibraryService) evictCachedScan(relPath string) {
109 s.scanMu.Lock()
110 delete(s.scanCache, relPath)
111 s.scanMu.Unlock()
112}
113
114// scannedItem returns a cached scan if fresh, otherwise scans and caches it.
115func (s *LibraryService) scannedItem(itemDir, relPath string) (*models.LibraryItem, error) {
116 if item, ok := s.getCachedScan(relPath); ok {
117 return item, nil
118 }
119 item, err := s.scanItem(itemDir, relPath)
120 if err != nil {
121 return nil, err
122 }
123 s.putCachedScan(relPath, item)
124 return item, nil
125}
126
127// ResolveWithinLibrary resolves a caller-supplied relative directory against the
128// library root and rejects anything that escapes it. The directory need not
129// exist yet, so it is safe to use when choosing a download's output location.
130func (s *LibraryService) ResolveWithinLibrary(relPath string) (string, error) {
131 return s.resolveItemDir(filepath.Clean(relPath))
132}
133
134// libraryRoot returns the symlink-resolved library root.
135func (s *LibraryService) libraryRoot() string {
136 if base, err := filepath.EvalSymlinks(s.libraryDir); err == nil {
137 return base
138 }
139 return filepath.Clean(s.libraryDir)
140}
141
142func (s *LibraryService) resolveItemDir(relPath string) (string, error) {
143 relPath = strings.Trim(relPath, string(filepath.Separator))
144 base := s.libraryRoot()
145 if relPath == "" || relPath == "." {
146 return base, nil
147 }
148 itemDir := filepath.Join(base, relPath)
149 cleanDir, err := filepath.EvalSymlinks(itemDir)
150 if err != nil {
151 cleanDir = filepath.Clean(itemDir)
152 }
153 if !strings.HasPrefix(cleanDir, base+string(filepath.Separator)) && cleanDir != base {
154 return "", fmt.Errorf("invalid path")
155 }
156 // Return the cleaned/symlink-resolved path we just validated, so callers do
157 // I/O on exactly the path that passed the boundary check.
158 return cleanDir, nil
159}
160
161func (s *LibraryService) GetAll(path, sortBy, filter string) ([]*models.LibraryItem, []string, error) {
162 if path != "" && !strings.HasSuffix(path, "/") {
163 path += "/"
164 }
165
166 dir, err := s.resolveItemDir(path)
167 if err != nil {
168 log.Printf("GetAll: invalid path %q: %v", path, err)
169 return nil, nil, nil
170 }
171 entries, err := os.ReadDir(dir)
172 if err != nil {
173 if os.IsNotExist(err) {
174 return nil, nil, nil
175 }
176 return nil, nil, err
177 }
178
179 var items []*models.LibraryItem
180 var folders []string
181
182 for _, entry := range entries {
183 if !entry.IsDir() {
184 continue
185 }
186 name := entry.Name()
187 if name == subtitlesDirName {
188 continue
189 }
190 itemDir := filepath.Join(dir, name)
191 relPath := filepath.ToSlash(filepath.Join(path, name))
192
193 markerPath := filepath.Join(itemDir, itemMarkerName)
194 if _, err := os.Stat(markerPath); err == nil {
195 item, err := s.scannedItem(itemDir, relPath)
196 if err != nil {
197 log.Printf("warning: failed to scan item %s: %v", relPath, err)
198 continue
199 }
200 if filter != "" && !strings.Contains(strings.ToLower(item.Name), strings.ToLower(filter)) && !strings.Contains(strings.ToLower(item.RelPath), strings.ToLower(filter)) {
201 continue
202 }
203 items = append(items, item)
204 } else {
205 folders = append(folders, name)
206 }
207 }
208
209 switch sortBy {
210 case "title":
211 sort.Slice(items, func(i, j int) bool { return strings.ToLower(items[i].Name) < strings.ToLower(items[j].Name) })
212 case "duration":
213 sort.Slice(items, func(i, j int) bool { return items[i].Duration > items[j].Duration })
214 case "date":
215 fallthrough
216 default:
217 // Stat each item once up front rather than twice per comparison.
218 modTime := make(map[string]int64, len(items))
219 for _, it := range items {
220 if info, err := os.Stat(it.DirPath); err == nil {
221 modTime[it.RelPath] = info.ModTime().UnixNano()
222 }
223 }
224 sort.Slice(items, func(i, j int) bool {
225 return modTime[items[i].RelPath] > modTime[items[j].RelPath]
226 })
227 }
228
229 sort.Strings(folders)
230
231 return items, folders, nil
232}
233
234// GetByRelPath returns the item at relPath, reusing a recent cached scan when
235// available (see scanCache).
236func (s *LibraryService) GetByRelPath(relPath string) (*models.LibraryItem, error) {
237 relPath = strings.Trim(relPath, "/")
238
239 if item, ok := s.getCachedScan(relPath); ok {
240 return item, nil
241 }
242
243 itemDir, err := s.resolveItemDir(relPath)
244 if err != nil {
245 return nil, fmt.Errorf("item not found")
246 }
247 markerPath := filepath.Join(itemDir, itemMarkerName)
248 if _, err := os.Stat(markerPath); err != nil {
249 return nil, fmt.Errorf("item not found")
250 }
251 item, err := s.scanItem(itemDir, relPath)
252 if err != nil {
253 return nil, err
254 }
255 s.putCachedScan(relPath, item)
256 return item, nil
257}
258
259func (s *LibraryService) scanItem(itemDir, relPath string) (*models.LibraryItem, error) {
260 metadata, err := s.readMetadata(itemDir)
261 if err != nil {
262 return nil, err
263 }
264
265 mediaFiles, infoJSONPath, err := s.listItemFiles(itemDir)
266 if err != nil {
267 return nil, err
268 }
269
270 var info map[string]interface{}
271 if infoJSONPath != "" {
272 data, err := os.ReadFile(infoJSONPath)
273 if err == nil {
274 if err := json.Unmarshal(data, &info); err != nil {
275 log.Printf("scanItem: ignoring malformed %s: %v", infoJSONPath, err)
276 }
277 }
278 }
279
280 // dirty tracks whether we derived any new metadata worth persisting, so a
281 // plain listing or detail view doesn't rewrite the marker file on every read.
282 dirty := false
283
284 if metadata.Name == "" {
285 if title, ok := infoString(info, "title"); ok && title != "" {
286 metadata.Name = title
287 } else if len(mediaFiles) > 0 {
288 metadata.Name = mediaFileStem(mediaFiles[0])
289 } else {
290 metadata.Name = filepath.Base(itemDir)
291 }
292 dirty = true
293 }
294
295 if metadata.SourceURL == "" {
296 dirty = backfillString(&metadata.SourceURL, info, "webpage_url", "url") || dirty
297 }
298
299 if metadata.Description == "" {
300 dirty = backfillString(&metadata.Description, info, "description") || dirty
301 }
302
303 // Backfill the stable identity (yt-dlp's video id) from info.json so
304 // pre-existing items gain an identity on their next scan. Subscriptions match
305 // and prune items by this id (see FindByVideoID / PruneToIDSet).
306 if metadata.VideoID == "" {
307 dirty = backfillString(&metadata.VideoID, info, "id") || dirty
308 }
309
310 if metadata.FileDurations == nil {
311 metadata.FileDurations = make(map[string]int)
312 }
313
314 // Per-file durations come from the marker's file_durations map (populated at
315 // import time). For a single-file item we also seed it from info.json's
316 // duration, which covers the common case without a probe. We deliberately do
317 // not run ffprobe here — keeping it off the scan/listing path is the point of
318 // the marker cache. Files without a known duration simply show no badge.
319 for i := range mediaFiles {
320 mf := &mediaFiles[i]
321 if d, ok := metadata.FileDurations[mf.Filename]; ok {
322 mf.Duration = d
323 continue
324 }
325 if len(mediaFiles) == 1 {
326 if d, ok := infoDuration(info); ok && d > 0 {
327 mf.Duration = d
328 metadata.FileDurations[mf.Filename] = d
329 dirty = true
330 }
331 }
332 }
333
334 // The item-level duration is the sum of known per-file durations, used only
335 // for the "duration" sort — there is no single "overall" duration shown.
336 total := 0
337 for _, mf := range mediaFiles {
338 if mf.Duration > 0 {
339 total += mf.Duration
340 }
341 }
342
343 item := &models.LibraryItem{
344 Name: metadata.Name,
345 RelPath: relPath,
346 DirPath: itemDir,
347 SourceURL: metadata.SourceURL,
348 Duration: total,
349 Description: metadata.Description,
350 YtdlpFlags: metadata.YtdlpFlags,
351 MediaFiles: mediaFiles,
352 }
353
354 if dirty {
355 if err := s.writeMetadata(itemDir, metadata); err != nil {
356 log.Printf("scanItem: failed to persist derived metadata for %s: %v", itemDir, err)
357 }
358 }
359
360 return item, nil
361}
362
363// backfillString sets *field from the first non-empty string value among
364// info's keys, reporting whether it changed anything. An empty value in
365// info.json must not count as a change: the marker would be rewritten on every
366// scan forever without ever gaining a value.
367func backfillString(field *string, info map[string]interface{}, keys ...string) bool {
368 for _, k := range keys {
369 if v, ok := infoString(info, k); ok && v != "" {
370 *field = v
371 return true
372 }
373 }
374 return false
375}
376
377func (s *LibraryService) readMetadata(itemDir string) (models.ItemMetadata, error) {
378 markerPath := filepath.Join(itemDir, itemMarkerName)
379 var metadata models.ItemMetadata
380 data, err := os.ReadFile(markerPath)
381 if err == nil {
382 if _, err := toml.Decode(string(data), &metadata); err != nil {
383 log.Printf("warning: failed to parse %s: %v", markerPath, err)
384 }
385 }
386 return metadata, nil
387}
388
389func (s *LibraryService) writeMetadata(itemDir string, metadata models.ItemMetadata) error {
390 markerPath := filepath.Join(itemDir, itemMarkerName)
391 // Write to a uniquely-named temp file and rename so a crash mid-encode can't
392 // leave a truncated marker, and two concurrent writers never collide on a
393 // shared temp path.
394 f, err := os.CreateTemp(itemDir, ".vidarchive-item-*.tmp")
395 if err != nil {
396 return err
397 }
398 tmpPath := f.Name()
399 if err := toml.NewEncoder(f).Encode(metadata); err != nil {
400 f.Close()
401 os.Remove(tmpPath)
402 return err
403 }
404 if err := f.Close(); err != nil {
405 os.Remove(tmpPath)
406 return err
407 }
408 return os.Rename(tmpPath, markerPath)
409}
410
411func (s *LibraryService) listItemFiles(itemDir string) ([]models.MediaFile, string, error) {
412 entries, err := os.ReadDir(itemDir)
413 if err != nil {
414 return nil, "", err
415 }
416
417 var mediaFiles []models.MediaFile
418 var infoJSONFiles []string
419
420 for _, entry := range entries {
421 if entry.IsDir() {
422 continue
423 }
424 name := entry.Name()
425 path := filepath.Join(itemDir, name)
426 ext := strings.ToLower(filepath.Ext(name))
427
428 if name == itemMarkerName {
429 continue
430 }
431 if name == "info.json" || strings.HasSuffix(name, ".info.json") {
432 infoJSONFiles = append(infoJSONFiles, path)
433 continue
434 }
435 if _, ok := imageExts[ext]; ok {
436 continue
437 }
438 if _, ok := mediaExts[ext]; !ok {
439 continue
440 }
441
442 _, isAudio := audioExts[ext]
443
444 mediaFiles = append(mediaFiles, models.MediaFile{
445 Filename: name,
446 Filepath: path,
447 IsAudio: isAudio,
448 Duration: -1,
449 })
450 }
451
452 sort.Slice(mediaFiles, func(i, j int) bool {
453 return mediaFiles[i].Filepath < mediaFiles[j].Filepath
454 })
455
456 var infoJSONPath string
457 if len(infoJSONFiles) > 0 {
458 sort.Strings(infoJSONFiles)
459 infoJSONPath = infoJSONFiles[0]
460 if len(infoJSONFiles) > 1 {
461 log.Printf("warning: multiple info.json files in %s, using %s", itemDir, infoJSONPath)
462 }
463 }
464
465 return mediaFiles, infoJSONPath, nil
466}
467
468func infoString(info map[string]interface{}, key string) (string, bool) {
469 if info == nil {
470 return "", false
471 }
472 // Only accept genuine strings: title/url/description are always strings in
473 // yt-dlp output, and stringifying an arbitrary JSON value (map, slice) would
474 // store junk like "map[...]" into the field.
475 if s, ok := info[key].(string); ok {
476 return s, true
477 }
478 return "", false
479}
480
481func infoDuration(info map[string]interface{}) (int, bool) {
482 if info == nil {
483 return 0, false
484 }
485 v, ok := info["duration"]
486 if !ok {
487 return 0, false
488 }
489 switch n := v.(type) {
490 case float64:
491 return int(n + 0.5), true
492 case string:
493 if f, err := strconv.ParseFloat(n, 64); err == nil {
494 return int(f + 0.5), true
495 }
496 }
497 return 0, false
498}
499
500func mediaFileStem(mf models.MediaFile) string {
501 return strings.TrimSuffix(filepath.Base(mf.Filename), filepath.Ext(mf.Filename))
502}
503
504// primaryMediaFile picks the representative file for an item: the largest video
505// file, or — if there are none — the largest file overall. Returns nil for an
506// item with no media files. Used for the item-level thumbnail and as the default
507// target for metadata, keeping those two consistent.
508func primaryMediaFile(item *models.LibraryItem) *models.MediaFile {
509 size := func(mf *models.MediaFile) int64 {
510 if info, err := os.Stat(mf.Filepath); err == nil {
511 return info.Size()
512 }
513 return 0
514 }
515 var best *models.MediaFile
516 for i := range item.MediaFiles {
517 mf := &item.MediaFiles[i]
518 switch {
519 case best == nil:
520 best = mf
521 case best.IsAudio && !mf.IsAudio:
522 // Prefer any video over audio.
523 best = mf
524 case best.IsAudio == mf.IsAudio && size(mf) > size(best):
525 best = mf
526 }
527 }
528 return best
529}
530
531func (s *LibraryService) GetMediaFile(relPath, filename string) (string, error) {
532 item, err := s.GetByRelPath(relPath)
533 if err != nil {
534 return "", err
535 }
536 for _, mf := range item.MediaFiles {
537 if mf.Filename == filename {
538 return mf.Filepath, nil
539 }
540 }
541 return "", fmt.Errorf("media file not found")
542}
543
544func (s *LibraryService) Delete(relPath string) error {
545 itemDir, err := s.resolveItemDir(relPath)
546 if err != nil {
547 return err
548 }
549 // resolveItemDir maps ""/"." to the library root and resolves symlinks, so a
550 // result equal to the root (reachable via a URL-encoded slash, "sub/..", or a
551 // symlink pointing back at the root) must be refused — deleting it would
552 // wipe the entire library.
553 if itemDir == s.libraryRoot() {
554 return fmt.Errorf("refusing to delete library root")
555 }
556 // Evict the cached scan so the deletion is reflected immediately rather than
557 // lingering until the TTL expires.
558 s.evictCachedScan(strings.Trim(relPath, "/"))
559 return os.RemoveAll(itemDir)
560}
561
562// FindByVideoID returns the absolute directory of the item under baseDir whose
563// marker matches the given yt-dlp video id, scanning only that directory (the
564// subscription's owned folder). Matching on the id alone is safe here because
565// each subscription owns a single source, so ids don't collide across
566// extractors within the folder. ok is false when no match is found or id is
567// empty.
568func (s *LibraryService) FindByVideoID(baseDir, id string) (string, bool) {
569 if id == "" {
570 return "", false
571 }
572
573 var match string
574 // An unreadable base dir simply means no match here.
575 _ = s.eachItemDir(baseDir, "FindByVideoID", func(itemDir string, meta models.ItemMetadata) bool {
576 if meta.VideoID != id {
577 return true
578 }
579 match = itemDir
580 return false
581 })
582
583 return match, match != ""
584}
585
586// eachItemDir walks the marked library items directly under baseDir, reading
587// each one's metadata, and calls fn until it returns false. Entries that aren't
588// items, or whose metadata can't be read, are skipped — a single bad item must
589// not abort a scan. logLabel names the caller in those skip messages.
590func (s *LibraryService) eachItemDir(baseDir, logLabel string, fn func(itemDir string, meta models.ItemMetadata) bool) error {
591 entries, err := os.ReadDir(baseDir)
592 if err != nil {
593 return err
594 }
595
596 for _, entry := range entries {
597 if !entry.IsDir() || entry.Name() == subtitlesDirName {
598 continue
599 }
600 itemDir := filepath.Join(baseDir, entry.Name())
601 if _, err := os.Stat(filepath.Join(itemDir, itemMarkerName)); err != nil {
602 continue
603 }
604 meta, err := s.readMetadata(itemDir)
605 if err != nil {
606 log.Printf("%s: skipping %s: %v", logLabel, itemDir, err)
607 continue
608 }
609 if !fn(itemDir, meta) {
610 return nil
611 }
612 }
613
614 return nil
615}
616
617// PruneToIDSet deletes items directly under baseDir whose identity key is not in
618// keep. It is used to mirror a subscription's source: entries removed upstream
619// are removed locally. Items without a known identity key are left untouched (we
620// never delete something we can't positively identify). Returns the number
621// removed.
622func (s *LibraryService) PruneToIDSet(baseDir string, keep map[string]bool) (int, error) {
623 removed := 0
624 err := s.eachItemDir(baseDir, "PruneToIDSet", func(itemDir string, meta models.ItemMetadata) bool {
625 if meta.VideoID == "" || keep[meta.VideoID] {
626 return true
627 }
628 if rel, err := filepath.Rel(s.libraryDir, itemDir); err == nil {
629 s.evictCachedScan(filepath.ToSlash(rel))
630 }
631 if err := os.RemoveAll(itemDir); err != nil {
632 log.Printf("warning: prune failed to remove %s: %v", itemDir, err)
633 return true
634 }
635 removed++
636 return true
637 })
638
639 return removed, err
640}
641