vidarchive

vidarchive/internal/service/download.go
download.go
⎇
Raw
1package service
2
3import (
4 "bufio"
5 "bytes"
6 "cmp"
7 "context"
8 "database/sql"
9 "encoding/json"
10 "errors"
11 "fmt"
12 "io"
13 "log"
14 "os"
15 "os/exec"
16 "path/filepath"
17 "slices"
18 "sort"
19 "strconv"
20 "strings"
21 "sync"
22 "syscall"
23 "time"
24
25 "github.com/BurntSushi/toml"
26 "github.com/gabriel-vasile/mimetype"
27
28 "vidarchive/internal/config"
29 "vidarchive/internal/models"
30 "vidarchive/internal/repository"
31)
32
33type DownloadService struct {
34 repo *repository.DownloadRepository
35 librarySvc *LibraryService
36 presetSvc *PresetService
37 settingsSvc *SettingsService
38 subscriptionSvc *SubscriptionService
39 cfg *config.Config
40 cache *ProgressCache
41 activeMu sync.Mutex
42 active map[int64]context.CancelFunc
43}
44
45func NewDownloadService(repo *repository.DownloadRepository, librarySvc *LibraryService, presetSvc *PresetService, settingsSvc *SettingsService, subscriptionSvc *SubscriptionService, cfg *config.Config) *DownloadService {
46 return &DownloadService{
47 repo: repo,
48 librarySvc: librarySvc,
49 presetSvc: presetSvc,
50 settingsSvc: settingsSvc,
51 subscriptionSvc: subscriptionSvc,
52 cfg: cfg,
53 cache: NewProgressCache(),
54 active: make(map[int64]context.CancelFunc),
55 }
56}
57
58func (s *DownloadService) Create(url string, presetID *int64, formatOverride, customFlags, outputDir string) (*models.Download, error) {
59 d := &models.Download{
60 URL: url,
61 Status: "queued",
62 FormatOverride: formatOverride,
63 CustomFlags: customFlags,
64 OutputDir: sql.NullString{String: outputDir, Valid: outputDir != ""},
65 }
66
67 if presetID != nil {
68 d.PresetID = sqlNullInt64(*presetID)
69 }
70
71 if err := s.repo.Create(d); err != nil {
72 return nil, err
73 }
74 return d, nil
75}
76
77// CreateForSubscription queues a download for a subscription run, copying its
78// download options and tagging it with the subscription id so ExecuteDownload
79// applies the right refresh mode and pruning.
80func (s *DownloadService) CreateForSubscription(sub *models.Subscription) (*models.Download, error) {
81 d := &models.Download{
82 URL: sub.URL,
83 Status: "queued",
84 FormatOverride: sub.FormatOverride,
85 CustomFlags: sub.CustomFlags,
86 OutputDir: sql.NullString{String: sub.OutputDir, Valid: sub.OutputDir != ""},
87 PresetID: sub.PresetID,
88 SubscriptionID: sqlNullInt64(sub.ID),
89 }
90 if err := s.repo.Create(d); err != nil {
91 return nil, err
92 }
93 return d, nil
94}
95
96func (s *DownloadService) GetByID(id int64) (*models.Download, error) {
97 d, err := s.repo.GetByID(id)
98 if err != nil {
99 return nil, err
100 }
101 if logs := s.cache.Snapshot(id); logs != "" {
102 d.Logs = sql.NullString{String: logs, Valid: true}
103 }
104 return d, nil
105}
106
107func (s *DownloadService) GetAll(status, sortBy string) ([]*models.Download, error) {
108 downloads, err := s.repo.GetAll(status, sortBy)
109 if err != nil {
110 return nil, err
111 }
112
113 for _, d := range downloads {
114 if logs := s.cache.Snapshot(d.ID); logs != "" {
115 d.Logs = sql.NullString{String: logs, Valid: true}
116 }
117 }
118
119 return downloads, nil
120}
121
122func (s *DownloadService) GetQueued(limit int) ([]*models.Download, error) {
123 return s.repo.GetQueued(limit)
124}
125
126// HasActiveForSubscription reports whether the subscription already has a queued
127// or in-progress download, so the scheduler can skip stacking another run.
128func (s *DownloadService) HasActiveForSubscription(subID int64) (bool, error) {
129 return s.repo.HasActiveForSubscription(subID)
130}
131
132func (s *DownloadService) Delete(id int64) error {
133 s.cancelDownload(id)
134 s.cache.Delete(id)
135 return s.repo.Delete(id)
136}
137
138// registerActive records the cancel func for a claimed download and returns a
139// release func. Registration happens at claim time rather than after the process
140// spawns, so a delete arriving during setup, between yt-dlp and the import, or
141// mid-import still stops the work instead of silently letting it finish.
142func (s *DownloadService) registerActive(id int64, cancel context.CancelFunc) func() {
143 s.activeMu.Lock()
144 s.active[id] = cancel
145 s.activeMu.Unlock()
146
147 return func() {
148 s.activeMu.Lock()
149 delete(s.active, id)
150 s.activeMu.Unlock()
151 }
152}
153
154func (s *DownloadService) cancelDownload(id int64) {
155 s.activeMu.Lock()
156 cancel, ok := s.active[id]
157 delete(s.active, id)
158 s.activeMu.Unlock()
159
160 if ok {
161 cancel()
162 }
163}
164
165// CancelAll stops every download currently in flight. Used on shutdown and when
166// clearing the queue, so no yt-dlp child outlives the rows that described it.
167func (s *DownloadService) CancelAll() {
168 s.activeMu.Lock()
169 cancels := make([]context.CancelFunc, 0, len(s.active))
170 for id, cancel := range s.active {
171 cancels = append(cancels, cancel)
172 delete(s.active, id)
173 }
174 s.activeMu.Unlock()
175
176 for _, cancel := range cancels {
177 cancel()
178 }
179}
180
181func (s *DownloadService) DeleteAll() error {
182 // Clearing the queue must also stop what is running; otherwise yt-dlp keeps
183 // going and imports into the library after its row is gone.
184 s.CancelAll()
185 return s.repo.DeleteAll()
186}
187
188func (s *DownloadService) CountByStatus(ctx context.Context) (map[string]int, error) {
189 return s.repo.CountByStatus(ctx)
190}
191
192// ResetStalledDownloads re-queues downloads left mid-flight by a previous run and
193// discards their temp directories. Without the cleanup the re-run imports into a
194// fresh uniqueDir and the library ends up with a duplicate of the same item.
195func (s *DownloadService) ResetStalledDownloads() error {
196 ids, err := s.repo.IDsByStatus("downloading")
197 if err != nil {
198 return err
199 }
200
201 for _, id := range ids {
202 for _, dir := range s.tempDirsFor(id) {
203 if err := os.RemoveAll(dir); err != nil {
204 log.Printf("warning: failed to remove stale temp dir %s: %v", dir, err)
205 }
206 }
207 }
208
209 return s.repo.UpdateStatusWhere("downloading", "queued")
210}
211
212// tempDirFor returns the scratch directory a download writes into.
213func (s *DownloadService) tempDirFor(id int64) string {
214 return filepath.Join(s.cfg.TempDir, strconv.FormatInt(id, 10))
215}
216
217// tempNewDirFor returns the second-pass scratch directory used by metadata mode.
218func (s *DownloadService) tempNewDirFor(id int64) string {
219 return s.tempDirFor(id) + "-new"
220}
221
222// tempDirsFor returns every scratch directory a download owns. ResetStalledDownloads
223// clears these, so the two builders above must stay the only places that name them.
224func (s *DownloadService) tempDirsFor(id int64) []string {
225 return []string{s.tempDirFor(id), s.tempNewDirFor(id)}
226}
227
228func (s *DownloadService) ListFormats(url string) ([]*models.FormatInfo, error) {
229 // Use machine-readable JSON (-J) rather than scraping the human "-F" table,
230 // whose columns/separators shift between yt-dlp versions. stderr is captured
231 // separately so warnings can't corrupt the JSON on stdout.
232 cmd := exec.Command(s.cfg.YTDLPPath, "-J", "--no-warnings", url)
233 var stderr bytes.Buffer
234 cmd.Stderr = &stderr
235 output, err := cmd.Output()
236 if err != nil {
237 return nil, fmt.Errorf("yt-dlp -J failed: %w\n%s", err, stderr.String())
238 }
239
240 return parseFormatJSON(output)
241}
242
243// ExecuteDownload runs the download for d. The bool reports whether this call
244// actually processed it: false means another worker already claimed it (Submit
245// and the queue checker can both enqueue the same row within the 2s poll window),
246// so the caller should not log it as completed. A cancelled run returns
247// ErrCancelled.
248//
249// parent belongs to the worker pool: deriving from it means a shutdown cancels
250// the download even if it lands before this call registers its own cancel func.
251func (s *DownloadService) ExecuteDownload(parent context.Context, d *models.Download) (bool, error) {
252 // Atomically claim the download. If it's no longer queued, another worker
253 // already took it — bail rather than download it twice.
254 claimed, err := s.repo.MarkStarted(d.ID)
255 if err != nil {
256 return false, err
257 }
258 if !claimed {
259 return false, nil
260 }
261
262 // Registered before any work starts so Delete/CancelAll can interrupt every
263 // phase, not just the window where yt-dlp happens to be running.
264 ctx, cancel := context.WithCancel(parent)
265 defer cancel()
266 defer s.registerActive(d.ID, cancel)()
267
268 s.cache.Set(d.ID, &LiveDownload{LastUpdate: time.Now()})
269 defer s.cache.Delete(d.ID)
270
271 var preset *models.Preset
272 if d.PresetID.Valid {
273 preset, err = s.presetSvc.GetByID(d.PresetID.Int64)
274 if err != nil {
275 log.Printf("download %d: preset %d lookup failed (%v); falling back to default", d.ID, d.PresetID.Int64, err)
276 preset = nil
277 }
278 }
279 if preset == nil {
280 var derr error
281 if preset, derr = s.presetSvc.GetDefault(); derr != nil {
282 log.Printf("download %d: no default preset available (%v); using built-in defaults", d.ID, derr)
283 preset = &models.Preset{}
284 }
285 }
286
287 var sub *models.Subscription
288 if d.SubscriptionID.Valid && s.subscriptionSvc != nil {
289 var serr error
290 if sub, serr = s.subscriptionSvc.GetByID(d.SubscriptionID.Int64); serr != nil {
291 log.Printf("download %d: subscription %d lookup failed: %v", d.ID, d.SubscriptionID.Int64, serr)
292 }
293 }
294
295 // Reject custom flags that clash with options VidArchive sets itself, before
296 // spending any work — the download fails with a message naming the offender.
297 isSubscription := d.SubscriptionID.Valid
298 for _, flags := range []string{d.CustomFlags, preset.CustomFlags} {
299 if err := checkReservedFlags(flags, isSubscription); err != nil {
300 s.finalizeError(d.ID, err)
301 return false, err
302 }
303 }
304
305 tempDownloadDir := s.tempDirFor(d.ID)
306 if err := os.MkdirAll(tempDownloadDir, 0755); err != nil {
307 return false, fmt.Errorf("create temp download dir: %w", err)
308 }
309 // Own the temp dir's lifetime here, where it's created, so it's removed on
310 // every exit path — including a failed yt-dlp run or an early return that
311 // crashes mid-import. The import helpers below no longer clean it up.
312 defer os.RemoveAll(tempDownloadDir)
313
314 args := s.presetSvc.BuildArgs(preset, d.FormatOverride, d.CustomFlags)
315
316 // Record the meaningful flags (format/audio/subs/custom) that shaped this
317 // download, before the internal plumbing (cookies, -P/-o, URL) is appended,
318 // so each imported item can show how it was fetched.
319 ytdlpFlags := strings.Join(args, " ")
320
321 var cookieCleanup func()
322 args, cookieCleanup = s.appendCookies(args)
323 defer cookieCleanup()
324
325 if sub != nil {
326 // Always write info.json so the import step can read the stable identity
327 // (yt-dlp's video id) used to match/replace existing items.
328 args = append(args, "--write-info-json")
329 switch sub.RefreshMode {
330 case "skip":
331 // Let yt-dlp skip entries already recorded — no re-download.
332 archive := s.subscriptionSvc.ArchivePath(sub.ID)
333 if err := os.MkdirAll(filepath.Dir(archive), 0755); err == nil {
334 args = append(args, "--download-archive", archive)
335 }
336 case "metadata":
337 // Refresh metadata only; don't fetch media.
338 args = append(args, "--skip-download")
339 }
340 }
341
342 args = append(args, "-P", tempDownloadDir)
343 args = append(args, "-o", "item-%(autonumber)05d/%(title)s.%(ext)s")
344 args = append(args, d.URL)
345
346 runErr := s.runYTDLP(ctx, d, args)
347
348 // Cancellation is checked before the run error, and both are checked before
349 // the import: a cancel that lands just after yt-dlp exited 0 leaves runErr nil,
350 // and the item must not reach the library after the user removed it.
351 if ctx.Err() != nil {
352 return false, s.finalizeCancelled(parent, d.ID)
353 }
354 if runErr != nil {
355 s.finalizeError(d.ID, runErr)
356 return false, runErr
357 }
358
359 mode := ""
360 if sub != nil {
361 mode = sub.RefreshMode
362 }
363
364 // Post-process before marking completed, so the download stays "downloading"
365 // until everything is really done — including metadata mode's second pass,
366 // which downloads any genuinely new entries as full items.
367 var postErr error
368 if mode == "metadata" {
369 // The main pass ran with --skip-download, so the temp dir holds only
370 // info.json files: refresh existing items in place and fetch new ones.
371 postErr = s.refreshAndAddNew(ctx, d, preset, tempDownloadDir, ytdlpFlags)
372 } else {
373 imported, err := s.importDownloadedItems(ctx, d, tempDownloadDir, mode, ytdlpFlags)
374 switch {
375 case err != nil:
376 postErr = err
377 // A plain (non-subscription) download that yields nothing is a failure, not
378 // a silent "completed". Subscription modes legitimately import zero (skip
379 // mode, or a metadata refresh with no new entries), so only enforce this for
380 // plain runs.
381 case sub == nil && imported == 0:
382 postErr = fmt.Errorf("yt-dlp finished but no media files were downloaded")
383 }
384 }
385
386 if postErr == nil && sub != nil && sub.PruneRemoved {
387 s.pruneSubscription(ctx, d, sub)
388 }
389
390 // Check cancellation before postErr, not only when postErr is non-nil: a run
391 // stopped just after yt-dlp exited 0 leaves postErr nil, and without this the
392 // download falls through and is recorded as "completed".
393 if ctx.Err() != nil {
394 return false, s.finalizeCancelled(parent, d.ID)
395 }
396 if postErr != nil {
397 s.finalizeError(d.ID, postErr)
398 return false, postErr
399 }
400
401 if err := s.repo.MarkCompleted(d.ID, "completed"); err != nil {
402 return false, err
403 }
404
405 return true, nil
406}
407
408// runYTDLP executes yt-dlp with args, streaming combined output into the live
409// progress cache and periodically flushing it to the download's persisted log.
410// Cancelling ctx kills the whole process group and makes this return.
411func (s *DownloadService) runYTDLP(ctx context.Context, d *models.Download, args []string) error {
412 // --newline forces yt-dlp to emit each progress update on its own line. Without
413 // it, progress is rewritten in place with carriage returns, so a long download
414 // becomes one ever-growing line that overflows the reader's buffer and stalls
415 // the pipe — hanging the download. See the hardened scanner below.
416 fullArgs := append([]string{"--newline"}, args...)
417 cmd := exec.CommandContext(ctx, s.cfg.YTDLPPath, fullArgs...)
418 cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
419 // yt-dlp spawns helpers (ffmpeg, external downloaders). Kill the whole group
420 // rather than just the parent, which would leave those orphaned.
421 cmd.Cancel = func() error {
422 return syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL)
423 }
424
425 stdout, err := cmd.StdoutPipe()
426 if err != nil {
427 return err
428 }
429 cmd.Stderr = cmd.Stdout
430
431 if err := cmd.Start(); err != nil {
432 return err
433 }
434
435 ticker := time.NewTicker(10 * time.Second)
436 defer ticker.Stop()
437 done := make(chan struct{})
438 go func() {
439 for {
440 select {
441 case <-ticker.C:
442 s.flushLogs(d.ID)
443 case <-done:
444 return
445 }
446 }
447 }()
448
449 scanner := bufio.NewScanner(stdout)
450 // Allow long lines (a single yt-dlp message can exceed the 64 KiB default)
451 // rather than letting the scanner abort and leave the pipe unread.
452 scanner.Buffer(make([]byte, 0, 64*1024), 1024*1024)
453 for scanner.Scan() {
454 s.cache.AppendLog(d.ID, scanner.Text())
455 }
456 if err := scanner.Err(); err != nil {
457 log.Printf("download %d: error reading yt-dlp output: %v", d.ID, err)
458 }
459 close(done)
460
461 s.flushLogs(d.ID)
462
463 return cmd.Wait()
464}
465
466// appendCookies writes the saved cookies (if any) to a temp file and appends a
467// --cookies flag. The returned cleanup removes the temp file and is always safe
468// to call, even when no cookies were configured.
469func (s *DownloadService) appendCookies(args []string) ([]string, func()) {
470 cleanup := func() {}
471 cookies, err := s.settingsSvc.GetCookies()
472 if err != nil || strings.TrimSpace(cookies) == "" {
473 return args, cleanup
474 }
475 // Written into the app's own temp dir rather than the system one, so the
476 // cookies file lands on the same volume the rest of the run uses.
477 if err := os.MkdirAll(s.cfg.TempDir, 0755); err != nil {
478 return args, cleanup
479 }
480 tmpFile, err := os.CreateTemp(s.cfg.TempDir, "cookies-*.txt")
481 if err != nil {
482 return args, cleanup
483 }
484 // A short write would hand yt-dlp a truncated cookies file; on any write/close
485 // failure, drop the temp file and proceed without cookies rather than silently
486 // using a broken one.
487 if _, err := tmpFile.WriteString(cookies); err != nil {
488 tmpFile.Close()
489 os.Remove(tmpFile.Name())
490 return args, cleanup
491 }
492 if err := tmpFile.Close(); err != nil {
493 os.Remove(tmpFile.Name())
494 return args, cleanup
495 }
496 return append(args, "--cookies", tmpFile.Name()), func() { os.Remove(tmpFile.Name()) }
497}
498
499func (s *DownloadService) finalizeError(id int64, err error) {
500 s.flushLogs(id)
501 if markErr := s.repo.MarkError(id, err.Error()); markErr != nil {
502 log.Printf("download %d: failed to record error: %v", id, markErr)
503 }
504}
505
506// ErrCancelled reports that a download was deliberately stopped (deleted, queue
507// cleared, or shutdown) rather than having failed. Callers distinguish it so a
508// cancellation isn't logged as an error.
509var ErrCancelled = errors.New("download cancelled")
510
511// finalizeCancelled records a stopped download.
512//
513// A shutdown (parent already cancelled) deliberately leaves the row
514// "downloading": ResetStalledDownloads re-queues it on the next start, so
515// stopping the server resumes the download instead of losing it. Only a
516// user-initiated cancel is terminal. The row may already be deleted in that
517// case — cancellation usually arrives via Delete — so a missing row is fine.
518func (s *DownloadService) finalizeCancelled(parent context.Context, id int64) error {
519 s.flushLogs(id)
520
521 if parent.Err() != nil {
522 return ErrCancelled
523 }
524
525 if err := s.repo.MarkCompleted(id, "cancelled"); err != nil {
526 log.Printf("download %d: failed to record cancellation: %v", id, err)
527 }
528 return ErrCancelled
529}
530
531// flushLogs persists whatever output has accumulated for a download.
532func (s *DownloadService) flushLogs(id int64) {
533 logs := s.cache.FlushLogs(id)
534 if logs == "" {
535 return
536 }
537 if err := s.repo.AppendLogs(id, logs); err != nil {
538 log.Printf("download %d: failed to persist logs: %v", id, err)
539 }
540}
541
542// resolveBaseLibraryDir returns the absolute library directory a download writes
543// into, applying the optional per-download OutputDir while rejecting any path
544// that escapes the library root.
545func (s *DownloadService) resolveBaseLibraryDir(d *models.Download) (string, error) {
546 if !d.OutputDir.Valid || d.OutputDir.String == "" {
547 return s.cfg.LibraryDir, nil
548 }
549 // Reuse the library service's guard so both entry points enforce the boundary
550 // the same way — it resolves symlinks, which a plain prefix check does not.
551 dir, err := s.librarySvc.ResolveWithinLibrary(d.OutputDir.String)
552 if err != nil {
553 return "", fmt.Errorf("invalid output directory: %w", err)
554 }
555 return dir, nil
556}
557
558// importDownloadedItems moves each downloaded item from the temp dir into the
559// library and returns the number of items successfully imported. Per-item
560// failures are logged and skipped (a playlist with a few bad entries still
561// imports the rest); a non-nil error means the import couldn't even start.
562//
563// A cancel stops the import between items and returns ctx.Err() with the count
564// imported so far. Importing a long playlist takes real time (a move plus an
565// ffprobe per file), so a deleted download must not keep filling the library.
566func (s *DownloadService) importDownloadedItems(ctx context.Context, d *models.Download, tempDownloadDir, mode, ytdlpFlags string) (int, error) {
567 entries, err := os.ReadDir(tempDownloadDir)
568 if err != nil {
569 return 0, err
570 }
571
572 baseLibraryDir, err := s.resolveBaseLibraryDir(d)
573 if err != nil {
574 return 0, err
575 }
576 if err := os.MkdirAll(baseLibraryDir, 0755); err != nil {
577 return 0, err
578 }
579
580 var itemDirs []string
581 for _, entry := range entries {
582 if !entry.IsDir() {
583 continue
584 }
585 name := entry.Name()
586 if strings.HasPrefix(name, "item-") {
587 itemDirs = append(itemDirs, filepath.Join(tempDownloadDir, name))
588 }
589 }
590 sort.Strings(itemDirs)
591
592 imported := 0
593 for _, itemDir := range itemDirs {
594 if err := ctx.Err(); err != nil {
595 return imported, err
596 }
597 if err := s.importItemDir(ctx, d.URL, itemDir, baseLibraryDir, mode, ytdlpFlags); err != nil {
598 // A cancelled item isn't a bad item: stop instead of logging a warning
599 // for it and every one that follows.
600 if ctx.Err() != nil {
601 return imported, ctx.Err()
602 }
603 log.Printf("warning: failed to import item %s: %v", itemDir, err)
604 continue
605 }
606 imported++
607 }
608
609 // The temp dir (and any leftovers from failed imports) is removed by the
610 // caller's deferred cleanup, so partial state never leaks even on a crash.
611 return imported, nil
612}
613
614func (s *DownloadService) importItemDir(ctx context.Context, url, itemDir, baseLibraryDir, mode, ytdlpFlags string) error {
615 entries, err := os.ReadDir(itemDir)
616 if err != nil {
617 return err
618 }
619
620 var mediaFiles []os.DirEntry
621 var infoJSONPath string
622 var subtitleFiles []string
623
624 for _, entry := range entries {
625 if entry.IsDir() {
626 continue
627 }
628 name := entry.Name()
629 path := filepath.Join(itemDir, name)
630 ext := strings.ToLower(filepath.Ext(name))
631
632 if isInfoJSON(name) {
633 infoJSONPath = path
634 continue
635 }
636 if ext == ".vtt" || ext == ".srt" || ext == ".ass" || ext == ".ssa" {
637 subtitleFiles = append(subtitleFiles, path)
638 continue
639 }
640
641 mtype, err := mimetype.DetectFile(path)
642 if err == nil && mtype != nil && (strings.HasPrefix(mtype.String(), "audio/") || strings.HasPrefix(mtype.String(), "video/")) {
643 mediaFiles = append(mediaFiles, entry)
644 }
645 }
646
647 if len(mediaFiles) == 0 {
648 return fmt.Errorf("no media files found in %s", itemDir)
649 }
650
651 info := readInfoJSON(infoJSONPath)
652 name := s.deriveItemName(itemDir, info, mediaFiles)
653 videoID := info.ID
654
655 // Last point at which nothing has been written to the library yet: give up
656 // here on a cancel rather than part-way through, which would leave a folder
657 // with some of its files and no marker — or, in overwrite mode, delete the
658 // existing item and not replace it.
659 if err := ctx.Err(); err != nil {
660 return err
661 }
662
663 // Overwrite mode: replace the existing copy of this video in place rather than
664 // creating a duplicate folder. Removing the old dir lets uniqueDir reuse its
665 // name (or land on the new title if it changed upstream).
666 if mode == "overwrite" && videoID != "" {
667 if existing, ok := s.librarySvc.FindByVideoID(baseLibraryDir, videoID); ok {
668 if rel, err := filepath.Rel(s.cfg.LibraryDir, existing); err == nil {
669 s.librarySvc.evictCachedScan(filepath.ToSlash(rel))
670 }
671 os.RemoveAll(existing)
672 }
673 }
674
675 targetDir := s.uniqueDir(baseLibraryDir, name)
676 if err := os.MkdirAll(targetDir, 0755); err != nil {
677 return err
678 }
679
680 if infoJSONPath != "" {
681 if err := moveFile(infoJSONPath, filepath.Join(targetDir, "info.json")); err != nil {
682 return err
683 }
684 }
685
686 for _, entry := range mediaFiles {
687 if err := moveFile(filepath.Join(itemDir, entry.Name()), filepath.Join(targetDir, entry.Name())); err != nil {
688 return err
689 }
690 }
691
692 // Probe each media file's duration once, here in the worker (off the request
693 // path), and cache it in the marker so the library never has to probe while
694 // serving pages. Files we can't probe simply get no duration.
695 fileDurations := make(map[string]int)
696 for _, entry := range mediaFiles {
697 if d, ok := probeDuration(ctx, s.cfg.FFprobePath, filepath.Join(targetDir, entry.Name())); ok {
698 fileDurations[entry.Name()] = d
699 }
700 }
701
702 if len(subtitleFiles) > 0 {
703 subtitlesDir := filepath.Join(targetDir, subtitlesDirName)
704 if err := os.MkdirAll(subtitlesDir, 0755); err != nil {
705 return err
706 }
707 for _, sf := range subtitleFiles {
708 if err := moveFile(sf, filepath.Join(subtitlesDir, filepath.Base(sf))); err != nil {
709 return err
710 }
711 }
712 }
713
714 metadata := models.ItemMetadata{
715 Name: name,
716 SourceURL: url,
717 VideoID: videoID,
718 YtdlpFlags: ytdlpFlags,
719 FileDurations: fileDurations,
720 }
721
722 markerPath := filepath.Join(targetDir, itemMarkerName)
723 f, err := os.Create(markerPath)
724 if err != nil {
725 return err
726 }
727 defer f.Close()
728 if err := toml.NewEncoder(f).Encode(metadata); err != nil {
729 return err
730 }
731
732 return nil
733}
734
735// infoJSON is the subset of yt-dlp's info.json VidArchive reads. ID is the
736// stable item identity; within a single subscription's own directory it is
737// enough to match items, so the extractor is not needed.
738type infoJSON struct {
739 ID string `json:"id"`
740 Title string `json:"title"`
741 Description string `json:"description"`
742 WebpageURL string `json:"webpage_url"`
743}
744
745// readInfoJSON parses an info.json. A missing, unreadable or malformed file
746// yields a zero-value struct: every caller treats absent fields as "unknown"
747// and falls back, so there is nothing to distinguish.
748func readInfoJSON(infoJSONPath string) infoJSON {
749 var info infoJSON
750 if infoJSONPath == "" {
751 return info
752 }
753 data, err := os.ReadFile(infoJSONPath)
754 if err != nil {
755 return info
756 }
757 if err := json.Unmarshal(data, &info); err != nil {
758 log.Printf("ignoring malformed %s: %v", infoJSONPath, err)
759 return infoJSON{}
760 }
761 return info
762}
763
764// refreshAndAddNew handles a metadata-mode run. The main pass used
765// --skip-download, so tempDownloadDir holds only info.json files. Existing
766// library items have their markers refreshed in place; entries with no existing
767// match are genuinely new and are downloaded as full items in a second pass.
768func (s *DownloadService) refreshAndAddNew(ctx context.Context, d *models.Download, preset *models.Preset, tempDownloadDir, ytdlpFlags string) error {
769 // The library dir is not created here: a refresh that matches everything
770 // writes nothing, and importDownloadedItems creates it when a second pass
771 // actually has an item to add.
772 baseLibraryDir, err := s.resolveBaseLibraryDir(d)
773 if err != nil {
774 return err
775 }
776
777 entries, err := os.ReadDir(tempDownloadDir)
778 if err != nil {
779 return err
780 }
781
782 var newURLs []string
783 for _, entry := range entries {
784 if err := ctx.Err(); err != nil {
785 return err
786 }
787 if !entry.IsDir() || !strings.HasPrefix(entry.Name(), "item-") {
788 continue
789 }
790 itemDir := filepath.Join(tempDownloadDir, entry.Name())
791 infoJSONPath := findInfoJSON(itemDir)
792 if infoJSONPath == "" {
793 continue
794 }
795 info := readInfoJSON(infoJSONPath)
796 if info.ID == "" {
797 continue
798 }
799 if existing, ok := s.librarySvc.FindByVideoID(baseLibraryDir, info.ID); ok {
800 if err := s.applyMetadata(existing, info); err != nil {
801 log.Printf("warning: failed to refresh metadata for %s: %v", itemDir, err)
802 }
803 continue
804 }
805 if info.WebpageURL != "" {
806 newURLs = append(newURLs, info.WebpageURL)
807 }
808 }
809
810 if len(newURLs) == 0 {
811 return nil
812 }
813 return s.downloadFresh(ctx, d, preset, newURLs, ytdlpFlags)
814}
815
816// downloadFresh fetches the given item URLs as full downloads (media + info.json)
817// and imports them into the download's library directory. Metadata mode uses this
818// to add entries that don't exist in the library yet.
819func (s *DownloadService) downloadFresh(ctx context.Context, d *models.Download, preset *models.Preset, urls []string, ytdlpFlags string) error {
820 tempDir := s.tempNewDirFor(d.ID)
821 if err := os.MkdirAll(tempDir, 0755); err != nil {
822 return err
823 }
824 defer os.RemoveAll(tempDir)
825
826 args := s.presetSvc.BuildArgs(preset, d.FormatOverride, d.CustomFlags)
827 args, cleanup := s.appendCookies(args)
828 defer cleanup()
829 args = append(args, "--write-info-json")
830 args = append(args, "-P", tempDir)
831 args = append(args, "-o", "item-%(autonumber)05d/%(title)s.%(ext)s")
832 args = append(args, urls...)
833
834 runErr := s.runYTDLP(ctx, d, args)
835 // A cancelled second pass has nothing worth importing.
836 if ctx.Err() != nil {
837 return runErr
838 }
839 // Import whatever succeeded even if some entries errored.
840 if _, err := s.importDownloadedItems(ctx, d, tempDir, "", ytdlpFlags); err != nil {
841 log.Printf("warning: failed to import new metadata-mode items: %v", err)
842 }
843 return runErr
844}
845
846// applyMetadata rewrites an existing item's marker (name/description/identity)
847// from a fresh info.json without touching its media.
848func (s *DownloadService) applyMetadata(existing string, info infoJSON) error {
849 meta, err := s.librarySvc.readOrCreateMetadata(existing)
850 if err != nil {
851 return fmt.Errorf("read metadata for %s: %w", existing, err)
852 }
853 if info.Title != "" {
854 meta.Name = info.Title
855 }
856 if info.Description != "" {
857 meta.Description = info.Description
858 }
859 if meta.SourceURL == "" && info.WebpageURL != "" {
860 meta.SourceURL = info.WebpageURL
861 }
862 meta.VideoID = info.ID
863
864 if err := s.librarySvc.writeMetadata(existing, meta); err != nil {
865 return err
866 }
867 if rel, err := filepath.Rel(s.cfg.LibraryDir, existing); err == nil {
868 s.librarySvc.evictCachedScan(filepath.ToSlash(rel))
869 }
870 return nil
871}
872
873// isInfoJSON reports whether a file name is yt-dlp's metadata sidecar. yt-dlp
874// writes "<title>.info.json" next to the media, but a bare "info.json" is what
875// an already-imported item holds.
876func isInfoJSON(name string) bool {
877 return name == "info.json" || strings.HasSuffix(name, ".info.json")
878}
879
880// findInfoJSON returns the path to an info.json directly inside itemDir, or "".
881func findInfoJSON(itemDir string) string {
882 entries, err := os.ReadDir(itemDir)
883 if err != nil {
884 return ""
885 }
886 for _, entry := range entries {
887 if entry.IsDir() {
888 continue
889 }
890 name := entry.Name()
891 if isInfoJSON(name) {
892 return filepath.Join(itemDir, name)
893 }
894 }
895 return ""
896}
897
898// pruneSubscription mirrors the source by deleting items in the subscription's
899// directory that are no longer present upstream. It enumerates the current id
900// set with a cheap flat-playlist listing; it never prunes when that enumeration
901// fails or returns nothing, so a dead URL or network error can't wipe the dir.
902func (s *DownloadService) pruneSubscription(ctx context.Context, d *models.Download, sub *models.Subscription) {
903 baseLibraryDir, err := s.resolveBaseLibraryDir(d)
904 if err != nil {
905 log.Printf("subscription %d prune skipped: %v", sub.ID, err)
906 return
907 }
908
909 keep, err := s.enumeratePlaylistIDs(ctx, sub.URL)
910 if err != nil {
911 log.Printf("subscription %d prune skipped: enumeration failed: %v", sub.ID, err)
912 return
913 }
914 if len(keep) == 0 {
915 log.Printf("subscription %d prune skipped: source returned no entries", sub.ID)
916 return
917 }
918
919 removed, err := s.librarySvc.PruneToIDSet(baseLibraryDir, keep)
920 if err != nil {
921 log.Printf("subscription %d prune error: %v", sub.ID, err)
922 return
923 }
924 if removed > 0 {
925 log.Printf("subscription %d pruned %d item(s) removed upstream", sub.ID, removed)
926 }
927}
928
929// enumeratePlaylistIDs lists the current video-id set for a URL without
930// downloading, using yt-dlp --flat-playlist. Cookies are applied so private
931// playlists enumerate correctly. Ids alone are sufficient to match items within
932// a subscription's own directory (see FindByVideoID / PruneToIDSet).
933func (s *DownloadService) enumeratePlaylistIDs(ctx context.Context, url string) (map[string]bool, error) {
934 args := []string{"--flat-playlist", "--no-warnings", "--print", "%(id)s"}
935
936 args, cleanup := s.appendCookies(args)
937 defer cleanup()
938 args = append(args, url)
939
940 out, err := exec.CommandContext(ctx, s.cfg.YTDLPPath, args...).Output()
941 if err != nil {
942 return nil, err
943 }
944
945 keep := make(map[string]bool)
946 for _, line := range strings.Split(string(out), "\n") {
947 id := strings.TrimSpace(line)
948 // yt-dlp prints "NA" for a missing field; never treat that as a real id.
949 if id == "" || id == "NA" {
950 continue
951 }
952 keep[id] = true
953 }
954 return keep, nil
955}
956
957// deriveItemName names the imported item after its title, falling back to the
958// largest media file's base name when there is no usable info.json.
959func (s *DownloadService) deriveItemName(itemDir string, info infoJSON, mediaFiles []os.DirEntry) string {
960 if info.Title != "" {
961 return sanitizeDirName(info.Title)
962 }
963
964 // Stat once per file up front: doing it inside the comparator re-stats the
965 // same files O(n log n) times.
966 sizes := make(map[string]int64, len(mediaFiles))
967 for _, f := range mediaFiles {
968 if st, err := os.Stat(filepath.Join(itemDir, f.Name())); err == nil {
969 sizes[f.Name()] = st.Size()
970 }
971 }
972 // Sort a copy: the caller iterates mediaFiles again to move and probe the
973 // files, and a naming helper must not reorder its caller's data.
974 byName := slices.Clone(mediaFiles)
975 slices.SortFunc(byName, func(a, b os.DirEntry) int {
976 return cmp.Compare(sizes[b.Name()], sizes[a.Name()])
977 })
978
979 base := strings.TrimSuffix(byName[0].Name(), filepath.Ext(byName[0].Name()))
980 return sanitizeDirName(base)
981}
982
983func (s *DownloadService) uniqueDir(base, name string) string {
984 dir := filepath.Join(base, name)
985 if _, err := os.Stat(dir); os.IsNotExist(err) {
986 return dir
987 }
988 for i := 1; ; i++ {
989 candidate := fmt.Sprintf("%s-%d", dir, i)
990 if _, err := os.Stat(candidate); os.IsNotExist(err) {
991 return candidate
992 }
993 }
994}
995
996// moveFile moves src to dst, falling back to copy-and-delete when the two are on
997// different filesystems. The temp and library directories are independently
998// configurable, so they can legitimately live on separate mounts — where a plain
999// rename fails with EXDEV.
1000func moveFile(src, dst string) error {
1001 if err := os.Rename(src, dst); err == nil {
1002 return nil
1003 } else if !errors.Is(err, syscall.EXDEV) {
1004 return err
1005 }
1006
1007 in, err := os.Open(src)
1008 if err != nil {
1009 return err
1010 }
1011 defer in.Close()
1012
1013 info, err := in.Stat()
1014 if err != nil {
1015 return err
1016 }
1017
1018 out, err := os.OpenFile(dst, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, info.Mode())
1019 if err != nil {
1020 return err
1021 }
1022
1023 if _, err := io.Copy(out, in); err != nil {
1024 out.Close()
1025 os.Remove(dst)
1026 return err
1027 }
1028 // Close explicitly: a deferred close would hide a flush error on the copy.
1029 if err := out.Close(); err != nil {
1030 os.Remove(dst)
1031 return err
1032 }
1033
1034 return os.Remove(src)
1035}
1036
1037func sanitizeDirName(name string) string {
1038 name = strings.TrimSpace(name)
1039 replacer := strings.NewReplacer(
1040 "/", "-",
1041 "\\", "-",
1042 ":", "-",
1043 "*", "-",
1044 "?", "-",
1045 "\"", "-",
1046 "<", "-",
1047 ">", "-",
1048 "|", "-",
1049 )
1050 name = replacer.Replace(name)
1051 name = strings.TrimSpace(name)
1052 if name == "" {
1053 name = "untitled"
1054 }
1055 return name
1056}
1057
1058// probeDuration returns the duration of a media file in whole seconds. The bool
1059// is false when ffprobe is unavailable or the file has no usable duration.
1060func probeDuration(ctx context.Context, ffprobePath, path string) (int, bool) {
1061 out, err := exec.CommandContext(ctx, ffprobePath, "-v", "error",
1062 "-show_entries", "format=duration",
1063 "-of", "default=nw=1:nk=1", path).Output()
1064 if err != nil {
1065 return 0, false
1066 }
1067 f, err := strconv.ParseFloat(strings.TrimSpace(string(out)), 64)
1068 if err != nil || f <= 0 {
1069 return 0, false
1070 }
1071 return int(f + 0.5), true
1072}
1073
1074// ytFormat mirrors the subset of yt-dlp's per-format JSON (-J) we surface.
1075// Numeric fields are pointers so an absent value (null/omitted) is distinct
1076// from a real zero.
1077type ytFormat struct {
1078 FormatID string `json:"format_id"`
1079 Ext string `json:"ext"`
1080 Resolution string `json:"resolution"`
1081 Width *int `json:"width"`
1082 Height *int `json:"height"`
1083 FPS *float64 `json:"fps"`
1084 VCodec string `json:"vcodec"`
1085 ACodec string `json:"acodec"`
1086 AudioChannels *int `json:"audio_channels"`
1087 Filesize *int64 `json:"filesize"`
1088 FilesizeApprox *int64 `json:"filesize_approx"`
1089 FormatNote string `json:"format_note"`
1090}
1091
1092// parseFormatJSON reads yt-dlp's single-JSON dump (-J) and returns the available
1093// formats. For a single video the formats live at the top level; for a playlist
1094// URL we fall back to the first entry's formats so the picker still shows
1095// something useful.
1096func parseFormatJSON(data []byte) ([]*models.FormatInfo, error) {
1097 var top struct {
1098 Formats []ytFormat `json:"formats"`
1099 Entries []struct {
1100 Formats []ytFormat `json:"formats"`
1101 } `json:"entries"`
1102 }
1103 if err := json.Unmarshal(data, &top); err != nil {
1104 return nil, fmt.Errorf("parse yt-dlp JSON: %w", err)
1105 }
1106
1107 raw := top.Formats
1108 if len(raw) == 0 && len(top.Entries) > 0 {
1109 raw = top.Entries[0].Formats
1110 }
1111
1112 formats := make([]*models.FormatInfo, 0, len(raw))
1113 for _, f := range raw {
1114 formats = append(formats, f.toFormatInfo())
1115 }
1116 return formats, nil
1117}
1118
1119func (f ytFormat) toFormatInfo() *models.FormatInfo {
1120 fi := &models.FormatInfo{
1121 ID: f.FormatID,
1122 Ext: f.Ext,
1123 Note: f.FormatNote,
1124 }
1125
1126 switch {
1127 case f.Resolution != "":
1128 fi.Resolution = f.Resolution
1129 case f.Width != nil && f.Height != nil && *f.Width > 0 && *f.Height > 0:
1130 fi.Resolution = fmt.Sprintf("%dx%d", *f.Width, *f.Height)
1131 }
1132
1133 if f.FPS != nil && *f.FPS > 0 {
1134 fi.FPS = strconv.FormatFloat(*f.FPS, 'f', -1, 64)
1135 }
1136 if f.AudioChannels != nil && *f.AudioChannels > 0 {
1137 fi.Channels = strconv.Itoa(*f.AudioChannels)
1138 }
1139
1140 // Prefer the video codec; fall back to the audio codec for audio-only formats.
1141 if f.VCodec != "" && f.VCodec != "none" {
1142 fi.Codec = f.VCodec
1143 } else if f.ACodec != "" && f.ACodec != "none" {
1144 fi.Codec = f.ACodec
1145 }
1146
1147 if f.Filesize != nil && *f.Filesize > 0 {
1148 fi.FileSize = humanizeBytes(*f.Filesize)
1149 } else if f.FilesizeApprox != nil && *f.FilesizeApprox > 0 {
1150 fi.FileSize = "~" + humanizeBytes(*f.FilesizeApprox)
1151 }
1152
1153 return fi
1154}
1155
1156// humanizeBytes renders a byte count as a compact human-readable size.
1157func humanizeBytes(n int64) string {
1158 const unit = 1024
1159 if n < unit {
1160 return fmt.Sprintf("%dB", n)
1161 }
1162 div, exp := int64(unit), 0
1163 for m := n / unit; m >= unit; m /= unit {
1164 div *= unit
1165 exp++
1166 }
1167 return fmt.Sprintf("%.1f%ciB", float64(n)/float64(div), "KMGTPE"[exp])
1168}
1169
1170func sqlNullInt64(v int64) sql.NullInt64 {
1171 return sql.NullInt64{Int64: v, Valid: true}
1172}
1173
1174// reservedFlags are yt-dlp options VidArchive always sets itself; user custom
1175// flags must not pass them (or a conflicting inverse). The value describes what
1176// the option controls, for the failure message.
1177var reservedFlags = map[string]string{
1178 "-o": "the output template",
1179 "--output": "the output template",
1180 "-P": "the download path",
1181 "--paths": "the download path",
1182 "--cookies": "cookies (set these in Settings instead)",
1183 "--no-cookies": "cookies (set these in Settings instead)",
1184 "--newline": "progress output formatting (VidArchive sets this to stream logs)",
1185
1186 // These hand yt-dlp an arbitrary command or binary to run. VidArchive passes
1187 // custom flags through verbatim, so allowing them would turn the preset form
1188 // into remote command execution.
1189 "--exec": "running external commands (not permitted)",
1190 "--exec-before-download": "running external commands (not permitted)",
1191 "--postprocessor-args": "post-processor arguments (not permitted)",
1192 "--ppa": "post-processor arguments (not permitted)",
1193 "--downloader": "selecting an external downloader (not permitted)",
1194 "--external-downloader": "selecting an external downloader (not permitted)",
1195 "--downloader-args": "external downloader arguments (not permitted)",
1196 "--external-downloader-args": "external downloader arguments (not permitted)",
1197}
1198
1199// reservedSubscriptionFlags are additionally reserved for subscription runs,
1200// where VidArchive drives info-json writing and the refresh mode.
1201var reservedSubscriptionFlags = map[string]string{
1202 "--write-info-json": "info-json writing (needed to track item identity)",
1203 "--no-write-info-json": "info-json writing (needed to track item identity)",
1204 "--download-archive": "the download archive (managed by Skip mode)",
1205 "--no-download-archive": "the download archive (managed by Skip mode)",
1206 "--skip-download": "media downloading (managed by Metadata mode)",
1207 "--no-skip-download": "media downloading (managed by Metadata mode)",
1208}
1209
1210// checkReservedFlags rejects custom flags that clash with options VidArchive
1211// controls, naming the offender. It matches both "--flag" and "--flag=value".
1212func checkReservedFlags(customFlags string, isSubscription bool) error {
1213 for _, tok := range strings.Fields(customFlags) {
1214 // Both "--flag value" and "--flag=value" name the same option.
1215 name, _, _ := strings.Cut(tok, "=")
1216
1217 desc, ok := reservedFlags[name]
1218 if !ok && isSubscription {
1219 desc, ok = reservedSubscriptionFlags[name]
1220 }
1221 if ok {
1222 return fmt.Errorf("custom flag %q conflicts with VidArchive's handling of %s; remove it and try again", tok, desc)
1223 }
1224 }
1225 return nil
1226}
1227