vidarchive

vidarchive/internal/service/download.go
download.go
⎇
Raw
1package service
2
3import (
4 "bufio"
5 "bytes"
6 "context"
7 "database/sql"
8 "encoding/json"
9 "errors"
10 "fmt"
11 "io"
12 "log"
13 "os"
14 "os/exec"
15 "path/filepath"
16 "sort"
17 "strconv"
18 "strings"
19 "sync"
20 "syscall"
21 "time"
22
23 "github.com/gabriel-vasile/mimetype"
24
25 "vidarchive/internal/config"
26 "vidarchive/internal/models"
27 "vidarchive/internal/repository"
28 "vidarchive/internal/util"
29)
30
31type DownloadService struct {
32 repo *repository.DownloadRepository
33 librarySvc *LibraryService
34 presetSvc *PresetService
35 settingsSvc *SettingsService
36 subscriptionSvc *SubscriptionService
37 cfg *config.Config
38 cache *ProgressCache
39 activeMu sync.Mutex
40 active map[int64]context.CancelFunc
41}
42
43func NewDownloadService(repo *repository.DownloadRepository, librarySvc *LibraryService, presetSvc *PresetService, settingsSvc *SettingsService, subscriptionSvc *SubscriptionService, cfg *config.Config) *DownloadService {
44 return &DownloadService{
45 repo: repo,
46 librarySvc: librarySvc,
47 presetSvc: presetSvc,
48 settingsSvc: settingsSvc,
49 subscriptionSvc: subscriptionSvc,
50 cfg: cfg,
51 cache: NewProgressCache(),
52 active: make(map[int64]context.CancelFunc),
53 }
54}
55
56func (s *DownloadService) Create(url string, presetID *int64, formatOverride, customFlags, outputDir string) (*models.Download, error) {
57 d := &models.Download{
58 URL: url,
59 Status: "queued",
60 FormatOverride: formatOverride,
61 CustomFlags: customFlags,
62 OutputDir: sql.NullString{String: outputDir, Valid: outputDir != ""},
63 }
64
65 if presetID != nil {
66 d.PresetID = sqlNullInt64(*presetID)
67 }
68
69 if err := s.repo.Create(d); err != nil {
70 return nil, err
71 }
72 return d, nil
73}
74
75// CreateForSubscription queues a download for a subscription run, copying its
76// download options and tagging it with the subscription id so ExecuteDownload
77// applies the right refresh mode and pruning.
78func (s *DownloadService) CreateForSubscription(sub *models.Subscription) (*models.Download, error) {
79 d := &models.Download{
80 URL: sub.URL,
81 Status: "queued",
82 FormatOverride: sub.FormatOverride,
83 CustomFlags: sub.CustomFlags,
84 OutputDir: sql.NullString{String: sub.OutputDir, Valid: sub.OutputDir != ""},
85 PresetID: sub.PresetID,
86 SubscriptionID: sqlNullInt64(sub.ID),
87 }
88 if err := s.repo.Create(d); err != nil {
89 return nil, err
90 }
91 return d, nil
92}
93
94func (s *DownloadService) GetByID(id int64) (*models.Download, error) {
95 d, err := s.repo.GetByID(id)
96 if err != nil {
97 return nil, err
98 }
99 if logs := s.cache.Snapshot(id); logs != "" {
100 d.Logs = sql.NullString{String: logs, Valid: true}
101 }
102 return d, nil
103}
104
105func (s *DownloadService) GetAll(status, sortBy string) ([]*models.Download, error) {
106 downloads, err := s.repo.GetAll(status, sortBy)
107 if err != nil {
108 return nil, err
109 }
110
111 for _, d := range downloads {
112 if logs := s.cache.Snapshot(d.ID); logs != "" {
113 d.Logs = sql.NullString{String: logs, Valid: true}
114 }
115 }
116
117 return downloads, nil
118}
119
120func (s *DownloadService) GetQueued(limit int) ([]*models.Download, error) {
121 return s.repo.GetQueued(limit)
122}
123
124// HasActiveForSubscription reports whether the subscription already has a queued
125// or in-progress download, so the scheduler can skip stacking another run.
126func (s *DownloadService) HasActiveForSubscription(subID int64) (bool, error) {
127 return s.repo.HasActiveForSubscription(subID)
128}
129
130func (s *DownloadService) Delete(id int64) error {
131 s.cancelDownload(id)
132 s.cache.Delete(id)
133 return s.repo.Delete(id)
134}
135
136// registerActive records the cancel func for a claimed download and returns a
137// release func. Registration happens at claim time rather than after the process
138// spawns, so a delete arriving during setup, between yt-dlp and the import, or
139// mid-import still stops the work instead of silently letting it finish.
140func (s *DownloadService) registerActive(id int64, cancel context.CancelFunc) func() {
141 s.activeMu.Lock()
142 s.active[id] = cancel
143 s.activeMu.Unlock()
144
145 return func() {
146 s.activeMu.Lock()
147 delete(s.active, id)
148 s.activeMu.Unlock()
149 }
150}
151
152func (s *DownloadService) cancelDownload(id int64) {
153 s.activeMu.Lock()
154 cancel, ok := s.active[id]
155 delete(s.active, id)
156 s.activeMu.Unlock()
157
158 if ok {
159 cancel()
160 }
161}
162
163// CancelAll stops every download currently in flight. Used on shutdown and when
164// clearing the queue, so no yt-dlp child outlives the rows that described it.
165func (s *DownloadService) CancelAll() {
166 s.activeMu.Lock()
167 cancels := make([]context.CancelFunc, 0, len(s.active))
168 for id, cancel := range s.active {
169 cancels = append(cancels, cancel)
170 delete(s.active, id)
171 }
172 s.activeMu.Unlock()
173
174 for _, cancel := range cancels {
175 cancel()
176 }
177}
178
179func (s *DownloadService) DeleteAll() error {
180 // Clearing the queue must also stop what is running; otherwise yt-dlp keeps
181 // going and imports into the library after its row is gone.
182 s.CancelAll()
183 return s.repo.DeleteAll()
184}
185
186func (s *DownloadService) CountByStatus(ctx context.Context) (map[string]int, error) {
187 return s.repo.CountByStatus(ctx)
188}
189
190// ResetStalledDownloads re-queues downloads left mid-flight by a previous run and
191// discards their temp directories. Without the cleanup the re-run imports into a
192// fresh uniqueDir and the library ends up with a duplicate of the same item.
193func (s *DownloadService) ResetStalledDownloads() error {
194 ids, err := s.repo.IDsByStatus("downloading")
195 if err != nil {
196 return err
197 }
198
199 for _, id := range ids {
200 for _, dir := range s.tempDirsFor(id) {
201 if err := os.RemoveAll(dir); err != nil {
202 log.Printf("warning: failed to remove stale temp dir %s: %v", dir, err)
203 }
204 }
205 }
206
207 return s.repo.UpdateStatusWhere("downloading", "queued")
208}
209
210// tempDirFor returns the scratch directory a download writes into.
211func (s *DownloadService) tempDirFor(id int64) string {
212 return filepath.Join(s.cfg.TempDir, strconv.FormatInt(id, 10))
213}
214
215// tempNewDirFor returns the second-pass scratch directory used by metadata mode.
216func (s *DownloadService) tempNewDirFor(id int64) string {
217 return s.tempDirFor(id) + "-new"
218}
219
220// tempDirsFor returns every scratch directory a download owns. ResetStalledDownloads
221// clears these, so the two builders above must stay the only places that name them.
222func (s *DownloadService) tempDirsFor(id int64) []string {
223 return []string{s.tempDirFor(id), s.tempNewDirFor(id)}
224}
225
226func (s *DownloadService) ListFormats(url string) ([]*models.FormatInfo, error) {
227 // Use machine-readable JSON (-J) rather than scraping the human "-F" table,
228 // whose columns/separators shift between yt-dlp versions. stderr is captured
229 // separately so warnings can't corrupt the JSON on stdout.
230 cmd := exec.Command(s.cfg.YTDLPPath, "-J", "--no-warnings", url)
231 var stderr bytes.Buffer
232 cmd.Stderr = &stderr
233 output, err := cmd.Output()
234 if err != nil {
235 return nil, fmt.Errorf("yt-dlp -J failed: %w\n%s", err, stderr.String())
236 }
237
238 return parseFormatJSON(output)
239}
240
241// ExecuteDownload runs the download for d. The bool reports whether this call
242// actually processed it: false means another worker already claimed it (Submit
243// and the queue checker can both enqueue the same row within the 2s poll window),
244// so the caller should not log it as completed. A cancelled run returns
245// ErrCancelled.
246//
247// parent belongs to the worker pool: deriving from it means a shutdown cancels
248// the download even if it lands before this call registers its own cancel func.
249func (s *DownloadService) ExecuteDownload(parent context.Context, d *models.Download) (bool, error) {
250 claimed, err := s.repo.MarkStarted(d.ID)
251 if err != nil {
252 return false, err
253 }
254 if !claimed {
255 return false, nil
256 }
257
258 // Registered before any work starts so Delete/CancelAll can interrupt every
259 // phase, not just the window where yt-dlp happens to be running.
260 ctx, cancel := context.WithCancel(parent)
261 defer cancel()
262 defer s.registerActive(d.ID, cancel)()
263
264 s.cache.Set(d.ID, &LiveDownload{LastUpdate: time.Now()})
265 defer s.cache.Delete(d.ID)
266
267 var preset *models.Preset
268 if d.PresetID.Valid {
269 preset, err = s.presetSvc.GetByID(d.PresetID.Int64)
270 if err != nil {
271 log.Printf("download %d: preset %d lookup failed (%v); falling back to default", d.ID, d.PresetID.Int64, err)
272 preset = nil
273 }
274 }
275 if preset == nil {
276 var derr error
277 if preset, derr = s.presetSvc.GetDefault(); derr != nil {
278 log.Printf("download %d: no default preset available (%v); using built-in defaults", d.ID, derr)
279 preset = &models.Preset{}
280 }
281 }
282
283 var sub *models.Subscription
284 if d.SubscriptionID.Valid && s.subscriptionSvc != nil {
285 var serr error
286 if sub, serr = s.subscriptionSvc.GetByID(d.SubscriptionID.Int64); serr != nil {
287 log.Printf("download %d: subscription %d lookup failed: %v", d.ID, d.SubscriptionID.Int64, serr)
288 }
289 }
290
291 // Reject custom flags that clash with options VidArchive sets itself, before
292 // spending any work — the download fails with a message naming the offender.
293 isSubscription := d.SubscriptionID.Valid
294 for _, flags := range []string{d.CustomFlags, preset.CustomFlags} {
295 if err := checkReservedFlags(flags, isSubscription); err != nil {
296 s.finalizeError(d.ID, err)
297 return false, err
298 }
299 }
300
301 tempDownloadDir := s.tempDirFor(d.ID)
302 if err := os.MkdirAll(tempDownloadDir, 0755); err != nil {
303 return false, fmt.Errorf("create temp download dir: %w", err)
304 }
305 // Own the temp dir's lifetime here, where it's created, so it's removed on
306 // every exit path — including a failed yt-dlp run or an early return that
307 // crashes mid-import. The import helpers below no longer clean it up.
308 defer os.RemoveAll(tempDownloadDir)
309
310 args := s.presetSvc.BuildArgs(preset, d.FormatOverride, d.CustomFlags)
311
312 // Record the meaningful flags (format/audio/subs/custom) that shaped this
313 // download, before the internal plumbing (cookies, -P/-o, URL) is appended,
314 // so each imported item can show how it was fetched.
315 ytdlpFlags := strings.Join(args, " ")
316
317 var cookieCleanup func()
318 args, cookieCleanup = s.appendCookies(args)
319 defer cookieCleanup()
320
321 if sub != nil {
322 // Always write info.json so the import step can read the stable identity
323 // (yt-dlp's video id) used to match/replace existing items.
324 args = append(args, "--write-info-json")
325 switch sub.RefreshMode {
326 case "skip":
327 // Let yt-dlp skip entries already recorded — no re-download.
328 archive := s.subscriptionSvc.ArchivePath(sub.ID)
329 if err := os.MkdirAll(filepath.Dir(archive), 0755); err == nil {
330 args = append(args, "--download-archive", archive)
331 }
332 case "metadata":
333 // Refresh metadata only; don't fetch media.
334 args = append(args, "--skip-download")
335 }
336 }
337
338 args = append(args, "-P", tempDownloadDir)
339 args = append(args, "-o", "item-%(autonumber)05d/%(title)s.%(ext)s")
340 args = append(args, d.URL)
341
342 runErr := s.runYTDLP(ctx, d, args)
343
344 // Cancellation wins over both the run error and the import: a cancel that
345 // lands just after yt-dlp exited 0 leaves runErr nil, and the item must not
346 // reach the library after the user removed it.
347 if ctx.Err() != nil {
348 return false, s.finalizeCancelled(parent, d.ID)
349 }
350 if runErr != nil {
351 s.finalizeError(d.ID, runErr)
352 return false, runErr
353 }
354
355 mode := ""
356 if sub != nil {
357 mode = sub.RefreshMode
358 }
359
360 // Post-process before marking completed, so the download stays "downloading"
361 // until everything is really done — including metadata mode's second pass,
362 // which downloads any genuinely new entries as full items.
363 var postErr error
364 if mode == "metadata" {
365 // The main pass ran with --skip-download, so the temp dir holds only
366 // info.json files: refresh existing items in place and fetch new ones.
367 postErr = s.refreshAndAddNew(ctx, d, preset, tempDownloadDir, ytdlpFlags)
368 } else {
369 imported, err := s.importDownloadedItems(ctx, d, tempDownloadDir, mode, ytdlpFlags)
370 switch {
371 case err != nil:
372 postErr = err
373 // A plain (non-subscription) download that yields nothing is a failure, not
374 // a silent "completed". Subscription modes legitimately import zero (skip
375 // mode, or a metadata refresh with no new entries), so only enforce this for
376 // plain runs.
377 case sub == nil && imported == 0:
378 postErr = fmt.Errorf("yt-dlp finished but no media files were downloaded")
379 }
380 }
381
382 if postErr == nil && sub != nil && sub.PruneRemoved {
383 s.pruneSubscription(ctx, d, sub)
384 }
385
386 // Same cancellation check as above: a stop during post-processing must not
387 // be recorded as "completed".
388 if ctx.Err() != nil {
389 return false, s.finalizeCancelled(parent, d.ID)
390 }
391 if postErr != nil {
392 s.finalizeError(d.ID, postErr)
393 return false, postErr
394 }
395
396 if err := s.repo.MarkCompleted(d.ID, "completed"); err != nil {
397 return false, err
398 }
399
400 return true, nil
401}
402
403// runYTDLP executes yt-dlp with args, streaming combined output into the live
404// progress cache and periodically flushing it to the download's persisted log.
405// Cancelling ctx kills the whole process group and makes this return.
406func (s *DownloadService) runYTDLP(ctx context.Context, d *models.Download, args []string) error {
407 // --newline forces yt-dlp to emit each progress update on its own line. Without
408 // it, progress is rewritten in place with carriage returns, so a long download
409 // becomes one ever-growing line that overflows the reader's buffer and stalls
410 // the pipe — hanging the download. See the hardened scanner below.
411 fullArgs := append([]string{"--newline"}, args...)
412 cmd := exec.CommandContext(ctx, s.cfg.YTDLPPath, fullArgs...)
413 cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
414 // yt-dlp spawns helpers (ffmpeg, external downloaders). Kill the whole group
415 // rather than just the parent, which would leave those orphaned.
416 cmd.Cancel = func() error {
417 return syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL)
418 }
419
420 stdout, err := cmd.StdoutPipe()
421 if err != nil {
422 return err
423 }
424 cmd.Stderr = cmd.Stdout
425
426 if err := cmd.Start(); err != nil {
427 return err
428 }
429
430 ticker := time.NewTicker(10 * time.Second)
431 defer ticker.Stop()
432 done := make(chan struct{})
433 go func() {
434 for {
435 select {
436 case <-ticker.C:
437 s.flushLogs(d.ID)
438 case <-done:
439 return
440 }
441 }
442 }()
443
444 scanner := bufio.NewScanner(stdout)
445 // Allow long lines (a single yt-dlp message can exceed the 64 KiB default)
446 // rather than letting the scanner abort and leave the pipe unread.
447 scanner.Buffer(make([]byte, 0, 64*1024), 1024*1024)
448 for scanner.Scan() {
449 s.cache.AppendLog(d.ID, scanner.Text())
450 }
451 if err := scanner.Err(); err != nil {
452 log.Printf("download %d: error reading yt-dlp output: %v", d.ID, err)
453 }
454 close(done)
455
456 s.flushLogs(d.ID)
457
458 return cmd.Wait()
459}
460
461// appendCookies writes the saved cookies (if any) to a temp file and appends a
462// --cookies flag. The returned cleanup removes the temp file and is always safe
463// to call, even when no cookies were configured.
464func (s *DownloadService) appendCookies(args []string) ([]string, func()) {
465 cookies, err := s.settingsSvc.GetCookies()
466 if err != nil || strings.TrimSpace(cookies) == "" {
467 return args, func() {}
468 }
469 path, err := s.writeCookiesFile(cookies)
470 if err != nil {
471 return args, func() {}
472 }
473 return append(args, "--cookies", path), func() { os.Remove(path) }
474}
475
476// writeCookiesFile writes cookies to a temp file in the app's own temp dir (the
477// same volume the rest of the run uses). On any failure the partial file is
478// removed — a truncated cookies file must not be handed to yt-dlp.
479func (s *DownloadService) writeCookiesFile(cookies string) (string, error) {
480 if err := os.MkdirAll(s.cfg.TempDir, 0755); err != nil {
481 return "", err
482 }
483 tmpFile, err := os.CreateTemp(s.cfg.TempDir, "cookies-*.txt")
484 if err != nil {
485 return "", err
486 }
487 if _, err := tmpFile.WriteString(cookies); err != nil {
488 tmpFile.Close()
489 os.Remove(tmpFile.Name())
490 return "", err
491 }
492 if err := tmpFile.Close(); err != nil {
493 os.Remove(tmpFile.Name())
494 return "", err
495 }
496 return tmpFile.Name(), nil
497}
498
499func (s *DownloadService) finalizeError(id int64, err error) {
500 s.flushLogs(id)
501 if markErr := s.repo.MarkError(id, err.Error()); markErr != nil {
502 log.Printf("download %d: failed to record error: %v", id, markErr)
503 }
504}
505
506// ErrCancelled reports that a download was deliberately stopped (deleted, queue
507// cleared, or shutdown) rather than having failed. Callers distinguish it so a
508// cancellation isn't logged as an error.
509var ErrCancelled = errors.New("download cancelled")
510
511// finalizeCancelled records a stopped download.
512//
513// A shutdown (parent already cancelled) deliberately leaves the row
514// "downloading": ResetStalledDownloads re-queues it on the next start, so
515// stopping the server resumes the download instead of losing it. Only a
516// user-initiated cancel is terminal. The row may already be deleted in that
517// case — cancellation usually arrives via Delete — so a missing row is fine.
518func (s *DownloadService) finalizeCancelled(parent context.Context, id int64) error {
519 s.flushLogs(id)
520
521 if parent.Err() != nil {
522 return ErrCancelled
523 }
524
525 if err := s.repo.MarkCompleted(id, "cancelled"); err != nil {
526 log.Printf("download %d: failed to record cancellation: %v", id, err)
527 }
528 return ErrCancelled
529}
530
531// flushLogs persists whatever output has accumulated for a download.
532func (s *DownloadService) flushLogs(id int64) {
533 logs := s.cache.FlushLogs(id)
534 if logs == "" {
535 return
536 }
537 if err := s.repo.AppendLogs(id, logs); err != nil {
538 log.Printf("download %d: failed to persist logs: %v", id, err)
539 }
540}
541
542// resolveBaseLibraryDir returns the absolute library directory a download writes
543// into, applying the optional per-download OutputDir while rejecting any path
544// that escapes the library root.
545func (s *DownloadService) resolveBaseLibraryDir(d *models.Download) (string, error) {
546 if !d.OutputDir.Valid || d.OutputDir.String == "" {
547 return s.cfg.LibraryDir, nil
548 }
549 // Reuse the library service's guard so both entry points enforce the boundary
550 // the same way — it resolves symlinks, which a plain prefix check does not.
551 dir, err := s.librarySvc.ResolveWithinLibrary(d.OutputDir.String)
552 if err != nil {
553 return "", fmt.Errorf("invalid output directory: %w", err)
554 }
555 return dir, nil
556}
557
558// importDownloadedItems moves each downloaded item from the temp dir into the
559// library and returns the number of items successfully imported. Per-item
560// failures are logged and skipped (a playlist with a few bad entries still
561// imports the rest); a non-nil error means the import couldn't even start.
562//
563// A cancel stops the import between items and returns ctx.Err() with the count
564// imported so far. Importing a long playlist takes real time (a move plus an
565// ffprobe per file), so a deleted download must not keep filling the library.
566func (s *DownloadService) importDownloadedItems(ctx context.Context, d *models.Download, tempDownloadDir, mode, ytdlpFlags string) (int, error) {
567 entries, err := os.ReadDir(tempDownloadDir)
568 if err != nil {
569 return 0, err
570 }
571
572 baseLibraryDir, err := s.resolveBaseLibraryDir(d)
573 if err != nil {
574 return 0, err
575 }
576 if err := os.MkdirAll(baseLibraryDir, 0755); err != nil {
577 return 0, err
578 }
579
580 var itemDirs []string
581 for _, entry := range entries {
582 if !entry.IsDir() {
583 continue
584 }
585 name := entry.Name()
586 if strings.HasPrefix(name, "item-") {
587 itemDirs = append(itemDirs, filepath.Join(tempDownloadDir, name))
588 }
589 }
590 sort.Strings(itemDirs)
591
592 imported := 0
593 for _, itemDir := range itemDirs {
594 if err := ctx.Err(); err != nil {
595 return imported, err
596 }
597 if err := s.importItemDir(ctx, d.URL, itemDir, baseLibraryDir, mode, ytdlpFlags); err != nil {
598 // A cancelled item isn't a bad item: stop instead of logging a warning
599 // for it and every one that follows.
600 if ctx.Err() != nil {
601 return imported, ctx.Err()
602 }
603 log.Printf("warning: failed to import item %s: %v", itemDir, err)
604 continue
605 }
606 imported++
607 }
608
609 // The temp dir (and any leftovers from failed imports) is removed by the
610 // caller's deferred cleanup, so partial state never leaks even on a crash.
611 return imported, nil
612}
613
614func (s *DownloadService) importItemDir(ctx context.Context, url, itemDir, baseLibraryDir, mode, ytdlpFlags string) error {
615 entries, err := os.ReadDir(itemDir)
616 if err != nil {
617 return err
618 }
619
620 var mediaFiles []os.DirEntry
621 var infoJSONPath string
622 var subtitleFiles []string
623
624 for _, entry := range entries {
625 if entry.IsDir() {
626 continue
627 }
628 name := entry.Name()
629 path := filepath.Join(itemDir, name)
630 ext := strings.ToLower(filepath.Ext(name))
631
632 if isInfoJSON(name) {
633 infoJSONPath = path
634 continue
635 }
636 if ext == ".vtt" || ext == ".srt" || ext == ".ass" || ext == ".ssa" {
637 subtitleFiles = append(subtitleFiles, path)
638 continue
639 }
640
641 mtype, err := mimetype.DetectFile(path)
642 if err == nil && mtype != nil && (strings.HasPrefix(mtype.String(), "audio/") || strings.HasPrefix(mtype.String(), "video/")) {
643 mediaFiles = append(mediaFiles, entry)
644 }
645 }
646
647 if len(mediaFiles) == 0 {
648 return fmt.Errorf("no media files found in %s", itemDir)
649 }
650
651 info := readInfoJSON(infoJSONPath)
652 name := s.deriveItemName(itemDir, info, mediaFiles)
653 videoID := info.ID
654
655 // Last point at which nothing has been written to the library yet: give up
656 // here on a cancel rather than part-way through, which would leave a folder
657 // with some of its files and no marker — or, in overwrite mode, delete the
658 // existing item and not replace it.
659 if err := ctx.Err(); err != nil {
660 return err
661 }
662
663 // Overwrite mode: replace the existing copy of this video in place rather than
664 // creating a duplicate folder. Removing the old dir lets uniqueDir reuse its
665 // name (or land on the new title if it changed upstream).
666 if mode == "overwrite" && videoID != "" {
667 if existing, ok := s.librarySvc.FindByVideoID(baseLibraryDir, videoID); ok {
668 if rel, err := filepath.Rel(s.cfg.LibraryDir, existing); err == nil {
669 s.librarySvc.evictCachedScan(filepath.ToSlash(rel))
670 }
671 os.RemoveAll(existing)
672 }
673 }
674
675 targetDir := s.uniqueDir(baseLibraryDir, name)
676 if err := os.MkdirAll(targetDir, 0755); err != nil {
677 return err
678 }
679
680 if infoJSONPath != "" {
681 if err := moveFile(infoJSONPath, filepath.Join(targetDir, "info.json")); err != nil {
682 return err
683 }
684 }
685
686 for _, entry := range mediaFiles {
687 if err := moveFile(filepath.Join(itemDir, entry.Name()), filepath.Join(targetDir, entry.Name())); err != nil {
688 return err
689 }
690 }
691
692 // Probe each media file's duration once, here in the worker (off the request
693 // path), and cache it in the marker so the library never has to probe while
694 // serving pages. Files we can't probe simply get no duration.
695 fileDurations := make(map[string]int)
696 for _, entry := range mediaFiles {
697 if d, ok := probeDuration(ctx, s.cfg.FFprobePath, filepath.Join(targetDir, entry.Name())); ok {
698 fileDurations[entry.Name()] = d
699 }
700 }
701
702 if len(subtitleFiles) > 0 {
703 subtitlesDir := filepath.Join(targetDir, subtitlesDirName)
704 if err := os.MkdirAll(subtitlesDir, 0755); err != nil {
705 return err
706 }
707 for _, sf := range subtitleFiles {
708 if err := moveFile(sf, filepath.Join(subtitlesDir, filepath.Base(sf))); err != nil {
709 return err
710 }
711 }
712 }
713
714 metadata := models.ItemMetadata{
715 Name: name,
716 SourceURL: url,
717 VideoID: videoID,
718 YtdlpFlags: ytdlpFlags,
719 FileDurations: fileDurations,
720 }
721
722 return s.librarySvc.writeMetadata(targetDir, metadata)
723}
724
725// infoJSON is the subset of yt-dlp's info.json VidArchive reads. ID is the
726// stable item identity; within a single subscription's own directory it is
727// enough to match items, so the extractor is not needed.
728type infoJSON struct {
729 ID string `json:"id"`
730 Title string `json:"title"`
731 Description string `json:"description"`
732 WebpageURL string `json:"webpage_url"`
733}
734
735// readInfoJSON parses an info.json. A missing, unreadable or malformed file
736// yields a zero-value struct: every caller treats absent fields as "unknown"
737// and falls back, so there is nothing to distinguish.
738func readInfoJSON(infoJSONPath string) infoJSON {
739 var info infoJSON
740 if infoJSONPath == "" {
741 return info
742 }
743 data, err := os.ReadFile(infoJSONPath)
744 if err != nil {
745 return info
746 }
747 if err := json.Unmarshal(data, &info); err != nil {
748 log.Printf("ignoring malformed %s: %v", infoJSONPath, err)
749 return infoJSON{}
750 }
751 return info
752}
753
754// refreshAndAddNew handles a metadata-mode run. The main pass used
755// --skip-download, so tempDownloadDir holds only info.json files. Existing
756// library items have their markers refreshed in place; entries with no existing
757// match are genuinely new and are downloaded as full items in a second pass.
758func (s *DownloadService) refreshAndAddNew(ctx context.Context, d *models.Download, preset *models.Preset, tempDownloadDir, ytdlpFlags string) error {
759 // The library dir is not created here: a refresh that matches everything
760 // writes nothing, and importDownloadedItems creates it when a second pass
761 // actually has an item to add.
762 baseLibraryDir, err := s.resolveBaseLibraryDir(d)
763 if err != nil {
764 return err
765 }
766
767 entries, err := os.ReadDir(tempDownloadDir)
768 if err != nil {
769 return err
770 }
771
772 var newURLs []string
773 for _, entry := range entries {
774 if err := ctx.Err(); err != nil {
775 return err
776 }
777 if !entry.IsDir() || !strings.HasPrefix(entry.Name(), "item-") {
778 continue
779 }
780 itemDir := filepath.Join(tempDownloadDir, entry.Name())
781 infoJSONPath := findInfoJSON(itemDir)
782 if infoJSONPath == "" {
783 continue
784 }
785 info := readInfoJSON(infoJSONPath)
786 if info.ID == "" {
787 continue
788 }
789 if existing, ok := s.librarySvc.FindByVideoID(baseLibraryDir, info.ID); ok {
790 if err := s.applyMetadata(existing, info, infoJSONPath); err != nil {
791 log.Printf("warning: failed to refresh metadata for %s: %v", itemDir, err)
792 }
793 continue
794 }
795 if info.WebpageURL != "" {
796 newURLs = append(newURLs, info.WebpageURL)
797 }
798 }
799
800 if len(newURLs) == 0 {
801 return nil
802 }
803 return s.downloadFresh(ctx, d, preset, newURLs, ytdlpFlags)
804}
805
806// downloadFresh fetches the given item URLs as full downloads (media + info.json)
807// and imports them into the download's library directory. Metadata mode uses this
808// to add entries that don't exist in the library yet.
809func (s *DownloadService) downloadFresh(ctx context.Context, d *models.Download, preset *models.Preset, urls []string, ytdlpFlags string) error {
810 tempDir := s.tempNewDirFor(d.ID)
811 if err := os.MkdirAll(tempDir, 0755); err != nil {
812 return err
813 }
814 defer os.RemoveAll(tempDir)
815
816 args := s.presetSvc.BuildArgs(preset, d.FormatOverride, d.CustomFlags)
817 args, cleanup := s.appendCookies(args)
818 defer cleanup()
819 args = append(args, "--write-info-json")
820 args = append(args, "-P", tempDir)
821 args = append(args, "-o", "item-%(autonumber)05d/%(title)s.%(ext)s")
822 args = append(args, urls...)
823
824 runErr := s.runYTDLP(ctx, d, args)
825 // A cancelled second pass has nothing worth importing.
826 if ctx.Err() != nil {
827 return runErr
828 }
829 // Import whatever succeeded even if some entries errored.
830 if _, err := s.importDownloadedItems(ctx, d, tempDir, "", ytdlpFlags); err != nil {
831 log.Printf("warning: failed to import new metadata-mode items: %v", err)
832 }
833 return runErr
834}
835
836// mergeInfoJSON keeps fields from the existing sidecar that are absent from a
837// metadata-only refresh. In particular, comments and heatmap data are expensive
838// to reacquire and must not disappear just because the refresh preset does not
839// request them.
840func mergeInfoJSON(oldData, newData []byte) ([]byte, error) {
841 var oldObject, newObject map[string]json.RawMessage
842 if err := json.Unmarshal(newData, &newObject); err != nil {
843 return nil, err
844 }
845 if err := json.Unmarshal(oldData, &oldObject); err != nil {
846 return newData, nil
847 }
848 if newObject == nil {
849 return newData, nil
850 }
851
852 merged := make(map[string]json.RawMessage, len(oldObject)+len(newObject))
853 for key, value := range oldObject {
854 merged[key] = value
855 }
856 for key, value := range newObject {
857 merged[key] = value
858 }
859 for _, key := range []string{"comments", "heatmap"} {
860 oldValue, hadOldValue := oldObject[key]
861 newValue, hasNewValue := newObject[key]
862 if hadOldValue && (!hasNewValue || isEmptyJSONArray(newValue)) {
863 merged[key] = oldValue
864 }
865 }
866 return json.Marshal(merged)
867}
868
869func isEmptyJSONArray(value json.RawMessage) bool {
870 var values []json.RawMessage
871 if err := json.Unmarshal(value, &values); err != nil {
872 return false
873 }
874 return len(values) == 0
875}
876
877// restoreFileAtomically puts data back at path without exposing a partial file.
878// It is used to roll back the marker if installing the staged info sidecar fails.
879func restoreFileAtomically(path string, data []byte) error {
880 tmp, err := os.CreateTemp(filepath.Dir(path), ".vidarchive-restore-*.tmp")
881 if err != nil {
882 return err
883 }
884 tmpPath := tmp.Name()
885 defer os.Remove(tmpPath)
886 if err := tmp.Chmod(0644); err != nil {
887 tmp.Close()
888 return err
889 }
890 if _, err := tmp.Write(data); err != nil {
891 tmp.Close()
892 return err
893 }
894 if err := tmp.Close(); err != nil {
895 return err
896 }
897 if err := os.Rename(tmpPath, path); err != nil {
898 return err
899 }
900 return nil
901}
902
903// applyMetadata refreshes an existing item's marker and info sidecar from a
904// fresh info.json without touching its media.
905func (s *DownloadService) applyMetadata(existing string, info infoJSON, sourceInfoJSON string) error {
906 meta, err := s.librarySvc.readMetadata(existing)
907 if err != nil {
908 return fmt.Errorf("read metadata for %s: %w", existing, err)
909 }
910 if info.Title != "" {
911 meta.Name = info.Title
912 }
913 if info.Description != "" {
914 meta.Description = info.Description
915 }
916 if meta.SourceURL == "" && info.WebpageURL != "" {
917 meta.SourceURL = info.WebpageURL
918 }
919 meta.VideoID = info.ID
920
921 markerPath := filepath.Join(existing, itemMarkerName)
922 oldMarker, err := os.ReadFile(markerPath)
923 if err != nil {
924 return fmt.Errorf("read existing marker: %w", err)
925 }
926
927 // Stage the sidecar before changing the marker. The marker is committed first;
928 // if installing the sidecar then fails, restore the old marker so an ordinary
929 // I/O error cannot leave the two metadata files out of sync.
930 stagedInfo := ""
931 defer func() {
932 if stagedInfo != "" {
933 _ = os.Remove(stagedInfo)
934 }
935 }()
936 if sourceInfoJSON != "" {
937 data, err := os.ReadFile(sourceInfoJSON)
938 if err != nil {
939 return fmt.Errorf("read refreshed info JSON: %w", err)
940 }
941 if oldInfoJSON := findInfoJSON(existing); oldInfoJSON != "" {
942 oldData, err := os.ReadFile(oldInfoJSON)
943 if err != nil {
944 return fmt.Errorf("read existing info JSON: %w", err)
945 }
946 data, err = mergeInfoJSON(oldData, data)
947 if err != nil {
948 return fmt.Errorf("merge refreshed info JSON: %w", err)
949 }
950 }
951 tmp, err := os.CreateTemp(existing, ".info-json-*.tmp")
952 if err != nil {
953 return fmt.Errorf("create refreshed info JSON: %w", err)
954 }
955 stagedInfo = tmp.Name()
956 if err := tmp.Chmod(0644); err != nil {
957 tmp.Close()
958 return fmt.Errorf("set refreshed info JSON permissions: %w", err)
959 }
960 if _, err := tmp.Write(data); err != nil {
961 tmp.Close()
962 return fmt.Errorf("write refreshed info JSON: %w", err)
963 }
964 if err := tmp.Close(); err != nil {
965 return fmt.Errorf("close refreshed info JSON: %w", err)
966 }
967 }
968
969 if err := s.librarySvc.writeMetadata(existing, meta); err != nil {
970 return err
971 }
972 if stagedInfo != "" {
973 if err := os.Rename(stagedInfo, filepath.Join(existing, "info.json")); err != nil {
974 if restoreErr := restoreFileAtomically(markerPath, oldMarker); restoreErr != nil {
975 return fmt.Errorf("install refreshed info JSON: %v; restore marker: %w", err, restoreErr)
976 }
977 return fmt.Errorf("install refreshed info JSON: %w", err)
978 }
979 stagedInfo = ""
980 }
981 if rel, err := filepath.Rel(s.cfg.LibraryDir, existing); err == nil {
982 s.librarySvc.evictCachedScan(filepath.ToSlash(rel))
983 }
984 return nil
985}
986
987// isInfoJSON reports whether a file name is yt-dlp's metadata sidecar. yt-dlp
988// writes "<title>.info.json" next to the media, but a bare "info.json" is what
989// an already-imported item holds.
990func isInfoJSON(name string) bool {
991 return name == "info.json" || strings.HasSuffix(name, ".info.json")
992}
993
994// findInfoJSON returns the path to an info.json directly inside itemDir, or "".
995func findInfoJSON(itemDir string) string {
996 entries, err := os.ReadDir(itemDir)
997 if err != nil {
998 return ""
999 }
1000 for _, entry := range entries {
1001 if entry.IsDir() {
1002 continue
1003 }
1004 name := entry.Name()
1005 if isInfoJSON(name) {
1006 return filepath.Join(itemDir, name)
1007 }
1008 }
1009 return ""
1010}
1011
1012// pruneSubscription mirrors the source by deleting items in the subscription's
1013// directory that are no longer present upstream. It enumerates the current id
1014// set with a cheap flat-playlist listing; it never prunes when that enumeration
1015// fails or returns nothing, so a dead URL or network error can't wipe the dir.
1016func (s *DownloadService) pruneSubscription(ctx context.Context, d *models.Download, sub *models.Subscription) {
1017 baseLibraryDir, err := s.resolveBaseLibraryDir(d)
1018 if err != nil {
1019 log.Printf("subscription %d prune skipped: %v", sub.ID, err)
1020 return
1021 }
1022
1023 keep, err := s.enumeratePlaylistIDs(ctx, sub.URL)
1024 if err != nil {
1025 log.Printf("subscription %d prune skipped: enumeration failed: %v", sub.ID, err)
1026 return
1027 }
1028 if len(keep) == 0 {
1029 log.Printf("subscription %d prune skipped: source returned no entries", sub.ID)
1030 return
1031 }
1032
1033 removed, err := s.librarySvc.PruneToIDSet(baseLibraryDir, keep)
1034 if err != nil {
1035 log.Printf("subscription %d prune error: %v", sub.ID, err)
1036 return
1037 }
1038 if removed > 0 {
1039 log.Printf("subscription %d pruned %d item(s) removed upstream", sub.ID, removed)
1040 }
1041}
1042
1043// enumeratePlaylistIDs lists the current video-id set for a URL without
1044// downloading, using yt-dlp --flat-playlist. Cookies are applied so private
1045// playlists enumerate correctly. Ids alone are sufficient to match items within
1046// a subscription's own directory (see FindByVideoID / PruneToIDSet).
1047func (s *DownloadService) enumeratePlaylistIDs(ctx context.Context, url string) (map[string]bool, error) {
1048 args := []string{"--flat-playlist", "--no-warnings", "--print", "%(id)s"}
1049
1050 args, cleanup := s.appendCookies(args)
1051 defer cleanup()
1052 args = append(args, url)
1053
1054 out, err := exec.CommandContext(ctx, s.cfg.YTDLPPath, args...).Output()
1055 if err != nil {
1056 return nil, err
1057 }
1058
1059 keep := make(map[string]bool)
1060 for _, line := range strings.Split(string(out), "\n") {
1061 id := strings.TrimSpace(line)
1062 // yt-dlp prints "NA" for a missing field; never treat that as a real id.
1063 if id == "" || id == "NA" {
1064 continue
1065 }
1066 keep[id] = true
1067 }
1068 return keep, nil
1069}
1070
1071// deriveItemName names the imported item after its title, falling back to the
1072// largest media file's base name when there is no usable info.json.
1073func (s *DownloadService) deriveItemName(itemDir string, info infoJSON, mediaFiles []os.DirEntry) string {
1074 if info.Title != "" {
1075 return sanitizeDirName(info.Title)
1076 }
1077
1078 var largest os.DirEntry
1079 var maxSize int64
1080 for _, f := range mediaFiles {
1081 st, err := os.Stat(filepath.Join(itemDir, f.Name()))
1082 if err == nil && (largest == nil || st.Size() > maxSize) {
1083 largest, maxSize = f, st.Size()
1084 }
1085 }
1086 if largest == nil {
1087 largest = mediaFiles[0]
1088 }
1089 base := strings.TrimSuffix(largest.Name(), filepath.Ext(largest.Name()))
1090 return sanitizeDirName(base)
1091}
1092
1093func (s *DownloadService) uniqueDir(base, name string) string {
1094 dir := filepath.Join(base, name)
1095 if _, err := os.Stat(dir); os.IsNotExist(err) {
1096 return dir
1097 }
1098 for i := 1; ; i++ {
1099 candidate := fmt.Sprintf("%s-%d", dir, i)
1100 if _, err := os.Stat(candidate); os.IsNotExist(err) {
1101 return candidate
1102 }
1103 }
1104}
1105
1106// moveFile moves src to dst, falling back to copy-and-delete when the two are on
1107// different filesystems. The temp and library directories are independently
1108// configurable, so they can legitimately live on separate mounts — where a plain
1109// rename fails with EXDEV.
1110func moveFile(src, dst string) error {
1111 if err := os.Rename(src, dst); err == nil {
1112 return nil
1113 } else if !errors.Is(err, syscall.EXDEV) {
1114 return err
1115 }
1116
1117 in, err := os.Open(src)
1118 if err != nil {
1119 return err
1120 }
1121 defer in.Close()
1122
1123 info, err := in.Stat()
1124 if err != nil {
1125 return err
1126 }
1127
1128 out, err := os.OpenFile(dst, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, info.Mode())
1129 if err != nil {
1130 return err
1131 }
1132
1133 if _, err := io.Copy(out, in); err != nil {
1134 out.Close()
1135 os.Remove(dst)
1136 return err
1137 }
1138 // Close explicitly: a deferred close would hide a flush error on the copy.
1139 if err := out.Close(); err != nil {
1140 os.Remove(dst)
1141 return err
1142 }
1143
1144 return os.Remove(src)
1145}
1146
1147func sanitizeDirName(name string) string {
1148 name = strings.TrimSpace(name)
1149 replacer := strings.NewReplacer(
1150 "/", "-",
1151 "\\", "-",
1152 ":", "-",
1153 "*", "-",
1154 "?", "-",
1155 "\"", "-",
1156 "<", "-",
1157 ">", "-",
1158 "|", "-",
1159 )
1160 name = replacer.Replace(name)
1161 name = strings.TrimSpace(name)
1162 if name == "" {
1163 name = "untitled"
1164 }
1165 return name
1166}
1167
1168// probeDuration returns the duration of a media file in whole seconds. The bool
1169// is false when ffprobe is unavailable or the file has no usable duration.
1170func probeDuration(ctx context.Context, ffprobePath, path string) (int, bool) {
1171 out, err := exec.CommandContext(ctx, ffprobePath, "-v", "error",
1172 "-show_entries", "format=duration",
1173 "-of", "default=nw=1:nk=1", path).Output()
1174 if err != nil {
1175 return 0, false
1176 }
1177 f, err := strconv.ParseFloat(strings.TrimSpace(string(out)), 64)
1178 if err != nil || f <= 0 {
1179 return 0, false
1180 }
1181 return int(f + 0.5), true
1182}
1183
1184// ytFormat mirrors the subset of yt-dlp's per-format JSON (-J) we surface.
1185// Numeric fields are pointers so an absent value (null/omitted) is distinct
1186// from a real zero.
1187type ytFormat struct {
1188 FormatID string `json:"format_id"`
1189 Ext string `json:"ext"`
1190 Resolution string `json:"resolution"`
1191 Width *int `json:"width"`
1192 Height *int `json:"height"`
1193 FPS *float64 `json:"fps"`
1194 VCodec string `json:"vcodec"`
1195 ACodec string `json:"acodec"`
1196 AudioChannels *int `json:"audio_channels"`
1197 Filesize *int64 `json:"filesize"`
1198 FilesizeApprox *int64 `json:"filesize_approx"`
1199 FormatNote string `json:"format_note"`
1200}
1201
1202// parseFormatJSON reads yt-dlp's single-JSON dump (-J) and returns the available
1203// formats. For a single video the formats live at the top level; for a playlist
1204// URL we fall back to the first entry's formats so the picker still shows
1205// something useful.
1206func parseFormatJSON(data []byte) ([]*models.FormatInfo, error) {
1207 var top struct {
1208 Formats []ytFormat `json:"formats"`
1209 Entries []struct {
1210 Formats []ytFormat `json:"formats"`
1211 } `json:"entries"`
1212 }
1213 if err := json.Unmarshal(data, &top); err != nil {
1214 return nil, fmt.Errorf("parse yt-dlp JSON: %w", err)
1215 }
1216
1217 raw := top.Formats
1218 if len(raw) == 0 && len(top.Entries) > 0 {
1219 raw = top.Entries[0].Formats
1220 }
1221
1222 formats := make([]*models.FormatInfo, 0, len(raw))
1223 for _, f := range raw {
1224 formats = append(formats, f.toFormatInfo())
1225 }
1226 return formats, nil
1227}
1228
1229func (f ytFormat) toFormatInfo() *models.FormatInfo {
1230 fi := &models.FormatInfo{
1231 ID: f.FormatID,
1232 Ext: f.Ext,
1233 Note: f.FormatNote,
1234 }
1235
1236 switch {
1237 case f.Resolution != "":
1238 fi.Resolution = f.Resolution
1239 case f.Width != nil && f.Height != nil && *f.Width > 0 && *f.Height > 0:
1240 fi.Resolution = fmt.Sprintf("%dx%d", *f.Width, *f.Height)
1241 }
1242
1243 if f.FPS != nil && *f.FPS > 0 {
1244 fi.FPS = strconv.FormatFloat(*f.FPS, 'f', -1, 64)
1245 }
1246 if f.AudioChannels != nil && *f.AudioChannels > 0 {
1247 fi.Channels = strconv.Itoa(*f.AudioChannels)
1248 }
1249
1250 // Prefer the video codec; fall back to the audio codec for audio-only formats.
1251 if f.VCodec != "" && f.VCodec != "none" {
1252 fi.Codec = f.VCodec
1253 } else if f.ACodec != "" && f.ACodec != "none" {
1254 fi.Codec = f.ACodec
1255 }
1256
1257 if f.Filesize != nil && *f.Filesize > 0 {
1258 fi.FileSize = util.FormatBytes(*f.Filesize)
1259 } else if f.FilesizeApprox != nil && *f.FilesizeApprox > 0 {
1260 fi.FileSize = "~" + util.FormatBytes(*f.FilesizeApprox)
1261 }
1262
1263 return fi
1264}
1265
1266func sqlNullInt64(v int64) sql.NullInt64 {
1267 return sql.NullInt64{Int64: v, Valid: true}
1268}
1269
1270// reservedFlags are yt-dlp options VidArchive always sets itself; user custom
1271// flags must not pass them (or a conflicting inverse). The value describes what
1272// the option controls, for the failure message.
1273var reservedFlags = map[string]string{
1274 "-o": "the output template",
1275 "--output": "the output template",
1276 "-P": "the download path",
1277 "--paths": "the download path",
1278 "--cookies": "cookies (set these in Settings instead)",
1279 "--no-cookies": "cookies (set these in Settings instead)",
1280 "--newline": "progress output formatting (VidArchive sets this to stream logs)",
1281
1282 // These hand yt-dlp an arbitrary command or binary to run. VidArchive passes
1283 // custom flags through verbatim, so allowing them would turn the preset form
1284 // into remote command execution.
1285 "--exec": "running external commands (not permitted)",
1286 "--exec-before-download": "running external commands (not permitted)",
1287 "--postprocessor-args": "post-processor arguments (not permitted)",
1288 "--ppa": "post-processor arguments (not permitted)",
1289 "--downloader": "selecting an external downloader (not permitted)",
1290 "--external-downloader": "selecting an external downloader (not permitted)",
1291 "--downloader-args": "external downloader arguments (not permitted)",
1292 "--external-downloader-args": "external downloader arguments (not permitted)",
1293}
1294
1295// reservedSubscriptionFlags are additionally reserved for subscription runs,
1296// where VidArchive drives info-json writing and the refresh mode.
1297var reservedSubscriptionFlags = map[string]string{
1298 "--write-info-json": "info-json writing (needed to track item identity)",
1299 "--no-write-info-json": "info-json writing (needed to track item identity)",
1300 "--download-archive": "the download archive (managed by Skip mode)",
1301 "--no-download-archive": "the download archive (managed by Skip mode)",
1302 "--skip-download": "media downloading (managed by Metadata mode)",
1303 "--no-skip-download": "media downloading (managed by Metadata mode)",
1304}
1305
1306// checkReservedFlags rejects custom flags that clash with options VidArchive
1307// controls, naming the offender. It matches both "--flag" and "--flag=value".
1308func checkReservedFlags(customFlags string, isSubscription bool) error {
1309 for _, tok := range strings.Fields(customFlags) {
1310 // Both "--flag value" and "--flag=value" name the same option.
1311 name, _, _ := strings.Cut(tok, "=")
1312
1313 desc, ok := reservedFlags[name]
1314 if !ok && isSubscription {
1315 desc, ok = reservedSubscriptionFlags[name]
1316 }
1317 if ok {
1318 return fmt.Errorf("custom flag %q conflicts with VidArchive's handling of %s; remove it and try again", tok, desc)
1319 }
1320 }
1321 return nil
1322}
1323