vidarchive

vidarchive/internal/service/download.go
download.go
⎇
Raw
1package service
2
3import (
4 "bufio"
5 "bytes"
6 "database/sql"
7 "encoding/json"
8 "fmt"
9 "log"
10 "os"
11 "os/exec"
12 "path/filepath"
13 "sort"
14 "strconv"
15 "strings"
16 "sync"
17 "syscall"
18 "time"
19
20 "github.com/BurntSushi/toml"
21 "github.com/gabriel-vasile/mimetype"
22
23 "vidarchive/internal/config"
24 "vidarchive/internal/models"
25 "vidarchive/internal/repository"
26)
27
28type DownloadService struct {
29 repo *repository.DownloadRepository
30 librarySvc *LibraryService
31 presetSvc *PresetService
32 settingsSvc *SettingsService
33 subscriptionSvc *SubscriptionService
34 cfg *config.Config
35 cache *ProgressCache
36 processMu sync.Mutex
37 processes map[int64]*os.Process
38}
39
40func NewDownloadService(repo *repository.DownloadRepository, librarySvc *LibraryService, presetSvc *PresetService, settingsSvc *SettingsService, subscriptionSvc *SubscriptionService, cfg *config.Config) *DownloadService {
41 return &DownloadService{
42 repo: repo,
43 librarySvc: librarySvc,
44 presetSvc: presetSvc,
45 settingsSvc: settingsSvc,
46 subscriptionSvc: subscriptionSvc,
47 cfg: cfg,
48 cache: NewProgressCache(),
49 processes: make(map[int64]*os.Process),
50 }
51}
52
53func (s *DownloadService) Create(url string, presetID *int64, formatOverride, customFlags, outputDir string) (*models.Download, error) {
54 d := &models.Download{
55 URL: url,
56 Status: "queued",
57 FormatOverride: formatOverride,
58 CustomFlags: customFlags,
59 OutputDir: sql.NullString{String: outputDir, Valid: outputDir != ""},
60 }
61
62 if presetID != nil {
63 d.PresetID = sqlNullInt64(*presetID)
64 }
65
66 if err := s.repo.Create(d); err != nil {
67 return nil, err
68 }
69 return d, nil
70}
71
72// CreateForSubscription queues a download for a subscription run, copying its
73// download options and tagging it with the subscription id so ExecuteDownload
74// applies the right refresh mode and pruning.
75func (s *DownloadService) CreateForSubscription(sub *models.Subscription) (*models.Download, error) {
76 d := &models.Download{
77 URL: sub.URL,
78 Status: "queued",
79 FormatOverride: sub.FormatOverride,
80 CustomFlags: sub.CustomFlags,
81 OutputDir: sql.NullString{String: sub.OutputDir, Valid: sub.OutputDir != ""},
82 PresetID: sub.PresetID,
83 SubscriptionID: sqlNullInt64(sub.ID),
84 }
85 if err := s.repo.Create(d); err != nil {
86 return nil, err
87 }
88 return d, nil
89}
90
91func (s *DownloadService) GetByID(id int64) (*models.Download, error) {
92 d, err := s.repo.GetByID(id)
93 if err != nil {
94 return nil, err
95 }
96 if logs := s.cache.Snapshot(id); logs != "" {
97 d.Logs = sql.NullString{String: logs, Valid: true}
98 }
99 return d, nil
100}
101
102func (s *DownloadService) GetAll(status, sortBy string) ([]*models.Download, error) {
103 downloads, err := s.repo.GetAll(status, sortBy)
104 if err != nil {
105 return nil, err
106 }
107
108 for _, d := range downloads {
109 if logs := s.cache.Snapshot(d.ID); logs != "" {
110 d.Logs = sql.NullString{String: logs, Valid: true}
111 }
112 }
113
114 return downloads, nil
115}
116
117func (s *DownloadService) GetQueued(limit int) ([]*models.Download, error) {
118 return s.repo.GetQueued(limit)
119}
120
121// HasActiveForSubscription reports whether the subscription already has a queued
122// or in-progress download, so the scheduler can skip stacking another run.
123func (s *DownloadService) HasActiveForSubscription(subID int64) (bool, error) {
124 return s.repo.HasActiveForSubscription(subID)
125}
126
127func (s *DownloadService) Delete(id int64) error {
128 s.killProcess(id)
129 s.cache.Delete(id)
130 return s.repo.Delete(id)
131}
132
133func (s *DownloadService) killProcess(id int64) {
134 s.processMu.Lock()
135 proc, ok := s.processes[id]
136 delete(s.processes, id)
137 s.processMu.Unlock()
138
139 if !ok || proc == nil {
140 return
141 }
142
143 _ = syscall.Kill(-proc.Pid, syscall.SIGKILL)
144}
145
146func (s *DownloadService) DeleteAll() error {
147 return s.repo.DeleteAll()
148}
149
150func (s *DownloadService) ResetStalledDownloads() error {
151 return s.repo.UpdateStatusWhere("downloading", "queued")
152}
153
154func (s *DownloadService) ListFormats(url string) ([]*models.FormatInfo, error) {
155 // Use machine-readable JSON (-J) rather than scraping the human "-F" table,
156 // whose columns/separators shift between yt-dlp versions. stderr is captured
157 // separately so warnings can't corrupt the JSON on stdout.
158 cmd := exec.Command(s.cfg.YTDLPPath, "-J", "--no-warnings", url)
159 var stderr bytes.Buffer
160 cmd.Stderr = &stderr
161 output, err := cmd.Output()
162 if err != nil {
163 return nil, fmt.Errorf("yt-dlp -J failed: %w\n%s", err, stderr.String())
164 }
165
166 return parseFormatJSON(output)
167}
168
169// ExecuteDownload runs the download for d. The bool reports whether this call
170// actually processed it: false means another worker already claimed it (Submit
171// and the queue checker can both enqueue the same row within the 2s poll window),
172// so the caller should not log it as completed.
173func (s *DownloadService) ExecuteDownload(d *models.Download) (bool, error) {
174 // Atomically claim the download. If it's no longer queued, another worker
175 // already took it — bail rather than download it twice.
176 claimed, err := s.repo.MarkStarted(d.ID)
177 if err != nil {
178 return false, err
179 }
180 if !claimed {
181 return false, nil
182 }
183
184 s.cache.Set(d.ID, &LiveDownload{LastUpdate: time.Now()})
185 defer s.cache.Delete(d.ID)
186
187 var preset *models.Preset
188 if d.PresetID.Valid {
189 preset, err = s.presetSvc.GetByID(d.PresetID.Int64)
190 if err != nil {
191 log.Printf("download %d: preset %d lookup failed (%v); falling back to default", d.ID, d.PresetID.Int64, err)
192 preset = nil
193 }
194 }
195 if preset == nil {
196 var derr error
197 if preset, derr = s.presetSvc.GetDefault(); derr != nil {
198 log.Printf("download %d: no default preset available (%v); using built-in defaults", d.ID, derr)
199 preset = &models.Preset{}
200 }
201 }
202
203 var sub *models.Subscription
204 if d.SubscriptionID.Valid && s.subscriptionSvc != nil {
205 var serr error
206 if sub, serr = s.subscriptionSvc.GetByID(d.SubscriptionID.Int64); serr != nil {
207 log.Printf("download %d: subscription %d lookup failed: %v", d.ID, d.SubscriptionID.Int64, serr)
208 }
209 }
210
211 // Reject custom flags that clash with options VidArchive sets itself, before
212 // spending any work — the download fails with a message naming the offender.
213 isSubscription := d.SubscriptionID.Valid
214 if err := checkReservedFlags(d.CustomFlags, isSubscription); err != nil {
215 s.finalizeError(d.ID, err)
216 return false, err
217 }
218 if err := checkReservedFlags(preset.CustomFlags, isSubscription); err != nil {
219 s.finalizeError(d.ID, err)
220 return false, err
221 }
222
223 tempDownloadDir := filepath.Join(s.cfg.TempDir, fmt.Sprintf("%d", d.ID))
224 if err := os.MkdirAll(tempDownloadDir, 0755); err != nil {
225 return false, fmt.Errorf("create temp download dir: %w", err)
226 }
227 // Own the temp dir's lifetime here, where it's created, so it's removed on
228 // every exit path — including a failed yt-dlp run or an early return that
229 // crashes mid-import. The import helpers below no longer clean it up.
230 defer os.RemoveAll(tempDownloadDir)
231
232 args := s.presetSvc.BuildArgs(preset, d.FormatOverride, d.CustomFlags)
233
234 // Record the meaningful flags (format/audio/subs/custom) that shaped this
235 // download, before the internal plumbing (cookies, -P/-o, URL) is appended,
236 // so each imported item can show how it was fetched.
237 ytdlpFlags := strings.Join(args, " ")
238
239 var cookieCleanup func()
240 args, cookieCleanup = s.appendCookies(args)
241 defer cookieCleanup()
242
243 if sub != nil {
244 // Always write info.json so the import step can read the stable identity
245 // (yt-dlp's video id) used to match/replace existing items.
246 args = append(args, "--write-info-json")
247 switch sub.RefreshMode {
248 case "skip":
249 // Let yt-dlp skip entries already recorded — no re-download.
250 archive := s.subscriptionSvc.ArchivePath(sub.ID)
251 if err := os.MkdirAll(filepath.Dir(archive), 0755); err == nil {
252 args = append(args, "--download-archive", archive)
253 }
254 case "metadata":
255 // Refresh metadata only; don't fetch media.
256 args = append(args, "--skip-download")
257 }
258 }
259
260 args = append(args, "-P", tempDownloadDir)
261 args = append(args, "-o", "item-%(autonumber)05d/%(title)s.%(ext)s")
262 args = append(args, d.URL)
263
264 if err := s.runYTDLP(d, args); err != nil {
265 s.finalizeError(d.ID, err)
266 return false, err
267 }
268
269 mode := ""
270 if sub != nil {
271 mode = sub.RefreshMode
272 }
273
274 // Post-process before marking completed, so the download stays "downloading"
275 // until everything is really done — including metadata mode's second pass,
276 // which downloads any genuinely new entries as full items.
277 if mode == "metadata" {
278 // The main pass ran with --skip-download, so the temp dir holds only
279 // info.json files: refresh existing items in place and fetch new ones.
280 if err := s.refreshAndAddNew(d, preset, tempDownloadDir, ytdlpFlags); err != nil {
281 s.finalizeError(d.ID, err)
282 return false, err
283 }
284 } else {
285 imported, err := s.importDownloadedItems(d, tempDownloadDir, mode, ytdlpFlags)
286 if err != nil {
287 s.finalizeError(d.ID, err)
288 return false, err
289 }
290 // A plain (non-subscription) download that yields nothing is a failure, not
291 // a silent "completed". Subscription modes legitimately import zero (skip
292 // mode, or a metadata refresh with no new entries), so only enforce this for
293 // plain runs.
294 if sub == nil && imported == 0 {
295 err := fmt.Errorf("yt-dlp finished but no media files were downloaded")
296 s.finalizeError(d.ID, err)
297 return false, err
298 }
299 }
300
301 if sub != nil && sub.PruneRemoved {
302 s.pruneSubscription(d, sub)
303 }
304
305 if err := s.repo.MarkCompleted(d.ID, "completed"); err != nil {
306 return false, err
307 }
308
309 return true, nil
310}
311
312// runYTDLP executes yt-dlp with args, streaming combined output into the live
313// progress cache and periodically flushing it to the download's persisted log.
314// It registers the process so Delete can kill it, and returns the exit error.
315func (s *DownloadService) runYTDLP(d *models.Download, args []string) error {
316 // --newline forces yt-dlp to emit each progress update on its own line. Without
317 // it, progress is rewritten in place with carriage returns, so a long download
318 // becomes one ever-growing line that overflows the reader's buffer and stalls
319 // the pipe — hanging the download. See the hardened scanner below.
320 fullArgs := append([]string{"--newline"}, args...)
321 cmd := exec.Command(s.cfg.YTDLPPath, fullArgs...)
322 cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
323
324 stdout, err := cmd.StdoutPipe()
325 if err != nil {
326 return err
327 }
328 cmd.Stderr = cmd.Stdout
329
330 if err := cmd.Start(); err != nil {
331 return err
332 }
333
334 s.processMu.Lock()
335 s.processes[d.ID] = cmd.Process
336 s.processMu.Unlock()
337 defer func() {
338 s.processMu.Lock()
339 delete(s.processes, d.ID)
340 s.processMu.Unlock()
341 }()
342
343 ticker := time.NewTicker(10 * time.Second)
344 defer ticker.Stop()
345 done := make(chan struct{})
346 go func() {
347 for {
348 select {
349 case <-ticker.C:
350 if logs := s.cache.FlushLogs(d.ID); logs != "" {
351 s.repo.AppendLogs(d.ID, logs)
352 }
353 case <-done:
354 return
355 }
356 }
357 }()
358
359 scanner := bufio.NewScanner(stdout)
360 // Allow long lines (a single yt-dlp message can exceed the 64 KiB default)
361 // rather than letting the scanner abort and leave the pipe unread.
362 scanner.Buffer(make([]byte, 0, 64*1024), 1024*1024)
363 for scanner.Scan() {
364 s.cache.AppendLog(d.ID, scanner.Text())
365 }
366 if err := scanner.Err(); err != nil {
367 log.Printf("download %d: error reading yt-dlp output: %v", d.ID, err)
368 }
369 close(done)
370
371 if logs := s.cache.FlushLogs(d.ID); logs != "" {
372 s.repo.AppendLogs(d.ID, logs)
373 }
374
375 return cmd.Wait()
376}
377
378// appendCookies writes the saved cookies (if any) to a temp file and appends a
379// --cookies flag. The returned cleanup removes the temp file and is always safe
380// to call, even when no cookies were configured.
381func (s *DownloadService) appendCookies(args []string) ([]string, func()) {
382 cleanup := func() {}
383 cookies, err := s.settingsSvc.GetCookies()
384 if err != nil || strings.TrimSpace(cookies) == "" {
385 return args, cleanup
386 }
387 tmpFile, err := os.CreateTemp("", "cookies-*.txt")
388 if err != nil {
389 return args, cleanup
390 }
391 // A short write would hand yt-dlp a truncated cookies file; on any write/close
392 // failure, drop the temp file and proceed without cookies rather than silently
393 // using a broken one.
394 if _, err := tmpFile.WriteString(cookies); err != nil {
395 tmpFile.Close()
396 os.Remove(tmpFile.Name())
397 return args, cleanup
398 }
399 if err := tmpFile.Close(); err != nil {
400 os.Remove(tmpFile.Name())
401 return args, cleanup
402 }
403 return append(args, "--cookies", tmpFile.Name()), func() { os.Remove(tmpFile.Name()) }
404}
405
406func (s *DownloadService) finalizeError(id int64, err error) {
407 logs := s.cache.FlushLogs(id)
408 if logs != "" {
409 s.repo.AppendLogs(id, logs)
410 }
411 s.repo.MarkError(id, err.Error())
412}
413
414// resolveBaseLibraryDir returns the absolute library directory a download writes
415// into, applying the optional per-download OutputDir while rejecting any path
416// that escapes the library root.
417func (s *DownloadService) resolveBaseLibraryDir(d *models.Download) (string, error) {
418 baseLibraryDir := s.cfg.LibraryDir
419 if d.OutputDir.Valid && d.OutputDir.String != "" {
420 cleanDir := filepath.Clean(d.OutputDir.String)
421 fullPath := filepath.Join(baseLibraryDir, cleanDir)
422 resolvedPath, err := filepath.Abs(fullPath)
423 if err != nil {
424 return "", fmt.Errorf("invalid output directory: %w", err)
425 }
426 resolvedLibraryDir, _ := filepath.Abs(baseLibraryDir)
427 if !strings.HasPrefix(resolvedPath, resolvedLibraryDir+string(filepath.Separator)) && resolvedPath != resolvedLibraryDir {
428 return "", fmt.Errorf("invalid output directory: path traversal attempt detected")
429 }
430 baseLibraryDir = fullPath
431 }
432 return baseLibraryDir, nil
433}
434
435// importDownloadedItems moves each downloaded item from the temp dir into the
436// library and returns the number of items successfully imported. Per-item
437// failures are logged and skipped (a playlist with a few bad entries still
438// imports the rest); a non-nil error means the import couldn't even start.
439func (s *DownloadService) importDownloadedItems(d *models.Download, tempDownloadDir, mode, ytdlpFlags string) (int, error) {
440 entries, err := os.ReadDir(tempDownloadDir)
441 if err != nil {
442 return 0, err
443 }
444
445 baseLibraryDir, err := s.resolveBaseLibraryDir(d)
446 if err != nil {
447 return 0, err
448 }
449 if err := os.MkdirAll(baseLibraryDir, 0755); err != nil {
450 return 0, err
451 }
452
453 var itemDirs []string
454 for _, entry := range entries {
455 if !entry.IsDir() {
456 continue
457 }
458 name := entry.Name()
459 if strings.HasPrefix(name, "item-") {
460 itemDirs = append(itemDirs, filepath.Join(tempDownloadDir, name))
461 }
462 }
463 sort.Strings(itemDirs)
464
465 imported := 0
466 for _, itemDir := range itemDirs {
467 if err := s.importItemDir(d.URL, itemDir, baseLibraryDir, mode, ytdlpFlags); err != nil {
468 log.Printf("warning: failed to import item %s: %v", itemDir, err)
469 continue
470 }
471 imported++
472 }
473
474 // The temp dir (and any leftovers from failed imports) is removed by the
475 // caller's deferred cleanup, so partial state never leaks even on a crash.
476 return imported, nil
477}
478
479func (s *DownloadService) importItemDir(url, itemDir, baseLibraryDir, mode, ytdlpFlags string) error {
480 entries, err := os.ReadDir(itemDir)
481 if err != nil {
482 return err
483 }
484
485 var mediaFiles []os.DirEntry
486 var infoJSONPath string
487 var subtitleFiles []string
488
489 for _, entry := range entries {
490 if entry.IsDir() {
491 continue
492 }
493 name := entry.Name()
494 path := filepath.Join(itemDir, name)
495 ext := strings.ToLower(filepath.Ext(name))
496
497 if name == "info.json" || strings.HasSuffix(name, ".info.json") {
498 infoJSONPath = path
499 continue
500 }
501 if ext == ".vtt" || ext == ".srt" || ext == ".ass" || ext == ".ssa" {
502 subtitleFiles = append(subtitleFiles, path)
503 continue
504 }
505
506 mtype, err := mimetype.DetectFile(path)
507 if err == nil && mtype != nil && (strings.HasPrefix(mtype.String(), "audio/") || strings.HasPrefix(mtype.String(), "video/")) {
508 mediaFiles = append(mediaFiles, entry)
509 }
510 }
511
512 if len(mediaFiles) == 0 {
513 return fmt.Errorf("no media files found in %s", itemDir)
514 }
515
516 name := s.deriveItemName(itemDir, infoJSONPath, mediaFiles)
517 videoID := readInfoID(infoJSONPath)
518
519 // Overwrite mode: replace the existing copy of this video in place rather than
520 // creating a duplicate folder. Removing the old dir lets uniqueDir reuse its
521 // name (or land on the new title if it changed upstream).
522 if mode == "overwrite" && videoID != "" {
523 if existing, ok := s.librarySvc.FindByVideoID(baseLibraryDir, videoID); ok {
524 if rel, err := filepath.Rel(s.cfg.LibraryDir, existing); err == nil {
525 s.librarySvc.evictCachedScan(filepath.ToSlash(rel))
526 }
527 os.RemoveAll(existing)
528 }
529 }
530
531 targetDir := s.uniqueDir(baseLibraryDir, name)
532 if err := os.MkdirAll(targetDir, 0755); err != nil {
533 return err
534 }
535
536 if infoJSONPath != "" {
537 if err := os.Rename(infoJSONPath, filepath.Join(targetDir, "info.json")); err != nil {
538 return err
539 }
540 }
541
542 for _, entry := range mediaFiles {
543 if err := os.Rename(filepath.Join(itemDir, entry.Name()), filepath.Join(targetDir, entry.Name())); err != nil {
544 return err
545 }
546 }
547
548 // Probe each media file's duration once, here in the worker (off the request
549 // path), and cache it in the marker so the library never has to probe while
550 // serving pages. Files we can't probe simply get no duration.
551 fileDurations := make(map[string]int)
552 for _, entry := range mediaFiles {
553 if d, ok := probeDuration(s.cfg.FFprobePath, filepath.Join(targetDir, entry.Name())); ok {
554 fileDurations[entry.Name()] = d
555 }
556 }
557
558 if len(subtitleFiles) > 0 {
559 subtitlesDir := filepath.Join(targetDir, subtitlesDirName)
560 if err := os.MkdirAll(subtitlesDir, 0755); err != nil {
561 return err
562 }
563 for _, sf := range subtitleFiles {
564 if err := os.Rename(sf, filepath.Join(subtitlesDir, filepath.Base(sf))); err != nil {
565 return err
566 }
567 }
568 }
569
570 metadata := models.ItemMetadata{
571 Name: name,
572 SourceURL: url,
573 VideoID: videoID,
574 YtdlpFlags: ytdlpFlags,
575 FileDurations: fileDurations,
576 }
577
578 markerPath := filepath.Join(targetDir, itemMarkerName)
579 f, err := os.Create(markerPath)
580 if err != nil {
581 return err
582 }
583 defer f.Close()
584 if err := toml.NewEncoder(f).Encode(metadata); err != nil {
585 return err
586 }
587
588 return nil
589}
590
591// readInfoID extracts the stable item identity (yt-dlp's video id) from an
592// info.json. The id alone is sufficient to match items within a single
593// subscription's owned directory, so the extractor is not used. Returns an
594// empty string when the file is absent or unreadable.
595func readInfoID(infoJSONPath string) string {
596 if infoJSONPath == "" {
597 return ""
598 }
599 data, err := os.ReadFile(infoJSONPath)
600 if err != nil {
601 return ""
602 }
603 var info struct {
604 ID string `json:"id"`
605 }
606 if err := json.Unmarshal(data, &info); err != nil {
607 return ""
608 }
609 return info.ID
610}
611
612// refreshAndAddNew handles a metadata-mode run. The main pass used
613// --skip-download, so tempDownloadDir holds only info.json files. Existing
614// library items have their markers refreshed in place; entries with no existing
615// match are genuinely new and are downloaded as full items in a second pass.
616func (s *DownloadService) refreshAndAddNew(d *models.Download, preset *models.Preset, tempDownloadDir, ytdlpFlags string) error {
617 baseLibraryDir, err := s.resolveBaseLibraryDir(d)
618 if err != nil {
619 return err
620 }
621 if err := os.MkdirAll(baseLibraryDir, 0755); err != nil {
622 return err
623 }
624
625 entries, err := os.ReadDir(tempDownloadDir)
626 if err != nil {
627 return err
628 }
629
630 var newURLs []string
631 for _, entry := range entries {
632 if !entry.IsDir() || !strings.HasPrefix(entry.Name(), "item-") {
633 continue
634 }
635 itemDir := filepath.Join(tempDownloadDir, entry.Name())
636 infoJSONPath := findInfoJSON(itemDir)
637 if infoJSONPath == "" {
638 continue
639 }
640 videoID := readInfoID(infoJSONPath)
641 if videoID == "" {
642 continue
643 }
644 if existing, ok := s.librarySvc.FindByVideoID(baseLibraryDir, videoID); ok {
645 if err := s.applyMetadata(existing, infoJSONPath, videoID); err != nil {
646 log.Printf("warning: failed to refresh metadata for %s: %v", itemDir, err)
647 }
648 continue
649 }
650 if u := readWebpageURL(infoJSONPath); u != "" {
651 newURLs = append(newURLs, u)
652 }
653 }
654
655 if len(newURLs) == 0 {
656 return nil
657 }
658 return s.downloadFresh(d, preset, newURLs, ytdlpFlags)
659}
660
661// downloadFresh fetches the given item URLs as full downloads (media + info.json)
662// and imports them into the download's library directory. Metadata mode uses this
663// to add entries that don't exist in the library yet.
664func (s *DownloadService) downloadFresh(d *models.Download, preset *models.Preset, urls []string, ytdlpFlags string) error {
665 tempDir := filepath.Join(s.cfg.TempDir, fmt.Sprintf("%d-new", d.ID))
666 if err := os.MkdirAll(tempDir, 0755); err != nil {
667 return err
668 }
669 defer os.RemoveAll(tempDir)
670
671 args := s.presetSvc.BuildArgs(preset, d.FormatOverride, d.CustomFlags)
672 args, cleanup := s.appendCookies(args)
673 defer cleanup()
674 args = append(args, "--write-info-json")
675 args = append(args, "-P", tempDir)
676 args = append(args, "-o", "item-%(autonumber)05d/%(title)s.%(ext)s")
677 args = append(args, urls...)
678
679 runErr := s.runYTDLP(d, args)
680 // Import whatever succeeded even if some entries errored.
681 if _, err := s.importDownloadedItems(d, tempDir, "", ytdlpFlags); err != nil {
682 log.Printf("warning: failed to import new metadata-mode items: %v", err)
683 }
684 return runErr
685}
686
687// applyMetadata rewrites an existing item's marker (name/description/identity)
688// from a fresh info.json without touching its media.
689func (s *DownloadService) applyMetadata(existing, infoJSONPath, videoID string) error {
690 data, err := os.ReadFile(infoJSONPath)
691 if err != nil {
692 return err
693 }
694 var info struct {
695 Title string `json:"title"`
696 Description string `json:"description"`
697 WebpageURL string `json:"webpage_url"`
698 }
699 if err := json.Unmarshal(data, &info); err != nil {
700 log.Printf("applyMetadata: ignoring malformed %s: %v", infoJSONPath, err)
701 }
702
703 meta, err := s.librarySvc.readOrCreateMetadata(existing)
704 if err != nil {
705 return fmt.Errorf("read metadata for %s: %w", existing, err)
706 }
707 if info.Title != "" {
708 meta.Name = info.Title
709 }
710 if info.Description != "" {
711 meta.Description = info.Description
712 }
713 if meta.SourceURL == "" && info.WebpageURL != "" {
714 meta.SourceURL = info.WebpageURL
715 }
716 meta.VideoID = videoID
717
718 if err := s.librarySvc.writeMetadata(existing, meta); err != nil {
719 return err
720 }
721 if rel, err := filepath.Rel(s.cfg.LibraryDir, existing); err == nil {
722 s.librarySvc.evictCachedScan(filepath.ToSlash(rel))
723 }
724 return nil
725}
726
727// readWebpageURL returns the canonical entry URL from an info.json, or "".
728func readWebpageURL(infoJSONPath string) string {
729 data, err := os.ReadFile(infoJSONPath)
730 if err != nil {
731 return ""
732 }
733 var info struct {
734 WebpageURL string `json:"webpage_url"`
735 }
736 if err := json.Unmarshal(data, &info); err != nil {
737 return ""
738 }
739 return info.WebpageURL
740}
741
742// findInfoJSON returns the path to an info.json directly inside itemDir, or "".
743func findInfoJSON(itemDir string) string {
744 entries, err := os.ReadDir(itemDir)
745 if err != nil {
746 return ""
747 }
748 for _, entry := range entries {
749 if entry.IsDir() {
750 continue
751 }
752 name := entry.Name()
753 if name == "info.json" || strings.HasSuffix(name, ".info.json") {
754 return filepath.Join(itemDir, name)
755 }
756 }
757 return ""
758}
759
760// pruneSubscription mirrors the source by deleting items in the subscription's
761// directory that are no longer present upstream. It enumerates the current id
762// set with a cheap flat-playlist listing; it never prunes when that enumeration
763// fails or returns nothing, so a dead URL or network error can't wipe the dir.
764func (s *DownloadService) pruneSubscription(d *models.Download, sub *models.Subscription) {
765 baseLibraryDir, err := s.resolveBaseLibraryDir(d)
766 if err != nil {
767 log.Printf("subscription %d prune skipped: %v", sub.ID, err)
768 return
769 }
770
771 keep, err := s.enumeratePlaylistIDs(sub.URL)
772 if err != nil {
773 log.Printf("subscription %d prune skipped: enumeration failed: %v", sub.ID, err)
774 return
775 }
776 if len(keep) == 0 {
777 log.Printf("subscription %d prune skipped: source returned no entries", sub.ID)
778 return
779 }
780
781 removed, err := s.librarySvc.PruneToIDSet(baseLibraryDir, keep)
782 if err != nil {
783 log.Printf("subscription %d prune error: %v", sub.ID, err)
784 return
785 }
786 if removed > 0 {
787 log.Printf("subscription %d pruned %d item(s) removed upstream", sub.ID, removed)
788 }
789}
790
791// enumeratePlaylistIDs lists the current video-id set for a URL without
792// downloading, using yt-dlp --flat-playlist. Cookies are applied so private
793// playlists enumerate correctly. Ids alone are sufficient to match items within
794// a subscription's own directory (see FindByVideoID / PruneToIDSet).
795func (s *DownloadService) enumeratePlaylistIDs(url string) (map[string]bool, error) {
796 args := []string{"--flat-playlist", "--no-warnings", "--print", "%(id)s"}
797
798 args, cleanup := s.appendCookies(args)
799 defer cleanup()
800 args = append(args, url)
801
802 out, err := exec.Command(s.cfg.YTDLPPath, args...).Output()
803 if err != nil {
804 return nil, err
805 }
806
807 keep := make(map[string]bool)
808 for _, line := range strings.Split(string(out), "\n") {
809 id := strings.TrimSpace(line)
810 // yt-dlp prints "NA" for a missing field; never treat that as a real id.
811 if id == "" || id == "NA" {
812 continue
813 }
814 keep[id] = true
815 }
816 return keep, nil
817}
818
819func (s *DownloadService) deriveItemName(itemDir, infoJSONPath string, mediaFiles []os.DirEntry) string {
820 if infoJSONPath != "" {
821 data, err := os.ReadFile(infoJSONPath)
822 if err == nil {
823 var info struct {
824 Title string `json:"title"`
825 }
826 if err := json.Unmarshal(data, &info); err == nil && info.Title != "" {
827 return sanitizeDirName(info.Title)
828 }
829 }
830 }
831
832 sort.Slice(mediaFiles, func(i, j int) bool {
833 ii, _ := os.Stat(filepath.Join(itemDir, mediaFiles[i].Name()))
834 jj, _ := os.Stat(filepath.Join(itemDir, mediaFiles[j].Name()))
835 if ii == nil || jj == nil {
836 return false
837 }
838 return ii.Size() > jj.Size()
839 })
840
841 base := strings.TrimSuffix(mediaFiles[0].Name(), filepath.Ext(mediaFiles[0].Name()))
842 return sanitizeDirName(base)
843}
844
845func (s *DownloadService) uniqueDir(base, name string) string {
846 dir := filepath.Join(base, name)
847 if _, err := os.Stat(dir); os.IsNotExist(err) {
848 return dir
849 }
850 for i := 1; ; i++ {
851 candidate := fmt.Sprintf("%s-%d", dir, i)
852 if _, err := os.Stat(candidate); os.IsNotExist(err) {
853 return candidate
854 }
855 }
856}
857
858func sanitizeDirName(name string) string {
859 name = strings.TrimSpace(name)
860 replacer := strings.NewReplacer(
861 "/", "-",
862 "\\", "-",
863 ":", "-",
864 "*", "-",
865 "?", "-",
866 "\"", "-",
867 "<", "-",
868 ">", "-",
869 "|", "-",
870 )
871 name = replacer.Replace(name)
872 name = strings.TrimSpace(name)
873 if name == "" {
874 name = "untitled"
875 }
876 return name
877}
878
879// probeDuration returns the duration of a media file in whole seconds. The bool
880// is false when ffprobe is unavailable or the file has no usable duration.
881func probeDuration(ffprobePath, path string) (int, bool) {
882 out, err := exec.Command(ffprobePath, "-v", "error",
883 "-show_entries", "format=duration",
884 "-of", "default=nw=1:nk=1", path).Output()
885 if err != nil {
886 return 0, false
887 }
888 f, err := strconv.ParseFloat(strings.TrimSpace(string(out)), 64)
889 if err != nil || f <= 0 {
890 return 0, false
891 }
892 return int(f + 0.5), true
893}
894
895// ytFormat mirrors the subset of yt-dlp's per-format JSON (-J) we surface.
896// Numeric fields are pointers so an absent value (null/omitted) is distinct
897// from a real zero.
898type ytFormat struct {
899 FormatID string `json:"format_id"`
900 Ext string `json:"ext"`
901 Resolution string `json:"resolution"`
902 Width *int `json:"width"`
903 Height *int `json:"height"`
904 FPS *float64 `json:"fps"`
905 VCodec string `json:"vcodec"`
906 ACodec string `json:"acodec"`
907 AudioChannels *int `json:"audio_channels"`
908 Filesize *int64 `json:"filesize"`
909 FilesizeApprox *int64 `json:"filesize_approx"`
910 FormatNote string `json:"format_note"`
911}
912
913// parseFormatJSON reads yt-dlp's single-JSON dump (-J) and returns the available
914// formats. For a single video the formats live at the top level; for a playlist
915// URL we fall back to the first entry's formats so the picker still shows
916// something useful.
917func parseFormatJSON(data []byte) ([]*models.FormatInfo, error) {
918 var top struct {
919 Formats []ytFormat `json:"formats"`
920 Entries []struct {
921 Formats []ytFormat `json:"formats"`
922 } `json:"entries"`
923 }
924 if err := json.Unmarshal(data, &top); err != nil {
925 return nil, fmt.Errorf("parse yt-dlp JSON: %w", err)
926 }
927
928 raw := top.Formats
929 if len(raw) == 0 && len(top.Entries) > 0 {
930 raw = top.Entries[0].Formats
931 }
932
933 formats := make([]*models.FormatInfo, 0, len(raw))
934 for _, f := range raw {
935 formats = append(formats, f.toFormatInfo())
936 }
937 return formats, nil
938}
939
940func (f ytFormat) toFormatInfo() *models.FormatInfo {
941 fi := &models.FormatInfo{
942 ID: f.FormatID,
943 Ext: f.Ext,
944 Note: f.FormatNote,
945 }
946
947 switch {
948 case f.Resolution != "":
949 fi.Resolution = f.Resolution
950 case f.Width != nil && f.Height != nil && *f.Width > 0 && *f.Height > 0:
951 fi.Resolution = fmt.Sprintf("%dx%d", *f.Width, *f.Height)
952 }
953
954 if f.FPS != nil && *f.FPS > 0 {
955 fi.FPS = strconv.FormatFloat(*f.FPS, 'f', -1, 64)
956 }
957 if f.AudioChannels != nil && *f.AudioChannels > 0 {
958 fi.Channels = strconv.Itoa(*f.AudioChannels)
959 }
960
961 // Prefer the video codec; fall back to the audio codec for audio-only formats.
962 if f.VCodec != "" && f.VCodec != "none" {
963 fi.Codec = f.VCodec
964 } else if f.ACodec != "" && f.ACodec != "none" {
965 fi.Codec = f.ACodec
966 }
967
968 if f.Filesize != nil && *f.Filesize > 0 {
969 fi.FileSize = humanizeBytes(*f.Filesize)
970 } else if f.FilesizeApprox != nil && *f.FilesizeApprox > 0 {
971 fi.FileSize = "~" + humanizeBytes(*f.FilesizeApprox)
972 }
973
974 return fi
975}
976
977// humanizeBytes renders a byte count as a compact human-readable size.
978func humanizeBytes(n int64) string {
979 const unit = 1024
980 if n < unit {
981 return fmt.Sprintf("%dB", n)
982 }
983 div, exp := int64(unit), 0
984 for m := n / unit; m >= unit; m /= unit {
985 div *= unit
986 exp++
987 }
988 return fmt.Sprintf("%.1f%ciB", float64(n)/float64(div), "KMGTPE"[exp])
989}
990
991func sqlNullInt64(v int64) sql.NullInt64 {
992 return sql.NullInt64{Int64: v, Valid: true}
993}
994
995// reservedFlags are yt-dlp options VidArchive always sets itself; user custom
996// flags must not pass them (or a conflicting inverse). The value describes what
997// the option controls, for the failure message.
998var reservedFlags = map[string]string{
999 "-o": "the output template",
1000 "--output": "the output template",
1001 "-P": "the download path",
1002 "--paths": "the download path",
1003 "--cookies": "cookies (set these in Settings instead)",
1004 "--no-cookies": "cookies (set these in Settings instead)",
1005 "--newline": "progress output formatting (VidArchive sets this to stream logs)",
1006}
1007
1008// reservedSubscriptionFlags are additionally reserved for subscription runs,
1009// where VidArchive drives info-json writing and the refresh mode.
1010var reservedSubscriptionFlags = map[string]string{
1011 "--write-info-json": "info-json writing (needed to track item identity)",
1012 "--no-write-info-json": "info-json writing (needed to track item identity)",
1013 "--download-archive": "the download archive (managed by Skip mode)",
1014 "--no-download-archive": "the download archive (managed by Skip mode)",
1015 "--skip-download": "media downloading (managed by Metadata mode)",
1016 "--no-skip-download": "media downloading (managed by Metadata mode)",
1017}
1018
1019// checkReservedFlags rejects custom flags that clash with options VidArchive
1020// controls, naming the offender. It matches both "--flag" and "--flag=value".
1021func checkReservedFlags(customFlags string, isSubscription bool) error {
1022 for _, tok := range strings.Fields(customFlags) {
1023 name := tok
1024 if i := strings.IndexByte(name, '='); i >= 0 {
1025 name = name[:i]
1026 }
1027 if desc, ok := reservedFlags[name]; ok {
1028 return fmt.Errorf("custom flag %q conflicts with VidArchive's handling of %s; remove it and try again", tok, desc)
1029 }
1030 if isSubscription {
1031 if desc, ok := reservedSubscriptionFlags[name]; ok {
1032 return fmt.Errorf("custom flag %q conflicts with VidArchive's handling of %s; remove it and try again", tok, desc)
1033 }
1034 }
1035 }
1036 return nil
1037}
1038