vidarchive

vidarchive/internal/service/download.go
download.go
⎇
Raw
1package service
2
3import (
4 "bufio"
5 "bytes"
6 "database/sql"
7 "encoding/json"
8 "fmt"
9 "log"
10 "os"
11 "os/exec"
12 "path/filepath"
13 "sort"
14 "strconv"
15 "strings"
16 "sync"
17 "syscall"
18 "time"
19
20 "github.com/BurntSushi/toml"
21 "github.com/gabriel-vasile/mimetype"
22
23 "vidarchive/internal/config"
24 "vidarchive/internal/models"
25 "vidarchive/internal/repository"
26)
27
28type DownloadService struct {
29 repo *repository.DownloadRepository
30 librarySvc *LibraryService
31 presetSvc *PresetService
32 settingsSvc *SettingsService
33 subscriptionSvc *SubscriptionService
34 cfg *config.Config
35 cache *ProgressCache
36 processMu sync.Mutex
37 processes map[int64]*os.Process
38}
39
40func NewDownloadService(repo *repository.DownloadRepository, librarySvc *LibraryService, presetSvc *PresetService, settingsSvc *SettingsService, subscriptionSvc *SubscriptionService, cfg *config.Config) *DownloadService {
41 return &DownloadService{
42 repo: repo,
43 librarySvc: librarySvc,
44 presetSvc: presetSvc,
45 settingsSvc: settingsSvc,
46 subscriptionSvc: subscriptionSvc,
47 cfg: cfg,
48 cache: NewProgressCache(),
49 processes: make(map[int64]*os.Process),
50 }
51}
52
53func (s *DownloadService) Create(url string, presetID *int64, formatOverride, customFlags, outputDir string) (*models.Download, error) {
54 d := &models.Download{
55 URL: url,
56 Status: "queued",
57 FormatOverride: formatOverride,
58 CustomFlags: customFlags,
59 OutputDir: sql.NullString{String: outputDir, Valid: outputDir != ""},
60 }
61
62 if presetID != nil {
63 d.PresetID = sqlNullInt64(*presetID)
64 }
65
66 if err := s.repo.Create(d); err != nil {
67 return nil, err
68 }
69 return d, nil
70}
71
72// CreateForSubscription queues a download for a subscription run, copying its
73// download options and tagging it with the subscription id so ExecuteDownload
74// applies the right refresh mode and pruning.
75func (s *DownloadService) CreateForSubscription(sub *models.Subscription) (*models.Download, error) {
76 d := &models.Download{
77 URL: sub.URL,
78 Status: "queued",
79 FormatOverride: sub.FormatOverride,
80 CustomFlags: sub.CustomFlags,
81 OutputDir: sql.NullString{String: sub.OutputDir, Valid: sub.OutputDir != ""},
82 PresetID: sub.PresetID,
83 SubscriptionID: sqlNullInt64(sub.ID),
84 }
85 if err := s.repo.Create(d); err != nil {
86 return nil, err
87 }
88 return d, nil
89}
90
91func (s *DownloadService) GetByID(id int64) (*models.Download, error) {
92 d, err := s.repo.GetByID(id)
93 if err != nil {
94 return nil, err
95 }
96 if logs := s.cache.Snapshot(id); logs != "" {
97 d.Logs = sql.NullString{String: logs, Valid: true}
98 }
99 return d, nil
100}
101
102func (s *DownloadService) GetAll(status, sortBy string) ([]*models.Download, error) {
103 downloads, err := s.repo.GetAll(status, sortBy)
104 if err != nil {
105 return nil, err
106 }
107
108 for _, d := range downloads {
109 if logs := s.cache.Snapshot(d.ID); logs != "" {
110 d.Logs = sql.NullString{String: logs, Valid: true}
111 }
112 }
113
114 return downloads, nil
115}
116
117func (s *DownloadService) GetQueued(limit int) ([]*models.Download, error) {
118 return s.repo.GetQueued(limit)
119}
120
121// HasActiveForSubscription reports whether the subscription already has a queued
122// or in-progress download, so the scheduler can skip stacking another run.
123func (s *DownloadService) HasActiveForSubscription(subID int64) (bool, error) {
124 return s.repo.HasActiveForSubscription(subID)
125}
126
127func (s *DownloadService) Delete(id int64) error {
128 s.killProcess(id)
129 s.cache.Delete(id)
130 return s.repo.Delete(id)
131}
132
133func (s *DownloadService) killProcess(id int64) {
134 s.processMu.Lock()
135 proc, ok := s.processes[id]
136 delete(s.processes, id)
137 s.processMu.Unlock()
138
139 if !ok || proc == nil {
140 return
141 }
142
143 _ = syscall.Kill(-proc.Pid, syscall.SIGKILL)
144}
145
146func (s *DownloadService) DeleteAll() error {
147 return s.repo.DeleteAll()
148}
149
150func (s *DownloadService) ResetStalledDownloads() error {
151 return s.repo.UpdateStatusWhere("downloading", "queued")
152}
153
154func (s *DownloadService) ListFormats(url string) ([]*models.FormatInfo, error) {
155 // Use machine-readable JSON (-J) rather than scraping the human "-F" table,
156 // whose columns/separators shift between yt-dlp versions. stderr is captured
157 // separately so warnings can't corrupt the JSON on stdout.
158 cmd := exec.Command(s.cfg.YTDLPPath, "-J", "--no-warnings", url)
159 var stderr bytes.Buffer
160 cmd.Stderr = &stderr
161 output, err := cmd.Output()
162 if err != nil {
163 return nil, fmt.Errorf("yt-dlp -J failed: %w\n%s", err, stderr.String())
164 }
165
166 return parseFormatJSON(output)
167}
168
169// ExecuteDownload runs the download for d. The bool reports whether this call
170// actually processed it: false means another worker already claimed it (Submit
171// and the queue checker can both enqueue the same row within the 2s poll window),
172// so the caller should not log it as completed.
173func (s *DownloadService) ExecuteDownload(d *models.Download) (bool, error) {
174 // Atomically claim the download. If it's no longer queued, another worker
175 // already took it — bail rather than download it twice.
176 claimed, err := s.repo.MarkStarted(d.ID)
177 if err != nil {
178 return false, err
179 }
180 if !claimed {
181 return false, nil
182 }
183
184 s.cache.Set(d.ID, &LiveDownload{LastUpdate: time.Now()})
185 defer s.cache.Delete(d.ID)
186
187 var preset *models.Preset
188
189 if d.PresetID.Valid {
190 preset, err = s.presetSvc.GetByID(d.PresetID.Int64)
191 if err != nil {
192 preset, _ = s.presetSvc.GetDefault()
193 }
194 } else {
195 preset, _ = s.presetSvc.GetDefault()
196 }
197
198 if preset == nil {
199 preset = &models.Preset{}
200 }
201
202 var sub *models.Subscription
203 if d.SubscriptionID.Valid && s.subscriptionSvc != nil {
204 sub, _ = s.subscriptionSvc.GetByID(d.SubscriptionID.Int64)
205 }
206
207 // Reject custom flags that clash with options VidArchive sets itself, before
208 // spending any work — the download fails with a message naming the offender.
209 isSubscription := d.SubscriptionID.Valid
210 if err := checkReservedFlags(d.CustomFlags, isSubscription); err != nil {
211 s.finalizeError(d.ID, err)
212 return false, err
213 }
214 if err := checkReservedFlags(preset.CustomFlags, isSubscription); err != nil {
215 s.finalizeError(d.ID, err)
216 return false, err
217 }
218
219 tempDownloadDir := filepath.Join(s.cfg.TempDir, fmt.Sprintf("%d", d.ID))
220 if err := os.MkdirAll(tempDownloadDir, 0755); err != nil {
221 return false, fmt.Errorf("create temp download dir: %w", err)
222 }
223
224 args := s.presetSvc.BuildArgs(preset, d.FormatOverride, d.CustomFlags)
225
226 // Record the meaningful flags (format/audio/subs/custom) that shaped this
227 // download, before the internal plumbing (cookies, -P/-o, URL) is appended,
228 // so each imported item can show how it was fetched.
229 ytdlpFlags := strings.Join(args, " ")
230
231 var cookieCleanup func()
232 args, cookieCleanup = s.appendCookies(args)
233 defer cookieCleanup()
234
235 if sub != nil {
236 // Always write info.json so the import step can read the stable identity
237 // (yt-dlp's video id) used to match/replace existing items.
238 args = append(args, "--write-info-json")
239 switch sub.RefreshMode {
240 case "skip":
241 // Let yt-dlp skip entries already recorded — no re-download.
242 archive := s.subscriptionSvc.ArchivePath(sub.ID)
243 if err := os.MkdirAll(filepath.Dir(archive), 0755); err == nil {
244 args = append(args, "--download-archive", archive)
245 }
246 case "metadata":
247 // Refresh metadata only; don't fetch media.
248 args = append(args, "--skip-download")
249 }
250 }
251
252 args = append(args, "-P", tempDownloadDir)
253 args = append(args, "-o", "item-%(autonumber)05d/%(title)s.%(ext)s")
254 args = append(args, d.URL)
255
256 if err := s.runYTDLP(d, args); err != nil {
257 s.finalizeError(d.ID, err)
258 return false, err
259 }
260
261 mode := ""
262 if sub != nil {
263 mode = sub.RefreshMode
264 }
265
266 // Post-process before marking completed, so the download stays "downloading"
267 // until everything is really done — including metadata mode's second pass,
268 // which downloads any genuinely new entries as full items.
269 if mode == "metadata" {
270 // The main pass ran with --skip-download, so the temp dir holds only
271 // info.json files: refresh existing items in place and fetch new ones.
272 if err := s.refreshAndAddNew(d, preset, tempDownloadDir, ytdlpFlags); err != nil {
273 s.finalizeError(d.ID, err)
274 return false, err
275 }
276 } else {
277 imported, err := s.importDownloadedItems(d, tempDownloadDir, mode, ytdlpFlags)
278 if err != nil {
279 s.finalizeError(d.ID, err)
280 return false, err
281 }
282 // A plain (non-subscription) download that yields nothing is a failure, not
283 // a silent "completed". Subscription modes legitimately import zero (skip
284 // mode, or a metadata refresh with no new entries), so only enforce this for
285 // plain runs.
286 if sub == nil && imported == 0 {
287 err := fmt.Errorf("yt-dlp finished but no media files were downloaded")
288 s.finalizeError(d.ID, err)
289 return false, err
290 }
291 }
292
293 if sub != nil && sub.PruneRemoved {
294 s.pruneSubscription(d, sub)
295 }
296
297 if err := s.repo.MarkCompleted(d.ID, "completed"); err != nil {
298 return false, err
299 }
300
301 return true, nil
302}
303
304// runYTDLP executes yt-dlp with args, streaming combined output into the live
305// progress cache and periodically flushing it to the download's persisted log.
306// It registers the process so Delete can kill it, and returns the exit error.
307func (s *DownloadService) runYTDLP(d *models.Download, args []string) error {
308 // --newline forces yt-dlp to emit each progress update on its own line. Without
309 // it, progress is rewritten in place with carriage returns, so a long download
310 // becomes one ever-growing line that overflows the reader's buffer and stalls
311 // the pipe — hanging the download. See the hardened scanner below.
312 fullArgs := append([]string{"--newline"}, args...)
313 cmd := exec.Command(s.cfg.YTDLPPath, fullArgs...)
314 cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
315
316 stdout, err := cmd.StdoutPipe()
317 if err != nil {
318 return err
319 }
320 cmd.Stderr = cmd.Stdout
321
322 if err := cmd.Start(); err != nil {
323 return err
324 }
325
326 s.processMu.Lock()
327 s.processes[d.ID] = cmd.Process
328 s.processMu.Unlock()
329 defer func() {
330 s.processMu.Lock()
331 delete(s.processes, d.ID)
332 s.processMu.Unlock()
333 }()
334
335 ticker := time.NewTicker(10 * time.Second)
336 defer ticker.Stop()
337 done := make(chan struct{})
338 go func() {
339 for {
340 select {
341 case <-ticker.C:
342 if logs := s.cache.FlushLogs(d.ID); logs != "" {
343 s.repo.AppendLogs(d.ID, logs)
344 }
345 case <-done:
346 return
347 }
348 }
349 }()
350
351 scanner := bufio.NewScanner(stdout)
352 // Allow long lines (a single yt-dlp message can exceed the 64 KiB default)
353 // rather than letting the scanner abort and leave the pipe unread.
354 scanner.Buffer(make([]byte, 0, 64*1024), 1024*1024)
355 for scanner.Scan() {
356 s.cache.AppendLog(d.ID, scanner.Text())
357 }
358 if err := scanner.Err(); err != nil {
359 log.Printf("download %d: error reading yt-dlp output: %v", d.ID, err)
360 }
361 close(done)
362
363 if logs := s.cache.FlushLogs(d.ID); logs != "" {
364 s.repo.AppendLogs(d.ID, logs)
365 }
366
367 return cmd.Wait()
368}
369
370// appendCookies writes the saved cookies (if any) to a temp file and appends a
371// --cookies flag. The returned cleanup removes the temp file and is always safe
372// to call, even when no cookies were configured.
373func (s *DownloadService) appendCookies(args []string) ([]string, func()) {
374 cleanup := func() {}
375 cookies, err := s.settingsSvc.GetCookies()
376 if err != nil || strings.TrimSpace(cookies) == "" {
377 return args, cleanup
378 }
379 tmpFile, err := os.CreateTemp("", "cookies-*.txt")
380 if err != nil {
381 return args, cleanup
382 }
383 // A short write would hand yt-dlp a truncated cookies file; on any write/close
384 // failure, drop the temp file and proceed without cookies rather than silently
385 // using a broken one.
386 if _, err := tmpFile.WriteString(cookies); err != nil {
387 tmpFile.Close()
388 os.Remove(tmpFile.Name())
389 return args, cleanup
390 }
391 if err := tmpFile.Close(); err != nil {
392 os.Remove(tmpFile.Name())
393 return args, cleanup
394 }
395 return append(args, "--cookies", tmpFile.Name()), func() { os.Remove(tmpFile.Name()) }
396}
397
398func (s *DownloadService) finalizeError(id int64, err error) {
399 logs := s.cache.FlushLogs(id)
400 if logs != "" {
401 s.repo.AppendLogs(id, logs)
402 }
403 s.repo.MarkError(id, err.Error())
404}
405
406// resolveBaseLibraryDir returns the absolute library directory a download writes
407// into, applying the optional per-download OutputDir while rejecting any path
408// that escapes the library root.
409func (s *DownloadService) resolveBaseLibraryDir(d *models.Download) (string, error) {
410 baseLibraryDir := s.cfg.LibraryDir
411 if d.OutputDir.Valid && d.OutputDir.String != "" {
412 cleanDir := filepath.Clean(d.OutputDir.String)
413 fullPath := filepath.Join(baseLibraryDir, cleanDir)
414 resolvedPath, err := filepath.Abs(fullPath)
415 if err != nil {
416 return "", fmt.Errorf("invalid output directory: %w", err)
417 }
418 resolvedLibraryDir, _ := filepath.Abs(baseLibraryDir)
419 if !strings.HasPrefix(resolvedPath, resolvedLibraryDir+string(filepath.Separator)) && resolvedPath != resolvedLibraryDir {
420 return "", fmt.Errorf("invalid output directory: path traversal attempt detected")
421 }
422 baseLibraryDir = fullPath
423 }
424 return baseLibraryDir, nil
425}
426
427// importDownloadedItems moves each downloaded item from the temp dir into the
428// library and returns the number of items successfully imported. Per-item
429// failures are logged and skipped (a playlist with a few bad entries still
430// imports the rest); a non-nil error means the import couldn't even start.
431func (s *DownloadService) importDownloadedItems(d *models.Download, tempDownloadDir, mode, ytdlpFlags string) (int, error) {
432 entries, err := os.ReadDir(tempDownloadDir)
433 if err != nil {
434 return 0, err
435 }
436
437 baseLibraryDir, err := s.resolveBaseLibraryDir(d)
438 if err != nil {
439 return 0, err
440 }
441 if err := os.MkdirAll(baseLibraryDir, 0755); err != nil {
442 return 0, err
443 }
444
445 var itemDirs []string
446 for _, entry := range entries {
447 if !entry.IsDir() {
448 continue
449 }
450 name := entry.Name()
451 if strings.HasPrefix(name, "item-") {
452 itemDirs = append(itemDirs, filepath.Join(tempDownloadDir, name))
453 }
454 }
455 sort.Strings(itemDirs)
456
457 imported := 0
458 for _, itemDir := range itemDirs {
459 if err := s.importItemDir(d.URL, itemDir, baseLibraryDir, mode, ytdlpFlags); err != nil {
460 log.Printf("warning: failed to import item %s: %v", itemDir, err)
461 continue
462 }
463 imported++
464 }
465
466 // RemoveAll (not Remove): leftover item dirs from failed imports, plus any
467 // orphaned thumbnail/image files yt-dlp left behind, would otherwise keep the
468 // temp dir non-empty and leak it forever.
469 os.RemoveAll(tempDownloadDir)
470 return imported, nil
471}
472
473func (s *DownloadService) importItemDir(url, itemDir, baseLibraryDir, mode, ytdlpFlags string) error {
474 entries, err := os.ReadDir(itemDir)
475 if err != nil {
476 return err
477 }
478
479 var mediaFiles []os.DirEntry
480 var infoJSONPath string
481 var subtitleFiles []string
482
483 for _, entry := range entries {
484 if entry.IsDir() {
485 continue
486 }
487 name := entry.Name()
488 path := filepath.Join(itemDir, name)
489 ext := strings.ToLower(filepath.Ext(name))
490
491 if name == "info.json" || strings.HasSuffix(name, ".info.json") {
492 infoJSONPath = path
493 continue
494 }
495 if ext == ".vtt" || ext == ".srt" || ext == ".ass" || ext == ".ssa" {
496 subtitleFiles = append(subtitleFiles, path)
497 continue
498 }
499
500 mtype, err := mimetype.DetectFile(path)
501 if err == nil && mtype != nil && (strings.HasPrefix(mtype.String(), "audio/") || strings.HasPrefix(mtype.String(), "video/")) {
502 mediaFiles = append(mediaFiles, entry)
503 }
504 }
505
506 if len(mediaFiles) == 0 {
507 return fmt.Errorf("no media files found in %s", itemDir)
508 }
509
510 name := s.deriveItemName(itemDir, infoJSONPath, mediaFiles)
511 videoID := readInfoID(infoJSONPath)
512
513 // Overwrite mode: replace the existing copy of this video in place rather than
514 // creating a duplicate folder. Removing the old dir lets uniqueDir reuse its
515 // name (or land on the new title if it changed upstream).
516 if mode == "overwrite" && videoID != "" {
517 if existing, ok := s.librarySvc.FindByVideoID(baseLibraryDir, videoID); ok {
518 if rel, err := filepath.Rel(s.cfg.LibraryDir, existing); err == nil {
519 s.librarySvc.evictCachedScan(filepath.ToSlash(rel))
520 }
521 os.RemoveAll(existing)
522 }
523 }
524
525 targetDir := s.uniqueDir(baseLibraryDir, name)
526 if err := os.MkdirAll(targetDir, 0755); err != nil {
527 return err
528 }
529
530 if infoJSONPath != "" {
531 if err := os.Rename(infoJSONPath, filepath.Join(targetDir, "info.json")); err != nil {
532 return err
533 }
534 }
535
536 for _, entry := range mediaFiles {
537 if err := os.Rename(filepath.Join(itemDir, entry.Name()), filepath.Join(targetDir, entry.Name())); err != nil {
538 return err
539 }
540 }
541
542 // Probe each media file's duration once, here in the worker (off the request
543 // path), and cache it in the marker so the library never has to probe while
544 // serving pages. Files we can't probe simply get no duration.
545 fileDurations := make(map[string]int)
546 for _, entry := range mediaFiles {
547 if d, ok := probeDuration(s.cfg.FFprobePath, filepath.Join(targetDir, entry.Name())); ok {
548 fileDurations[entry.Name()] = d
549 }
550 }
551
552 if len(subtitleFiles) > 0 {
553 subtitlesDir := filepath.Join(targetDir, subtitlesDirName)
554 if err := os.MkdirAll(subtitlesDir, 0755); err != nil {
555 return err
556 }
557 for _, sf := range subtitleFiles {
558 if err := os.Rename(sf, filepath.Join(subtitlesDir, filepath.Base(sf))); err != nil {
559 return err
560 }
561 }
562 }
563
564 metadata := models.ItemMetadata{
565 Name: name,
566 SourceURL: url,
567 VideoID: videoID,
568 YtdlpFlags: ytdlpFlags,
569 FileDurations: fileDurations,
570 }
571
572 markerPath := filepath.Join(targetDir, itemMarkerName)
573 f, err := os.Create(markerPath)
574 if err != nil {
575 return err
576 }
577 defer f.Close()
578 if err := toml.NewEncoder(f).Encode(metadata); err != nil {
579 return err
580 }
581
582 return nil
583}
584
585// readInfoID extracts the stable item identity (yt-dlp's video id) from an
586// info.json. The id alone is sufficient to match items within a single
587// subscription's owned directory, so the extractor is not used. Returns an
588// empty string when the file is absent or unreadable.
589func readInfoID(infoJSONPath string) string {
590 if infoJSONPath == "" {
591 return ""
592 }
593 data, err := os.ReadFile(infoJSONPath)
594 if err != nil {
595 return ""
596 }
597 var info struct {
598 ID string `json:"id"`
599 }
600 if err := json.Unmarshal(data, &info); err != nil {
601 return ""
602 }
603 return info.ID
604}
605
606// refreshAndAddNew handles a metadata-mode run. The main pass used
607// --skip-download, so tempDownloadDir holds only info.json files. Existing
608// library items have their markers refreshed in place; entries with no existing
609// match are genuinely new and are downloaded as full items in a second pass.
610func (s *DownloadService) refreshAndAddNew(d *models.Download, preset *models.Preset, tempDownloadDir, ytdlpFlags string) error {
611 baseLibraryDir, err := s.resolveBaseLibraryDir(d)
612 if err != nil {
613 return err
614 }
615 if err := os.MkdirAll(baseLibraryDir, 0755); err != nil {
616 return err
617 }
618
619 entries, err := os.ReadDir(tempDownloadDir)
620 if err != nil {
621 return err
622 }
623
624 var newURLs []string
625 for _, entry := range entries {
626 if !entry.IsDir() || !strings.HasPrefix(entry.Name(), "item-") {
627 continue
628 }
629 itemDir := filepath.Join(tempDownloadDir, entry.Name())
630 infoJSONPath := findInfoJSON(itemDir)
631 if infoJSONPath == "" {
632 continue
633 }
634 videoID := readInfoID(infoJSONPath)
635 if videoID == "" {
636 continue
637 }
638 if existing, ok := s.librarySvc.FindByVideoID(baseLibraryDir, videoID); ok {
639 if err := s.applyMetadata(existing, infoJSONPath, videoID); err != nil {
640 log.Printf("warning: failed to refresh metadata for %s: %v", itemDir, err)
641 }
642 continue
643 }
644 if u := readWebpageURL(infoJSONPath); u != "" {
645 newURLs = append(newURLs, u)
646 }
647 }
648
649 os.RemoveAll(tempDownloadDir)
650
651 if len(newURLs) == 0 {
652 return nil
653 }
654 return s.downloadFresh(d, preset, newURLs, ytdlpFlags)
655}
656
657// downloadFresh fetches the given item URLs as full downloads (media + info.json)
658// and imports them into the download's library directory. Metadata mode uses this
659// to add entries that don't exist in the library yet.
660func (s *DownloadService) downloadFresh(d *models.Download, preset *models.Preset, urls []string, ytdlpFlags string) error {
661 tempDir := filepath.Join(s.cfg.TempDir, fmt.Sprintf("%d-new", d.ID))
662 if err := os.MkdirAll(tempDir, 0755); err != nil {
663 return err
664 }
665
666 args := s.presetSvc.BuildArgs(preset, d.FormatOverride, d.CustomFlags)
667 args, cleanup := s.appendCookies(args)
668 defer cleanup()
669 args = append(args, "--write-info-json")
670 args = append(args, "-P", tempDir)
671 args = append(args, "-o", "item-%(autonumber)05d/%(title)s.%(ext)s")
672 args = append(args, urls...)
673
674 runErr := s.runYTDLP(d, args)
675 // Import whatever succeeded even if some entries errored.
676 if _, err := s.importDownloadedItems(d, tempDir, "", ytdlpFlags); err != nil {
677 log.Printf("warning: failed to import new metadata-mode items: %v", err)
678 }
679 return runErr
680}
681
682// applyMetadata rewrites an existing item's marker (name/description/identity)
683// from a fresh info.json without touching its media.
684func (s *DownloadService) applyMetadata(existing, infoJSONPath, videoID string) error {
685 data, err := os.ReadFile(infoJSONPath)
686 if err != nil {
687 return err
688 }
689 var info struct {
690 Title string `json:"title"`
691 Description string `json:"description"`
692 WebpageURL string `json:"webpage_url"`
693 }
694 _ = json.Unmarshal(data, &info)
695
696 meta, _ := s.librarySvc.readOrCreateMetadata(existing)
697 if info.Title != "" {
698 meta.Name = info.Title
699 }
700 if info.Description != "" {
701 meta.Description = info.Description
702 }
703 if meta.SourceURL == "" && info.WebpageURL != "" {
704 meta.SourceURL = info.WebpageURL
705 }
706 meta.VideoID = videoID
707
708 if err := s.librarySvc.writeMetadata(existing, meta); err != nil {
709 return err
710 }
711 if rel, err := filepath.Rel(s.cfg.LibraryDir, existing); err == nil {
712 s.librarySvc.evictCachedScan(filepath.ToSlash(rel))
713 }
714 return nil
715}
716
717// readWebpageURL returns the canonical entry URL from an info.json, or "".
718func readWebpageURL(infoJSONPath string) string {
719 data, err := os.ReadFile(infoJSONPath)
720 if err != nil {
721 return ""
722 }
723 var info struct {
724 WebpageURL string `json:"webpage_url"`
725 }
726 if err := json.Unmarshal(data, &info); err != nil {
727 return ""
728 }
729 return info.WebpageURL
730}
731
732// findInfoJSON returns the path to an info.json directly inside itemDir, or "".
733func findInfoJSON(itemDir string) string {
734 entries, err := os.ReadDir(itemDir)
735 if err != nil {
736 return ""
737 }
738 for _, entry := range entries {
739 if entry.IsDir() {
740 continue
741 }
742 name := entry.Name()
743 if name == "info.json" || strings.HasSuffix(name, ".info.json") {
744 return filepath.Join(itemDir, name)
745 }
746 }
747 return ""
748}
749
750// pruneSubscription mirrors the source by deleting items in the subscription's
751// directory that are no longer present upstream. It enumerates the current id
752// set with a cheap flat-playlist listing; it never prunes when that enumeration
753// fails or returns nothing, so a dead URL or network error can't wipe the dir.
754func (s *DownloadService) pruneSubscription(d *models.Download, sub *models.Subscription) {
755 baseLibraryDir, err := s.resolveBaseLibraryDir(d)
756 if err != nil {
757 log.Printf("subscription %d prune skipped: %v", sub.ID, err)
758 return
759 }
760
761 keep, err := s.enumeratePlaylistIDs(sub.URL)
762 if err != nil {
763 log.Printf("subscription %d prune skipped: enumeration failed: %v", sub.ID, err)
764 return
765 }
766 if len(keep) == 0 {
767 log.Printf("subscription %d prune skipped: source returned no entries", sub.ID)
768 return
769 }
770
771 removed, err := s.librarySvc.PruneToIDSet(baseLibraryDir, keep)
772 if err != nil {
773 log.Printf("subscription %d prune error: %v", sub.ID, err)
774 return
775 }
776 if removed > 0 {
777 log.Printf("subscription %d pruned %d item(s) removed upstream", sub.ID, removed)
778 }
779}
780
781// enumeratePlaylistIDs lists the current video-id set for a URL without
782// downloading, using yt-dlp --flat-playlist. Cookies are applied so private
783// playlists enumerate correctly. Ids alone are sufficient to match items within
784// a subscription's own directory (see FindByVideoID / PruneToIDSet).
785func (s *DownloadService) enumeratePlaylistIDs(url string) (map[string]bool, error) {
786 args := []string{"--flat-playlist", "--no-warnings", "--print", "%(id)s"}
787
788 args, cleanup := s.appendCookies(args)
789 defer cleanup()
790 args = append(args, url)
791
792 out, err := exec.Command(s.cfg.YTDLPPath, args...).Output()
793 if err != nil {
794 return nil, err
795 }
796
797 keep := make(map[string]bool)
798 for _, line := range strings.Split(string(out), "\n") {
799 id := strings.TrimSpace(line)
800 // yt-dlp prints "NA" for a missing field; never treat that as a real id.
801 if id == "" || id == "NA" {
802 continue
803 }
804 keep[id] = true
805 }
806 return keep, nil
807}
808
809func (s *DownloadService) deriveItemName(itemDir, infoJSONPath string, mediaFiles []os.DirEntry) string {
810 if infoJSONPath != "" {
811 data, err := os.ReadFile(infoJSONPath)
812 if err == nil {
813 var info struct {
814 Title string `json:"title"`
815 }
816 if err := json.Unmarshal(data, &info); err == nil && info.Title != "" {
817 return sanitizeDirName(info.Title)
818 }
819 }
820 }
821
822 sort.Slice(mediaFiles, func(i, j int) bool {
823 ii, _ := os.Stat(filepath.Join(itemDir, mediaFiles[i].Name()))
824 jj, _ := os.Stat(filepath.Join(itemDir, mediaFiles[j].Name()))
825 if ii == nil || jj == nil {
826 return false
827 }
828 return ii.Size() > jj.Size()
829 })
830
831 base := strings.TrimSuffix(mediaFiles[0].Name(), filepath.Ext(mediaFiles[0].Name()))
832 return sanitizeDirName(base)
833}
834
835func (s *DownloadService) uniqueDir(base, name string) string {
836 dir := filepath.Join(base, name)
837 if _, err := os.Stat(dir); os.IsNotExist(err) {
838 return dir
839 }
840 for i := 1; ; i++ {
841 candidate := fmt.Sprintf("%s-%d", dir, i)
842 if _, err := os.Stat(candidate); os.IsNotExist(err) {
843 return candidate
844 }
845 }
846}
847
848func sanitizeDirName(name string) string {
849 name = strings.TrimSpace(name)
850 replacer := strings.NewReplacer(
851 "/", "-",
852 "\\", "-",
853 ":", "-",
854 "*", "-",
855 "?", "-",
856 "\"", "-",
857 "<", "-",
858 ">", "-",
859 "|", "-",
860 )
861 name = replacer.Replace(name)
862 name = strings.TrimSpace(name)
863 if name == "" {
864 name = "untitled"
865 }
866 return name
867}
868
869// probeDuration returns the duration of a media file in whole seconds. The bool
870// is false when ffprobe is unavailable or the file has no usable duration.
871func probeDuration(ffprobePath, path string) (int, bool) {
872 out, err := exec.Command(ffprobePath, "-v", "error",
873 "-show_entries", "format=duration",
874 "-of", "default=nw=1:nk=1", path).Output()
875 if err != nil {
876 return 0, false
877 }
878 f, err := strconv.ParseFloat(strings.TrimSpace(string(out)), 64)
879 if err != nil || f <= 0 {
880 return 0, false
881 }
882 return int(f + 0.5), true
883}
884
885// ytFormat mirrors the subset of yt-dlp's per-format JSON (-J) we surface.
886// Numeric fields are pointers so an absent value (null/omitted) is distinct
887// from a real zero.
888type ytFormat struct {
889 FormatID string `json:"format_id"`
890 Ext string `json:"ext"`
891 Resolution string `json:"resolution"`
892 Width *int `json:"width"`
893 Height *int `json:"height"`
894 FPS *float64 `json:"fps"`
895 VCodec string `json:"vcodec"`
896 ACodec string `json:"acodec"`
897 AudioChannels *int `json:"audio_channels"`
898 Filesize *int64 `json:"filesize"`
899 FilesizeApprox *int64 `json:"filesize_approx"`
900 FormatNote string `json:"format_note"`
901}
902
903// parseFormatJSON reads yt-dlp's single-JSON dump (-J) and returns the available
904// formats. For a single video the formats live at the top level; for a playlist
905// URL we fall back to the first entry's formats so the picker still shows
906// something useful.
907func parseFormatJSON(data []byte) ([]*models.FormatInfo, error) {
908 var top struct {
909 Formats []ytFormat `json:"formats"`
910 Entries []struct {
911 Formats []ytFormat `json:"formats"`
912 } `json:"entries"`
913 }
914 if err := json.Unmarshal(data, &top); err != nil {
915 return nil, fmt.Errorf("parse yt-dlp JSON: %w", err)
916 }
917
918 raw := top.Formats
919 if len(raw) == 0 && len(top.Entries) > 0 {
920 raw = top.Entries[0].Formats
921 }
922
923 formats := make([]*models.FormatInfo, 0, len(raw))
924 for _, f := range raw {
925 formats = append(formats, f.toFormatInfo())
926 }
927 return formats, nil
928}
929
930func (f ytFormat) toFormatInfo() *models.FormatInfo {
931 fi := &models.FormatInfo{
932 ID: f.FormatID,
933 Ext: f.Ext,
934 Note: f.FormatNote,
935 }
936
937 switch {
938 case f.Resolution != "":
939 fi.Resolution = f.Resolution
940 case f.Width != nil && f.Height != nil && *f.Width > 0 && *f.Height > 0:
941 fi.Resolution = fmt.Sprintf("%dx%d", *f.Width, *f.Height)
942 }
943
944 if f.FPS != nil && *f.FPS > 0 {
945 fi.FPS = strconv.FormatFloat(*f.FPS, 'f', -1, 64)
946 }
947 if f.AudioChannels != nil && *f.AudioChannels > 0 {
948 fi.Channels = strconv.Itoa(*f.AudioChannels)
949 }
950
951 // Prefer the video codec; fall back to the audio codec for audio-only formats.
952 if f.VCodec != "" && f.VCodec != "none" {
953 fi.Codec = f.VCodec
954 } else if f.ACodec != "" && f.ACodec != "none" {
955 fi.Codec = f.ACodec
956 }
957
958 if f.Filesize != nil && *f.Filesize > 0 {
959 fi.FileSize = humanizeBytes(*f.Filesize)
960 } else if f.FilesizeApprox != nil && *f.FilesizeApprox > 0 {
961 fi.FileSize = "~" + humanizeBytes(*f.FilesizeApprox)
962 }
963
964 return fi
965}
966
967// humanizeBytes renders a byte count as a compact human-readable size.
968func humanizeBytes(n int64) string {
969 const unit = 1024
970 if n < unit {
971 return fmt.Sprintf("%dB", n)
972 }
973 div, exp := int64(unit), 0
974 for m := n / unit; m >= unit; m /= unit {
975 div *= unit
976 exp++
977 }
978 return fmt.Sprintf("%.1f%ciB", float64(n)/float64(div), "KMGTPE"[exp])
979}
980
981func sqlNullInt64(v int64) sql.NullInt64 {
982 return sql.NullInt64{Int64: v, Valid: true}
983}
984
985// reservedFlags are yt-dlp options VidArchive always sets itself; user custom
986// flags must not pass them (or a conflicting inverse). The value describes what
987// the option controls, for the failure message.
988var reservedFlags = map[string]string{
989 "-o": "the output template",
990 "--output": "the output template",
991 "-P": "the download path",
992 "--paths": "the download path",
993 "--cookies": "cookies (set these in Settings instead)",
994 "--no-cookies": "cookies (set these in Settings instead)",
995 "--newline": "progress output formatting (VidArchive sets this to stream logs)",
996}
997
998// reservedSubscriptionFlags are additionally reserved for subscription runs,
999// where VidArchive drives info-json writing and the refresh mode.
1000var reservedSubscriptionFlags = map[string]string{
1001 "--write-info-json": "info-json writing (needed to track item identity)",
1002 "--no-write-info-json": "info-json writing (needed to track item identity)",
1003 "--download-archive": "the download archive (managed by Skip mode)",
1004 "--no-download-archive": "the download archive (managed by Skip mode)",
1005 "--skip-download": "media downloading (managed by Metadata mode)",
1006 "--no-skip-download": "media downloading (managed by Metadata mode)",
1007}
1008
1009// checkReservedFlags rejects custom flags that clash with options VidArchive
1010// controls, naming the offender. It matches both "--flag" and "--flag=value".
1011func checkReservedFlags(customFlags string, isSubscription bool) error {
1012 for _, tok := range strings.Fields(customFlags) {
1013 name := tok
1014 if i := strings.IndexByte(name, '='); i >= 0 {
1015 name = name[:i]
1016 }
1017 if desc, ok := reservedFlags[name]; ok {
1018 return fmt.Errorf("custom flag %q conflicts with VidArchive's handling of %s; remove it and try again", tok, desc)
1019 }
1020 if isSubscription {
1021 if desc, ok := reservedSubscriptionFlags[name]; ok {
1022 return fmt.Errorf("custom flag %q conflicts with VidArchive's handling of %s; remove it and try again", tok, desc)
1023 }
1024 }
1025 }
1026 return nil
1027}
1028