vidarchive

vidarchive/internal/service/download.go
download.go
⎇
Raw
1package service
2
3import (
4 "bufio"
5 "database/sql"
6 "encoding/json"
7 "fmt"
8 "log"
9 "os"
10 "os/exec"
11 "path/filepath"
12 "regexp"
13 "sort"
14 "strconv"
15 "strings"
16 "sync"
17 "syscall"
18 "time"
19
20 "github.com/BurntSushi/toml"
21 "github.com/gabriel-vasile/mimetype"
22
23 "vidarchive/internal/config"
24 "vidarchive/internal/models"
25 "vidarchive/internal/repository"
26)
27
28type DownloadService struct {
29 repo *repository.DownloadRepository
30 librarySvc *LibraryService
31 presetSvc *PresetService
32 settingsSvc *SettingsService
33 subscriptionSvc *SubscriptionService
34 cfg *config.Config
35 cache *ProgressCache
36 processMu sync.Mutex
37 processes map[int64]*os.Process
38}
39
40func NewDownloadService(repo *repository.DownloadRepository, librarySvc *LibraryService, presetSvc *PresetService, settingsSvc *SettingsService, subscriptionSvc *SubscriptionService, cfg *config.Config) *DownloadService {
41 return &DownloadService{
42 repo: repo,
43 librarySvc: librarySvc,
44 presetSvc: presetSvc,
45 settingsSvc: settingsSvc,
46 subscriptionSvc: subscriptionSvc,
47 cfg: cfg,
48 cache: NewProgressCache(),
49 processes: make(map[int64]*os.Process),
50 }
51}
52
53func (s *DownloadService) Create(url string, presetID *int64, formatOverride, customFlags, outputDir string) (*models.Download, error) {
54 d := &models.Download{
55 URL: url,
56 Status: "queued",
57 FormatOverride: formatOverride,
58 CustomFlags: customFlags,
59 OutputDir: sql.NullString{String: outputDir, Valid: outputDir != ""},
60 }
61
62 if presetID != nil {
63 d.PresetID = sqlNullInt64(*presetID)
64 }
65
66 if err := s.repo.Create(d); err != nil {
67 return nil, err
68 }
69 return d, nil
70}
71
72// CreateForSubscription queues a download for a subscription run, copying its
73// download options and tagging it with the subscription id so ExecuteDownload
74// applies the right refresh mode and pruning.
75func (s *DownloadService) CreateForSubscription(sub *models.Subscription) (*models.Download, error) {
76 d := &models.Download{
77 URL: sub.URL,
78 Status: "queued",
79 FormatOverride: sub.FormatOverride,
80 CustomFlags: sub.CustomFlags,
81 OutputDir: sql.NullString{String: sub.OutputDir, Valid: sub.OutputDir != ""},
82 PresetID: sub.PresetID,
83 SubscriptionID: sqlNullInt64(sub.ID),
84 }
85 if err := s.repo.Create(d); err != nil {
86 return nil, err
87 }
88 return d, nil
89}
90
91func (s *DownloadService) GetByID(id int64) (*models.Download, error) {
92 d, err := s.repo.GetByID(id)
93 if err != nil {
94 return nil, err
95 }
96 if logs := s.cache.Snapshot(id); logs != "" {
97 d.Logs = sql.NullString{String: logs, Valid: true}
98 }
99 return d, nil
100}
101
102func (s *DownloadService) GetAll(status, sortBy string) ([]*models.Download, error) {
103 downloads, err := s.repo.GetAll(status, sortBy)
104 if err != nil {
105 return nil, err
106 }
107
108 for _, d := range downloads {
109 if logs := s.cache.Snapshot(d.ID); logs != "" {
110 d.Logs = sql.NullString{String: logs, Valid: true}
111 }
112 }
113
114 return downloads, nil
115}
116
117func (s *DownloadService) GetQueued(limit int) ([]*models.Download, error) {
118 return s.repo.GetQueued(limit)
119}
120
121func (s *DownloadService) Delete(id int64) error {
122 s.killProcess(id)
123 s.cache.Delete(id)
124 return s.repo.Delete(id)
125}
126
127func (s *DownloadService) killProcess(id int64) {
128 s.processMu.Lock()
129 proc, ok := s.processes[id]
130 delete(s.processes, id)
131 s.processMu.Unlock()
132
133 if !ok || proc == nil {
134 return
135 }
136
137 _ = syscall.Kill(-proc.Pid, syscall.SIGKILL)
138}
139
140func (s *DownloadService) DeleteAll() error {
141 return s.repo.DeleteAll()
142}
143
144func (s *DownloadService) ResetStalledDownloads() error {
145 return s.repo.UpdateStatusWhere("downloading", "queued")
146}
147
148func (s *DownloadService) ListFormats(url string) ([]*models.FormatInfo, error) {
149 cmd := exec.Command(s.cfg.YTDLPPath, "-F", "--no-warnings", url)
150 output, err := cmd.CombinedOutput()
151 if err != nil {
152 return nil, fmt.Errorf("yt-dlp -F failed: %w\nOutput: %s", err, string(output))
153 }
154
155 return parseFormatList(string(output)), nil
156}
157
158// ExecuteDownload runs the download for d. The bool reports whether this call
159// actually processed it: false means another worker already claimed it (Submit
160// and the queue checker can both enqueue the same row within the 2s poll window),
161// so the caller should not log it as completed.
162func (s *DownloadService) ExecuteDownload(d *models.Download) (bool, error) {
163 // Atomically claim the download. If it's no longer queued, another worker
164 // already took it — bail rather than download it twice.
165 claimed, err := s.repo.MarkStarted(d.ID)
166 if err != nil {
167 return false, err
168 }
169 if !claimed {
170 return false, nil
171 }
172
173 s.cache.Set(d.ID, &LiveDownload{LastUpdate: time.Now()})
174 defer s.cache.Delete(d.ID)
175
176 var preset *models.Preset
177
178 if d.PresetID.Valid {
179 preset, err = s.presetSvc.GetByID(d.PresetID.Int64)
180 if err != nil {
181 preset, _ = s.presetSvc.GetDefault()
182 }
183 } else {
184 preset, _ = s.presetSvc.GetDefault()
185 }
186
187 if preset == nil {
188 preset = &models.Preset{}
189 }
190
191 var sub *models.Subscription
192 if d.SubscriptionID.Valid && s.subscriptionSvc != nil {
193 sub, _ = s.subscriptionSvc.GetByID(d.SubscriptionID.Int64)
194 }
195
196 // Reject custom flags that clash with options VidArchive sets itself, before
197 // spending any work — the download fails with a message naming the offender.
198 isSubscription := d.SubscriptionID.Valid
199 if err := checkReservedFlags(d.CustomFlags, isSubscription); err != nil {
200 s.finalizeError(d.ID, err)
201 return false, err
202 }
203 if err := checkReservedFlags(preset.CustomFlags, isSubscription); err != nil {
204 s.finalizeError(d.ID, err)
205 return false, err
206 }
207
208 tempDownloadDir := filepath.Join(s.cfg.TempDir, fmt.Sprintf("%d", d.ID))
209 if err := os.MkdirAll(tempDownloadDir, 0755); err != nil {
210 return false, fmt.Errorf("create temp download dir: %w", err)
211 }
212
213 args := s.presetSvc.BuildArgs(preset, d.FormatOverride, d.CustomFlags)
214
215 // Record the meaningful flags (format/audio/subs/custom) that shaped this
216 // download, before the internal plumbing (cookies, -P/-o, URL) is appended,
217 // so each imported item can show how it was fetched.
218 ytdlpFlags := strings.Join(args, " ")
219
220 var cookieCleanup func()
221 args, cookieCleanup = s.appendCookies(args)
222 defer cookieCleanup()
223
224 if sub != nil {
225 // Always write info.json so the import step can read the stable identity
226 // (id + extractor) used to match/replace existing items.
227 args = append(args, "--write-info-json")
228 switch sub.RefreshMode {
229 case "skip":
230 // Let yt-dlp skip entries already recorded — no re-download.
231 archive := s.subscriptionSvc.ArchivePath(sub.ID)
232 if err := os.MkdirAll(filepath.Dir(archive), 0755); err == nil {
233 args = append(args, "--download-archive", archive)
234 }
235 case "metadata":
236 // Refresh metadata only; don't fetch media.
237 args = append(args, "--skip-download")
238 }
239 }
240
241 args = append(args, "-P", tempDownloadDir)
242 args = append(args, "-o", "item-%(autonumber)05d/%(title)s.%(ext)s")
243 args = append(args, d.URL)
244
245 if err := s.runYTDLP(d, args); err != nil {
246 s.finalizeError(d.ID, err)
247 return false, err
248 }
249
250 mode := ""
251 if sub != nil {
252 mode = sub.RefreshMode
253 }
254
255 // Post-process before marking completed, so the download stays "downloading"
256 // until everything is really done — including metadata mode's second pass,
257 // which downloads any genuinely new entries as full items.
258 if mode == "metadata" {
259 // The main pass ran with --skip-download, so the temp dir holds only
260 // info.json files: refresh existing items in place and fetch new ones.
261 if err := s.refreshAndAddNew(d, preset, tempDownloadDir, ytdlpFlags); err != nil {
262 log.Printf("Download %d metadata refresh failed: %v", d.ID, err)
263 }
264 } else if err := s.importDownloadedItems(d, tempDownloadDir, mode, ytdlpFlags); err != nil {
265 log.Printf("Download %d import failed: %v", d.ID, err)
266 }
267
268 if sub != nil && sub.PruneRemoved {
269 s.pruneSubscription(d, sub)
270 }
271
272 if err := s.repo.MarkCompleted(d.ID, "completed"); err != nil {
273 return false, err
274 }
275
276 return true, nil
277}
278
279// runYTDLP executes yt-dlp with args, streaming combined output into the live
280// progress cache and periodically flushing it to the download's persisted log.
281// It registers the process so Delete can kill it, and returns the exit error.
282func (s *DownloadService) runYTDLP(d *models.Download, args []string) error {
283 cmd := exec.Command(s.cfg.YTDLPPath, args...)
284 cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
285
286 stdout, err := cmd.StdoutPipe()
287 if err != nil {
288 return err
289 }
290 cmd.Stderr = cmd.Stdout
291
292 if err := cmd.Start(); err != nil {
293 return err
294 }
295
296 s.processMu.Lock()
297 s.processes[d.ID] = cmd.Process
298 s.processMu.Unlock()
299 defer func() {
300 s.processMu.Lock()
301 delete(s.processes, d.ID)
302 s.processMu.Unlock()
303 }()
304
305 ticker := time.NewTicker(10 * time.Second)
306 defer ticker.Stop()
307 done := make(chan struct{})
308 go func() {
309 for {
310 select {
311 case <-ticker.C:
312 if logs := s.cache.FlushLogs(d.ID); logs != "" {
313 s.repo.AppendLogs(d.ID, logs)
314 }
315 case <-done:
316 return
317 }
318 }
319 }()
320
321 scanner := bufio.NewScanner(stdout)
322 for scanner.Scan() {
323 s.cache.AppendLog(d.ID, scanner.Text())
324 }
325 close(done)
326
327 if logs := s.cache.FlushLogs(d.ID); logs != "" {
328 s.repo.AppendLogs(d.ID, logs)
329 }
330
331 return cmd.Wait()
332}
333
334// appendCookies writes the saved cookies (if any) to a temp file and appends a
335// --cookies flag. The returned cleanup removes the temp file and is always safe
336// to call, even when no cookies were configured.
337func (s *DownloadService) appendCookies(args []string) ([]string, func()) {
338 cleanup := func() {}
339 cookies, err := s.settingsSvc.GetCookies()
340 if err != nil || strings.TrimSpace(cookies) == "" {
341 return args, cleanup
342 }
343 tmpFile, err := os.CreateTemp("", "cookies-*.txt")
344 if err != nil {
345 return args, cleanup
346 }
347 tmpFile.WriteString(cookies)
348 tmpFile.Close()
349 return append(args, "--cookies", tmpFile.Name()), func() { os.Remove(tmpFile.Name()) }
350}
351
352func (s *DownloadService) finalizeError(id int64, err error) {
353 logs := s.cache.FlushLogs(id)
354 if logs != "" {
355 s.repo.AppendLogs(id, logs)
356 }
357 s.repo.MarkError(id, err.Error())
358}
359
360// resolveBaseLibraryDir returns the absolute library directory a download writes
361// into, applying the optional per-download OutputDir while rejecting any path
362// that escapes the library root.
363func (s *DownloadService) resolveBaseLibraryDir(d *models.Download) (string, error) {
364 baseLibraryDir := s.cfg.LibraryDir
365 if d.OutputDir.Valid && d.OutputDir.String != "" {
366 cleanDir := filepath.Clean(d.OutputDir.String)
367 fullPath := filepath.Join(baseLibraryDir, cleanDir)
368 resolvedPath, err := filepath.Abs(fullPath)
369 if err != nil {
370 return "", fmt.Errorf("invalid output directory: %w", err)
371 }
372 resolvedLibraryDir, _ := filepath.Abs(baseLibraryDir)
373 if !strings.HasPrefix(resolvedPath, resolvedLibraryDir+string(filepath.Separator)) && resolvedPath != resolvedLibraryDir {
374 return "", fmt.Errorf("invalid output directory: path traversal attempt detected")
375 }
376 baseLibraryDir = fullPath
377 }
378 return baseLibraryDir, nil
379}
380
381func (s *DownloadService) importDownloadedItems(d *models.Download, tempDownloadDir, mode, ytdlpFlags string) error {
382 entries, err := os.ReadDir(tempDownloadDir)
383 if err != nil {
384 return err
385 }
386
387 baseLibraryDir, err := s.resolveBaseLibraryDir(d)
388 if err != nil {
389 return err
390 }
391 if err := os.MkdirAll(baseLibraryDir, 0755); err != nil {
392 return err
393 }
394
395 var itemDirs []string
396 for _, entry := range entries {
397 if !entry.IsDir() {
398 continue
399 }
400 name := entry.Name()
401 if strings.HasPrefix(name, "item-") {
402 itemDirs = append(itemDirs, filepath.Join(tempDownloadDir, name))
403 }
404 }
405 sort.Strings(itemDirs)
406
407 for _, itemDir := range itemDirs {
408 if err := s.importItemDir(d.URL, itemDir, baseLibraryDir, mode, ytdlpFlags); err != nil {
409 log.Printf("warning: failed to import item %s: %v", itemDir, err)
410 }
411 }
412
413 os.Remove(tempDownloadDir)
414 return nil
415}
416
417func (s *DownloadService) importItemDir(url, itemDir, baseLibraryDir, mode, ytdlpFlags string) error {
418 entries, err := os.ReadDir(itemDir)
419 if err != nil {
420 return err
421 }
422
423 var mediaFiles []os.DirEntry
424 var infoJSONPath string
425 var subtitleFiles []string
426
427 for _, entry := range entries {
428 if entry.IsDir() {
429 continue
430 }
431 name := entry.Name()
432 path := filepath.Join(itemDir, name)
433 ext := strings.ToLower(filepath.Ext(name))
434
435 if name == "info.json" || strings.HasSuffix(name, ".info.json") {
436 infoJSONPath = path
437 continue
438 }
439 if ext == ".vtt" || ext == ".srt" || ext == ".ass" || ext == ".ssa" {
440 subtitleFiles = append(subtitleFiles, path)
441 continue
442 }
443
444 mtype, err := mimetype.DetectFile(path)
445 if err == nil && mtype != nil && (strings.HasPrefix(mtype.String(), "audio/") || strings.HasPrefix(mtype.String(), "video/")) {
446 mediaFiles = append(mediaFiles, entry)
447 }
448 }
449
450 if len(mediaFiles) == 0 {
451 return fmt.Errorf("no media files found in %s", itemDir)
452 }
453
454 name := s.deriveItemName(itemDir, infoJSONPath, mediaFiles)
455 videoID, extractor := readInfoIdentity(infoJSONPath)
456
457 // Overwrite mode: replace the existing copy of this video in place rather than
458 // creating a duplicate folder. Removing the old dir lets uniqueDir reuse its
459 // name (or land on the new title if it changed upstream).
460 if mode == "overwrite" && videoID != "" {
461 if existing, ok := s.librarySvc.FindByVideoID(baseLibraryDir, extractor, videoID); ok {
462 if rel, err := filepath.Rel(s.cfg.LibraryDir, existing); err == nil {
463 s.librarySvc.evictCachedScan(filepath.ToSlash(rel))
464 }
465 os.RemoveAll(existing)
466 }
467 }
468
469 targetDir := s.uniqueDir(baseLibraryDir, name)
470 if err := os.MkdirAll(targetDir, 0755); err != nil {
471 return err
472 }
473
474 if infoJSONPath != "" {
475 if err := os.Rename(infoJSONPath, filepath.Join(targetDir, "info.json")); err != nil {
476 return err
477 }
478 }
479
480 for _, entry := range mediaFiles {
481 if err := os.Rename(filepath.Join(itemDir, entry.Name()), filepath.Join(targetDir, entry.Name())); err != nil {
482 return err
483 }
484 }
485
486 // Probe each media file's duration once, here in the worker (off the request
487 // path), and cache it in the marker so the library never has to probe while
488 // serving pages. Files we can't probe simply get no duration.
489 fileDurations := make(map[string]int)
490 for _, entry := range mediaFiles {
491 if d, ok := probeDuration(filepath.Join(targetDir, entry.Name())); ok {
492 fileDurations[entry.Name()] = d
493 }
494 }
495
496 if len(subtitleFiles) > 0 {
497 subtitlesDir := filepath.Join(targetDir, subtitlesDirName)
498 if err := os.MkdirAll(subtitlesDir, 0755); err != nil {
499 return err
500 }
501 for _, sf := range subtitleFiles {
502 if err := os.Rename(sf, filepath.Join(subtitlesDir, filepath.Base(sf))); err != nil {
503 return err
504 }
505 }
506 }
507
508 metadata := models.ItemMetadata{
509 Name: name,
510 SourceURL: url,
511 VideoID: videoID,
512 Extractor: extractor,
513 YtdlpFlags: ytdlpFlags,
514 FileDurations: fileDurations,
515 }
516
517 markerPath := filepath.Join(targetDir, itemMarkerName)
518 f, err := os.Create(markerPath)
519 if err != nil {
520 return err
521 }
522 defer f.Close()
523 if err := toml.NewEncoder(f).Encode(metadata); err != nil {
524 return err
525 }
526
527 return nil
528}
529
530// readInfoIdentity extracts the stable identity (yt-dlp id + extractor) from an
531// info.json. Returns empty strings when the file is absent or unreadable.
532func readInfoIdentity(infoJSONPath string) (id, extractor string) {
533 if infoJSONPath == "" {
534 return "", ""
535 }
536 data, err := os.ReadFile(infoJSONPath)
537 if err != nil {
538 return "", ""
539 }
540 var info struct {
541 ID string `json:"id"`
542 Extractor string `json:"extractor"`
543 }
544 if err := json.Unmarshal(data, &info); err != nil {
545 return "", ""
546 }
547 return info.ID, info.Extractor
548}
549
550// refreshAndAddNew handles a metadata-mode run. The main pass used
551// --skip-download, so tempDownloadDir holds only info.json files. Existing
552// library items have their markers refreshed in place; entries with no existing
553// match are genuinely new and are downloaded as full items in a second pass.
554func (s *DownloadService) refreshAndAddNew(d *models.Download, preset *models.Preset, tempDownloadDir, ytdlpFlags string) error {
555 baseLibraryDir, err := s.resolveBaseLibraryDir(d)
556 if err != nil {
557 return err
558 }
559 if err := os.MkdirAll(baseLibraryDir, 0755); err != nil {
560 return err
561 }
562
563 entries, err := os.ReadDir(tempDownloadDir)
564 if err != nil {
565 return err
566 }
567
568 var newURLs []string
569 for _, entry := range entries {
570 if !entry.IsDir() || !strings.HasPrefix(entry.Name(), "item-") {
571 continue
572 }
573 itemDir := filepath.Join(tempDownloadDir, entry.Name())
574 infoJSONPath := findInfoJSON(itemDir)
575 if infoJSONPath == "" {
576 continue
577 }
578 videoID, extractor := readInfoIdentity(infoJSONPath)
579 if videoID == "" {
580 continue
581 }
582 if existing, ok := s.librarySvc.FindByVideoID(baseLibraryDir, extractor, videoID); ok {
583 if err := s.applyMetadata(existing, infoJSONPath, videoID, extractor); err != nil {
584 log.Printf("warning: failed to refresh metadata for %s: %v", itemDir, err)
585 }
586 continue
587 }
588 if u := readWebpageURL(infoJSONPath); u != "" {
589 newURLs = append(newURLs, u)
590 }
591 }
592
593 os.RemoveAll(tempDownloadDir)
594
595 if len(newURLs) == 0 {
596 return nil
597 }
598 return s.downloadFresh(d, preset, newURLs, ytdlpFlags)
599}
600
601// downloadFresh fetches the given item URLs as full downloads (media + info.json)
602// and imports them into the download's library directory. Metadata mode uses this
603// to add entries that don't exist in the library yet.
604func (s *DownloadService) downloadFresh(d *models.Download, preset *models.Preset, urls []string, ytdlpFlags string) error {
605 tempDir := filepath.Join(s.cfg.TempDir, fmt.Sprintf("%d-new", d.ID))
606 if err := os.MkdirAll(tempDir, 0755); err != nil {
607 return err
608 }
609
610 args := s.presetSvc.BuildArgs(preset, d.FormatOverride, d.CustomFlags)
611 args, cleanup := s.appendCookies(args)
612 defer cleanup()
613 args = append(args, "--write-info-json")
614 args = append(args, "-P", tempDir)
615 args = append(args, "-o", "item-%(autonumber)05d/%(title)s.%(ext)s")
616 args = append(args, urls...)
617
618 runErr := s.runYTDLP(d, args)
619 // Import whatever succeeded even if some entries errored.
620 if err := s.importDownloadedItems(d, tempDir, "", ytdlpFlags); err != nil {
621 log.Printf("warning: failed to import new metadata-mode items: %v", err)
622 }
623 return runErr
624}
625
626// applyMetadata rewrites an existing item's marker (name/description/identity)
627// from a fresh info.json without touching its media.
628func (s *DownloadService) applyMetadata(existing, infoJSONPath, videoID, extractor string) error {
629 data, err := os.ReadFile(infoJSONPath)
630 if err != nil {
631 return err
632 }
633 var info struct {
634 Title string `json:"title"`
635 Description string `json:"description"`
636 WebpageURL string `json:"webpage_url"`
637 }
638 _ = json.Unmarshal(data, &info)
639
640 meta, _ := s.librarySvc.readOrCreateMetadata(existing)
641 if info.Title != "" {
642 meta.Name = info.Title
643 }
644 if info.Description != "" {
645 meta.Description = info.Description
646 }
647 if meta.SourceURL == "" && info.WebpageURL != "" {
648 meta.SourceURL = info.WebpageURL
649 }
650 meta.VideoID = videoID
651 meta.Extractor = extractor
652
653 if err := s.librarySvc.writeMetadata(existing, meta); err != nil {
654 return err
655 }
656 if rel, err := filepath.Rel(s.cfg.LibraryDir, existing); err == nil {
657 s.librarySvc.evictCachedScan(filepath.ToSlash(rel))
658 }
659 return nil
660}
661
662// readWebpageURL returns the canonical entry URL from an info.json, or "".
663func readWebpageURL(infoJSONPath string) string {
664 data, err := os.ReadFile(infoJSONPath)
665 if err != nil {
666 return ""
667 }
668 var info struct {
669 WebpageURL string `json:"webpage_url"`
670 }
671 if err := json.Unmarshal(data, &info); err != nil {
672 return ""
673 }
674 return info.WebpageURL
675}
676
677// findInfoJSON returns the path to an info.json directly inside itemDir, or "".
678func findInfoJSON(itemDir string) string {
679 entries, err := os.ReadDir(itemDir)
680 if err != nil {
681 return ""
682 }
683 for _, entry := range entries {
684 if entry.IsDir() {
685 continue
686 }
687 name := entry.Name()
688 if name == "info.json" || strings.HasSuffix(name, ".info.json") {
689 return filepath.Join(itemDir, name)
690 }
691 }
692 return ""
693}
694
695// pruneSubscription mirrors the source by deleting items in the subscription's
696// directory that are no longer present upstream. It enumerates the current id
697// set with a cheap flat-playlist listing; it never prunes when that enumeration
698// fails or returns nothing, so a dead URL or network error can't wipe the dir.
699func (s *DownloadService) pruneSubscription(d *models.Download, sub *models.Subscription) {
700 baseLibraryDir, err := s.resolveBaseLibraryDir(d)
701 if err != nil {
702 log.Printf("subscription %d prune skipped: %v", sub.ID, err)
703 return
704 }
705
706 keep, err := s.enumeratePlaylistIDs(sub.URL)
707 if err != nil {
708 log.Printf("subscription %d prune skipped: enumeration failed: %v", sub.ID, err)
709 return
710 }
711 if len(keep) == 0 {
712 log.Printf("subscription %d prune skipped: source returned no entries", sub.ID)
713 return
714 }
715
716 removed, err := s.librarySvc.PruneToIDSet(baseLibraryDir, keep)
717 if err != nil {
718 log.Printf("subscription %d prune error: %v", sub.ID, err)
719 return
720 }
721 if removed > 0 {
722 log.Printf("subscription %d pruned %d item(s) removed upstream", sub.ID, removed)
723 }
724}
725
726// enumeratePlaylistIDs lists the current "extractor:id" set for a URL without
727// downloading, using yt-dlp --flat-playlist. Cookies are applied so private
728// playlists enumerate correctly.
729func (s *DownloadService) enumeratePlaylistIDs(url string) (map[string]bool, error) {
730 args := []string{"--flat-playlist", "--no-warnings", "--print", "%(extractor)s %(id)s"}
731
732 args, cleanup := s.appendCookies(args)
733 defer cleanup()
734 args = append(args, url)
735
736 out, err := exec.Command(s.cfg.YTDLPPath, args...).Output()
737 if err != nil {
738 return nil, err
739 }
740
741 keep := make(map[string]bool)
742 for _, line := range strings.Split(string(out), "\n") {
743 fields := strings.Fields(line)
744 if len(fields) != 2 {
745 continue
746 }
747 if key := videoKey(fields[0], fields[1]); key != "" {
748 keep[key] = true
749 }
750 }
751 return keep, nil
752}
753
754func (s *DownloadService) deriveItemName(itemDir, infoJSONPath string, mediaFiles []os.DirEntry) string {
755 if infoJSONPath != "" {
756 data, err := os.ReadFile(infoJSONPath)
757 if err == nil {
758 var info struct {
759 Title string `json:"title"`
760 }
761 if err := json.Unmarshal(data, &info); err == nil && info.Title != "" {
762 return sanitizeDirName(info.Title)
763 }
764 }
765 }
766
767 sort.Slice(mediaFiles, func(i, j int) bool {
768 ii, _ := os.Stat(filepath.Join(itemDir, mediaFiles[i].Name()))
769 jj, _ := os.Stat(filepath.Join(itemDir, mediaFiles[j].Name()))
770 if ii == nil || jj == nil {
771 return false
772 }
773 return ii.Size() > jj.Size()
774 })
775
776 base := strings.TrimSuffix(mediaFiles[0].Name(), filepath.Ext(mediaFiles[0].Name()))
777 return sanitizeDirName(base)
778}
779
780func (s *DownloadService) uniqueDir(base, name string) string {
781 dir := filepath.Join(base, name)
782 if _, err := os.Stat(dir); os.IsNotExist(err) {
783 return dir
784 }
785 for i := 1; ; i++ {
786 candidate := fmt.Sprintf("%s-%d", dir, i)
787 if _, err := os.Stat(candidate); os.IsNotExist(err) {
788 return candidate
789 }
790 }
791}
792
793func sanitizeDirName(name string) string {
794 name = strings.TrimSpace(name)
795 replacer := strings.NewReplacer(
796 "/", "-",
797 "\\", "-",
798 ":", "-",
799 "*", "-",
800 "?", "-",
801 "\"", "-",
802 "<", "-",
803 ">", "-",
804 "|", "-",
805 )
806 name = replacer.Replace(name)
807 name = strings.TrimSpace(name)
808 if name == "" {
809 name = "untitled"
810 }
811 return name
812}
813
814// resolutionRe matches a yt-dlp resolution column like "1920x1080".
815var resolutionRe = regexp.MustCompile(`^\d+x\d+$`)
816
817// probeDuration returns the duration of a media file in whole seconds. The bool
818// is false when ffprobe is unavailable or the file has no usable duration.
819func probeDuration(path string) (int, bool) {
820 out, err := exec.Command("ffprobe", "-v", "error",
821 "-show_entries", "format=duration",
822 "-of", "default=nw=1:nk=1", path).Output()
823 if err != nil {
824 return 0, false
825 }
826 f, err := strconv.ParseFloat(strings.TrimSpace(string(out)), 64)
827 if err != nil || f <= 0 {
828 return 0, false
829 }
830 return int(f + 0.5), true
831}
832
833func parseFormatList(output string) []*models.FormatInfo {
834 lines := strings.Split(output, "\n")
835 var formats []*models.FormatInfo
836
837 inFormats := false
838 for _, line := range lines {
839 line = strings.TrimSpace(line)
840 if line == "" {
841 continue
842 }
843
844 if strings.Contains(line, "ID") && strings.Contains(line, "EXT") {
845 inFormats = true
846 continue
847 }
848
849 if !inFormats {
850 continue
851 }
852
853 parts := strings.Fields(line)
854 if len(parts) >= 4 {
855 format := &models.FormatInfo{
856 ID: parts[0],
857 Ext: parts[1],
858 }
859
860 for i, part := range parts {
861 // A resolution token is strictly <digits>x<digits> (e.g. 1920x1080);
862 // matching on a literal "x" anywhere misclassified notes/codecs.
863 if resolutionRe.MatchString(part) {
864 format.Resolution = part
865 if i+1 < len(parts) {
866 format.FPS = parts[i+1]
867 }
868 break
869 }
870 }
871
872 if len(parts) > 3 {
873 format.Note = strings.Join(parts[3:], " ")
874 }
875
876 formats = append(formats, format)
877 }
878 }
879
880 return formats
881}
882
883func sqlNullInt64(v int64) sql.NullInt64 {
884 return sql.NullInt64{Int64: v, Valid: true}
885}
886
887// reservedFlags are yt-dlp options VidArchive always sets itself; user custom
888// flags must not pass them (or a conflicting inverse). The value describes what
889// the option controls, for the failure message.
890var reservedFlags = map[string]string{
891 "-o": "the output template",
892 "--output": "the output template",
893 "-P": "the download path",
894 "--paths": "the download path",
895 "--cookies": "cookies (set these in Settings instead)",
896 "--no-cookies": "cookies (set these in Settings instead)",
897}
898
899// reservedSubscriptionFlags are additionally reserved for subscription runs,
900// where VidArchive drives info-json writing and the refresh mode.
901var reservedSubscriptionFlags = map[string]string{
902 "--write-info-json": "info-json writing (needed to track item identity)",
903 "--no-write-info-json": "info-json writing (needed to track item identity)",
904 "--download-archive": "the download archive (managed by Skip mode)",
905 "--no-download-archive": "the download archive (managed by Skip mode)",
906 "--skip-download": "media downloading (managed by Metadata mode)",
907 "--no-skip-download": "media downloading (managed by Metadata mode)",
908}
909
910// checkReservedFlags rejects custom flags that clash with options VidArchive
911// controls, naming the offender. It matches both "--flag" and "--flag=value".
912func checkReservedFlags(customFlags string, isSubscription bool) error {
913 for _, tok := range strings.Fields(customFlags) {
914 name := tok
915 if i := strings.IndexByte(name, '='); i >= 0 {
916 name = name[:i]
917 }
918 if desc, ok := reservedFlags[name]; ok {
919 return fmt.Errorf("custom flag %q conflicts with VidArchive's handling of %s; remove it and try again", tok, desc)
920 }
921 if isSubscription {
922 if desc, ok := reservedSubscriptionFlags[name]; ok {
923 return fmt.Errorf("custom flag %q conflicts with VidArchive's handling of %s; remove it and try again", tok, desc)
924 }
925 }
926 }
927 return nil
928}
929