package service import ( "context" "encoding/json" "fmt" "log" "os" "os/exec" "path/filepath" "strings" "vidarchive/internal/models" ) // refreshAndAddNew handles a metadata-mode run. The main pass used // --skip-download, so tempDownloadDir holds only info.json files. Existing // library items have their markers refreshed in place; entries with no existing // match are genuinely new and are downloaded as full items in a second pass. func (s *DownloadService) refreshAndAddNew(ctx context.Context, d *models.Download, preset *models.Preset, tempDownloadDir, ytdlpFlags string) error { // The library dir is not created here: a refresh that matches everything // writes nothing, and importDownloadedItems creates it when a second pass // actually has an item to add. baseLibraryDir, err := s.resolveBaseLibraryDir(d) if err != nil { return err } entries, err := os.ReadDir(tempDownloadDir) if err != nil { return err } var newURLs []string for _, entry := range entries { if err := ctx.Err(); err != nil { return err } if !entry.IsDir() || !strings.HasPrefix(entry.Name(), "item-") { continue } itemDir := filepath.Join(tempDownloadDir, entry.Name()) infoJSONPath := findInfoJSON(itemDir) if infoJSONPath == "" { continue } info := readInfoJSON(infoJSONPath) if info.ID == "" { continue } // A playlist-level info.json describes the source, not an item. Its id // never matches a library item, so it would look new and trigger a // download of the whole playlist. --no-write-playlist-metafiles already // prevents it; this also covers sidecars written by older runs. if info.Type == "playlist" { continue } if existing, ok := s.librarySvc.FindByVideoID(baseLibraryDir, info.ID); ok { if err := s.applyMetadata(existing, info, infoJSONPath); err != nil { log.Printf("warning: failed to refresh metadata for %s: %v", itemDir, err) } continue } if info.WebpageURL != "" { newURLs = append(newURLs, info.WebpageURL) } } if len(newURLs) == 0 { return nil } return s.downloadFresh(ctx, d, preset, newURLs, ytdlpFlags) } // downloadFresh fetches the given item URLs as full downloads (media + info.json) // and imports them into the download's library directory. Metadata mode uses this // to add entries that don't exist in the library yet. func (s *DownloadService) downloadFresh(ctx context.Context, d *models.Download, preset *models.Preset, urls []string, ytdlpFlags string) error { tempDir := s.tempNewDirFor(d.ID) if err := os.MkdirAll(tempDir, 0755); err != nil { return err } defer os.RemoveAll(tempDir) args := s.presetSvc.BuildArgs(preset, d.FormatOverride, d.CustomFlags) args, cleanup := s.appendCookies(args) defer cleanup() args = append(args, "--write-info-json") args = append(args, "-P", tempDir) args = append(args, "-o", "item-%(autonumber)05d/%(title)s.%(ext)s") args = append(args, urls...) runErr := s.runYTDLP(ctx, d, args) // A cancelled second pass has nothing worth importing. if ctx.Err() != nil { return runErr } // Import whatever succeeded even if some entries errored. yt-dlp exits // non-zero when one entry fails, so its error only counts as a failure when // nothing at all was imported. imported, err := s.importDownloadedItems(ctx, d, tempDir, "", ytdlpFlags) if err != nil { log.Printf("warning: failed to import new metadata-mode items: %v", err) } if imported > 0 { return nil } return runErr } // mergeInfoJSON keeps fields from the existing sidecar that are absent from a // metadata-only refresh. In particular, comments and heatmap data are expensive // to reacquire and must not disappear just because the refresh preset does not // request them. func mergeInfoJSON(oldData, newData []byte) ([]byte, error) { var oldObject, newObject map[string]json.RawMessage if err := json.Unmarshal(newData, &newObject); err != nil { return nil, err } if err := json.Unmarshal(oldData, &oldObject); err != nil { return newData, nil } if newObject == nil { return newData, nil } merged := make(map[string]json.RawMessage, len(oldObject)+len(newObject)) for key, value := range oldObject { merged[key] = value } for key, value := range newObject { merged[key] = value } for _, key := range []string{"comments", "heatmap"} { oldValue, hadOldValue := oldObject[key] newValue, hasNewValue := newObject[key] if hadOldValue && (!hasNewValue || isEmptyJSONArray(newValue)) { merged[key] = oldValue } } return json.Marshal(merged) } func isEmptyJSONArray(value json.RawMessage) bool { var values []json.RawMessage if err := json.Unmarshal(value, &values); err != nil { return false } return len(values) == 0 } // restoreFileAtomically puts data back at path without exposing a partial file. // It is used to roll back the marker if installing the staged info sidecar fails. func restoreFileAtomically(path string, data []byte) error { tmp, err := os.CreateTemp(filepath.Dir(path), ".vidarchive-restore-*.tmp") if err != nil { return err } tmpPath := tmp.Name() defer os.Remove(tmpPath) if err := tmp.Chmod(0644); err != nil { tmp.Close() return err } if _, err := tmp.Write(data); err != nil { tmp.Close() return err } if err := tmp.Close(); err != nil { return err } if err := os.Rename(tmpPath, path); err != nil { return err } return nil } // applyMetadata refreshes an existing item's marker and info sidecar from a // fresh info.json without touching its media. func (s *DownloadService) applyMetadata(existing string, info infoJSON, sourceInfoJSON string) error { meta, err := s.librarySvc.readMetadata(existing) if err != nil { return fmt.Errorf("read metadata for %s: %w", existing, err) } if info.Title != "" { meta.Name = info.Title } if info.Description != "" { meta.Description = info.Description } if meta.SourceURL == "" && info.WebpageURL != "" { meta.SourceURL = info.WebpageURL } meta.VideoID = info.ID markerPath := filepath.Join(existing, itemMarkerName) oldMarker, err := os.ReadFile(markerPath) if err != nil { return fmt.Errorf("read existing marker: %w", err) } // Stage the sidecar before changing the marker. The marker is committed first; // if installing the sidecar then fails, restore the old marker so an ordinary // I/O error cannot leave the two metadata files out of sync. stagedInfo := "" defer func() { if stagedInfo != "" { _ = os.Remove(stagedInfo) } }() if sourceInfoJSON != "" { data, err := os.ReadFile(sourceInfoJSON) if err != nil { return fmt.Errorf("read refreshed info JSON: %w", err) } if oldInfoJSON := findInfoJSON(existing); oldInfoJSON != "" { oldData, err := os.ReadFile(oldInfoJSON) if err != nil { return fmt.Errorf("read existing info JSON: %w", err) } data, err = mergeInfoJSON(oldData, data) if err != nil { return fmt.Errorf("merge refreshed info JSON: %w", err) } } tmp, err := os.CreateTemp(existing, ".info-json-*.tmp") if err != nil { return fmt.Errorf("create refreshed info JSON: %w", err) } stagedInfo = tmp.Name() if err := tmp.Chmod(0644); err != nil { tmp.Close() return fmt.Errorf("set refreshed info JSON permissions: %w", err) } if _, err := tmp.Write(data); err != nil { tmp.Close() return fmt.Errorf("write refreshed info JSON: %w", err) } if err := tmp.Close(); err != nil { return fmt.Errorf("close refreshed info JSON: %w", err) } } if err := s.librarySvc.writeMetadata(existing, meta); err != nil { return err } if stagedInfo != "" { if err := os.Rename(stagedInfo, filepath.Join(existing, "info.json")); err != nil { if restoreErr := restoreFileAtomically(markerPath, oldMarker); restoreErr != nil { return fmt.Errorf("install refreshed info JSON: %v; restore marker: %w", err, restoreErr) } return fmt.Errorf("install refreshed info JSON: %w", err) } stagedInfo = "" } if rel, err := filepath.Rel(s.cfg.LibraryDir, existing); err == nil { s.librarySvc.evictCachedScan(filepath.ToSlash(rel)) } return nil } // pruneSubscription mirrors the source by deleting items in the subscription's // directory that are no longer present upstream. It enumerates the current id // set with a cheap flat-playlist listing; it never prunes when that enumeration // fails or returns nothing, so a dead URL or network error can't wipe the dir. func (s *DownloadService) pruneSubscription(ctx context.Context, d *models.Download, sub *models.Subscription) { baseLibraryDir, err := s.resolveBaseLibraryDir(d) if err != nil { log.Printf("subscription %d prune skipped: %v", sub.ID, err) return } keep, err := s.enumeratePlaylistIDs(ctx, sub.URL) if err != nil { log.Printf("subscription %d prune skipped: enumeration failed: %v", sub.ID, err) return } if len(keep) == 0 { log.Printf("subscription %d prune skipped: source returned no entries", sub.ID) return } removed, err := s.librarySvc.PruneToIDSet(baseLibraryDir, keep) if err != nil { log.Printf("subscription %d prune error: %v", sub.ID, err) return } if removed > 0 { log.Printf("subscription %d pruned %d item(s) removed upstream", sub.ID, removed) } } // enumeratePlaylistIDs lists the current video-id set for a URL without // downloading, using yt-dlp --flat-playlist. Cookies are applied so private // playlists enumerate correctly. Ids alone are sufficient to match items within // a subscription's own directory (see FindByVideoID / PruneToIDSet). func (s *DownloadService) enumeratePlaylistIDs(ctx context.Context, url string) (map[string]bool, error) { args := []string{"--flat-playlist", "--no-warnings", "--print", "%(id)s"} args, cleanup := s.appendCookies(args) defer cleanup() args = append(args, url) out, err := exec.CommandContext(ctx, s.cfg.YTDLPPath, args...).Output() if err != nil { return nil, err } keep := make(map[string]bool) for _, line := range strings.Split(string(out), "\n") { id := strings.TrimSpace(line) // yt-dlp prints "NA" for a missing field; never treat that as a real id. if id == "" || id == "NA" { continue } keep[id] = true } return keep, nil }