package service import ( "context" "encoding/json" "fmt" "log/slog" "os" "path/filepath" "slices" "strings" "vidarchive/internal/models" "vidarchive/internal/util" ) // refreshAndAddNew handles a metadata-mode run, where tempDownloadDir holds only // info.json files. Matched items have their markers refreshed in place; the rest // are genuinely new and are downloaded as full items in a second pass. func (s *DownloadService) refreshAndAddNew(ctx context.Context, d *models.Download, preset *models.Preset, tempDownloadDir, ytdlpFlags string) error { // The library dir is left uncreated: a refresh that matches everything writes // nothing, and importDownloadedItems creates it when a second pass adds one. baseLibraryDir, err := s.resolveBaseLibraryDir(d) if err != nil { return err } entries, err := os.ReadDir(tempDownloadDir) if err != nil { return err } var newURLs []string for _, entry := range entries { if err := ctx.Err(); err != nil { return err } if !entry.IsDir() || !strings.HasPrefix(entry.Name(), "item-") { continue } itemDir := filepath.Join(tempDownloadDir, entry.Name()) infoJSONPath := findInfoJSON(itemDir) if infoJSONPath == "" { continue } info := readInfoJSON(infoJSONPath) if info.ID == "" { continue } // A playlist-level info.json describes the source, not an item: its id // never matches, so it would look new and re-download the whole playlist. // --no-write-playlist-metafiles prevents it now; older runs left some. if info.Type == "playlist" { continue } if existing, ok := s.librarySvc.FindByVideoID(baseLibraryDir, info.ID); ok { if err := s.applyMetadata(existing, info, infoJSONPath); err != nil { slog.Warn("metadata refresh failed", "dir", itemDir, "err", err) } continue } if info.WebpageURL != "" { newURLs = append(newURLs, info.WebpageURL) } } if len(newURLs) == 0 { return nil } return s.downloadFresh(ctx, d, preset, newURLs, ytdlpFlags) } // downloadFresh fetches the given URLs as full downloads, media included. It is // how metadata mode adds entries the library does not have yet. func (s *DownloadService) downloadFresh(ctx context.Context, d *models.Download, preset *models.Preset, urls []string, ytdlpFlags string) error { tempDir := s.tempNewDirFor(d.ID) if err := os.MkdirAll(tempDir, 0o755); err != nil { return err } defer os.RemoveAll(tempDir) args := s.presetSvc.BuildArgs(preset, d.FormatOverride, d.CustomFlags) args, cleanup := s.appendCookies(args) defer cleanup() if !slices.Contains(args, "--write-info-json") { args = append(args, "--write-info-json") } args = append(args, "-P", tempDir) args = append(args, "-o", "item-%(autonumber)05d/%(title)s.%(ext)s") args = append(args, urls...) runErr := s.runYTDLP(ctx, d, args) // A cancelled second pass has nothing worth importing. if ctx.Err() != nil { return runErr } // yt-dlp exits non-zero when a single entry fails, so its error only counts // as a failure when nothing at all was imported. imported, err := s.importDownloadedItems(ctx, d, tempDir, "", ytdlpFlags) if err != nil { slog.Warn("failed to import new metadata-mode items", "err", err) } if imported > 0 { return nil } return runErr } // mergeInfoJSON keeps fields the refresh did not fetch. Comments and heatmap // data are expensive to reacquire and must not disappear just because the // refresh preset does not request them. func mergeInfoJSON(oldData, newData []byte) ([]byte, error) { var oldObject, newObject map[string]json.RawMessage if err := json.Unmarshal(newData, &newObject); err != nil { return nil, err } if err := json.Unmarshal(oldData, &oldObject); err != nil { return newData, nil } if newObject == nil { return newData, nil } merged := make(map[string]json.RawMessage, len(oldObject)+len(newObject)) for key, value := range oldObject { merged[key] = value } for key, value := range newObject { merged[key] = value } for _, key := range []string{"comments", "heatmap"} { oldValue, hadOldValue := oldObject[key] newValue, hasNewValue := newObject[key] if hadOldValue && (!hasNewValue || isEmptyJSONArray(newValue)) { merged[key] = oldValue } } return json.Marshal(merged) } func isEmptyJSONArray(value json.RawMessage) bool { var values []json.RawMessage if err := json.Unmarshal(value, &values); err != nil { return false } return len(values) == 0 } // applyMetadata refreshes an item's marker and sidecar without touching media. func (s *DownloadService) applyMetadata(existing string, info infoJSON, sourceInfoJSON string) error { meta, err := s.librarySvc.readMetadata(existing) if err != nil { return fmt.Errorf("read metadata for %s: %w", existing, err) } if info.Title != "" { meta.Name = info.Title } if info.Description != "" { meta.Description = info.Description } if meta.SourceURL == "" && info.WebpageURL != "" { meta.SourceURL = info.WebpageURL } meta.VideoID = info.ID markerPath := filepath.Join(existing, itemMarkerName) oldMarker, err := os.ReadFile(markerPath) if err != nil { return fmt.Errorf("read existing marker: %w", err) } // Merge the sidecar before touching the marker, so a bad source file fails // before anything is committed. If installing the sidecar still fails, the old // marker is restored: the two files must not end up out of sync. var infoData []byte if sourceInfoJSON != "" { data, err := os.ReadFile(sourceInfoJSON) if err != nil { return fmt.Errorf("read refreshed info JSON: %w", err) } if oldInfoJSON := findInfoJSON(existing); oldInfoJSON != "" { oldData, err := os.ReadFile(oldInfoJSON) if err != nil { return fmt.Errorf("read existing info JSON: %w", err) } data, err = mergeInfoJSON(oldData, data) if err != nil { return fmt.Errorf("merge refreshed info JSON: %w", err) } } infoData = data } if err := s.librarySvc.writeMetadata(existing, meta); err != nil { return err } if infoData != nil { if err := atomicWrite(filepath.Join(existing, "info.json"), infoData, markerFileMode); err != nil { if restoreErr := atomicWrite(markerPath, oldMarker, markerFileMode); restoreErr != nil { return fmt.Errorf("install refreshed info JSON: %v; restore marker: %w", err, restoreErr) } return fmt.Errorf("install refreshed info JSON: %w", err) } } s.librarySvc.evictCachedDir(existing) return nil } // pruneSubscription deletes items no longer present upstream. It never prunes // when the enumeration fails or comes back empty, so a dead URL or network error // can't wipe the directory. func (s *DownloadService) pruneSubscription(ctx context.Context, d *models.Download, sub *models.Subscription) { baseLibraryDir, err := s.resolveBaseLibraryDir(d) if err != nil { slog.Warn("prune skipped", "subscription_id", sub.ID, "err", err) return } keep, err := s.enumeratePlaylistIDs(ctx, sub.URL) if err != nil { slog.Warn("prune skipped, enumeration failed", "subscription_id", sub.ID, "err", err) return } if len(keep) == 0 { slog.Warn("prune skipped, source returned no entries", "subscription_id", sub.ID) return } removed, err := s.librarySvc.PruneToIDSet(baseLibraryDir, keep) if err != nil { slog.Error("prune failed", "subscription_id", sub.ID, "err", err) return } if removed > 0 { slog.Info("pruned items removed upstream", "subscription_id", sub.ID, "removed", removed) } } // enumeratePlaylistIDs lists the current video-id set without downloading. // Cookies are applied so private playlists enumerate correctly. func (s *DownloadService) enumeratePlaylistIDs(ctx context.Context, url string) (map[string]bool, error) { args := []string{"--flat-playlist", "--no-warnings", "--print", "%(id)s"} args, cleanup := s.appendCookies(args) defer cleanup() args = append(args, url) out, err := util.KillableCommand(ctx, s.cfg.YTDLPPath, args...).Output() if err != nil { return nil, err } keep := make(map[string]bool) for _, line := range strings.Split(string(out), "\n") { id := strings.TrimSpace(line) // yt-dlp prints "NA" for a missing field; never treat that as a real id. if id == "" || id == "NA" { continue } keep[id] = true } return keep, nil }