harden yt-dlp interop, add live tests and Hearthforge CI
yt-dlp interop: - import downloaded items even when yt-dlp exits non-zero; a single failed playlist entry no longer discards the rest. Zero imported items plus an error is still an error, otherwise the run completes with a warning line in the log - always pass --no-write-playlist-metafiles and skip "_type: playlist" sidecars in metadata mode; the playlist-level info.json was treated as a new item and re-downloaded the whole playlist - join youtube extractor args with ";" and merge the raw extractor-args field into the same flag; max_comments was silently dropped before - pass --socket-timeout 30 so a stalled connection cannot pin a worker - format listing uses -I 1 and the saved cookies - save the cookie jar yt-dlp writes back, so rotated session cookies do not go stale - split custom flags like a shell (go-shellwords) so quoted values work; malformed quoting fails the download early - document that the reserved-flag list is a misuse guard, not a security boundary tests and CI: - opt-in live tests against the real yt-dlp, gated by VIDARCHIVE_ONLINE_TESTS=1, to catch version drift - .hearthforge-ci.toml: vet, test, optional online tests via the ONLINE_TESTS variable, static build, and an image step that pushes to the built-in registry - Containerfile: prebuilt stage selected by BIN_STAGE so CI ships the tested binary; drop the dead COPY of web/, assets are embedded misc: - exit with an error when the binary is given arguments; it is configured by environment variables only
M.gitignore
@@ -33,3 +33,6 @@ go.work.sum
data/
todo.txt
# CI drops the prebuilt binary here for the image step
ci-bin/
A.hearthforge-ci.toml
@@ -0,0 +1,105 @@
# Hearthforge CI for VidArchive.
# Steps share one container and run in file order, so what "setup" installs
# stays available to every later step.
image = "docker.io/golang:1.26"
work_dir = "/ci/build"
clone_project_to = "/ci/build/project"
shell_setup = "set -euo pipefail"
timeout = 1800
cpu_limit = 2.0
memory_limit = "2g"
# Go's build and module caches. Both live outside clone_project_to.
cache = [
{ path = "/root/.cache/go-build", max_size = "2g" },
{ path = "/go/pkg/mod", max_size = "2g" },
]
[on]
push = ["*"]
tag = true
[variables]
[variables.ONLINE_TESTS]
default = "1"
description = "Set to 1 to also run the live yt-dlp tests against YouTube (needs network)"
# ffmpeg/ffprobe unlock the media tests; yt-dlp is the standalone Linux build,
# which needs no Python. podman-remote is for the image step.
[[steps]]
name = "setup"
timeout = 600
run_sh = """
apt-get update -qq && apt-get install -y -qq --no-install-recommends ffmpeg podman-remote > /dev/null
curl -fsSL https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp_linux -o /usr/local/bin/yt-dlp
chmod +x /usr/local/bin/yt-dlp
yt-dlp --version
"""
# gofmt has no failure exit code, so an empty report is the pass condition.
[[steps]]
name = "vet"
run_sh = "cd project && gofmt -l . | tee /ci/build/gofmt.txt && test ! -s /ci/build/gofmt.txt && go vet ./..."
[[steps]]
name = "test"
run_sh = "cd project && go test ./..."
# The live tests hit YouTube, which may block CI IPs. A failure must stay
# visible but must not fail the run.
[[steps]]
name = "test-online"
run_if = 'test "$ONLINE_TESTS" = "1"'
run_sh = "cd project && VIDARCHIVE_ONLINE_TESTS=1 go test ./internal/service -run Live -v"
warn_on_fail = true
[[steps]]
name = "build"
run_sh = "cd project && CGO_ENABLED=0 go build -trimpath -ldflags='-s -w' -o /ci/build/dist/vidarchive ./cmd/vidarchive"
publish_file = ["/ci/build/dist/vidarchive"]
# ── image ────────────────────────────────────────────────────────────────────
# Packages the binary the build step made, so the image ships exactly what was
# tested. The Containerfile's build stage is skipped via BIN_STAGE.
# engine_socket hands this step the host engine, which is Podman on this
# server (needs CI_ENGINE_SOCKET=1). REGISTRY_PASSWORD is a CI secret with the
# admin password. CI_REGISTRY is "<host>/<repo>" on the built-in registry.
# Tags: every run pushes the short sha and "edge"; a tag run also pushes the
# tag and "latest".
[[steps]]
name = "image"
engine_socket = true
timeout = 900
run_sh = """
cd project
mkdir -p ci-bin
cp /ci/build/dist/vidarchive ci-bin/vidarchive
echo "$REGISTRY_PASSWORD" | podman-remote login "${CI_REGISTRY%%/*}" -u admin --password-stdin
# A remote build sends a seccomp profile path that the server opens. Ask the
# server for its own path. An empty answer means no profile can be named, so
# the build runs unconfined rather than failing.
prof=$(podman-remote info --format '{{.Host.Security.SECCOMPProfilePath}}' 2>/dev/null || true)
if [ -n "$prof" ]; then
seccomp="seccomp=$prof"
else
seccomp="seccomp=unconfined"
fi
img="$CI_REGISTRY:$CI_COMMIT_SHORT_SHA"
podman-remote build --security-opt "$seccomp" \
-f Containerfile -t "$img" --build-arg BIN_STAGE=prebuilt .
podman-remote push "$img"
if [ -n "${CI_COMMIT_TAG:-}" ]; then
tags="$CI_COMMIT_TAG latest"
else
tags="edge"
fi
for t in $tags; do
podman-remote tag "$img" "$CI_REGISTRY:$t"
podman-remote push "$CI_REGISTRY:$t"
done
"""
MContainerfile
@@ -1,14 +1,30 @@
FROM golang:1.26-alpine AS builder
# Which stage supplies the binary: "build" compiles it, "prebuilt" takes it
# from ci-bin/ (see below). Declared before the first FROM so the stage lookup
# in COPY --from can resolve it.
ARG BIN_STAGE=build
FROM golang:1.26-alpine AS build
WORKDIR /build
COPY go.mod go.sum ./
RUN go mod download
COPY . .
RUN CGO_ENABLED=0 go build -o vidarchive ./cmd/vidarchive
RUN CGO_ENABLED=0 go build -trimpath -ldflags='-s -w' -o /vidarchive ./cmd/vidarchive
# ── prebuilt ─────────────────────────────────────────────────────────────────
# CI has the binary already. It puts it at ci-bin/vidarchive and selects this
# stage with --build-arg BIN_STAGE=prebuilt. BuildKit and buildah do not build a
# stage nobody references, so a normal build needs no ci-bin/.
FROM scratch AS prebuilt
COPY ci-bin/vidarchive /vidarchive
# ── runtime ──────────────────────────────────────────────────────────────────
FROM alpine:3.24
# re-declare: args set before FROM do not carry into stages
ARG BIN_STAGE
RUN apk --no-cache add ca-certificates ffmpeg python3 py3-pip \
&& python3 -m venv /opt/ytdlp \
&& /opt/ytdlp/bin/pip install --no-cache-dir --upgrade yt-dlp
@@ -17,8 +33,8 @@ ENV PATH="/opt/ytdlp/bin:${PATH}"
WORKDIR /app
COPY --from=builder /build/vidarchive /app/vidarchive
COPY --from=builder /build/web /app/web
# Templates and static files are embedded in the binary (assets.go).
COPY --from=${BIN_STAGE} /vidarchive /app/vidarchive
ENV VIDARCHIVE_DATA_DIR=/data
ENV VIDARCHIVE_PORT=8080
MREADME.md
@@ -1,5 +1,7 @@
# VidArchive
A self-hosted web front end for [yt-dlp](https://github.com/yt-dlp/yt-dlp). It
queues downloads, keeps the results in a browsable library with playback,
metadata, subtitles and comments, and re-runs saved subscriptions on a schedule.
@@ -58,6 +60,27 @@ embedded in the binary.
| `VIDARCHIVE_SCHEDULER_INTERVAL` | `60` | Seconds between subscription checks |
| `VIDARCHIVE_BASE_URL` | — | External URL; an `https://` value enables HSTS |
## Tests
The offline suite needs no network and stubs yt-dlp with shell scripts:
```sh
go test ./...
```
Some tests build real media files. They are skipped unless `ffmpeg` and
`ffprobe` are on `PATH`.
The online suite runs the real yt-dlp against a real YouTube video. It checks
that a new yt-dlp release still behaves the way VidArchive expects. It needs
network access and `yt-dlp` on `PATH`, and is skipped otherwise:
```sh
VIDARCHIVE_ONLINE_TESTS=1 go test ./internal/service -run Live -v
```
Set `VIDARCHIVE_TEST_VIDEO_URL` to use a different video.
## Concepts
**Presets** collect the yt-dlp options for a download: format selection, audio
Mcmd/vidarchive/main.go
@@ -21,6 +21,13 @@ import (
)
func main() {
// All configuration is environment variables (see README). An argument is
// a mistake, so fail instead of silently starting the server.
if len(os.Args) > 1 {
fmt.Fprintf(os.Stderr, "vidarchive takes no arguments; configure it with VIDARCHIVE_* environment variables (got %q)\n", os.Args[1:])
os.Exit(2)
}
// Cancelled on SIGINT/SIGTERM so the server drains, workers stop, running
// yt-dlp children are killed and the database is checkpointed and closed.
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
Mgo.mod
@@ -6,6 +6,7 @@ require (
github.com/BurntSushi/toml v1.6.0
github.com/gabriel-vasile/mimetype v1.4.13
github.com/go-chi/chi/v5 v5.2.5
github.com/mattn/go-shellwords v1.0.15
github.com/robfig/cron/v3 v3.0.1
modernc.org/sqlite v1.50.0
)
Mgo.sum
@@ -14,6 +14,8 @@ github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs
github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
github.com/mattn/go-shellwords v1.0.15 h1:rx0n8+ZdM9JWZMlr2BMPAjtLU0rfluLNtwMC2FJOTtY=
github.com/mattn/go-shellwords v1.0.15/go.mod h1:EZzvwXDESEeg03EKmM+RmDnNOPKG4lLtQsUlTZDWQ8Y=
github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w=
github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
Minternal/service/download.go
@@ -329,10 +329,6 @@ func (s *DownloadService) ExecuteDownload(parent context.Context, d *models.Down
if ctx.Err() != nil {
return false, s.finalizeCancelled(parent, d)
}
if runErr != nil {
s.finalizeError(d, runErr)
return false, runErr
}
mode := ""
if sub != nil {
@@ -342,22 +338,35 @@ func (s *DownloadService) ExecuteDownload(parent context.Context, d *models.Down
// Post-process before marking completed, so the download stays "downloading"
// until everything is really done — including metadata mode's second pass,
// which downloads any genuinely new entries as full items.
//
// This runs even when runErr is set: yt-dlp exits non-zero if a single
// playlist entry fails, while the other entries downloaded fine. Skipping the
// import would throw those away with the temp dir. The outcome is decided
// below, once the imported count is known.
var postErr error
partial := ""
if mode == "metadata" {
// The main pass ran with --skip-download, so the temp dir holds only
// info.json files: refresh existing items in place and fetch new ones.
postErr = s.refreshAndAddNew(ctx, d, preset, tempDownloadDir, ytdlpFlags)
if postErr == nil && runErr != nil {
partial = fmt.Sprintf("VidArchive: yt-dlp exited with an error (%v); the metadata refresh finished anyway. Check the log above for failed entries.", runErr)
}
} else {
imported, err := s.importDownloadedItems(ctx, d, tempDownloadDir, mode, ytdlpFlags)
switch {
case err != nil:
postErr = err
case imported == 0 && runErr != nil:
postErr = runErr
// A plain (non-subscription) download that yields nothing is a failure, not
// a silent "completed". Subscription modes legitimately import zero (skip
// mode, or a metadata refresh with no new entries), so only enforce this for
// plain runs.
case sub == nil && imported == 0:
case imported == 0 && sub == nil:
postErr = fmt.Errorf("yt-dlp finished but no media files were downloaded")
case runErr != nil:
partial = fmt.Sprintf("VidArchive: yt-dlp exited with an error (%v); %d item(s) were imported anyway. Check the log above for failed entries.", runErr, imported)
}
}
@@ -375,6 +384,13 @@ func (s *DownloadService) ExecuteDownload(parent context.Context, d *models.Down
return false, postErr
}
// Record the partial failure in the download's own log. The run counts as
// completed, so nothing else would tell the user some entries failed.
if partial != "" {
s.cache.AppendLog(d.ID, partial)
s.flushLogs(d.ID)
}
if err := s.repo.MarkCompleted(d.ID, "completed"); err != nil {
return false, err
}
@@ -391,7 +407,14 @@ func (s *DownloadService) runYTDLP(ctx context.Context, d *models.Download, args
// it, progress is rewritten in place with carriage returns, so a long download
// becomes one ever-growing line that overflows the reader's buffer and stalls
// the pipe — hanging the download. See the hardened scanner below.
fullArgs := append([]string{"--newline"}, args...)
//
// --no-write-playlist-metafiles suppresses the playlist-level info.json that
// --write-info-json would also produce. It lands in the first item dir and
// would be imported as if it were an item.
//
// --socket-timeout bounds a stalled connection. Without it a dead socket pins
// a worker forever. There is no inactivity killer beyond this.
fullArgs := append([]string{"--newline", "--no-write-playlist-metafiles", "--socket-timeout", "30"}, args...)
cmd := exec.CommandContext(ctx, s.cfg.YTDLPPath, fullArgs...)
cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
// yt-dlp spawns helpers (ffmpeg, external downloaders). Kill the whole group
@@ -442,8 +465,9 @@ func (s *DownloadService) runYTDLP(ctx context.Context, d *models.Download, args
}
// appendCookies writes the saved cookies (if any) to a temp file and appends a
// --cookies flag. The returned cleanup removes the temp file and is always safe
// to call, even when no cookies were configured.
// --cookies flag. The returned cleanup saves the cookies yt-dlp left behind and
// removes the temp file. It is always safe to call, even when no cookies were
// configured.
func (s *DownloadService) appendCookies(args []string) ([]string, func()) {
cookies, err := s.settingsSvc.GetCookies()
if err != nil || strings.TrimSpace(cookies) == "" {
@@ -453,7 +477,28 @@ func (s *DownloadService) appendCookies(args []string) ([]string, func()) {
if err != nil {
return args, func() {}
}
return append(args, "--cookies", path), func() { os.Remove(path) }
return append(args, "--cookies", path), func() {
s.saveRefreshedCookies(path, cookies)
os.Remove(path)
}
}
// saveRefreshedCookies stores back what yt-dlp wrote to the cookie file.
//
// yt-dlp rewrites the jar on exit. YouTube rotates session cookies on use, so
// the snapshot we sent is stale once the run ends. Keeping the old snapshot and
// replaying it later gets the session invalidated, and the user has to export
// cookies again. sent is what we wrote, so an untouched file saves nothing.
func (s *DownloadService) saveRefreshedCookies(path, sent string) {
data, err := os.ReadFile(path)
// A missing file means yt-dlp never got that far. Empty content would wipe
// working cookies, so treat it as nothing to do.
if err != nil || strings.TrimSpace(string(data)) == "" || string(data) == sent {
return
}
if err := s.settingsSvc.SetCookies(string(data)); err != nil {
log.Printf("failed to save refreshed cookies: %v", err)
}
}
// writeCookiesFile writes cookies to a temp file in the app's own temp dir (the
Minternal/service/execute_download_test.go
@@ -16,10 +16,11 @@ import (
// execEnv is a DownloadService wired to a real database and temp directories,
// with yt-dlp and ffprobe replaced by scripts the test controls.
type execEnv struct {
svc *DownloadService
repo *repository.DownloadRepository
subRepo *repository.SubscriptionRepository
cfg *config.Config
svc *DownloadService
repo *repository.DownloadRepository
subRepo *repository.SubscriptionRepository
settings *SettingsService
cfg *config.Config
// scratch holds the marker/signal files the fake tools read and write.
scratch string
}
@@ -46,16 +47,17 @@ func newExecEnv(t *testing.T) *execEnv {
downloadRepo := repository.NewDownloadRepository(db)
subRepo := repository.NewSubscriptionRepository(db)
settingsSvc := NewSettingsService(repository.NewSettingsRepository(db))
svc := NewDownloadService(
downloadRepo,
NewLibraryService(cfg.LibraryDir, cfg.FFmpegPath, cfg.FFprobePath),
NewPresetService(repository.NewPresetRepository(db)),
NewSettingsService(repository.NewSettingsRepository(db)),
settingsSvc,
NewSubscriptionService(subRepo, cfg),
cfg,
)
return &execEnv{svc: svc, repo: downloadRepo, subRepo: subRepo, cfg: cfg, scratch: root}
return &execEnv{svc: svc, repo: downloadRepo, subRepo: subRepo, settings: settingsSvc, cfg: cfg, scratch: root}
}
// fakeYTDLP installs a stand-in for yt-dlp. body is shell run with $DEST set to
@@ -258,6 +260,51 @@ func TestExecuteDownloadEmptyResultIsFailure(t *testing.T) {
}
}
// yt-dlp exits non-zero when a single playlist entry fails. The entries that did
// download must still be imported, and the run must read as completed with a
// note in the log.
func TestExecuteDownloadPartialPlaylistFailureCompletes(t *testing.T) {
e := newExecEnv(t)
e.fakeYTDLP(t, e.writeItem(t, "item-00001", "My Clip", "abc123")+`
echo "ERROR: [youtube] bad2: Video unavailable" >&2
exit 1`)
d := e.queue(t, &models.Download{})
processed, err := e.svc.ExecuteDownload(context.Background(), d)
if err != nil {
t.Fatalf("ExecuteDownload: %v", err)
}
if !processed {
t.Error("processed = false, want true")
}
got := e.status(t, d.ID)
if got.Status != "completed" {
t.Errorf("status = %q, want completed", got.Status)
}
if names := libraryEntries(t, e.cfg.LibraryDir); len(names) != 1 || names[0] != "My Clip" {
t.Errorf("library = %v, want [My Clip], the entry that did download", names)
}
if !strings.Contains(got.Logs.String, "1 item(s) were imported anyway") {
t.Errorf("logs do not report the partial failure: %q", got.Logs.String)
}
}
// The same failure with nothing downloaded is a plain error.
func TestExecuteDownloadTotalFailureIsError(t *testing.T) {
e := newExecEnv(t)
e.fakeYTDLP(t, `echo "ERROR: [youtube] bad: Video unavailable" >&2; exit 1`)
d := e.queue(t, &models.Download{})
if _, err := e.svc.ExecuteDownload(context.Background(), d); err == nil {
t.Fatal("expected an error when nothing was downloaded")
}
if got := e.status(t, d.ID); got.Status != "error" {
t.Errorf("status = %q, want error", got.Status)
}
}
// A reserved flag must fail before yt-dlp is ever started, and a preset's flags
// are checked as well as the download's own.
func TestExecuteDownloadRejectsReservedPresetFlag(t *testing.T) {
@@ -565,3 +612,60 @@ func TestDeleteAllCancelsRunningDownload(t *testing.T) {
t.Fatal("clearing the queue did not stop the running download")
}
}
// yt-dlp rewrites the cookie file on exit with rotated session cookies. Those
// must be saved back, or the stored snapshot goes stale and the session dies.
func TestExecuteDownloadSavesRefreshedCookies(t *testing.T) {
e := newExecEnv(t)
if err := e.settings.SetCookies("# Netscape HTTP Cookie File\nold-session\n"); err != nil {
t.Fatalf("SetCookies: %v", err)
}
e.fakeYTDLP(t, `
cookies=""
prev=""
for a in "$@"; do
if [ "$prev" = "--cookies" ]; then cookies="$a"; fi
prev="$a"
done
[ -n "$cookies" ] || { echo "no --cookies argument" >&2; exit 1; }
printf '# Netscape HTTP Cookie File\nnew-session\n' > "$cookies"
`+e.writeItem(t, "item-00001", "My Clip", "abc123"))
d := e.queue(t, &models.Download{})
if _, err := e.svc.ExecuteDownload(context.Background(), d); err != nil {
t.Fatalf("ExecuteDownload: %v", err)
}
got, err := e.settings.GetCookies()
if err != nil {
t.Fatalf("GetCookies: %v", err)
}
if want := "# Netscape HTTP Cookie File\nnew-session\n"; got != want {
t.Errorf("cookies = %q, want %q", got, want)
}
}
// An untouched cookie file must leave the stored cookies exactly as they were.
func TestExecuteDownloadKeepsUntouchedCookies(t *testing.T) {
e := newExecEnv(t)
const stored = "# Netscape HTTP Cookie File\nold-session\n"
if err := e.settings.SetCookies(stored); err != nil {
t.Fatalf("SetCookies: %v", err)
}
e.fakeYTDLP(t, e.writeItem(t, "item-00001", "My Clip", "abc123"))
d := e.queue(t, &models.Download{})
if _, err := e.svc.ExecuteDownload(context.Background(), d); err != nil {
t.Fatalf("ExecuteDownload: %v", err)
}
got, err := e.settings.GetCookies()
if err != nil {
t.Fatalf("GetCookies: %v", err)
}
if got != stored {
t.Errorf("cookies = %q, want them unchanged (%q)", got, stored)
}
}
Minternal/service/formats.go
@@ -15,7 +15,14 @@ func (s *DownloadService) ListFormats(url string) ([]*models.FormatInfo, error)
// Use machine-readable JSON (-J) rather than scraping the human "-F" table,
// whose columns/separators shift between yt-dlp versions. stderr is captured
// separately so warnings can't corrupt the JSON on stdout.
cmd := exec.Command(s.cfg.YTDLPPath, "-J", "--no-warnings", url)
// -I 1 limits a playlist URL to its first entry, which is all the parsing
// below reads anyway.
args := []string{"-J", "--no-warnings", "-I", "1"}
args, cleanup := s.appendCookies(args)
defer cleanup()
args = append(args, url)
cmd := exec.Command(s.cfg.YTDLPPath, args...)
var stderr bytes.Buffer
cmd.Stderr = &stderr
output, err := cmd.Output()
@@ -46,8 +53,8 @@ type ytFormat struct {
// parseFormatJSON reads yt-dlp's single-JSON dump (-J) and returns the available
// formats. For a single video the formats live at the top level; for a playlist
// URL we fall back to the first entry's formats so the picker still shows
// something useful.
// URL the dump is restricted to one entry (-I 1), so we fall back to that
// entry's formats and the picker still shows something useful.
func parseFormatJSON(data []byte) ([]*models.FormatInfo, error) {
var top struct {
Formats []ytFormat `json:"formats"`
Minternal/service/import.go
@@ -207,7 +207,10 @@ func (s *DownloadService) importItemDir(ctx context.Context, url, itemDir, baseL
// stable item identity; within a single subscription's own directory it is
// enough to match items, so the extractor is not needed.
type infoJSON struct {
ID string `json:"id"`
ID string `json:"id"`
// Type is yt-dlp's "_type": "playlist" marks a playlist-level sidecar rather
// than an item.
Type string `json:"_type"`
Title string `json:"title"`
Description string `json:"description"`
WebpageURL string `json:"webpage_url"`
Minternal/service/preset.go
@@ -99,23 +99,53 @@ func (s *PresetService) BuildArgs(p *models.Preset, formatOverride, customFlags
if p.MaxComments > 0 {
commentArgs = append(commentArgs, "max_comments="+strconv.Itoa(p.MaxComments))
}
// yt-dlp's syntax is IE_KEY:ARG1=VAL1,VAL2;ARG2=VAL: arguments are separated
// by ";", only the values of one argument by ",". Passing --extractor-args
// twice for the same key makes the second replace the first, so all youtube
// arguments must go into a single flag.
extra := strings.TrimSpace(p.CommentExtractorArgs)
if rest, ok := cutYoutubePrefix(extra); ok {
if rest != "" {
commentArgs = append(commentArgs, rest)
}
extra = ""
}
if len(commentArgs) > 0 {
args = append(args, "--extractor-args", "youtube:"+strings.Join(commentArgs, ","))
args = append(args, "--extractor-args", "youtube:"+strings.Join(commentArgs, ";"))
}
if strings.TrimSpace(p.CommentExtractorArgs) != "" {
args = append(args, "--extractor-args", strings.TrimSpace(p.CommentExtractorArgs))
if extra != "" {
args = append(args, "--extractor-args", extra)
}
// Custom flags
if p.CustomFlags != "" {
args = append(args, strings.Fields(p.CustomFlags)...)
}
args = append(args, splitCustomFlags(p.CustomFlags)...)
args = append(args, splitCustomFlags(customFlags)...)
return args
}
if customFlags != "" {
args = append(args, strings.Fields(customFlags)...)
// cutYoutubePrefix strips a leading "youtube:" (any case) from extractor args.
// The bool reports whether the value targeted the youtube extractor at all.
func cutYoutubePrefix(s string) (string, bool) {
if len(s) < len("youtube:") || !strings.EqualFold(s[:len("youtube:")], "youtube:") {
return s, false
}
return strings.TrimSpace(s[len("youtube:"):]), true
}
return args
// splitCustomFlags splits a custom-flags string into arguments. BuildArgs cannot
// report an error, so malformed quoting falls back to whitespace splitting. The
// download path already rejected such input in checkReservedFlags, so only the
// EffectiveFlags display can reach the fallback.
func splitCustomFlags(s string) []string {
if strings.TrimSpace(s) == "" {
return nil
}
fields, err := splitFlags(s)
if err != nil {
return strings.Fields(s)
}
return fields
}
// applyHeightCap adds a max-height filter to a format selector. For a combined
Minternal/service/preset_test.go
@@ -95,26 +95,74 @@ func TestPresetServiceBuildArgsWithOverride(t *testing.T) {
}
}
func TestPresetServiceBuildArgsWithComments(t *testing.T) {
preset := &models.Preset{
WriteComments: true,
CommentSort: "top",
MaxComments: 25,
CommentExtractorArgs: "youtube:player_client=web",
}
args := (&PresetService{}).BuildArgs(preset, "", "")
want := []string{"--write-info-json", "--write-comments", "--extractor-args", "youtube:comment_sort=top,max_comments=25", "--extractor-args", "youtube:player_client=web"}
for _, value := range want {
found := false
for _, arg := range args {
if arg == value {
found = true
break
// yt-dlp separates extractor arguments with ";" — a "," separates the values of
// one argument, so joining with it makes max_comments part of the sort value.
func TestPresetServiceBuildArgsCommentExtractorArgs(t *testing.T) {
cases := []struct {
name string
preset *models.Preset
wantExtractor []string
}{
{
name: "comment args use a semicolon",
preset: &models.Preset{WriteComments: true, CommentSort: "top", MaxComments: 25},
wantExtractor: []string{"youtube:comment_sort=top;max_comments=25"},
},
{
// Two --extractor-args for the same key would make the second replace
// the first, so youtube arguments must be merged into one flag.
name: "youtube extra args are merged",
preset: &models.Preset{
WriteComments: true,
CommentSort: "top",
MaxComments: 25,
CommentExtractorArgs: "youtube:player_client=web",
},
wantExtractor: []string{"youtube:comment_sort=top;max_comments=25;player_client=web"},
},
{
name: "another extractor stays separate",
preset: &models.Preset{
WriteComments: true,
CommentSort: "top",
CommentExtractorArgs: "vimeo:foo=bar",
},
wantExtractor: []string{"youtube:comment_sort=top", "vimeo:foo=bar"},
},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
args := (&PresetService{}).BuildArgs(tc.preset, "", "")
var got []string
for i, arg := range args {
if arg == "--extractor-args" && i+1 < len(args) {
got = append(got, args[i+1])
}
}
}
if !found {
t.Errorf("BuildArgs missing %q: %v", value, args)
if len(got) != len(tc.wantExtractor) {
t.Fatalf("--extractor-args values = %v, want %v", got, tc.wantExtractor)
}
for i, want := range tc.wantExtractor {
if got[i] != want {
t.Errorf("--extractor-args[%d] = %q, want %q", i, got[i], want)
}
}
})
}
}
// A quoted value must survive as a single argument instead of being split on
// spaces, which would hand yt-dlp three broken arguments.
func TestPresetServiceBuildArgsKeepsQuotedCustomFlag(t *testing.T) {
args := (&PresetService{}).BuildArgs(&models.Preset{}, "", `--match-filter "duration > 60"`)
want := []string{"--match-filter", "duration > 60"}
if len(args) != len(want) {
t.Fatalf("args = %v, want %v", args, want)
}
for i := range want {
if args[i] != want[i] {
t.Errorf("args[%d] = %q, want %q", i, args[i], want[i])
}
}
}
Minternal/service/subscription_run.go
@@ -48,6 +48,13 @@ func (s *DownloadService) refreshAndAddNew(ctx context.Context, d *models.Downlo
if info.ID == "" {
continue
}
// A playlist-level info.json describes the source, not an item. Its id
// never matches a library item, so it would look new and trigger a
// download of the whole playlist. --no-write-playlist-metafiles already
// prevents it; this also covers sidecars written by older runs.
if info.Type == "playlist" {
continue
}
if existing, ok := s.librarySvc.FindByVideoID(baseLibraryDir, info.ID); ok {
if err := s.applyMetadata(existing, info, infoJSONPath); err != nil {
log.Printf("warning: failed to refresh metadata for %s: %v", itemDir, err)
@@ -88,10 +95,16 @@ func (s *DownloadService) downloadFresh(ctx context.Context, d *models.Download,
if ctx.Err() != nil {
return runErr
}
// Import whatever succeeded even if some entries errored.
if _, err := s.importDownloadedItems(ctx, d, tempDir, "", ytdlpFlags); err != nil {
// Import whatever succeeded even if some entries errored. yt-dlp exits
// non-zero when one entry fails, so its error only counts as a failure when
// nothing at all was imported.
imported, err := s.importDownloadedItems(ctx, d, tempDir, "", ytdlpFlags)
if err != nil {
log.Printf("warning: failed to import new metadata-mode items: %v", err)
}
if imported > 0 {
return nil
}
return runErr
}
Minternal/service/ytdlp_flags.go
@@ -3,8 +3,16 @@ package service
import (
"fmt"
"strings"
shellwords "github.com/mattn/go-shellwords"
)
// splitFlags splits a custom-flags string like a POSIX shell would, so quoted
// values such as --match-filter "duration > 60" stay one argument.
func splitFlags(s string) ([]string, error) {
return shellwords.Parse(s)
}
// reservedFlags are yt-dlp options VidArchive always sets itself; user custom
// flags must not pass them (or a conflicting inverse). The value describes what
// the option controls, for the failure message.
@@ -17,9 +25,14 @@ var reservedFlags = map[string]string{
"--no-cookies": "cookies (set these in Settings instead)",
"--newline": "progress output formatting (VidArchive sets this to stream logs)",
// These hand yt-dlp an arbitrary command or binary to run. VidArchive passes
// custom flags through verbatim, so allowing them would turn the preset form
// into remote command execution.
"--write-playlist-metafiles": "playlist metadata files (VidArchive imports per-item metadata only)",
"--no-write-playlist-metafiles": "playlist metadata files (VidArchive imports per-item metadata only)",
// These options are blocked as a guard against accidental misuse. The list is
// not a security boundary: yt-dlp accepts unambiguous option prefixes (e.g.
// --exec-b) and --alias can define new options, and options such as
// --ffmpeg-location or --plugin-dirs are not listed. Custom flags are
// operator-controlled by design.
"--exec": "running external commands (not permitted)",
"--exec-before-download": "running external commands (not permitted)",
"--postprocessor-args": "post-processor arguments (not permitted)",
@@ -44,7 +57,11 @@ var reservedSubscriptionFlags = map[string]string{
// checkReservedFlags rejects custom flags that clash with options VidArchive
// controls, naming the offender. It matches both "--flag" and "--flag=value".
func checkReservedFlags(customFlags string, isSubscription bool) error {
for _, tok := range strings.Fields(customFlags) {
tokens, err := splitFlags(customFlags)
if err != nil {
return fmt.Errorf("invalid custom flags: %w", err)
}
for _, tok := range tokens {
// Both "--flag value" and "--flag=value" name the same option.
name, _, _ := strings.Cut(tok, "=")
Minternal/service/ytdlp_flags_test.go
@@ -33,6 +33,12 @@ func TestCheckReservedFlags(t *testing.T) {
{"download-archive equals form sub", "--download-archive=a.txt", true, true},
// Base reserved flags still apply to subscriptions.
{"output sub", "-o x", true, true},
{"playlist metafiles", "--write-playlist-metafiles", false, true},
// A quoted value is one token, so its contents cannot look like a flag.
{"quoted value", `--match-filter "duration > 60"`, false, false},
{"quoted value hiding a reserved flag", `--match-filter "-o x"`, false, false},
// Malformed quoting must fail the download instead of being split anyway.
{"unterminated quote", `--match-filter "duration > 60`, false, true},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
Ainternal/service/ytdlp_live_test.go
@@ -0,0 +1,152 @@
// These tests run the real yt-dlp against a real video. They exist to catch
// yt-dlp version drift in the few places VidArchive depends on its behaviour:
// the -J JSON field names, --flat-playlist printing, and the output layout
// produced by -P/-o plus --write-info-json.
//
// They are opt-in because they need network access and take time. Run them with:
//
// VIDARCHIVE_ONLINE_TESTS=1 go test ./internal/service -run Live -v
//
// Override the video with VIDARCHIVE_TEST_VIDEO_URL.
package service
import (
"context"
"net/url"
"os"
"os/exec"
"path/filepath"
"testing"
"time"
"vidarchive/internal/models"
)
// defaultLiveVideoURL is a 19 second public video, small enough to download.
const defaultLiveVideoURL = "https://www.youtube.com/watch?v=jNQXAC9IVRw"
// liveEnv returns an execEnv wired to the real yt-dlp, the test video URL, and
// the video id expected for it. It skips the test unless the online tests are
// enabled and yt-dlp is installed.
func liveEnv(t *testing.T) (*execEnv, string, string) {
t.Helper()
if os.Getenv("VIDARCHIVE_ONLINE_TESTS") != "1" {
t.Skip("online test: set VIDARCHIVE_ONLINE_TESTS=1 to run it")
}
ytdlp, err := exec.LookPath("yt-dlp")
if err != nil {
t.Skipf("online test: yt-dlp not found on PATH: %v", err)
}
videoURL := os.Getenv("VIDARCHIVE_TEST_VIDEO_URL")
if videoURL == "" {
videoURL = defaultLiveVideoURL
}
// The id comes from the URL, so an override still knows what to expect.
parsed, err := url.Parse(videoURL)
if err != nil {
t.Fatalf("parse test video URL %q: %v", videoURL, err)
}
videoID := parsed.Query().Get("v")
if videoID == "" {
t.Fatalf("test video URL %q has no v parameter", videoURL)
}
e := newExecEnv(t)
e.cfg.YTDLPPath = ytdlp
return e, videoURL, videoID
}
// Checks the -J JSON field names the format picker reads.
func TestLiveListFormats(t *testing.T) {
e, videoURL, _ := liveEnv(t)
formats, err := e.svc.ListFormats(videoURL)
if err != nil {
t.Fatalf("ListFormats: %v", err)
}
if len(formats) == 0 {
t.Fatal("ListFormats returned no formats")
}
for _, f := range formats {
if f.ID == "" || f.Ext == "" {
t.Fatalf("format with empty id or ext: %+v", f)
}
}
}
// Checks --flat-playlist --print %(id)s, which subscription pruning relies on.
func TestLiveEnumeratePlaylistIDs(t *testing.T) {
e, videoURL, videoID := liveEnv(t)
ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
defer cancel()
ids, err := e.svc.enumeratePlaylistIDs(ctx, videoURL)
if err != nil {
t.Fatalf("enumeratePlaylistIDs: %v", err)
}
if !ids[videoID] {
t.Errorf("ids = %v, want them to contain %q", ids, videoID)
}
}
// Checks the full download path: --newline, -P/-o with %(autonumber), the
// info.json sidecar, and the import that turns it into a library item.
func TestLiveExecuteDownload(t *testing.T) {
e, videoURL, videoID := liveEnv(t)
preset := &models.Preset{Name: "Live", WriteInfoJSON: true}
if err := e.svc.presetSvc.Create(preset); err != nil {
t.Fatalf("create preset: %v", err)
}
// "wa/w" is worst audio, else worst overall: the smallest thing yt-dlp can
// give us, so the test stays cheap.
d := e.queue(t, &models.Download{
URL: videoURL,
PresetID: sqlNullInt64(preset.ID),
FormatOverride: "wa/w",
})
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute)
defer cancel()
processed, err := e.svc.ExecuteDownload(ctx, d)
if err != nil {
t.Fatalf("ExecuteDownload: %v", err)
}
if !processed {
t.Error("processed = false, want true")
}
if got := e.status(t, d.ID); got.Status != "completed" {
t.Fatalf("status = %q, want completed; logs:\n%s", got.Status, got.Logs.String)
}
names := libraryEntries(t, e.cfg.LibraryDir)
if len(names) != 1 {
t.Fatalf("library = %v, want exactly one item", names)
}
itemDir := filepath.Join(e.cfg.LibraryDir, names[0])
if _, err := os.Stat(filepath.Join(itemDir, itemMarkerName)); err != nil {
t.Errorf("item marker missing: %v", err)
}
infoPath := filepath.Join(itemDir, "info.json")
if _, err := os.Stat(infoPath); err != nil {
t.Fatalf("info.json missing: %v", err)
}
info := readInfoJSON(infoPath)
if info.ID != videoID {
t.Errorf("info.json id = %q, want %q", info.ID, videoID)
}
if info.Title == "" {
t.Error("info.json title is empty")
}
if info.WebpageURL == "" {
t.Error("info.json webpage_url is empty")
}
}
Mweb/templates/ytdlp_notes.html
@@ -26,6 +26,10 @@
<td><code>--cookies</code></td>
<td>Injected automatically from the cookies you save in <strong>Settings</strong> (when present).</td>
</tr>
<tr>
<td><code>--write-playlist-metafiles</code></td>
<td>Disabled so a playlist's own metadata file is not imported as if it were an item.</td>
</tr>
{{if .}}
<tr>
<td><code>--write-info-json</code></td>