feat: scheduled subscriptions, plus library/queue polish and fixes
Add a subscriptions feature: a saved URL that re-downloads on a schedule (hourly/daily/weekly/monthly or custom cron) without piling up duplicates. Each subscription owns a library directory and matches items by a stable identity (extractor:id) persisted in the marker.
MContainerfile
@@ -1,4 +1,4 @@
FROM golang:1.22-alpine AS builder
FROM golang:1.26-alpine AS builder
WORKDIR /build
COPY go.mod go.sum ./
@@ -7,7 +7,7 @@ RUN go mod download
COPY . .
RUN CGO_ENABLED=0 go build -o vidarchive ./cmd/vidarchive
FROM alpine:latest
FROM alpine:3.24
RUN apk --no-cache add ca-certificates ffmpeg yt-dlp
Mcmd/vidarchive/main.go
@@ -4,6 +4,8 @@ import (
"fmt"
"log"
"os"
"path/filepath"
"time"
"vidarchive/internal/config"
"vidarchive/internal/database"
@@ -26,6 +28,9 @@ func main() {
if err := os.MkdirAll(cfg.TempDir, 0755); err != nil {
log.Fatalf("Failed to create temp dir: %v", err)
}
if err := os.MkdirAll(filepath.Join(cfg.DataDir, "archives"), 0755); err != nil {
log.Fatalf("Failed to create archives dir: %v", err)
}
db, err := database.New(cfg)
if err != nil {
@@ -36,17 +41,23 @@ func main() {
presetRepo := repository.NewPresetRepository(db)
downloadRepo := repository.NewDownloadRepository(db)
settingsRepo := repository.NewSettingsRepository(db)
subscriptionRepo := repository.NewSubscriptionRepository(db)
presetSvc := service.NewPresetService(presetRepo)
librarySvc := service.NewLibraryService(cfg.LibraryDir)
settingsSvc := service.NewSettingsService(settingsRepo)
downloadSvc := service.NewDownloadService(downloadRepo, librarySvc, presetSvc, settingsSvc, cfg)
subscriptionSvc := service.NewSubscriptionService(subscriptionRepo, cfg)
downloadSvc := service.NewDownloadService(downloadRepo, librarySvc, presetSvc, settingsSvc, subscriptionSvc, cfg)
workerPool := worker.New(downloadSvc, cfg.Workers)
workerPool.Start()
defer workerPool.Stop()
h, err := handler.New(cfg, presetSvc, downloadSvc, librarySvc, settingsSvc, workerPool)
scheduler := worker.NewScheduler(subscriptionSvc, downloadSvc, workerPool, time.Duration(cfg.SchedulerInterval)*time.Second)
scheduler.Start()
defer scheduler.Stop()
h, err := handler.New(cfg, presetSvc, downloadSvc, librarySvc, settingsSvc, subscriptionSvc, workerPool)
if err != nil {
log.Fatalf("Failed to initialize handler: %v", err)
}
Mgo.mod
@@ -6,6 +6,7 @@ require (
github.com/BurntSushi/toml v1.6.0
github.com/gabriel-vasile/mimetype v1.4.13
github.com/go-chi/chi/v5 v5.2.5
github.com/robfig/cron/v3 v3.0.1
modernc.org/sqlite v1.50.0
)
Mgo.sum
@@ -18,6 +18,8 @@ github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOF
github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs=
github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=
golang.org/x/mod v0.33.0 h1:tHFzIWbBifEmbwtGz65eaWyGiGZatSrT9prnU8DbVL8=
golang.org/x/mod v0.33.0/go.mod h1:swjeQEj+6r7fODbD2cqrnje9PnziFuw4bmLbBZFrQ5w=
golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4=
Minternal/config/config.go
@@ -9,30 +9,32 @@ import (
)
type Config struct {
Port int
DataDir string
DBPath string
LibraryDir string
TempDir string
YTDLPPath string
BaseURL string
Workers int
RefreshInterval int
Port int
DataDir string
DBPath string
LibraryDir string
TempDir string
YTDLPPath string
BaseURL string
Workers int
RefreshInterval int
SchedulerInterval int
}
func New() *Config {
dataDir := getEnv("VIDARCHIVE_DATA_DIR", "./data")
return &Config{
Port: getEnvInt("VIDARCHIVE_PORT", 8080),
DataDir: dataDir,
DBPath: getEnv("VIDARCHIVE_DB_PATH", filepath.Join(dataDir, "vidarchive.db")),
LibraryDir: getEnv("VIDARCHIVE_LIBRARY_DIR", filepath.Join(dataDir, "library")),
TempDir: getEnv("VIDARCHIVE_TEMP_DIR", filepath.Join(dataDir, "temp")),
YTDLPPath: getEnv("VIDARCHIVE_YTDLP_PATH", "yt-dlp"),
BaseURL: getEnv("VIDARCHIVE_BASE_URL", ""),
Workers: getEnvInt("VIDARCHIVE_WORKERS", 2),
RefreshInterval: getEnvInt("VIDARCHIVE_REFRESH_INTERVAL", 5),
Port: getEnvInt("VIDARCHIVE_PORT", 8080),
DataDir: dataDir,
DBPath: getEnv("VIDARCHIVE_DB_PATH", filepath.Join(dataDir, "vidarchive.db")),
LibraryDir: getEnv("VIDARCHIVE_LIBRARY_DIR", filepath.Join(dataDir, "library")),
TempDir: getEnv("VIDARCHIVE_TEMP_DIR", filepath.Join(dataDir, "temp")),
YTDLPPath: getEnv("VIDARCHIVE_YTDLP_PATH", "yt-dlp"),
BaseURL: getEnv("VIDARCHIVE_BASE_URL", ""),
Workers: getEnvInt("VIDARCHIVE_WORKERS", 2),
RefreshInterval: getEnvInt("VIDARCHIVE_REFRESH_INTERVAL", 5),
SchedulerInterval: getEnvInt("VIDARCHIVE_SCHEDULER_INTERVAL", 60),
}
}
Minternal/database/database.go
@@ -96,6 +96,26 @@ func migrate(db *sql.DB) error {
{10, `ALTER TABLE downloads ADD COLUMN output_dir TEXT`},
{11, `DROP TABLE IF EXISTS media`},
{12, `ALTER TABLE presets DROP COLUMN output_template`},
{13, `CREATE TABLE IF NOT EXISTS subscriptions (
id INTEGER PRIMARY KEY AUTOINCREMENT,
name TEXT NOT NULL,
url TEXT NOT NULL,
enabled INTEGER DEFAULT 1,
refresh_mode TEXT DEFAULT 'overwrite',
schedule_kind TEXT DEFAULT 'daily',
cron_expr TEXT DEFAULT '0 3 * * *',
preset_id INTEGER,
format_override TEXT,
custom_flags TEXT,
output_dir TEXT NOT NULL,
prune_removed INTEGER DEFAULT 0,
last_run_at DATETIME,
next_run_at DATETIME,
last_status TEXT,
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
FOREIGN KEY (preset_id) REFERENCES presets(id)
)`},
{14, `ALTER TABLE downloads ADD COLUMN subscription_id INTEGER`},
}
for _, m := range migrations {
Minternal/handler/handler.go
@@ -21,16 +21,17 @@ import (
)
type Handler struct {
cfg *config.Config
templates *template.Template
presetSvc *service.PresetService
downloadSvc *service.DownloadService
librarySvc *service.LibraryService
settingsSvc *service.SettingsService
workerPool *worker.Pool
cfg *config.Config
templates *template.Template
presetSvc *service.PresetService
downloadSvc *service.DownloadService
librarySvc *service.LibraryService
settingsSvc *service.SettingsService
subscriptionSvc *service.SubscriptionService
workerPool *worker.Pool
}
func New(cfg *config.Config, presetSvc *service.PresetService, downloadSvc *service.DownloadService, librarySvc *service.LibraryService, settingsSvc *service.SettingsService, workerPool *worker.Pool) (*Handler, error) {
func New(cfg *config.Config, presetSvc *service.PresetService, downloadSvc *service.DownloadService, librarySvc *service.LibraryService, settingsSvc *service.SettingsService, subscriptionSvc *service.SubscriptionService, workerPool *worker.Pool) (*Handler, error) {
presetSvcGlobal = presetSvc
tmpl, err := loadTemplates()
if err != nil {
@@ -38,13 +39,14 @@ func New(cfg *config.Config, presetSvc *service.PresetService, downloadSvc *serv
}
return &Handler{
cfg: cfg,
templates: tmpl,
presetSvc: presetSvc,
downloadSvc: downloadSvc,
librarySvc: librarySvc,
settingsSvc: settingsSvc,
workerPool: workerPool,
cfg: cfg,
templates: tmpl,
presetSvc: presetSvc,
downloadSvc: downloadSvc,
librarySvc: librarySvc,
settingsSvc: settingsSvc,
subscriptionSvc: subscriptionSvc,
workerPool: workerPool,
}, nil
}
@@ -271,8 +273,12 @@ func (h *Handler) ServeMediaItem(w http.ResponseWriter, r *http.Request) {
h.serveThumbnail(strings.TrimSuffix(relPath, "/thumbnail"), w, r)
return
}
if strings.HasSuffix(relPath, "/subtitles") {
h.serveSubtitles(strings.TrimSuffix(relPath, "/subtitles"), w, r)
// Subtitle tracks are addressed as <item>/subtitles/<lang>, with the language
// as a trailing path segment (see LibraryService.GetSubtitles).
if i := strings.LastIndex(relPath, "/subtitles/"); i >= 0 {
item := relPath[:i]
lang := relPath[i+len("/subtitles/"):]
h.serveSubtitles(item, lang, w, r)
return
}
h.serveMedia(relPath, w, r)
@@ -335,9 +341,8 @@ func isAudioFile(item *models.LibraryItem, filename string) bool {
return len(item.MediaFiles) > 0 && item.MediaFiles[0].IsAudio
}
func (h *Handler) serveSubtitles(relPath string, w http.ResponseWriter, r *http.Request) {
func (h *Handler) serveSubtitles(relPath, lang string, w http.ResponseWriter, r *http.Request) {
lang := r.URL.Query().Get("lang")
if lang == "" {
http.Error(w, "Missing language", http.StatusBadRequest)
return
Ainternal/handler/subscription.go
@@ -0,0 +1,204 @@
package handler
import (
"database/sql"
"net/http"
"strconv"
"strings"
"time"
"github.com/go-chi/chi/v5"
"vidarchive/internal/models"
)
func (h *Handler) Subscriptions(w http.ResponseWriter, r *http.Request) {
subscriptions, err := h.subscriptionSvc.GetAll()
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
presets, err := h.presetSvc.GetAll()
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
h.renderWithRequest(w, r, "subscriptions", PageData{
Title: "Subscriptions",
ActiveTab: "subscriptions",
Data: struct {
Subscriptions []*models.Subscription
Presets []*models.Preset
}{
Subscriptions: subscriptions,
Presets: presets,
},
})
}
// subscriptionFromForm builds and validates a Subscription from form values,
// shared by create and update. It resolves/validates the schedule into a canonical
// cron expression and requires an output directory (the subscription owns it).
func (h *Handler) subscriptionFromForm(r *http.Request) (*models.Subscription, error) {
name := strings.TrimSpace(r.FormValue("name"))
url := strings.TrimSpace(r.FormValue("url"))
if name == "" {
return nil, errBadRequest("a name is required")
}
if url == "" {
return nil, errBadRequest("URL is required")
}
outputDir := strings.TrimSpace(r.FormValue("output_dir"))
if outputDir == "" {
return nil, errBadRequest("an output directory is required — the subscription owns this folder")
}
refreshMode := r.FormValue("refresh_mode")
switch refreshMode {
case "overwrite", "skip", "metadata":
default:
refreshMode = "overwrite"
}
scheduleKind := r.FormValue("schedule_kind")
cronExpr, err := h.subscriptionSvc.CronExprFor(scheduleKind, strings.TrimSpace(r.FormValue("cron_expr")))
if err != nil {
return nil, errBadRequest(err.Error())
}
sub := &models.Subscription{
Name: name,
URL: url,
Enabled: true,
RefreshMode: refreshMode,
ScheduleKind: scheduleKind,
CronExpr: cronExpr,
FormatOverride: r.FormValue("format_override"),
CustomFlags: r.FormValue("custom_flags"),
OutputDir: outputDir,
PruneRemoved: r.FormValue("prune_removed") == "1",
}
if pid := r.FormValue("preset_id"); pid != "" {
if id, err := strconv.ParseInt(pid, 10, 64); err == nil {
sub.PresetID = sql.NullInt64{Int64: id, Valid: true}
}
}
return sub, nil
}
func (h *Handler) CreateSubscription(w http.ResponseWriter, r *http.Request) {
if err := r.ParseForm(); err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
sub, err := h.subscriptionFromForm(r)
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
if next, err := h.subscriptionSvc.ComputeNextRun(sub, time.Now()); err == nil {
sub.NextRunAt = sql.NullTime{Time: next, Valid: true}
}
if err := h.subscriptionSvc.Create(sub); err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
http.Redirect(w, r, "/subscriptions", http.StatusSeeOther)
}
func (h *Handler) UpdateSubscription(w http.ResponseWriter, r *http.Request) {
id, err := strconv.ParseInt(chi.URLParam(r, "id"), 10, 64)
if err != nil {
http.Error(w, "Invalid ID", http.StatusBadRequest)
return
}
if err := r.ParseForm(); err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
existing, err := h.subscriptionSvc.GetByID(id)
if err != nil {
http.Error(w, "Not found", http.StatusNotFound)
return
}
sub, err := h.subscriptionFromForm(r)
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
sub.ID = id
sub.Enabled = existing.Enabled
// Recompute the next run from the (possibly changed) schedule.
if next, err := h.subscriptionSvc.ComputeNextRun(sub, time.Now()); err == nil {
sub.NextRunAt = sql.NullTime{Time: next, Valid: true}
}
if err := h.subscriptionSvc.Update(sub); err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
http.Redirect(w, r, "/subscriptions", http.StatusSeeOther)
}
func (h *Handler) ToggleSubscription(w http.ResponseWriter, r *http.Request) {
id, err := strconv.ParseInt(chi.URLParam(r, "id"), 10, 64)
if err != nil {
http.Error(w, "Invalid ID", http.StatusBadRequest)
return
}
sub, err := h.subscriptionSvc.GetByID(id)
if err != nil {
http.Error(w, "Not found", http.StatusNotFound)
return
}
if err := h.subscriptionSvc.SetEnabled(id, !sub.Enabled); err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
http.Redirect(w, r, "/subscriptions", http.StatusSeeOther)
}
func (h *Handler) RunSubscription(w http.ResponseWriter, r *http.Request) {
id, err := strconv.ParseInt(chi.URLParam(r, "id"), 10, 64)
if err != nil {
http.Error(w, "Invalid ID", http.StatusBadRequest)
return
}
sub, err := h.subscriptionSvc.GetByID(id)
if err != nil {
http.Error(w, "Not found", http.StatusNotFound)
return
}
download, err := h.downloadSvc.CreateForSubscription(sub)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
h.workerPool.Submit(download)
http.Redirect(w, r, "/queue", http.StatusSeeOther)
}
func (h *Handler) DeleteSubscription(w http.ResponseWriter, r *http.Request) {
id, err := strconv.ParseInt(chi.URLParam(r, "id"), 10, 64)
if err != nil {
http.Error(w, "Invalid ID", http.StatusBadRequest)
return
}
if err := h.subscriptionSvc.Delete(id); err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
http.Redirect(w, r, "/subscriptions", http.StatusSeeOther)
}
// errBadRequest is a small sentinel-style error carrying a user-facing message.
type errBadRequest string
func (e errBadRequest) Error() string { return string(e) }
Minternal/models/models.go
@@ -32,6 +32,7 @@ type LibraryItem struct {
SourceURL string
Duration int
Description string
YtdlpFlags string
MediaFiles []MediaFile
}
@@ -52,11 +53,34 @@ type Download struct {
FormatOverride string
CustomFlags string
OutputDir sql.NullString
SubscriptionID sql.NullInt64
StartedAt sql.NullTime
CompletedAt sql.NullTime
CreatedAt time.Time
}
// Subscription is a saved URL that the scheduler re-downloads on a recurring
// schedule. Each subscription owns its OutputDir: refresh/dedup and pruning are
// scoped to that directory (see service.DownloadService refresh modes).
type Subscription struct {
ID int64
Name string
URL string
Enabled bool
RefreshMode string // "overwrite" | "skip" | "metadata"
ScheduleKind string // "hourly" | "daily" | "weekly" | "monthly" | "cron"
CronExpr string
PresetID sql.NullInt64
FormatOverride string
CustomFlags string
OutputDir string
PruneRemoved bool
LastRunAt sql.NullTime
NextRunAt sql.NullTime
LastStatus sql.NullString
CreatedAt time.Time
}
type Settings struct {
RefreshInterval int
AutoRefreshLibrary bool
@@ -85,5 +109,8 @@ type ItemMetadata struct {
Name string `toml:"name"`
SourceURL string `toml:"source_url"`
Description string `toml:"description"`
VideoID string `toml:"video_id"`
Extractor string `toml:"extractor"`
YtdlpFlags string `toml:"ytdlp_flags"`
FileDurations map[string]int `toml:"file_durations"`
}
Minternal/repository/download.go
@@ -16,9 +16,9 @@ func NewDownloadRepository(db *sql.DB) *DownloadRepository {
func (r *DownloadRepository) Create(d *models.Download) error {
result, err := r.db.Exec(
`INSERT INTO downloads (url, status, preset_id, format_override, custom_flags, output_dir)
VALUES (?, ?, ?, ?, ?, ?)`,
d.URL, d.Status, d.PresetID, d.FormatOverride, d.CustomFlags, d.OutputDir,
`INSERT INTO downloads (url, status, preset_id, format_override, custom_flags, output_dir, subscription_id)
VALUES (?, ?, ?, ?, ?, ?, ?)`,
d.URL, d.Status, d.PresetID, d.FormatOverride, d.CustomFlags, d.OutputDir, d.SubscriptionID,
)
if err != nil {
return err
@@ -29,14 +29,14 @@ func (r *DownloadRepository) Create(d *models.Download) error {
func (r *DownloadRepository) GetByID(id int64) (*models.Download, error) {
row := r.db.QueryRow(
`SELECT id, url, status, logs, error_message, preset_id, format_override, custom_flags, output_dir, started_at, completed_at, created_at
`SELECT id, url, status, logs, error_message, preset_id, format_override, custom_flags, output_dir, subscription_id, started_at, completed_at, created_at
FROM downloads WHERE id = ?`, id,
)
return scanDownload(row)
}
func (r *DownloadRepository) GetAll(status, sortBy string) ([]*models.Download, error) {
query := `SELECT id, url, status, logs, error_message, preset_id, format_override, custom_flags, output_dir, started_at, completed_at, created_at
query := `SELECT id, url, status, logs, error_message, preset_id, format_override, custom_flags, output_dir, subscription_id, started_at, completed_at, created_at
FROM downloads WHERE 1=1`
var args []interface{}
@@ -73,7 +73,7 @@ func (r *DownloadRepository) GetAll(status, sortBy string) ([]*models.Download,
func (r *DownloadRepository) GetQueued(limit int) ([]*models.Download, error) {
rows, err := r.db.Query(
`SELECT id, url, status, logs, error_message, preset_id, format_override, custom_flags, output_dir, started_at, completed_at, created_at
`SELECT id, url, status, logs, error_message, preset_id, format_override, custom_flags, output_dir, subscription_id, started_at, completed_at, created_at
FROM downloads WHERE status = 'queued' ORDER BY created_at ASC LIMIT ?`,
limit,
)
@@ -162,7 +162,7 @@ func scanDownload(row interface{ Scan(...interface{}) error }) (*models.Download
var d models.Download
err := row.Scan(
&d.ID, &d.URL, &d.Status,
&d.Logs, &d.ErrorMessage, &d.PresetID, &d.FormatOverride, &d.CustomFlags, &d.OutputDir,
&d.Logs, &d.ErrorMessage, &d.PresetID, &d.FormatOverride, &d.CustomFlags, &d.OutputDir, &d.SubscriptionID,
&d.StartedAt, &d.CompletedAt, &d.CreatedAt,
)
if err != nil {
Ainternal/repository/subscription.go
@@ -0,0 +1,135 @@
package repository
import (
"database/sql"
"time"
"vidarchive/internal/models"
)
type SubscriptionRepository struct {
db *sql.DB
}
func NewSubscriptionRepository(db *sql.DB) *SubscriptionRepository {
return &SubscriptionRepository{db: db}
}
const subscriptionColumns = `id, name, url, enabled, refresh_mode, schedule_kind, cron_expr,
preset_id, format_override, custom_flags, output_dir, prune_removed,
last_run_at, next_run_at, last_status, created_at`
func (r *SubscriptionRepository) Create(s *models.Subscription) error {
result, err := r.db.Exec(
`INSERT INTO subscriptions
(name, url, enabled, refresh_mode, schedule_kind, cron_expr, preset_id,
format_override, custom_flags, output_dir, prune_removed, next_run_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
s.Name, s.URL, boolToInt(s.Enabled), s.RefreshMode, s.ScheduleKind, s.CronExpr,
s.PresetID, s.FormatOverride, s.CustomFlags, s.OutputDir, boolToInt(s.PruneRemoved),
s.NextRunAt,
)
if err != nil {
return err
}
s.ID, _ = result.LastInsertId()
return nil
}
func (r *SubscriptionRepository) GetByID(id int64) (*models.Subscription, error) {
row := r.db.QueryRow(`SELECT `+subscriptionColumns+` FROM subscriptions WHERE id = ?`, id)
return scanSubscription(row)
}
func (r *SubscriptionRepository) GetAll() ([]*models.Subscription, error) {
rows, err := r.db.Query(`SELECT ` + subscriptionColumns + ` FROM subscriptions ORDER BY created_at DESC`)
if err != nil {
return nil, err
}
defer rows.Close()
return scanSubscriptions(rows)
}
// GetDue returns enabled subscriptions whose next run is at or before now (or
// has never run). The scheduler uses this to decide what to enqueue.
func (r *SubscriptionRepository) GetDue(now time.Time) ([]*models.Subscription, error) {
rows, err := r.db.Query(
`SELECT `+subscriptionColumns+`
FROM subscriptions
WHERE enabled = 1 AND (next_run_at IS NULL OR next_run_at <= ?)
ORDER BY created_at ASC`,
now,
)
if err != nil {
return nil, err
}
defer rows.Close()
return scanSubscriptions(rows)
}
func (r *SubscriptionRepository) Update(s *models.Subscription) error {
_, err := r.db.Exec(
`UPDATE subscriptions SET name=?, url=?, enabled=?, refresh_mode=?, schedule_kind=?,
cron_expr=?, preset_id=?, format_override=?, custom_flags=?, output_dir=?,
prune_removed=?, next_run_at=?
WHERE id=?`,
s.Name, s.URL, boolToInt(s.Enabled), s.RefreshMode, s.ScheduleKind, s.CronExpr,
s.PresetID, s.FormatOverride, s.CustomFlags, s.OutputDir, boolToInt(s.PruneRemoved),
s.NextRunAt, s.ID,
)
return err
}
func (r *SubscriptionRepository) SetEnabled(id int64, enabled bool) error {
_, err := r.db.Exec(`UPDATE subscriptions SET enabled = ? WHERE id = ?`, boolToInt(enabled), id)
return err
}
// MarkRun records a run's outcome and the computed next run time.
func (r *SubscriptionRepository) MarkRun(id int64, lastRunAt, nextRunAt time.Time, status string) error {
_, err := r.db.Exec(
`UPDATE subscriptions SET last_run_at = ?, next_run_at = ?, last_status = ? WHERE id = ?`,
lastRunAt, nextRunAt, status, id,
)
return err
}
func (r *SubscriptionRepository) Delete(id int64) error {
_, err := r.db.Exec(`DELETE FROM subscriptions WHERE id = ?`, id)
return err
}
func scanSubscriptions(rows *sql.Rows) ([]*models.Subscription, error) {
var subs []*models.Subscription
for rows.Next() {
s, err := scanSubscription(rows)
if err != nil {
return nil, err
}
subs = append(subs, s)
}
return subs, rows.Err()
}
func scanSubscription(row interface{ Scan(...interface{}) error }) (*models.Subscription, error) {
var s models.Subscription
var enabled, pruneRemoved int
var refreshMode, scheduleKind, cronExpr, formatOverride, customFlags, outputDir sql.NullString
err := row.Scan(
&s.ID, &s.Name, &s.URL, &enabled, &refreshMode, &scheduleKind, &cronExpr,
&s.PresetID, &formatOverride, &customFlags, &outputDir, &pruneRemoved,
&s.LastRunAt, &s.NextRunAt, &s.LastStatus, &s.CreatedAt,
)
if err != nil {
return nil, err
}
s.Enabled = enabled == 1
s.PruneRemoved = pruneRemoved == 1
s.RefreshMode = refreshMode.String
s.ScheduleKind = scheduleKind.String
s.CronExpr = cronExpr.String
s.FormatOverride = formatOverride.String
s.CustomFlags = customFlags.String
s.OutputDir = outputDir.String
return &s, nil
}
Ainternal/repository/subscription_test.go
@@ -0,0 +1,127 @@
package repository
import (
"database/sql"
"testing"
"time"
"vidarchive/internal/models"
)
func TestSubscriptionRepositoryCRUD(t *testing.T) {
db := setupTestDB(t)
defer db.Close()
repo := NewSubscriptionRepository(db)
sub := &models.Subscription{
Name: "News",
URL: "https://example.com/playlist",
Enabled: true,
RefreshMode: "overwrite",
ScheduleKind: "daily",
CronExpr: "0 3 * * *",
OutputDir: "subscriptions/news",
PruneRemoved: true,
}
if err := repo.Create(sub); err != nil {
t.Fatalf("create: %v", err)
}
if sub.ID == 0 {
t.Fatal("expected an assigned ID after create")
}
got, err := repo.GetByID(sub.ID)
if err != nil {
t.Fatalf("get: %v", err)
}
if got.Name != "News" || got.URL != sub.URL || !got.Enabled || got.RefreshMode != "overwrite" ||
got.OutputDir != "subscriptions/news" || !got.PruneRemoved {
t.Errorf("round-trip mismatch: %+v", got)
}
got.Name = "Updated"
got.RefreshMode = "skip"
if err := repo.Update(got); err != nil {
t.Fatalf("update: %v", err)
}
again, _ := repo.GetByID(sub.ID)
if again.Name != "Updated" || again.RefreshMode != "skip" {
t.Errorf("update not persisted: %+v", again)
}
if err := repo.SetEnabled(sub.ID, false); err != nil {
t.Fatalf("set enabled: %v", err)
}
if again, _ = repo.GetByID(sub.ID); again.Enabled {
t.Error("expected disabled after SetEnabled(false)")
}
if err := repo.Delete(sub.ID); err != nil {
t.Fatalf("delete: %v", err)
}
if _, err := repo.GetByID(sub.ID); err == nil {
t.Error("expected error fetching deleted subscription")
}
}
func TestSubscriptionRepositoryGetDueAndMarkRun(t *testing.T) {
db := setupTestDB(t)
defer db.Close()
repo := NewSubscriptionRepository(db)
now := time.Now()
// Due: next run in the past.
due := &models.Subscription{
Name: "due", URL: "u1", Enabled: true, RefreshMode: "overwrite",
ScheduleKind: "daily", CronExpr: "0 3 * * *", OutputDir: "a",
NextRunAt: nullTime(now.Add(-time.Hour)),
}
// Not due: next run in the future.
future := &models.Subscription{
Name: "future", URL: "u2", Enabled: true, RefreshMode: "overwrite",
ScheduleKind: "daily", CronExpr: "0 3 * * *", OutputDir: "b",
NextRunAt: nullTime(now.Add(time.Hour)),
}
// Disabled: never due even though its next run is in the past.
disabled := &models.Subscription{
Name: "disabled", URL: "u3", Enabled: false, RefreshMode: "overwrite",
ScheduleKind: "daily", CronExpr: "0 3 * * *", OutputDir: "c",
NextRunAt: nullTime(now.Add(-time.Hour)),
}
for _, s := range []*models.Subscription{due, future, disabled} {
if err := repo.Create(s); err != nil {
t.Fatalf("create %s: %v", s.Name, err)
}
}
gotDue, err := repo.GetDue(now)
if err != nil {
t.Fatalf("get due: %v", err)
}
if len(gotDue) != 1 || gotDue[0].Name != "due" {
names := make([]string, len(gotDue))
for i, s := range gotDue {
names[i] = s.Name
}
t.Fatalf("expected only the due subscription, got %v", names)
}
next := now.Add(24 * time.Hour)
if err := repo.MarkRun(due.ID, now, next, "queued"); err != nil {
t.Fatalf("mark run: %v", err)
}
after, _ := repo.GetByID(due.ID)
if !after.LastRunAt.Valid || after.LastStatus.String != "queued" {
t.Errorf("MarkRun not persisted: %+v", after)
}
// After marking run, it should no longer be due.
if gotDue, _ = repo.GetDue(now); len(gotDue) != 0 {
t.Errorf("expected no due subscriptions after MarkRun, got %d", len(gotDue))
}
}
func nullTime(t time.Time) sql.NullTime {
return sql.NullTime{Time: t, Valid: true}
}
Minternal/server/server.go
@@ -56,6 +56,13 @@ func (s *Server) setupRoutes() {
s.router.Get("/download", s.handler.DownloadForm)
s.router.Post("/download", s.handler.CreateDownload)
s.router.Get("/subscriptions", s.handler.Subscriptions)
s.router.Post("/subscriptions", s.handler.CreateSubscription)
s.router.Post("/subscriptions/{id}", s.handler.UpdateSubscription)
s.router.Post("/subscriptions/{id}/delete", s.handler.DeleteSubscription)
s.router.Post("/subscriptions/{id}/toggle", s.handler.ToggleSubscription)
s.router.Post("/subscriptions/{id}/run", s.handler.RunSubscription)
s.router.Get("/settings", s.handler.Settings)
s.router.Post("/settings/presets", s.handler.CreatePreset)
s.router.Post("/settings/presets/{id}", s.handler.UpdatePreset)
Minternal/server/server_test.go
@@ -1,6 +1,7 @@
package server
import (
"html"
"io"
"net/http"
"net/http/httptest"
@@ -41,14 +42,16 @@ func setupTestServer(t *testing.T) (*Server, *config.Config, func()) {
presetRepo := repository.NewPresetRepository(db)
downloadRepo := repository.NewDownloadRepository(db)
settingsRepo := repository.NewSettingsRepository(db)
subscriptionRepo := repository.NewSubscriptionRepository(db)
presetSvc := service.NewPresetService(presetRepo)
librarySvc := service.NewLibraryService(cfg.LibraryDir)
settingsSvc := service.NewSettingsService(settingsRepo)
downloadSvc := service.NewDownloadService(downloadRepo, librarySvc, presetSvc, settingsSvc, cfg)
subscriptionSvc := service.NewSubscriptionService(subscriptionRepo, cfg)
downloadSvc := service.NewDownloadService(downloadRepo, librarySvc, presetSvc, settingsSvc, subscriptionSvc, cfg)
workerPool := worker.New(downloadSvc, cfg.Workers)
h, err := handler.New(cfg, presetSvc, downloadSvc, librarySvc, settingsSvc, workerPool)
h, err := handler.New(cfg, presetSvc, downloadSvc, librarySvc, settingsSvc, subscriptionSvc, workerPool)
if err != nil {
t.Fatalf("init handler: %v", err)
}
@@ -154,6 +157,53 @@ func TestMediaFileQueryDecoding(t *testing.T) {
}
}
// TestSubtitleServedByPathSegment guards the regression where subtitle tracks
// are linked as <item>/subtitles/<lang> (language as a trailing path segment),
// but the handler only recognized the "/subtitles" suffix with a ?lang= query.
// The path form fell through to media serving and returned 400 "Missing file".
func TestSubtitleServedByPathSegment(t *testing.T) {
srv, cfg, cleanup := setupTestServer(t)
defer cleanup()
// An item whose name contains non-ASCII + spaces, like the reported URL.
const item = "ずんだパーリナイ ⧸ なみぐる [ywXQ9SqsaBQ]"
createItem(t, cfg.LibraryDir, item, "Vid", map[string]string{
"video.mp4": "dummy video",
})
vtt := "WEBVTT\n\n00:00:00.000 --> 00:00:01.000\nhi\n"
subDir := filepath.Join(cfg.LibraryDir, item, "subtitles")
if err := os.MkdirAll(subDir, 0755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(subDir, "eng.vtt"), []byte(vtt), 0644); err != nil {
t.Fatal(err)
}
router := srv.Router()
reqURL := "/media/item/" + (&url.URL{Path: item}).EscapedPath() + "/subtitles/eng"
req := httptest.NewRequest("GET", reqURL, nil)
w := httptest.NewRecorder()
router.ServeHTTP(w, req)
if w.Code != http.StatusOK {
body, _ := io.ReadAll(w.Body)
t.Fatalf("expected 200 for %s, got %d: %s", reqURL, w.Code, string(body))
}
if ct := w.Header().Get("Content-Type"); !strings.HasPrefix(ct, "text/vtt") {
t.Errorf("expected text/vtt content-type, got %q", ct)
}
if body, _ := io.ReadAll(w.Body); !strings.Contains(string(body), "WEBVTT") {
t.Errorf("expected the .vtt contents, got %q", string(body))
}
// A traversal attempt in the language segment must be rejected, not served.
bad := httptest.NewRequest("GET", "/media/item/"+(&url.URL{Path: item}).EscapedPath()+"/subtitles/..%2f..%2fsecret", nil)
bw := httptest.NewRecorder()
router.ServeHTTP(bw, bad)
if bw.Code == http.StatusOK {
t.Errorf("traversal in language segment was served (status %d)", bw.Code)
}
}
func TestPathTraversalBlocked(t *testing.T) {
srv, cfg, cleanup := setupTestServer(t)
defer cleanup()
@@ -258,6 +308,72 @@ func TestMultiFileCardThumbnailURLsDecodeToFilenames(t *testing.T) {
}
}
// TestNestedFolderLinkRoundTrip guards the double-encoding regression: a folder
// whose name contains a space was linked with urlEncodePath *inside* a ?path=
// query, which html/template then re-escaped (%20 -> %2520). Clicking the link
// landed on a path the server decoded to "playlist%20test%202" — a directory
// that doesn't exist — so the folder rendered empty and the breadcrumb showed
// the literal "%20". The link must round-trip: its decoded ?path must be the
// real directory, the item inside must render, and the breadcrumb must show the
// human-readable name.
func TestNestedFolderLinkRoundTrip(t *testing.T) {
srv, cfg, cleanup := setupTestServer(t)
defer cleanup()
createItem(t, cfg.LibraryDir, "subs/playlist test 2/Vid One", "Vid One", map[string]string{
"video.mp4": "dummy video",
})
router := srv.Router()
// List the parent and pull out the generated folder link.
req := httptest.NewRequest("GET", "/library?path=subs", nil)
w := httptest.NewRecorder()
router.ServeHTTP(w, req)
if w.Code != http.StatusOK {
t.Fatalf("list /library?path=subs: got %d", w.Code)
}
body := w.Body.String()
folderHref := regexp.MustCompile(`href="(/library\?path=[^"]+)" class="folder-item"`).FindStringSubmatch(body)
if folderHref == nil {
t.Fatalf("no folder link rendered for nested folder; body:\n%s", body)
}
href := html.UnescapeString(folderHref[1])
// The link's decoded ?path must be the real directory, not a still-encoded one.
u, err := url.Parse(href)
if err != nil {
t.Fatalf("parse folder href %q: %v", href, err)
}
if got := u.Query().Get("path"); got != "subs/playlist test 2" {
t.Fatalf("folder link path decodes to %q, want %q (double-encoding regression)", got, "subs/playlist test 2")
}
// Follow the link exactly as a browser would. The folder must not be empty,
// and the breadcrumb must show the readable name (never the encoded form).
req = httptest.NewRequest("GET", href, nil)
w = httptest.NewRecorder()
router.ServeHTTP(w, req)
if w.Code != http.StatusOK {
t.Fatalf("follow folder link %q: got %d", href, w.Code)
}
nested := w.Body.String()
if !strings.Contains(nested, "Vid One") {
t.Errorf("nested folder rendered empty — item 'Vid One' missing; body:\n%s", nested)
}
// The breadcrumb must display the readable name, not the percent-encoded form.
if !strings.Contains(nested, ">playlist test 2<") {
t.Errorf("breadcrumb missing readable folder name 'playlist test 2'")
}
if strings.Contains(nested, ">playlist%20test%202<") {
t.Errorf("breadcrumb displays the encoded name instead of a space (regression)")
}
// No link may carry a double-encoded path (%2520 == %25 + 20 == re-escaped %20).
if strings.Contains(nested, "%2520") {
t.Errorf("a link is double-encoded (%%2520) — urlEncodePath inside a ?path= query (regression)")
}
}
func TestListingDoesNotExtractThumbnails(t *testing.T) {
srv, cfg, cleanup := setupTestServer(t)
defer cleanup()
Ainternal/server/subscription_test.go
@@ -0,0 +1,91 @@
package server
import (
"io"
"net/http"
"net/http/httptest"
"net/url"
"strings"
"testing"
)
// postForm posts an application/x-www-form-urlencoded body to the router.
func postForm(router http.Handler, path string, form url.Values) *httptest.ResponseRecorder {
req := httptest.NewRequest("POST", path, strings.NewReader(form.Encode()))
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
w := httptest.NewRecorder()
router.ServeHTTP(w, req)
return w
}
func TestSubscriptionsPageRenders(t *testing.T) {
srv, _, cleanup := setupTestServer(t)
defer cleanup()
req := httptest.NewRequest("GET", "/subscriptions", nil)
w := httptest.NewRecorder()
srv.Router().ServeHTTP(w, req)
if w.Code != http.StatusOK {
body, _ := io.ReadAll(w.Body)
t.Fatalf("expected 200, got %d: %s", w.Code, body)
}
if !strings.Contains(w.Body.String(), "Create Subscription") {
t.Error("subscriptions page missing create form")
}
}
func TestCreateSubscriptionLifecycle(t *testing.T) {
srv, _, cleanup := setupTestServer(t)
defer cleanup()
router := srv.Router()
// Valid create redirects (303) and persists.
w := postForm(router, "/subscriptions", url.Values{
"name": {"My Sub"},
"url": {"https://example.com/playlist"},
"output_dir": {"subscriptions/mine"},
"refresh_mode": {"overwrite"},
"schedule_kind": {"daily"},
})
if w.Code != http.StatusSeeOther {
t.Fatalf("create: expected 303, got %d", w.Code)
}
// It now appears on the listing with its details.
req := httptest.NewRequest("GET", "/subscriptions", nil)
lw := httptest.NewRecorder()
router.ServeHTTP(lw, req)
body := lw.Body.String()
for _, want := range []string{"My Sub", "subscriptions/mine", "Next run:"} {
if !strings.Contains(body, want) {
t.Errorf("listing missing %q", want)
}
}
}
func TestCreateSubscriptionValidation(t *testing.T) {
srv, _, cleanup := setupTestServer(t)
defer cleanup()
router := srv.Router()
// Missing output_dir is rejected.
w := postForm(router, "/subscriptions", url.Values{
"url": {"https://example.com/x"},
"schedule_kind": {"daily"},
"output_dir": {""},
})
if w.Code != http.StatusBadRequest {
t.Errorf("empty output_dir: expected 400, got %d", w.Code)
}
// Invalid custom cron is rejected.
w = postForm(router, "/subscriptions", url.Values{
"url": {"https://example.com/x"},
"output_dir": {"foo"},
"schedule_kind": {"cron"},
"cron_expr": {"definitely not cron"},
})
if w.Code != http.StatusBadRequest {
t.Errorf("bad cron: expected 400, got %d", w.Code)
}
}
Minternal/service/download.go
@@ -26,25 +26,27 @@ import (
)
type DownloadService struct {
repo *repository.DownloadRepository
librarySvc *LibraryService
presetSvc *PresetService
settingsSvc *SettingsService
cfg *config.Config
cache *ProgressCache
processMu sync.Mutex
processes map[int64]*os.Process
repo *repository.DownloadRepository
librarySvc *LibraryService
presetSvc *PresetService
settingsSvc *SettingsService
subscriptionSvc *SubscriptionService
cfg *config.Config
cache *ProgressCache
processMu sync.Mutex
processes map[int64]*os.Process
}
func NewDownloadService(repo *repository.DownloadRepository, librarySvc *LibraryService, presetSvc *PresetService, settingsSvc *SettingsService, cfg *config.Config) *DownloadService {
func NewDownloadService(repo *repository.DownloadRepository, librarySvc *LibraryService, presetSvc *PresetService, settingsSvc *SettingsService, subscriptionSvc *SubscriptionService, cfg *config.Config) *DownloadService {
return &DownloadService{
repo: repo,
librarySvc: librarySvc,
presetSvc: presetSvc,
settingsSvc: settingsSvc,
cfg: cfg,
cache: NewProgressCache(),
processes: make(map[int64]*os.Process),
repo: repo,
librarySvc: librarySvc,
presetSvc: presetSvc,
settingsSvc: settingsSvc,
subscriptionSvc: subscriptionSvc,
cfg: cfg,
cache: NewProgressCache(),
processes: make(map[int64]*os.Process),
}
}
@@ -67,6 +69,25 @@ func (s *DownloadService) Create(url string, presetID *int64, formatOverride, cu
return d, nil
}
// CreateForSubscription queues a download for a subscription run, copying its
// download options and tagging it with the subscription id so ExecuteDownload
// applies the right refresh mode and pruning.
func (s *DownloadService) CreateForSubscription(sub *models.Subscription) (*models.Download, error) {
d := &models.Download{
URL: sub.URL,
Status: "queued",
FormatOverride: sub.FormatOverride,
CustomFlags: sub.CustomFlags,
OutputDir: sql.NullString{String: sub.OutputDir, Valid: sub.OutputDir != ""},
PresetID: sub.PresetID,
SubscriptionID: sqlNullInt64(sub.ID),
}
if err := s.repo.Create(d); err != nil {
return nil, err
}
return d, nil
}
func (s *DownloadService) GetByID(id int64) (*models.Download, error) {
d, err := s.repo.GetByID(id)
if err != nil {
@@ -167,6 +188,23 @@ func (s *DownloadService) ExecuteDownload(d *models.Download) (bool, error) {
preset = &models.Preset{}
}
var sub *models.Subscription
if d.SubscriptionID.Valid && s.subscriptionSvc != nil {
sub, _ = s.subscriptionSvc.GetByID(d.SubscriptionID.Int64)
}
// Reject custom flags that clash with options VidArchive sets itself, before
// spending any work — the download fails with a message naming the offender.
isSubscription := d.SubscriptionID.Valid
if err := checkReservedFlags(d.CustomFlags, isSubscription); err != nil {
s.finalizeError(d.ID, err)
return false, err
}
if err := checkReservedFlags(preset.CustomFlags, isSubscription); err != nil {
s.finalizeError(d.ID, err)
return false, err
}
tempDownloadDir := filepath.Join(s.cfg.TempDir, fmt.Sprintf("%d", d.ID))
if err := os.MkdirAll(tempDownloadDir, 0755); err != nil {
return false, fmt.Errorf("create temp download dir: %w", err)
@@ -174,14 +212,29 @@ func (s *DownloadService) ExecuteDownload(d *models.Download) (bool, error) {
args := s.presetSvc.BuildArgs(preset, d.FormatOverride, d.CustomFlags)
cookies, err := s.settingsSvc.GetCookies()
if err == nil && strings.TrimSpace(cookies) != "" {
tmpFile, err := os.CreateTemp("", "cookies-*.txt")
if err == nil {
tmpFile.WriteString(cookies)
tmpFile.Close()
args = append(args, "--cookies", tmpFile.Name())
defer os.Remove(tmpFile.Name())
// Record the meaningful flags (format/audio/subs/custom) that shaped this
// download, before the internal plumbing (cookies, -P/-o, URL) is appended,
// so each imported item can show how it was fetched.
ytdlpFlags := strings.Join(args, " ")
var cookieCleanup func()
args, cookieCleanup = s.appendCookies(args)
defer cookieCleanup()
if sub != nil {
// Always write info.json so the import step can read the stable identity
// (id + extractor) used to match/replace existing items.
args = append(args, "--write-info-json")
switch sub.RefreshMode {
case "skip":
// Let yt-dlp skip entries already recorded — no re-download.
archive := s.subscriptionSvc.ArchivePath(sub.ID)
if err := os.MkdirAll(filepath.Dir(archive), 0755); err == nil {
args = append(args, "--download-archive", archive)
}
case "metadata":
// Refresh metadata only; don't fetch media.
args = append(args, "--skip-download")
}
}
@@ -189,19 +242,55 @@ func (s *DownloadService) ExecuteDownload(d *models.Download) (bool, error) {
args = append(args, "-o", "item-%(autonumber)05d/%(title)s.%(ext)s")
args = append(args, d.URL)
if err := s.runYTDLP(d, args); err != nil {
s.finalizeError(d.ID, err)
return false, err
}
mode := ""
if sub != nil {
mode = sub.RefreshMode
}
// Post-process before marking completed, so the download stays "downloading"
// until everything is really done — including metadata mode's second pass,
// which downloads any genuinely new entries as full items.
if mode == "metadata" {
// The main pass ran with --skip-download, so the temp dir holds only
// info.json files: refresh existing items in place and fetch new ones.
if err := s.refreshAndAddNew(d, preset, tempDownloadDir, ytdlpFlags); err != nil {
log.Printf("Download %d metadata refresh failed: %v", d.ID, err)
}
} else if err := s.importDownloadedItems(d, tempDownloadDir, mode, ytdlpFlags); err != nil {
log.Printf("Download %d import failed: %v", d.ID, err)
}
if sub != nil && sub.PruneRemoved {
s.pruneSubscription(d, sub)
}
if err := s.repo.MarkCompleted(d.ID, "completed"); err != nil {
return false, err
}
return true, nil
}
// runYTDLP executes yt-dlp with args, streaming combined output into the live
// progress cache and periodically flushing it to the download's persisted log.
// It registers the process so Delete can kill it, and returns the exit error.
func (s *DownloadService) runYTDLP(d *models.Download, args []string) error {
cmd := exec.Command(s.cfg.YTDLPPath, args...)
cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
stdout, err := cmd.StdoutPipe()
if err != nil {
s.finalizeError(d.ID, err)
return false, err
return err
}
cmd.Stderr = cmd.Stdout
if err := cmd.Start(); err != nil {
s.finalizeError(d.ID, err)
return false, err
return err
}
s.processMu.Lock()
@@ -213,8 +302,6 @@ func (s *DownloadService) ExecuteDownload(d *models.Download) (bool, error) {
s.processMu.Unlock()
}()
scanner := bufio.NewScanner(stdout)
ticker := time.NewTicker(10 * time.Second)
defer ticker.Stop()
done := make(chan struct{})
@@ -222,8 +309,7 @@ func (s *DownloadService) ExecuteDownload(d *models.Download) (bool, error) {
for {
select {
case <-ticker.C:
logs := s.cache.FlushLogs(d.ID)
if logs != "" {
if logs := s.cache.FlushLogs(d.ID); logs != "" {
s.repo.AppendLogs(d.ID, logs)
}
case <-done:
@@ -232,32 +318,35 @@ func (s *DownloadService) ExecuteDownload(d *models.Download) (bool, error) {
}
}()
scanner := bufio.NewScanner(stdout)
for scanner.Scan() {
line := scanner.Text()
s.cache.AppendLog(d.ID, line)
s.cache.AppendLog(d.ID, scanner.Text())
}
close(done)
logs := s.cache.FlushLogs(d.ID)
if logs != "" {
if logs := s.cache.FlushLogs(d.ID); logs != "" {
s.repo.AppendLogs(d.ID, logs)
}
if err := cmd.Wait(); err != nil {
s.finalizeError(d.ID, err)
return false, err
}
return cmd.Wait()
}
if err := s.repo.MarkCompleted(d.ID, "completed"); err != nil {
return false, err
// appendCookies writes the saved cookies (if any) to a temp file and appends a
// --cookies flag. The returned cleanup removes the temp file and is always safe
// to call, even when no cookies were configured.
func (s *DownloadService) appendCookies(args []string) ([]string, func()) {
cleanup := func() {}
cookies, err := s.settingsSvc.GetCookies()
if err != nil || strings.TrimSpace(cookies) == "" {
return args, cleanup
}
if err := s.importDownloadedItems(d, tempDownloadDir); err != nil {
log.Printf("Download %d completed but import failed: %v", d.ID, err)
tmpFile, err := os.CreateTemp("", "cookies-*.txt")
if err != nil {
return args, cleanup
}
return true, nil
tmpFile.WriteString(cookies)
tmpFile.Close()
return append(args, "--cookies", tmpFile.Name()), func() { os.Remove(tmpFile.Name()) }
}
func (s *DownloadService) finalizeError(id int64, err error) {
@@ -268,26 +357,37 @@ func (s *DownloadService) finalizeError(id int64, err error) {
s.repo.MarkError(id, err.Error())
}
func (s *DownloadService) importDownloadedItems(d *models.Download, tempDownloadDir string) error {
entries, err := os.ReadDir(tempDownloadDir)
if err != nil {
return err
}
// resolveBaseLibraryDir returns the absolute library directory a download writes
// into, applying the optional per-download OutputDir while rejecting any path
// that escapes the library root.
func (s *DownloadService) resolveBaseLibraryDir(d *models.Download) (string, error) {
baseLibraryDir := s.cfg.LibraryDir
if d.OutputDir.Valid && d.OutputDir.String != "" {
cleanDir := filepath.Clean(d.OutputDir.String)
fullPath := filepath.Join(baseLibraryDir, cleanDir)
resolvedPath, err := filepath.Abs(fullPath)
if err != nil {
return fmt.Errorf("invalid output directory: %w", err)
return "", fmt.Errorf("invalid output directory: %w", err)
}
resolvedLibraryDir, _ := filepath.Abs(baseLibraryDir)
if !strings.HasPrefix(resolvedPath, resolvedLibraryDir+string(filepath.Separator)) && resolvedPath != resolvedLibraryDir {
return fmt.Errorf("invalid output directory: path traversal attempt detected")
return "", fmt.Errorf("invalid output directory: path traversal attempt detected")
}
baseLibraryDir = fullPath
}
return baseLibraryDir, nil
}
func (s *DownloadService) importDownloadedItems(d *models.Download, tempDownloadDir, mode, ytdlpFlags string) error {
entries, err := os.ReadDir(tempDownloadDir)
if err != nil {
return err
}
baseLibraryDir, err := s.resolveBaseLibraryDir(d)
if err != nil {
return err
}
if err := os.MkdirAll(baseLibraryDir, 0755); err != nil {
return err
}
@@ -305,7 +405,7 @@ func (s *DownloadService) importDownloadedItems(d *models.Download, tempDownload
sort.Strings(itemDirs)
for _, itemDir := range itemDirs {
if err := s.importItemDir(d.URL, itemDir, baseLibraryDir); err != nil {
if err := s.importItemDir(d.URL, itemDir, baseLibraryDir, mode, ytdlpFlags); err != nil {
log.Printf("warning: failed to import item %s: %v", itemDir, err)
}
}
@@ -314,7 +414,7 @@ func (s *DownloadService) importDownloadedItems(d *models.Download, tempDownload
return nil
}
func (s *DownloadService) importItemDir(url, itemDir, baseLibraryDir string) error {
func (s *DownloadService) importItemDir(url, itemDir, baseLibraryDir, mode, ytdlpFlags string) error {
entries, err := os.ReadDir(itemDir)
if err != nil {
return err
@@ -352,6 +452,20 @@ func (s *DownloadService) importItemDir(url, itemDir, baseLibraryDir string) err
}
name := s.deriveItemName(itemDir, infoJSONPath, mediaFiles)
videoID, extractor := readInfoIdentity(infoJSONPath)
// Overwrite mode: replace the existing copy of this video in place rather than
// creating a duplicate folder. Removing the old dir lets uniqueDir reuse its
// name (or land on the new title if it changed upstream).
if mode == "overwrite" && videoID != "" {
if existing, ok := s.librarySvc.FindByVideoID(baseLibraryDir, extractor, videoID); ok {
if rel, err := filepath.Rel(s.cfg.LibraryDir, existing); err == nil {
s.librarySvc.evictCachedScan(filepath.ToSlash(rel))
}
os.RemoveAll(existing)
}
}
targetDir := s.uniqueDir(baseLibraryDir, name)
if err := os.MkdirAll(targetDir, 0755); err != nil {
return err
@@ -394,6 +508,9 @@ func (s *DownloadService) importItemDir(url, itemDir, baseLibraryDir string) err
metadata := models.ItemMetadata{
Name: name,
SourceURL: url,
VideoID: videoID,
Extractor: extractor,
YtdlpFlags: ytdlpFlags,
FileDurations: fileDurations,
}
@@ -410,6 +527,230 @@ func (s *DownloadService) importItemDir(url, itemDir, baseLibraryDir string) err
return nil
}
// readInfoIdentity extracts the stable identity (yt-dlp id + extractor) from an
// info.json. Returns empty strings when the file is absent or unreadable.
func readInfoIdentity(infoJSONPath string) (id, extractor string) {
if infoJSONPath == "" {
return "", ""
}
data, err := os.ReadFile(infoJSONPath)
if err != nil {
return "", ""
}
var info struct {
ID string `json:"id"`
Extractor string `json:"extractor"`
}
if err := json.Unmarshal(data, &info); err != nil {
return "", ""
}
return info.ID, info.Extractor
}
// refreshAndAddNew handles a metadata-mode run. The main pass used
// --skip-download, so tempDownloadDir holds only info.json files. Existing
// library items have their markers refreshed in place; entries with no existing
// match are genuinely new and are downloaded as full items in a second pass.
func (s *DownloadService) refreshAndAddNew(d *models.Download, preset *models.Preset, tempDownloadDir, ytdlpFlags string) error {
baseLibraryDir, err := s.resolveBaseLibraryDir(d)
if err != nil {
return err
}
if err := os.MkdirAll(baseLibraryDir, 0755); err != nil {
return err
}
entries, err := os.ReadDir(tempDownloadDir)
if err != nil {
return err
}
var newURLs []string
for _, entry := range entries {
if !entry.IsDir() || !strings.HasPrefix(entry.Name(), "item-") {
continue
}
itemDir := filepath.Join(tempDownloadDir, entry.Name())
infoJSONPath := findInfoJSON(itemDir)
if infoJSONPath == "" {
continue
}
videoID, extractor := readInfoIdentity(infoJSONPath)
if videoID == "" {
continue
}
if existing, ok := s.librarySvc.FindByVideoID(baseLibraryDir, extractor, videoID); ok {
if err := s.applyMetadata(existing, infoJSONPath, videoID, extractor); err != nil {
log.Printf("warning: failed to refresh metadata for %s: %v", itemDir, err)
}
continue
}
if u := readWebpageURL(infoJSONPath); u != "" {
newURLs = append(newURLs, u)
}
}
os.RemoveAll(tempDownloadDir)
if len(newURLs) == 0 {
return nil
}
return s.downloadFresh(d, preset, newURLs, ytdlpFlags)
}
// downloadFresh fetches the given item URLs as full downloads (media + info.json)
// and imports them into the download's library directory. Metadata mode uses this
// to add entries that don't exist in the library yet.
func (s *DownloadService) downloadFresh(d *models.Download, preset *models.Preset, urls []string, ytdlpFlags string) error {
tempDir := filepath.Join(s.cfg.TempDir, fmt.Sprintf("%d-new", d.ID))
if err := os.MkdirAll(tempDir, 0755); err != nil {
return err
}
args := s.presetSvc.BuildArgs(preset, d.FormatOverride, d.CustomFlags)
args, cleanup := s.appendCookies(args)
defer cleanup()
args = append(args, "--write-info-json")
args = append(args, "-P", tempDir)
args = append(args, "-o", "item-%(autonumber)05d/%(title)s.%(ext)s")
args = append(args, urls...)
runErr := s.runYTDLP(d, args)
// Import whatever succeeded even if some entries errored.
if err := s.importDownloadedItems(d, tempDir, "", ytdlpFlags); err != nil {
log.Printf("warning: failed to import new metadata-mode items: %v", err)
}
return runErr
}
// applyMetadata rewrites an existing item's marker (name/description/identity)
// from a fresh info.json without touching its media.
func (s *DownloadService) applyMetadata(existing, infoJSONPath, videoID, extractor string) error {
data, err := os.ReadFile(infoJSONPath)
if err != nil {
return err
}
var info struct {
Title string `json:"title"`
Description string `json:"description"`
WebpageURL string `json:"webpage_url"`
}
_ = json.Unmarshal(data, &info)
meta, _ := s.librarySvc.readOrCreateMetadata(existing)
if info.Title != "" {
meta.Name = info.Title
}
if info.Description != "" {
meta.Description = info.Description
}
if meta.SourceURL == "" && info.WebpageURL != "" {
meta.SourceURL = info.WebpageURL
}
meta.VideoID = videoID
meta.Extractor = extractor
if err := s.librarySvc.writeMetadata(existing, meta); err != nil {
return err
}
if rel, err := filepath.Rel(s.cfg.LibraryDir, existing); err == nil {
s.librarySvc.evictCachedScan(filepath.ToSlash(rel))
}
return nil
}
// readWebpageURL returns the canonical entry URL from an info.json, or "".
func readWebpageURL(infoJSONPath string) string {
data, err := os.ReadFile(infoJSONPath)
if err != nil {
return ""
}
var info struct {
WebpageURL string `json:"webpage_url"`
}
if err := json.Unmarshal(data, &info); err != nil {
return ""
}
return info.WebpageURL
}
// findInfoJSON returns the path to an info.json directly inside itemDir, or "".
func findInfoJSON(itemDir string) string {
entries, err := os.ReadDir(itemDir)
if err != nil {
return ""
}
for _, entry := range entries {
if entry.IsDir() {
continue
}
name := entry.Name()
if name == "info.json" || strings.HasSuffix(name, ".info.json") {
return filepath.Join(itemDir, name)
}
}
return ""
}
// pruneSubscription mirrors the source by deleting items in the subscription's
// directory that are no longer present upstream. It enumerates the current id
// set with a cheap flat-playlist listing; it never prunes when that enumeration
// fails or returns nothing, so a dead URL or network error can't wipe the dir.
func (s *DownloadService) pruneSubscription(d *models.Download, sub *models.Subscription) {
baseLibraryDir, err := s.resolveBaseLibraryDir(d)
if err != nil {
log.Printf("subscription %d prune skipped: %v", sub.ID, err)
return
}
keep, err := s.enumeratePlaylistIDs(sub.URL)
if err != nil {
log.Printf("subscription %d prune skipped: enumeration failed: %v", sub.ID, err)
return
}
if len(keep) == 0 {
log.Printf("subscription %d prune skipped: source returned no entries", sub.ID)
return
}
removed, err := s.librarySvc.PruneToIDSet(baseLibraryDir, keep)
if err != nil {
log.Printf("subscription %d prune error: %v", sub.ID, err)
return
}
if removed > 0 {
log.Printf("subscription %d pruned %d item(s) removed upstream", sub.ID, removed)
}
}
// enumeratePlaylistIDs lists the current "extractor:id" set for a URL without
// downloading, using yt-dlp --flat-playlist. Cookies are applied so private
// playlists enumerate correctly.
func (s *DownloadService) enumeratePlaylistIDs(url string) (map[string]bool, error) {
args := []string{"--flat-playlist", "--no-warnings", "--print", "%(extractor)s %(id)s"}
args, cleanup := s.appendCookies(args)
defer cleanup()
args = append(args, url)
out, err := exec.Command(s.cfg.YTDLPPath, args...).Output()
if err != nil {
return nil, err
}
keep := make(map[string]bool)
for _, line := range strings.Split(string(out), "\n") {
fields := strings.Fields(line)
if len(fields) != 2 {
continue
}
if key := videoKey(fields[0], fields[1]); key != "" {
keep[key] = true
}
}
return keep, nil
}
func (s *DownloadService) deriveItemName(itemDir, infoJSONPath string, mediaFiles []os.DirEntry) string {
if infoJSONPath != "" {
data, err := os.ReadFile(infoJSONPath)
@@ -542,3 +883,46 @@ func parseFormatList(output string) []*models.FormatInfo {
func sqlNullInt64(v int64) sql.NullInt64 {
return sql.NullInt64{Int64: v, Valid: true}
}
// reservedFlags are yt-dlp options VidArchive always sets itself; user custom
// flags must not pass them (or a conflicting inverse). The value describes what
// the option controls, for the failure message.
var reservedFlags = map[string]string{
"-o": "the output template",
"--output": "the output template",
"-P": "the download path",
"--paths": "the download path",
"--cookies": "cookies (set these in Settings instead)",
"--no-cookies": "cookies (set these in Settings instead)",
}
// reservedSubscriptionFlags are additionally reserved for subscription runs,
// where VidArchive drives info-json writing and the refresh mode.
var reservedSubscriptionFlags = map[string]string{
"--write-info-json": "info-json writing (needed to track item identity)",
"--no-write-info-json": "info-json writing (needed to track item identity)",
"--download-archive": "the download archive (managed by Skip mode)",
"--no-download-archive": "the download archive (managed by Skip mode)",
"--skip-download": "media downloading (managed by Metadata mode)",
"--no-skip-download": "media downloading (managed by Metadata mode)",
}
// checkReservedFlags rejects custom flags that clash with options VidArchive
// controls, naming the offender. It matches both "--flag" and "--flag=value".
func checkReservedFlags(customFlags string, isSubscription bool) error {
for _, tok := range strings.Fields(customFlags) {
name := tok
if i := strings.IndexByte(name, '='); i >= 0 {
name = name[:i]
}
if desc, ok := reservedFlags[name]; ok {
return fmt.Errorf("custom flag %q conflicts with VidArchive's handling of %s; remove it and try again", tok, desc)
}
if isSubscription {
if desc, ok := reservedSubscriptionFlags[name]; ok {
return fmt.Errorf("custom flag %q conflicts with VidArchive's handling of %s; remove it and try again", tok, desc)
}
}
}
return nil
}
Minternal/service/download_test.go
@@ -136,7 +136,7 @@ func TestImportItemDir(t *testing.T) {
t.Fatal(err)
}
if err := svc.importItemDir("https://example.com/v", src, libDir); err != nil {
if err := svc.importItemDir("https://example.com/v", src, libDir, "", "-f bestvideo+bestaudio"); err != nil {
t.Fatalf("importItemDir: %v", err)
}
@@ -163,6 +163,46 @@ func TestImportItemDir(t *testing.T) {
if !strings.Contains(string(markerData), "file_durations") || !strings.Contains(string(markerData), "raw.mp4") {
t.Errorf("marker missing probed per-file duration: %s", markerData)
}
if want := "-f bestvideo+bestaudio"; !strings.Contains(string(markerData), want) {
t.Errorf("marker missing yt-dlp flags %q: %s", want, markerData)
}
}
func TestCheckReservedFlags(t *testing.T) {
cases := []struct {
name string
flags string
subscription bool
wantErr bool
}{
{"empty", "", false, false},
{"harmless", "--no-playlist --write-thumbnail", false, false},
{"output short", "-o foo.mp4", false, true},
{"output long", "--output foo.mp4", false, true},
{"output equals form", "--output=foo.mp4", false, true},
{"paths short", "-P /tmp", false, true},
{"cookies", "--cookies x.txt", false, true},
{"cookies inverse", "--no-cookies", false, true},
// Subscription-only reserved flags pass for normal downloads...
{"skip-download non-sub", "--skip-download", false, false},
{"write-info-json non-sub", "--write-info-json", false, false},
// ...but are rejected for subscription runs (and their inverses).
{"skip-download sub", "--skip-download", true, true},
{"no-skip-download sub", "--no-skip-download", true, true},
{"write-info-json sub", "--write-info-json", true, true},
{"no-write-info-json sub", "--no-write-info-json", true, true},
{"download-archive sub", "--download-archive a.txt", true, true},
// Base reserved flags still apply to subscriptions.
{"output sub", "-o x", true, true},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
err := checkReservedFlags(tc.flags, tc.subscription)
if tc.wantErr != (err != nil) {
t.Errorf("checkReservedFlags(%q, %v) error = %v, wantErr %v", tc.flags, tc.subscription, err, tc.wantErr)
}
})
}
}
func TestImportDownloadedItemsRejectsOutputTraversal(t *testing.T) {
@@ -180,7 +220,7 @@ func TestImportDownloadedItemsRejectsOutputTraversal(t *testing.T) {
URL: "u",
OutputDir: sql.NullString{String: "../escape", Valid: true},
}
if err := svc.importDownloadedItems(d, tempDir); err == nil {
if err := svc.importDownloadedItems(d, tempDir, "", ""); err == nil {
t.Error("expected path-traversal output dir to be rejected")
}
}
Minternal/service/library.go
@@ -285,22 +285,43 @@ func (s *LibraryService) scanItem(itemDir, relPath string) (*models.LibraryItem,
}
if metadata.SourceURL == "" {
if url, ok := infoString(info, "webpage_url"); ok {
// Accept only a non-empty value: a present-but-empty field would otherwise
// leave SourceURL "" while flipping dirty on every scan, rewriting the
// marker forever.
if url, ok := infoString(info, "webpage_url"); ok && url != "" {
metadata.SourceURL = url
dirty = true
} else if url, ok := infoString(info, "url"); ok {
} else if url, ok := infoString(info, "url"); ok && url != "" {
metadata.SourceURL = url
dirty = true
}
}
if metadata.Description == "" {
if desc, ok := infoString(info, "description"); ok {
// Same guard: an empty "description" in info.json must not mark the marker
// dirty on every listing (items whose text lives only in the title).
if desc, ok := infoString(info, "description"); ok && desc != "" {
metadata.Description = desc
dirty = true
}
}
// Backfill the stable identity (yt-dlp id + extractor) from info.json so
// pre-existing items gain an identity on their next scan. Subscriptions match
// and prune items by this key (see videoKey / FindByVideoID).
if metadata.VideoID == "" {
if id, ok := infoString(info, "id"); ok && id != "" {
metadata.VideoID = id
dirty = true
}
}
if metadata.Extractor == "" {
if ex, ok := infoString(info, "extractor"); ok && ex != "" {
metadata.Extractor = ex
dirty = true
}
}
if metadata.FileDurations == nil {
metadata.FileDurations = make(map[string]int)
}
@@ -341,6 +362,7 @@ func (s *LibraryService) scanItem(itemDir, relPath string) (*models.LibraryItem,
SourceURL: metadata.SourceURL,
Duration: total,
Description: metadata.Description,
YtdlpFlags: metadata.YtdlpFlags,
MediaFiles: mediaFiles,
}
@@ -811,6 +833,81 @@ func (s *LibraryService) Delete(relPath string) error {
return os.RemoveAll(itemDir)
}
// videoKey is the stable per-item identity used to match items across runs:
// yt-dlp's extractor + id, matching the form used in --download-archive and the
// "%(extractor)s %(id)s" enumeration. Empty when either component is missing.
func videoKey(extractor, id string) string {
if extractor == "" || id == "" {
return ""
}
return extractor + ":" + id
}
// FindByVideoID returns the absolute directory of the item under baseDir whose
// marker matches the given extractor+id, scanning only that directory (the
// subscription's owned folder). ok is false when no match is found or the key
// is incomplete.
func (s *LibraryService) FindByVideoID(baseDir, extractor, id string) (string, bool) {
want := videoKey(extractor, id)
if want == "" {
return "", false
}
entries, err := os.ReadDir(baseDir)
if err != nil {
return "", false
}
for _, entry := range entries {
if !entry.IsDir() || entry.Name() == subtitlesDirName {
continue
}
itemDir := filepath.Join(baseDir, entry.Name())
if _, err := os.Stat(filepath.Join(itemDir, itemMarkerName)); err != nil {
continue
}
meta, _ := s.readOrCreateMetadata(itemDir)
if videoKey(meta.Extractor, meta.VideoID) == want {
return itemDir, true
}
}
return "", false
}
// PruneToIDSet deletes items directly under baseDir whose identity key is not in
// keep. It is used to mirror a subscription's source: entries removed upstream
// are removed locally. Items without a known identity key are left untouched (we
// never delete something we can't positively identify). Returns the number
// removed.
func (s *LibraryService) PruneToIDSet(baseDir string, keep map[string]bool) (int, error) {
entries, err := os.ReadDir(baseDir)
if err != nil {
return 0, err
}
removed := 0
for _, entry := range entries {
if !entry.IsDir() || entry.Name() == subtitlesDirName {
continue
}
itemDir := filepath.Join(baseDir, entry.Name())
if _, err := os.Stat(filepath.Join(itemDir, itemMarkerName)); err != nil {
continue
}
meta, _ := s.readOrCreateMetadata(itemDir)
key := videoKey(meta.Extractor, meta.VideoID)
if key == "" || keep[key] {
continue
}
if rel, err := filepath.Rel(s.libraryDir, itemDir); err == nil {
s.evictCachedScan(filepath.ToSlash(rel))
}
if err := os.RemoveAll(itemDir); err != nil {
log.Printf("warning: prune failed to remove %s: %v", itemDir, err)
continue
}
removed++
}
return removed, nil
}
func (s *LibraryService) SubtitleDir(relPath string) string {
itemDir, err := s.resolveItemDir(relPath)
if err != nil {
Minternal/service/library_test.go
@@ -450,6 +450,33 @@ func TestScanItemDoesNotRewriteConvergedMarker(t *testing.T) {
}
}
// TestScanItemEmptyInfoFieldsDoNotRewriteMarker is a regression test: an
// info.json with present-but-empty "description"/"webpage_url" must not flip the
// marker dirty on every scan. infoString returns ("", true) for those fields, so
// the old code reassigned "" and rewrote the marker forever (e.g. clips whose
// text lives only in the title). We detect any rewrite via an unknown TOML key,
// which scanItem's re-encode would drop.
func TestScanItemEmptyInfoFieldsDoNotRewriteMarker(t *testing.T) {
svc, dir := newLibrary(t)
svc.scanTTL = 0 // force a real rescan on every read
info := `{"title":"#hashtags only","description":"","webpage_url":""}`
marker := "name = \"#hashtags only\"\nsentinel = \"keep\"\n"
writeItem(t, dir, "z", marker, map[string]string{"z.mp4": "v", "info.json": info})
markerPath := filepath.Join(dir, "z", itemMarkerName)
for i := 0; i < 3; i++ {
if _, err := svc.GetByRelPath(context.Background(), "z"); err != nil {
t.Fatalf("GetByRelPath: %v", err)
}
}
data, _ := os.ReadFile(markerPath)
if !strings.Contains(string(data), "sentinel") {
t.Errorf("marker was rewritten on read (sentinel dropped): %s", data)
}
}
// --- scan cache ---
func TestScanCacheReusesWithinTTL(t *testing.T) {
Ainternal/service/subscription.go
@@ -0,0 +1,109 @@
package service
import (
"fmt"
"path/filepath"
"time"
"github.com/robfig/cron/v3"
"vidarchive/internal/config"
"vidarchive/internal/models"
"vidarchive/internal/repository"
)
type SubscriptionService struct {
repo *repository.SubscriptionRepository
cfg *config.Config
}
func NewSubscriptionService(repo *repository.SubscriptionRepository, cfg *config.Config) *SubscriptionService {
return &SubscriptionService{repo: repo, cfg: cfg}
}
// cronForKind maps a schedule kind to a standard 5-field cron expression. Preset
// kinds use canonical expressions; "cron" passes the user's custom expression
// through. Preset times default to 03:00 local to avoid the top-of-hour rush.
func cronForKind(kind, custom string) (string, error) {
switch kind {
case "hourly":
return "0 * * * *", nil
case "daily":
return "0 3 * * *", nil
case "weekly":
return "0 3 * * 0", nil
case "monthly":
return "0 3 1 * *", nil
case "cron":
if custom == "" {
return "", fmt.Errorf("custom schedule requires a cron expression")
}
return custom, nil
default:
return "", fmt.Errorf("unknown schedule kind %q", kind)
}
}
// CronExprFor resolves and validates the effective cron expression for a kind +
// custom expression, returning an error for unknown kinds or unparseable cron.
func (s *SubscriptionService) CronExprFor(kind, custom string) (string, error) {
expr, err := cronForKind(kind, custom)
if err != nil {
return "", err
}
if _, err := cron.ParseStandard(expr); err != nil {
return "", fmt.Errorf("invalid cron expression %q: %w", expr, err)
}
return expr, nil
}
// ComputeNextRun returns the next scheduled run time strictly after `from`.
func (s *SubscriptionService) ComputeNextRun(sub *models.Subscription, from time.Time) (time.Time, error) {
expr, err := s.CronExprFor(sub.ScheduleKind, sub.CronExpr)
if err != nil {
return time.Time{}, err
}
sched, err := cron.ParseStandard(expr)
if err != nil {
return time.Time{}, err
}
return sched.Next(from), nil
}
// ArchivePath is the per-subscription yt-dlp download-archive file used by
// "skip" refresh mode to avoid re-downloading already-fetched entries.
func (s *SubscriptionService) ArchivePath(id int64) string {
return filepath.Join(s.cfg.DataDir, "archives", fmt.Sprintf("sub-%d.txt", id))
}
func (s *SubscriptionService) GetAll() ([]*models.Subscription, error) {
return s.repo.GetAll()
}
func (s *SubscriptionService) GetByID(id int64) (*models.Subscription, error) {
return s.repo.GetByID(id)
}
func (s *SubscriptionService) GetDue(now time.Time) ([]*models.Subscription, error) {
return s.repo.GetDue(now)
}
func (s *SubscriptionService) Create(sub *models.Subscription) error {
return s.repo.Create(sub)
}
func (s *SubscriptionService) Update(sub *models.Subscription) error {
return s.repo.Update(sub)
}
func (s *SubscriptionService) SetEnabled(id int64, enabled bool) error {
return s.repo.SetEnabled(id, enabled)
}
func (s *SubscriptionService) MarkRun(id int64, lastRunAt, nextRunAt time.Time, status string) error {
return s.repo.MarkRun(id, lastRunAt, nextRunAt, status)
}
func (s *SubscriptionService) Delete(id int64) error {
return s.repo.Delete(id)
}
Ainternal/service/subscription_library_test.go
@@ -0,0 +1,56 @@
package service
import (
"os"
"path/filepath"
"testing"
)
func TestFindByVideoID(t *testing.T) {
lib, dir := newLibrary(t)
writeItem(t, dir, "alpha", "name = \"Alpha\"\nvideo_id = \"aaa\"\nextractor = \"youtube\"\n", nil)
writeItem(t, dir, "beta", "name = \"Beta\"\nvideo_id = \"bbb\"\nextractor = \"youtube\"\n", nil)
// Same id but different extractor must not match.
writeItem(t, dir, "gamma", "name = \"Gamma\"\nvideo_id = \"aaa\"\nextractor = \"vimeo\"\n", nil)
got, ok := lib.FindByVideoID(dir, "youtube", "aaa")
if !ok || filepath.Base(got) != "alpha" {
t.Fatalf("expected to find alpha, got %q ok=%v", got, ok)
}
if _, ok := lib.FindByVideoID(dir, "youtube", "zzz"); ok {
t.Error("expected no match for unknown id")
}
// Incomplete key never matches.
if _, ok := lib.FindByVideoID(dir, "", "aaa"); ok {
t.Error("expected no match for empty extractor")
}
}
func TestPruneToIDSet(t *testing.T) {
lib, dir := newLibrary(t)
writeItem(t, dir, "keep", "name = \"Keep\"\nvideo_id = \"k1\"\nextractor = \"youtube\"\n", nil)
writeItem(t, dir, "drop", "name = \"Drop\"\nvideo_id = \"d1\"\nextractor = \"youtube\"\n", nil)
// An item without identity must never be pruned (we can't positively id it).
writeItem(t, dir, "noid", "name = \"NoId\"\n", nil)
keep := map[string]bool{"youtube:k1": true}
removed, err := lib.PruneToIDSet(dir, keep)
if err != nil {
t.Fatalf("prune: %v", err)
}
if removed != 1 {
t.Errorf("expected 1 removed, got %d", removed)
}
if _, err := os.Stat(filepath.Join(dir, "drop")); !os.IsNotExist(err) {
t.Error("expected 'drop' to be removed")
}
if _, err := os.Stat(filepath.Join(dir, "keep")); err != nil {
t.Error("expected 'keep' to survive")
}
if _, err := os.Stat(filepath.Join(dir, "noid")); err != nil {
t.Error("expected unidentified item to survive")
}
}
Ainternal/service/subscription_test.go
@@ -0,0 +1,66 @@
package service
import (
"testing"
"time"
"vidarchive/internal/models"
)
func TestComputeNextRun(t *testing.T) {
svc := &SubscriptionService{}
// A fixed reference time: 2026-06-21 12:30 local.
from := time.Date(2026, 6, 21, 12, 30, 0, 0, time.Local)
cases := []struct {
kind string
custom string
wantErr bool
check func(t *testing.T, next time.Time)
}{
{kind: "hourly", check: func(t *testing.T, next time.Time) {
if !next.After(from) || next.Minute() != 0 {
t.Errorf("hourly: got %v", next)
}
}},
{kind: "daily", check: func(t *testing.T, next time.Time) {
if next.Hour() != 3 || next.Minute() != 0 || !next.After(from) {
t.Errorf("daily: got %v", next)
}
}},
{kind: "weekly", check: func(t *testing.T, next time.Time) {
if next.Weekday() != time.Sunday || !next.After(from) {
t.Errorf("weekly: got %v", next)
}
}},
{kind: "monthly", check: func(t *testing.T, next time.Time) {
if next.Day() != 1 || !next.After(from) {
t.Errorf("monthly: got %v", next)
}
}},
{kind: "cron", custom: "*/15 * * * *", check: func(t *testing.T, next time.Time) {
if next.Minute() != 45 || !next.After(from) {
t.Errorf("custom cron */15: got %v", next)
}
}},
{kind: "cron", custom: "not a cron", wantErr: true},
{kind: "cron", custom: "", wantErr: true},
{kind: "bogus", wantErr: true},
}
for _, c := range cases {
sub := &models.Subscription{ScheduleKind: c.kind, CronExpr: c.custom}
next, err := svc.ComputeNextRun(sub, from)
if c.wantErr {
if err == nil {
t.Errorf("kind=%q custom=%q: expected error, got %v", c.kind, c.custom, next)
}
continue
}
if err != nil {
t.Errorf("kind=%q custom=%q: unexpected error: %v", c.kind, c.custom, err)
continue
}
c.check(t, next)
}
}
Ainternal/worker/scheduler.go
@@ -0,0 +1,127 @@
package worker
import (
"context"
"database/sql"
"log"
"time"
"vidarchive/internal/models"
"vidarchive/internal/service"
)
// Scheduler periodically enqueues downloads for subscriptions whose next run is
// due. It reuses the worker Pool and the normal download pipeline; refresh-mode
// and pruning behaviour live in DownloadService.ExecuteDownload.
type Scheduler struct {
subscriptionSvc *service.SubscriptionService
downloadSvc *service.DownloadService
pool *Pool
interval time.Duration
ctx context.Context
cancel context.CancelFunc
}
func NewScheduler(subscriptionSvc *service.SubscriptionService, downloadSvc *service.DownloadService, pool *Pool, interval time.Duration) *Scheduler {
if interval <= 0 {
interval = time.Minute
}
ctx, cancel := context.WithCancel(context.Background())
return &Scheduler{
subscriptionSvc: subscriptionSvc,
downloadSvc: downloadSvc,
pool: pool,
interval: interval,
ctx: ctx,
cancel: cancel,
}
}
func (s *Scheduler) Start() {
s.backfillNextRuns()
go s.loop()
}
func (s *Scheduler) Stop() {
s.cancel()
}
func (s *Scheduler) loop() {
ticker := time.NewTicker(s.interval)
defer ticker.Stop()
// Check once promptly on startup so a subscription that came due during
// downtime doesn't wait a full interval.
s.checkDue()
for {
select {
case <-ticker.C:
s.checkDue()
case <-s.ctx.Done():
return
}
}
}
// backfillNextRuns gives any enabled subscription without a next run time one,
// without running it — so a fresh restart doesn't fire every subscription that
// happens to have a null next_run_at.
func (s *Scheduler) backfillNextRuns() {
subs, err := s.subscriptionSvc.GetAll()
if err != nil {
log.Printf("scheduler: backfill failed to list subscriptions: %v", err)
return
}
now := time.Now()
for _, sub := range subs {
if !sub.Enabled || sub.NextRunAt.Valid {
continue
}
next, err := s.subscriptionSvc.ComputeNextRun(sub, now)
if err != nil {
log.Printf("scheduler: subscription %d has invalid schedule: %v", sub.ID, err)
continue
}
sub.NextRunAt = sql.NullTime{Time: next, Valid: true}
if err := s.subscriptionSvc.Update(sub); err != nil {
log.Printf("scheduler: failed to set next run for subscription %d: %v", sub.ID, err)
}
}
}
func (s *Scheduler) checkDue() {
now := time.Now()
due, err := s.subscriptionSvc.GetDue(now)
if err != nil {
log.Printf("scheduler: failed to query due subscriptions: %v", err)
return
}
for _, sub := range due {
s.run(sub, now)
}
}
func (s *Scheduler) run(sub *models.Subscription, now time.Time) {
next, err := s.subscriptionSvc.ComputeNextRun(sub, now)
if err != nil {
// Don't keep retrying a broken schedule every tick; push it out a day.
log.Printf("scheduler: subscription %d has invalid schedule, deferring: %v", sub.ID, err)
next = now.Add(24 * time.Hour)
}
d, err := s.downloadSvc.CreateForSubscription(sub)
if err != nil {
log.Printf("scheduler: failed to queue subscription %d: %v", sub.ID, err)
if mErr := s.subscriptionSvc.MarkRun(sub.ID, now, next, "error"); mErr != nil {
log.Printf("scheduler: failed to mark subscription %d run: %v", sub.ID, mErr)
}
return
}
s.pool.Submit(d)
if err := s.subscriptionSvc.MarkRun(sub.ID, now, next, "queued"); err != nil {
log.Printf("scheduler: failed to mark subscription %d run: %v", sub.ID, err)
}
log.Printf("scheduler: queued subscription %d (download %d), next run %s", sub.ID, d.ID, next.Format(time.RFC3339))
}
Aweb/static/icons/logo.svg
@@ -0,0 +1,6 @@
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 32 32" role="img" aria-label="VidArchive">
<!-- archive folder (tab integrated into the body) -->
<path fill="#0172ad" d="M6.5 6 H12 L15 9.5 H25 a3 3 0 0 1 3 3 V23.5 a3 3 0 0 1 -3 3 H7.5 a3 3 0 0 1 -3 -3 V8 a2 2 0 0 1 2 -2 Z"/>
<!-- play button -->
<path fill="#ffffff" d="M12.5 13.5 L21 18 L12.5 22.5 Z"/>
</svg>
Mweb/static/style.css
@@ -37,6 +37,18 @@ header h1 {
font-size: 1.5rem;
}
/* Brand: logo + wordmark, no longer a link */
.brand {
display: flex;
align-items: center;
gap: 0.55rem;
}
.brand-logo {
display: block;
flex-shrink: 0;
}
nav {
display: flex;
gap: 1rem;
@@ -421,6 +433,17 @@ button:disabled {
display: grid;
grid-template-columns: 150px 1fr;
gap: 0.5rem 1rem;
align-items: baseline;
}
/* Render keys as understated labels so they read as keys — not just another
line of value text — in both the two-column and the stacked (narrow) layout. */
.detail-meta dt {
font-size: 0.75rem;
font-weight: 600;
text-transform: uppercase;
letter-spacing: 0.04em;
color: var(--pico-muted-color);
}
.detail-meta dd {
@@ -501,6 +524,30 @@ button:disabled {
.queue-item {
padding: 1rem;
margin-bottom: 0;
position: relative;
transition: background-color 0.15s ease-in-out, border-color 0.15s ease-in-out, box-shadow 0.15s ease-in-out;
}
/* Stretched link: the whole row navigates to the detail page, matching the
library card hover. Interactive children (Remove) sit above it via z-index. */
.queue-link {
position: absolute;
inset: 0;
z-index: 1;
}
.queue-item:hover {
background-color: var(--pico-primary-background);
border-color: var(--pico-primary-border);
box-shadow: var(--pico-card-box-shadow), 0 4px 12px rgba(0, 0, 0, 0.15);
cursor: pointer;
}
/* Keep text legible on the solid primary hover background (status badges and
the error message keep their own colors). */
.queue-item:hover .queue-url,
.queue-item:hover .queue-meta {
color: var(--pico-primary-inverse);
}
.queue-header {
@@ -515,9 +562,6 @@ button:disabled {
.queue-url {
flex: 1;
min-width: 0;
}
.queue-url a {
word-break: break-all;
}
@@ -528,6 +572,12 @@ button:disabled {
flex-wrap: wrap;
}
/* Keep the Remove action clickable above the stretched link overlay. */
.queue-action {
position: relative;
z-index: 2;
}
.status-badge {
padding: 0.2rem 0.6rem;
border-radius: 12px;
@@ -750,6 +800,17 @@ details[open] .cancel-text {
display: block;
}
/* Subscriptions: lock the cron field unless the schedule is "Custom (cron)".
Pure CSS — the :has() reacts live as the <select> changes, no JS needed. */
input[name="cron_expr"] {
opacity: 0.5;
pointer-events: none;
}
.form-row:has(select[name="schedule_kind"] option[value="cron"]:checked) input[name="cron_expr"] {
opacity: 1;
pointer-events: auto;
}
/* Format mode selector - hidden radio inputs with button-like labels */
.radio-group.format-mode {
display: flex;
@@ -785,6 +846,76 @@ details[open] .cancel-text {
border-color: var(--pico-primary);
}
/* Inline informational note paragraph */
.note {
font-size: 0.85rem;
color: var(--pico-muted-color);
}
/* Collapsible "managed yt-dlp options" notes component */
.ytdlp-notes {
margin: 0 0 1rem;
border: 1px solid var(--pico-muted-border-color);
border-radius: var(--pico-border-radius);
background: var(--pico-card-sectioning-background-color);
overflow: hidden;
}
.ytdlp-notes > summary {
cursor: pointer;
padding: 0.6rem 0.9rem;
font-size: 0.9rem;
font-weight: 600;
color: var(--pico-color);
user-select: none;
transition: background 0.15s ease-in-out;
}
.ytdlp-notes > summary:hover {
background: var(--pico-card-background-color);
}
.ytdlp-notes-icon {
color: var(--pico-primary);
font-style: normal;
font-size: 1rem;
margin-right: 0.4rem;
}
.ytdlp-notes-body {
padding: 0 0.9rem 0.6rem;
border-top: 1px solid var(--pico-muted-border-color);
}
.ytdlp-notes-body > p {
font-size: 0.85rem;
color: var(--pico-muted-color);
margin: 0.7rem 0;
}
.ytdlp-notes-table-wrap {
overflow-x: auto;
}
.ytdlp-notes-body table {
margin: 0;
font-size: 0.85rem;
}
.ytdlp-notes-body th,
.ytdlp-notes-body td {
padding: 0.4rem 0.6rem;
vertical-align: top;
}
.ytdlp-notes-body td:first-child {
white-space: nowrap;
}
.ytdlp-notes-body code {
font-size: 0.8rem;
}
/* Theme selector */
.theme-selector {
display: flex;
@@ -862,6 +993,21 @@ details[open] .cancel-text {
.detail-meta dl {
grid-template-columns: 1fr;
gap: 0;
}
/* Stacked layout: tighten key→value and space out each pair so the grouping
is obvious without the column alignment to lean on. */
.detail-meta dt {
margin-top: 0.85rem;
}
.detail-meta dt:first-child {
margin-top: 0;
}
.detail-meta dd {
margin-top: 0.15rem;
}
.queue-header {
Mweb/templates/download_form.html
@@ -5,8 +5,6 @@
<div class="download-form">
<article>
<p class="note"><strong>Note:</strong> The output template option (<code>-o</code>) is always overridden by VidArchive so that each downloaded item lives in its own self-contained directory.</p>
<form method="post" action="/download">
<label for="url">Video/Playlist URL</label>
<input type="url" id="url" name="url" placeholder="https://youtube.com/watch?v=..." required value="{{.Data.URL}}">
@@ -33,6 +31,8 @@
<textarea id="custom_flags" name="custom_flags" rows="3" placeholder="Any additional yt-dlp flags...">{{.Data.CustomFlags}}</textarea>
<small>Additional flags appended after preset flags. Example: --no-playlist --write-thumbnail</small>
{{template "ytdlp_notes" false}}
<div class="form-actions">
<button type="submit" class="primary">Start Download</button>
<button type="submit" class="secondary" formaction="/download" formmethod="get" name="list_formats" value="1">List Available Formats</button>
Mweb/templates/layout.html
@@ -5,6 +5,7 @@
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>{{.Title}} - VidArchive</title>
<link rel="icon" type="image/svg+xml" href="/static/icons/logo.svg">
<link rel="stylesheet" href="/static/pico.min.css">
<link rel="stylesheet" href="/static/style.css">
{{if .AutoRefresh}}
@@ -13,11 +14,18 @@
</head>
<body>
<header>
<h1><a href="/">VidArchive</a></h1>
<h1 class="brand">
<svg class="brand-logo" viewBox="0 0 32 32" width="30" height="30" aria-hidden="true" focusable="false">
<path fill="var(--pico-primary)" d="M6.5 6 H12 L15 9.5 H25 a3 3 0 0 1 3 3 V23.5 a3 3 0 0 1 -3 3 H7.5 a3 3 0 0 1 -3 -3 V8 a2 2 0 0 1 2 -2 Z"/>
<path fill="var(--pico-card-background-color)" d="M12.5 13.5 L21 18 L12.5 22.5 Z"/>
</svg>
<span>VidArchive</span>
</h1>
<nav>
<a href="/library" class="{{if eq .ActiveTab "library"}}active{{end}}">Library</a>
<a href="/queue" class="{{if eq .ActiveTab "queue"}}active{{end}}">Queue</a>
<a href="/download" class="{{if eq .ActiveTab "download"}}active{{end}}">Download</a>
<a href="/subscriptions" class="{{if eq .ActiveTab "subscriptions"}}active{{end}}">Subscriptions</a>
<a href="/settings" class="{{if eq .ActiveTab "settings"}}active{{end}}">Settings</a>
</nav>
</header>
Mweb/templates/library.html
@@ -25,9 +25,9 @@
{{$currentPath = printf "%s/%s" $currentPath $part}}
<span>/</span>
{{if eq $i $last}}
<a href="/library?path={{urlEncodePath $currentPath}}" aria-current="page" class="current">{{$part}}</a>
<a href="/library?path={{$currentPath}}" aria-current="page" class="current">{{$part}}</a>
{{else}}
<a href="/library?path={{urlEncodePath $currentPath}}">{{$part}}</a>
<a href="/library?path={{$currentPath}}">{{$part}}</a>
{{end}}
{{end}}
</nav>
@@ -36,7 +36,7 @@
{{$parentPath := .Data.Path}}
<div class="library-grid">
{{range .Data.Folders}}
<a href="/library?path={{urlEncodePath (printf "%s/%s" $parentPath .)}}" class="folder-item">
<a href="/library?path={{printf "%s/%s" $parentPath .}}" class="folder-item">
<div class="folder-icon">📁</div>
<div class="folder-name">{{baseName .}}</div>
</a>
Mweb/templates/library_detail.html
@@ -72,6 +72,11 @@
<dd>{{.Data.SelectedFilename}}</dd>
{{end}}
{{if .Data.Item.YtdlpFlags}}
<dt>yt-dlp flags</dt>
<dd><code>{{.Data.Item.YtdlpFlags}}</code></dd>
{{end}}
{{if .Data.Metadata}}
{{if .Data.Metadata.Container}}
<dt>Container</dt>
Mweb/templates/queue.html
@@ -25,12 +25,13 @@
<div class="queue-list">
{{range .Data.Items}}
<article class="queue-item {{.Status}}">
<a href="/queue/{{.ID}}" class="queue-link" aria-label="View download details"></a>
<div class="queue-header">
<div class="queue-url"><a href="/queue/{{.ID}}">{{.URL}}</a></div>
<div class="queue-url">{{.URL}}</div>
<div class="queue-status">
<span class="status-badge {{.Status}}">{{.Status}}</span>
</div>
<form method="post" action="/queue/{{.ID}}/delete" class="inline">
<form method="post" action="/queue/{{.ID}}/delete" class="inline queue-action">
<button type="submit" class="small danger">Remove</button>
</form>
</div>
Mweb/templates/queue_detail.html
@@ -1,7 +1,6 @@
{{define "queue_detail"}}
<div class="detail-view">
<div class="detail-header">
<a href="/queue" class="back-link">← Back to Queue</a>
<h2>Queue Details</h2>
<div class="download-status-header">
<span class="status-badge {{.Data.Status}}">{{.Data.Status}}</span>
Mweb/templates/settings.html
@@ -255,6 +255,7 @@
<div class="form-group">
<label for="custom_flags">Custom Flags</label>
<textarea name="custom_flags" rows="2" placeholder="Any additional yt-dlp flags...">{{if .}}{{.CustomFlags}}{{end}}</textarea>
{{template "ytdlp_notes" false}}
</div>
{{end}}
@@ -390,5 +391,6 @@
<div class="form-group">
<label for="custom_flags">Custom Flags</label>
<textarea name="custom_flags" rows="2" placeholder="Any additional yt-dlp flags..."></textarea>
{{template "ytdlp_notes" false}}
</div>
{{end}}
Aweb/templates/subscriptions.html
@@ -0,0 +1,208 @@
{{define "subscriptions"}}
<div class="page-header">
<h2>Subscriptions</h2>
</div>
<div class="settings-layout">
<article>
<h3>Your Subscriptions</h3>
<div class="subscriptions-list">
{{range .Data.Subscriptions}}
{{$sub := .}}
<article>
<div class="preset-header">
<h4>
{{.Name}}
{{if .Enabled}}<span class="badge">active</span>{{else}}<span class="badge">paused</span>{{end}}
</h4>
<details class="delete-overlay">
<summary role="button" class="small danger">Delete</summary>
<div class="overlay-panel">
<p>Delete subscription <strong>{{.Name}}</strong>? (Downloaded files are kept.)</p>
<form method="post" action="/subscriptions/{{.ID}}/delete" class="form-actions">
<button type="submit" class="danger">Yes, Delete</button>
</form>
</div>
</details>
</div>
<p class="preset-description">
<code>{{.URL}}</code><br>
Folder: <code>{{.OutputDir}}</code> ·
Mode: <strong>{{.RefreshMode}}</strong> ·
Schedule: <strong>{{.ScheduleKind}}</strong> (<code>{{.CronExpr}}</code>)
{{if .PruneRemoved}} · prunes removed{{end}}
</p>
<p class="preset-description">
Next run: {{if .NextRunAt.Valid}}{{.NextRunAt.Time.Format "2006-01-02 15:04"}}{{else}}—{{end}}
· Last run: {{if .LastRunAt.Valid}}{{.LastRunAt.Time.Format "2006-01-02 15:04"}}{{else}}never{{end}}
{{if .LastStatus.Valid}} ({{.LastStatus.String}}){{end}}
</p>
<div class="preset-actions">
<form method="post" action="/subscriptions/{{.ID}}/run" class="inline">
<button type="submit" class="small">Run now</button>
</form>
<form method="post" action="/subscriptions/{{.ID}}/toggle" class="inline">
<button type="submit" class="small secondary">{{if .Enabled}}Pause{{else}}Resume{{end}}</button>
</form>
<details>
<summary role="button" class="small"><span class="edit-text">Edit</span><span class="cancel-text">Cancel</span></summary>
<form method="post" action="/subscriptions/{{.ID}}" class="edit-form">
<div class="form-group">
<label>Name *</label>
<input type="text" name="name" required value="{{$sub.Name}}">
</div>
<div class="form-group">
<label>URL *</label>
<input type="text" name="url" required value="{{$sub.URL}}">
</div>
<div class="form-group">
<label>Library folder *</label>
<input type="text" name="output_dir" required value="{{$sub.OutputDir}}">
<small>This subscription owns this folder — don't download into it manually. May be nested, e.g. <code>subscriptions/news</code>.</small>
</div>
<div class="form-group">
<label>Refresh mode</label>
<div class="radio-group">
<label><input type="radio" name="refresh_mode" value="overwrite" {{if eq $sub.RefreshMode "overwrite"}}checked{{end}}> Overwrite (re-download & replace)</label>
<label><input type="radio" name="refresh_mode" value="skip" {{if eq $sub.RefreshMode "skip"}}checked{{end}}> Skip existing (add new only)</label>
<label><input type="radio" name="refresh_mode" value="metadata" {{if eq $sub.RefreshMode "metadata"}}checked{{end}}> Metadata refresh (update existing, add new)</label>
</div>
</div>
<div class="form-row">
<div class="form-group">
<label>Schedule</label>
<select name="schedule_kind">
<option value="hourly" {{if eq $sub.ScheduleKind "hourly"}}selected{{end}}>Hourly</option>
<option value="daily" {{if eq $sub.ScheduleKind "daily"}}selected{{end}}>Daily</option>
<option value="weekly" {{if eq $sub.ScheduleKind "weekly"}}selected{{end}}>Weekly</option>
<option value="monthly" {{if eq $sub.ScheduleKind "monthly"}}selected{{end}}>Monthly</option>
<option value="cron" {{if eq $sub.ScheduleKind "cron"}}selected{{end}}>Custom (cron)</option>
</select>
</div>
<div class="form-group">
<label>Cron expression</label>
<input type="text" name="cron_expr" value="{{$sub.CronExpr}}" placeholder="0 3 * * *">
<small>Used when schedule is "Custom". Standard 5-field cron.</small>
</div>
</div>
<div class="form-group">
<label>Preset</label>
<select name="preset_id">
<option value="">Default preset</option>
{{range $.Data.Presets}}
<option value="{{.ID}}" {{if and $sub.PresetID.Valid (eq $sub.PresetID.Int64 .ID)}}selected{{end}}>{{.Name}}</option>
{{end}}
</select>
</div>
<div class="form-group">
<label>Format override</label>
<input type="text" name="format_override" value="{{$sub.FormatOverride}}" placeholder="e.g., bestvideo+bestaudio/best or format ID like 22">
</div>
<div class="form-group">
<label>Custom flags</label>
<textarea name="custom_flags" rows="2">{{$sub.CustomFlags}}</textarea>
{{template "ytdlp_notes" true}}
</div>
<div class="form-group">
<label>
<input type="checkbox" name="prune_removed" value="1" {{if $sub.PruneRemoved}}checked{{end}}>
Delete items removed from the source (prune)
</label>
</div>
<div class="form-actions">
<button type="submit" class="primary">Update Subscription</button>
</div>
</form>
</details>
</div>
</article>
{{else}}
<p>No subscriptions yet. Create one below.</p>
{{end}}
</div>
</article>
<article>
<h3>Create Subscription</h3>
<form method="post" action="/subscriptions">
<div class="form-group">
<label>Name *</label>
<input type="text" name="name" required placeholder="A name for this subscription">
</div>
<div class="form-group">
<label>URL *</label>
<input type="text" name="url" required placeholder="https://youtube.com/watch?v=...">
</div>
<div class="form-group">
<label>Library folder *</label>
<input type="text" name="output_dir" required placeholder="e.g., videos/music (relative to library folder)">
<small>This subscription owns this folder — don't download into it manually. May be nested.</small>
</div>
<div class="form-group">
<label>Refresh mode</label>
<div class="radio-group">
<label><input type="radio" name="refresh_mode" value="overwrite" checked> Overwrite (re-download & replace)</label>
<label><input type="radio" name="refresh_mode" value="skip"> Skip existing (add new only)</label>
<label><input type="radio" name="refresh_mode" value="metadata"> Metadata refresh (update existing, add new)</label>
</div>
</div>
<div class="form-row">
<div class="form-group">
<label>Schedule</label>
<select name="schedule_kind">
<option value="hourly">Hourly</option>
<option value="daily" selected>Daily</option>
<option value="weekly">Weekly</option>
<option value="monthly">Monthly</option>
<option value="cron">Custom (cron)</option>
</select>
</div>
<div class="form-group">
<label>Cron expression</label>
<input type="text" name="cron_expr" placeholder="0 3 * * *">
<small>Used when schedule is "Custom". Standard 5-field cron.</small>
</div>
</div>
<div class="form-group">
<label>Preset</label>
<select name="preset_id">
<option value="">Default preset</option>
{{range .Data.Presets}}
<option value="{{.ID}}">{{.Name}}</option>
{{end}}
</select>
</div>
<div class="form-group">
<label>Format override</label>
<input type="text" name="format_override" placeholder="e.g., bestvideo+bestaudio/best or format ID like 22">
</div>
<div class="form-group">
<label>Custom flags</label>
<textarea name="custom_flags" rows="2" placeholder="Any additional yt-dlp flags..."></textarea>
{{template "ytdlp_notes" true}}
</div>
<div class="form-group">
<label>
<input type="checkbox" name="prune_removed" value="1">
Delete items removed from the source (prune)
</label>
</div>
<div class="form-actions">
<button type="submit" class="primary">Create Subscription</button>
</div>
</form>
</article>
</div>
{{end}}
Aweb/templates/ytdlp_notes.html
@@ -0,0 +1,48 @@
{{define "ytdlp_notes"}}
<details class="ytdlp-notes">
<summary>
<span class="ytdlp-notes-icon" aria-hidden="true">ⓘ</span>
yt-dlp options managed by VidArchive
</summary>
<div class="ytdlp-notes-body">
<p>VidArchive sets a few yt-dlp options itself so downloads stay self-contained
and trackable. Passing any of these — or a conflicting inverse — in
<strong>Custom Flags</strong> will make the download fail with an error.</p>
<div class="ytdlp-notes-table-wrap">
<table>
<thead>
<tr><th>Option</th><th>Why it's managed</th></tr>
</thead>
<tbody>
<tr>
<td><code>-o</code></td>
<td>Output template is always overridden so each item lands in its own self-contained directory.</td>
</tr>
<tr>
<td><code>-P</code></td>
<td>Download path is set to a per-download working directory before items are imported into the library.</td>
</tr>
<tr>
<td><code>--cookies</code></td>
<td>Injected automatically from the cookies you save in <strong>Settings</strong> (when present).</td>
</tr>
{{if .}}
<tr>
<td><code>--write-info-json</code></td>
<td>Always added for subscriptions so each item's stable identity (<code>id</code> + <code>extractor</code>) is recorded for refresh and de-duplication.</td>
</tr>
<tr>
<td><code>--download-archive</code></td>
<td>Added in <strong>Skip existing</strong> mode so entries already downloaded are skipped.</td>
</tr>
<tr>
<td><code>--skip-download</code></td>
<td>Added in <strong>Metadata only</strong> mode to refresh metadata without re-fetching media.</td>
</tr>
{{end}}
</tbody>
</table>
</div>
</div>
</details>
{{end}}