add mandatory authentication, pagination and queue actions; move to slog
Eight TODO items. New dependency: golang.org/x/crypto for bcrypt. - Auth: a login page, required. VIDARCHIVE_USERNAME and VIDARCHIVE_PASSWORD_HASH (bcrypt) must both be set or the server refuses to start, as it does for a malformed hash or a cost above 15. One RequireAuth middleware guards everything but /login, /healthz and /static, so a new route is protected by default. The session is a cookie signed with the credentials plus a secret in the settings table; sign-out rotates that secret, so a captured cookie stops working. VerifyPassword re-checks the cost, since deriving against a cost-31 hash would hold a login slot for hours, and at most two checks run at once so a flood on /login cannot pin every core. - Retry and cancel per queue row, plus Clear completed and Clear failed. Both transitions are guarded single-statement UPDATEs, and Cancel marks before cancelling the context so a racing worker skips the row instead of starting work that was just cancelled. Actions return to the queue page they came from, keeping the filter, sort and page. - Pagination at 50 items on the queue and the library. The queue reads one row past the page instead of a second COUNT. An overflowing ?page= used to slice the library out of range and 500; it is now page 1. - Logging moves to log/slog with levels, VIDARCHIVE_LOG_FORMAT=json, and per-record attributes. chi's Logger is replaced by one that emits the same facts as attributes. No request-id correlation yet. - Format listing says which playlist entry the formats belong to, rather than presenting the first entry's as the whole playlist's. - thumbFailed and scanCache are bounded by an LRU; thumbLocks is reference counted instead, because evicting a held lock would reintroduce the double-extraction race it exists to prevent. - /healthz caches the three tool versions for a minute. It is public, and it forked three processes per request.
MREADME.md
@@ -61,6 +61,31 @@ embedded in the binary.
| `VIDARCHIVE_WORKERS` | `2` | Concurrent downloads (minimum 1) |
| `VIDARCHIVE_SCHEDULER_INTERVAL` | `60` | Seconds between subscription checks |
| `VIDARCHIVE_BASE_URL` | — | External URL; an `https://` value enables HSTS |
| `VIDARCHIVE_LOG_LEVEL` | `info` | `debug`, `info`, `warn` or `error` |
| `VIDARCHIVE_LOG_FORMAT` | `text` | `text` or `json`; logs go to stderr |
| `VIDARCHIVE_USERNAME` | — | **Required.** Login user |
| `VIDARCHIVE_PASSWORD_HASH` | — | **Required.** bcrypt hash of that user's password |
## Authentication
Every route needs a login. Only `/login`, `/healthz` and the static assets are
public. The server refuses to start without `VIDARCHIVE_USERNAME` and
`VIDARCHIVE_PASSWORD_HASH`, and rejects a hash it cannot parse, so a typo fails
at startup instead of looking like a forgotten password later.
The password is stored as a bcrypt hash. Generate one with `htpasswd`, which
ships with Apache's tools (`apache2-utils` on Debian/Ubuntu, `httpd-tools` on
Fedora):
```sh
htpasswd -bnBC 12 "" 'your-password' | tr -d ':\n'
```
That prints one line, which is the value for `VIDARCHIVE_PASSWORD_HASH`:
```
$2y$12$c2FsdHNhbHRzYWx0c2FsdOhashhashhashhashhashhashhashhashhashhas
```
## Tests
Mcmd/vidarchive/main.go
@@ -3,7 +3,7 @@ package main
import (
"context"
"fmt"
"log"
"log/slog"
"os"
"os/exec"
"os/signal"
@@ -34,7 +34,9 @@ func main() {
defer stop()
cfg := config.New()
cfg.SetupLogging()
checkCredentials(cfg)
checkDependencies(cfg)
for _, dir := range []string{
@@ -44,13 +46,13 @@ func main() {
filepath.Join(cfg.DataDir, "archives"),
} {
if err := os.MkdirAll(dir, 0o755); err != nil {
log.Fatalf("Failed to create %s: %v", dir, err)
fatal("failed to create directory", "dir", dir, "err", err)
}
}
db, err := database.New(cfg)
if err != nil {
log.Fatalf("Failed to initialize database: %v", err)
fatal("failed to initialize database", "err", err)
}
presetRepo := repository.NewPresetRepository(db)
@@ -72,34 +74,56 @@ func main() {
h, err := handler.New(cfg, presetSvc, downloadSvc, librarySvc, settingsSvc, subscriptionSvc, workerPool)
if err != nil {
log.Fatalf("Failed to initialize handler: %v", err)
fatal("failed to initialize handler", "err", err)
}
srv := server.New(cfg, h)
fmt.Printf("VidArchive starting...\n")
fmt.Printf("Data directory: %s\n", cfg.DataDir)
fmt.Printf("Library directory: %s\n", cfg.LibraryDir)
fmt.Printf("Temp directory: %s\n", cfg.TempDir)
slog.Info("VidArchive starting",
"data_dir", cfg.DataDir, "library_dir", cfg.LibraryDir, "temp_dir", cfg.TempDir,
"user", cfg.Username)
srvErr := srv.Start(ctx)
// Shut down in dependency order: stop scheduling new runs, then stop the
// workers (killing any running yt-dlp), then checkpoint and close the DB.
// This runs on both a signal and a server error, so nothing is left orphaned.
log.Printf("Shutting down...")
slog.Info("shutting down")
scheduler.Stop()
workerPool.Stop()
if err := database.Checkpoint(db); err != nil {
log.Printf("Warning: %v", err)
slog.Warn("database checkpoint failed", "err", err)
}
if err := db.Close(); err != nil {
log.Printf("Warning: failed to close database: %v", err)
slog.Warn("failed to close database", "err", err)
}
log.Printf("Shutdown complete")
slog.Info("shutdown complete")
if srvErr != nil {
log.Fatalf("Server error: %v", srvErr)
fatal("server error", "err", srvErr)
}
}
// fatal logs a startup or shutdown failure and exits. slog has no Fatal, and
// log.Fatalf would report it at info level once slog owns the log package.
func fatal(msg string, args ...any) {
slog.Error(msg, args...)
os.Exit(1)
}
// hashHint shows the operator how to generate a usable hash. htpasswd ships
// with Apache's tools (apache2-utils on Debian, httpd-tools on Fedora).
const hashHint = `htpasswd -bnBC 12 "" 'your-password' | tr -d ':\n'`
// checkCredentials refuses to start without a usable login. A hash that only
// fails at the login form would look like a forgotten password rather than a
// misconfiguration.
func checkCredentials(cfg *config.Config) {
if cfg.Username == "" || cfg.PasswordHash == "" {
fatal("VIDARCHIVE_USERNAME and VIDARCHIVE_PASSWORD_HASH are required", "hint", hashHint)
}
if err := handler.ValidatePasswordHash(cfg.PasswordHash); err != nil {
fatal("VIDARCHIVE_PASSWORD_HASH is not usable", "err", err, "hint", hashHint)
}
}
@@ -114,7 +138,7 @@ func checkDependencies(cfg *config.Config) {
}
for _, dep := range deps {
if _, err := exec.LookPath(dep.path); err != nil {
log.Printf("WARNING: %s (%q) not found in PATH; related features will fail until it is installed", dep.label, dep.path)
slog.Warn("dependency not found in PATH, related features will fail until it is installed", "dependency", dep.label, "path", dep.path)
}
}
}
Mcompose.yml
@@ -14,8 +14,9 @@ services:
- VIDARCHIVE_PORT=8080
- VIDARCHIVE_DATA_DIR=/data
- VIDARCHIVE_WORKERS=2
# How often the subscription scheduler looks for due runs, in seconds. The
# page auto-refresh interval is a UI setting, not an environment variable.
- VIDARCHIVE_USERNAME=admin
- VIDARCHIVE_PASSWORD_HASH="$2y$12$..."
# How often the subscription scheduler looks for due runs, in seconds
- VIDARCHIVE_SCHEDULER_INTERVAL=60
# Uncomment to run behind HTTPS proxy:
# - VIDARCHIVE_BASE_URL=https://vidarchive.example.com
Mgo.mod
@@ -8,6 +8,7 @@ require (
github.com/go-chi/chi/v5 v5.2.5
github.com/mattn/go-shellwords v1.0.15
github.com/robfig/cron/v3 v3.0.1
golang.org/x/crypto v0.57.0
modernc.org/sqlite v1.50.0
)
@@ -17,7 +18,7 @@ require (
github.com/mattn/go-isatty v0.0.20 // indirect
github.com/ncruces/go-strftime v1.0.0 // indirect
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
golang.org/x/sys v0.42.0 // indirect
golang.org/x/sys v0.48.0 // indirect
modernc.org/libc v1.72.0 // indirect
modernc.org/mathutil v1.7.1 // indirect
modernc.org/memory v1.11.0 // indirect
Mgo.sum
@@ -22,13 +22,15 @@ github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs=
github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=
golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M=
golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA=
golang.org/x/mod v0.33.0 h1:tHFzIWbBifEmbwtGz65eaWyGiGZatSrT9prnU8DbVL8=
golang.org/x/mod v0.33.0/go.mod h1:swjeQEj+6r7fODbD2cqrnje9PnziFuw4bmLbBZFrQ5w=
golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4=
golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo=
golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
golang.org/x/tools v0.42.0 h1:uNgphsn75Tdz5Ji2q36v/nsFSfR/9BRFvqhGBaJGd5k=
golang.org/x/tools v0.42.0/go.mod h1:Ma6lCIwGZvHK6XtgbswSoWroEkhugApmsXyrUmBhfr0=
modernc.org/cc/v4 v4.27.3 h1:uNCgn37E5U09mTv1XgskEVUJ8ADKpmFMPxzGJ0TSo+U=
Minternal/config/config.go
@@ -2,6 +2,7 @@ package config
import (
"fmt"
"log/slog"
"os"
"path/filepath"
"strconv"
@@ -20,6 +21,10 @@ type Config struct {
BaseURL string
Workers int
SchedulerInterval int
LogLevel string
LogFormat string
Username string
PasswordHash string
}
func New() *Config {
@@ -37,9 +42,31 @@ func New() *Config {
BaseURL: getEnv("VIDARCHIVE_BASE_URL", ""),
Workers: getEnvInt("VIDARCHIVE_WORKERS", 2, 1),
SchedulerInterval: getEnvInt("VIDARCHIVE_SCHEDULER_INTERVAL", 60, 1),
LogLevel: getEnv("VIDARCHIVE_LOG_LEVEL", "info"),
LogFormat: getEnv("VIDARCHIVE_LOG_FORMAT", "text"),
Username: os.Getenv("VIDARCHIVE_USERNAME"),
PasswordHash: os.Getenv("VIDARCHIVE_PASSWORD_HASH"),
}
}
// SetupLogging installs the process-wide slog handler. slog.SetDefault also
// redirects the stdlib log package through it, so output from dependencies is
// formatted and levelled the same way.
func (c *Config) SetupLogging() {
var level slog.Level
if err := level.UnmarshalText([]byte(c.LogLevel)); err != nil {
fmt.Fprintf(os.Stderr, "invalid VIDARCHIVE_LOG_LEVEL %q: %v, using info\n", c.LogLevel, err)
level = slog.LevelInfo
}
opts := &slog.HandlerOptions{Level: level}
var h slog.Handler = slog.NewTextHandler(os.Stderr, opts)
if strings.EqualFold(c.LogFormat, "json") {
h = slog.NewJSONHandler(os.Stderr, opts)
}
slog.SetDefault(slog.New(h))
}
func (c *Config) IsHTTPS() bool {
return strings.HasPrefix(c.BaseURL, "https://")
}
Ainternal/handler/auth.go
@@ -0,0 +1,246 @@
package handler
import (
"crypto/hmac"
"crypto/rand"
"crypto/sha256"
"crypto/subtle"
"encoding/hex"
"fmt"
"log/slog"
"net/http"
"strconv"
"strings"
"time"
"golang.org/x/crypto/bcrypt"
"vidarchive/internal/service"
)
// Authentication is mandatory: nothing here has a disabled path, and the server
// refuses to start without credentials (see checkCredentials in main). The
// session is a signed cookie rather than server-side state; sessionKey explains
// what signs it.
const sessionCookie = "session"
const sessionTTL = 30 * 24 * time.Hour
// maxLoginCost bounds bcrypt's work factor. bcrypt itself only refuses a cost
// outside 4..31, and the top of that range takes hours per attempt. Cost 15 is
// roughly two seconds, the most a login can take and still fit in loginWait.
const maxLoginCost = 15
// maxConcurrentLogins caps how many password checks run at once. /login is
// public and bcrypt is deliberately slow, so without this an unauthenticated
// flood pins every core. loginWait bounds how long a request queues for a slot
// rather than parking a goroutine indefinitely.
const (
maxConcurrentLogins = 2
loginWait = 3 * time.Second
)
// maxLoginBody is generous for two form fields, and stops a large body from
// being read into memory before the credentials are even looked at.
const maxLoginBody = 4 << 10
// sessionKey derives the cookie signing key. The stored secret is mixed in so
// Logout can rotate it and make every cookie issued so far stop verifying.
func (h *Handler) sessionKey() []byte {
h.sessionMu.RLock()
secret := h.sessionSecret
h.sessionMu.RUnlock()
sum := sha256.Sum256([]byte(h.cfg.Username + ":" + h.cfg.PasswordHash + ":" + secret))
return sum[:]
}
// signExpiry returns the MAC binding a session to its expiry time. Signing the
// expiry is what stops a client from extending its own session.
func (h *Handler) signExpiry(exp int64) string {
mac := hmac.New(sha256.New, h.sessionKey())
fmt.Fprintf(mac, "%d", exp)
return hex.EncodeToString(mac.Sum(nil))
}
func (h *Handler) issueSession(w http.ResponseWriter) {
exp := time.Now().Add(sessionTTL).Unix()
http.SetCookie(w, &http.Cookie{
Name: sessionCookie,
Value: fmt.Sprintf("%d|%s", exp, h.signExpiry(exp)),
Path: "/",
MaxAge: int(sessionTTL.Seconds()),
HttpOnly: true,
Secure: h.cfg.IsHTTPS(),
SameSite: http.SameSiteLaxMode,
})
}
func (h *Handler) clearSession(w http.ResponseWriter) {
http.SetCookie(w, &http.Cookie{
Name: sessionCookie,
Value: "",
Path: "/",
MaxAge: -1,
HttpOnly: true,
Secure: h.cfg.IsHTTPS(),
SameSite: http.SameSiteLaxMode,
})
}
func (h *Handler) hasSession(r *http.Request) bool {
c, err := r.Cookie(sessionCookie)
if err != nil {
return false
}
rawExp, mac, ok := strings.Cut(c.Value, "|")
if !ok {
return false
}
exp, err := strconv.ParseInt(rawExp, 10, 64)
if err != nil {
return false
}
if !hmac.Equal([]byte(mac), []byte(h.signExpiry(exp))) {
return false
}
return time.Now().Unix() < exp
}
// publicPath reports the routes reachable without a session: the login form
// itself, the assets it needs to render, and the health check a monitor scrapes.
func publicPath(path string) bool {
return path == "/login" || path == "/healthz" || strings.HasPrefix(path, "/static/")
}
// RequireAuth guards every route. It is a single middleware rather than
// per-route wrapping, so a new route is protected by default. Forgetting to
// guard one is the failure mode that matters here.
func (h *Handler) RequireAuth(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if publicPath(r.URL.Path) || h.hasSession(r) {
next.ServeHTTP(w, r)
return
}
http.Redirect(w, r, "/login", http.StatusSeeOther)
})
}
func (h *Handler) LoginForm(w http.ResponseWriter, r *http.Request) {
if h.hasSession(r) {
http.Redirect(w, r, "/", http.StatusSeeOther)
return
}
h.renderWithRequest(w, r, "login", PageData{Title: "Sign in", ActiveTab: "login"})
}
func (h *Handler) Login(w http.ResponseWriter, r *http.Request) {
r.Body = http.MaxBytesReader(w, r.Body, maxLoginBody)
if err := r.ParseForm(); err != nil {
http.Error(w, "Invalid form", http.StatusBadRequest)
return
}
// Take a slot before hashing, so the work is bounded no matter how many
// requests arrive. Giving up after loginWait keeps a flood from queueing.
select {
case h.loginSem <- struct{}{}:
defer func() { <-h.loginSem }()
case <-time.After(loginWait):
flashError(w, "Too many sign-in attempts right now. Please try again.")
http.Redirect(w, r, "/login", http.StatusSeeOther)
return
}
// Verify the password even when the username is wrong, so a valid username
// can't be identified by how fast the request comes back.
userOK := subtle.ConstantTimeCompare([]byte(r.FormValue("username")), []byte(h.cfg.Username)) == 1
passOK := VerifyPassword(h.cfg.PasswordHash, r.FormValue("password"))
if !userOK || !passOK {
flashError(w, "Wrong username or password.")
http.Redirect(w, r, "/login", http.StatusSeeOther)
return
}
h.issueSession(w)
http.Redirect(w, r, "/", http.StatusSeeOther)
}
// Logout clears the browser's cookie and rotates the signing secret, so a copy
// of that cookie taken beforehand stops working too. There is one account, so
// invalidating every session is exactly the intent.
func (h *Handler) Logout(w http.ResponseWriter, r *http.Request) {
h.clearSession(w)
if err := h.rotateSessionSecret(); err != nil {
// This browser is signed out either way, since its cookie is gone. Only a
// copy taken elsewhere survives, which is worth a loud log.
slog.Error("failed to rotate the session secret; cookies issued earlier stay valid", "err", err)
}
http.Redirect(w, r, "/login", http.StatusSeeOther)
}
func (h *Handler) rotateSessionSecret() error {
secret, err := newSessionSecret()
if err != nil {
return err
}
if err := h.settingsSvc.SetSessionSecret(secret); err != nil {
return err
}
h.sessionMu.Lock()
h.sessionSecret = secret
h.sessionMu.Unlock()
return nil
}
func newSessionSecret() (string, error) {
var b [32]byte
if _, err := rand.Read(b[:]); err != nil {
return "", err
}
return hex.EncodeToString(b[:]), nil
}
// loadSessionSecret returns the stored signing secret, creating one on first
// run. Persisting it is what lets sessions survive a restart.
func loadSessionSecret(settingsSvc *service.SettingsService) (string, error) {
secret, err := settingsSvc.GetSessionSecret()
if err != nil {
return "", err
}
if secret != "" {
return secret, nil
}
if secret, err = newSessionSecret(); err != nil {
return "", err
}
return secret, settingsSvc.SetSessionSecret(secret)
}
// ValidatePasswordHash reports whether VIDARCHIVE_PASSWORD_HASH is a bcrypt
// hash this server can actually use. Startup calls it, so a malformed or
// absurdly expensive hash fails there instead of turning into a login that
// mysteriously fails or never returns.
func ValidatePasswordHash(encoded string) error {
cost, err := bcrypt.Cost([]byte(encoded))
if err != nil {
return fmt.Errorf("not a bcrypt hash: %w", err)
}
if cost > maxLoginCost {
return fmt.Errorf("bcrypt cost %d is above the usable maximum %d", cost, maxLoginCost)
}
return nil
}
// VerifyPassword reports whether password matches the stored bcrypt hash. The
// comparison is constant time.
//
// The hash is re-validated first. Startup already did that, but this check is
// what bounds the work: deriving against a cost-31 hash would hold a login slot
// for hours, and reading the cost header is free by comparison.
func VerifyPassword(encoded, password string) bool {
if ValidatePasswordHash(encoded) != nil {
return false
}
return bcrypt.CompareHashAndPassword([]byte(encoded), []byte(password)) == nil
}
Ainternal/handler/auth_test.go
@@ -0,0 +1,59 @@
package handler
import "testing"
// bcrypt hash of "test-password" at the minimum cost, so the test stays fast.
const testHash = "$2a$04$6h5pthPxQs2muU4eCZjhReZlMvwf7XP9HUKCEubLMQZsR/84fHs0i"
func TestVerifyPassword(t *testing.T) {
if !VerifyPassword(testHash, "test-password") {
t.Error("the right password did not verify")
}
for _, pw := range []string{"", "test-passwor", "test-password ", "Test-Password", testHash} {
if VerifyPassword(testHash, pw) {
t.Errorf("password %q verified against a hash of something else", pw)
}
}
}
func TestValidatePasswordHash(t *testing.T) {
if err := ValidatePasswordHash(testHash); err != nil {
t.Errorf("a valid hash was rejected: %v", err)
}
// htpasswd writes $2y$; the README tells operators to use it, so it has to work.
if err := ValidatePasswordHash("$2y$04$6h5pthPxQs2muU4eCZjhReZlMvwf7XP9HUKCEubLMQZsR/84fHs0i"); err != nil {
t.Errorf("a $2y$ hash was rejected: %v", err)
}
// Every one of these is a plausible misconfiguration that must fail at startup.
for name, encoded := range map[string]string{
"empty": "",
"plaintext": "test-password",
"argon2id": "$argon2id$v=19$m=4096,t=1,p=1$dmlkYXJjaGl2ZXRlc3QwMQ$+Y2TzOaB1vDgGkOgclcj+Q0xYIcRBZf5wh616yVaJCQ",
"truncated": "$2a$04$6h5pthPxQs2muU4eCZjhRe",
"no cost": "$2a$6h5pthPxQs2muU4eCZjhReZlMvwf7XP9HUKCEubLMQZsR/84fHs0i",
"bogus cost": "$2a$zz$6h5pthPxQs2muU4eCZjhReZlMvwf7XP9HUKCEubLMQZsR/84fHs0i",
// Accepted by bcrypt, but a login against it would take hours.
"absurd cost": "$2a$31$6h5pthPxQs2muU4eCZjhReZlMvwf7XP9HUKCEubLMQZsR/84fHs0i",
} {
if err := ValidatePasswordHash(encoded); err == nil {
t.Errorf("%s hash was accepted", name)
}
if VerifyPassword(encoded, "test-password") {
t.Errorf("%s hash verified a password", name)
}
}
}
func TestPublicPaths(t *testing.T) {
for _, path := range []string{"/login", "/healthz", "/static/style.css"} {
if !publicPath(path) {
t.Errorf("%s is guarded, so the login page cannot work", path)
}
}
for _, path := range []string{"/", "/library", "/queue", "/settings", "/media/item/x", "/logins"} {
if publicPath(path) {
t.Errorf("%s is reachable without a session", path)
}
}
}
Minternal/handler/handler.go
@@ -4,11 +4,12 @@ import (
"bytes"
"fmt"
"html/template"
"log"
"log/slog"
"net/http"
"net/url"
"path/filepath"
"strings"
"sync"
"github.com/gabriel-vasile/mimetype"
@@ -29,6 +30,14 @@ type Handler struct {
settingsSvc *service.SettingsService
subscriptionSvc *service.SubscriptionService
workerPool *worker.Pool
// loginSem caps concurrent password checks; see maxConcurrentLogins.
loginSem chan struct{}
// sessionSecret is the stored cookie signing secret, cached here because
// every guarded request reads it. Logout rotates it.
sessionMu sync.RWMutex
sessionSecret string
tools toolCache
}
func New(cfg *config.Config, presetSvc *service.PresetService, downloadSvc *service.DownloadService, librarySvc *service.LibraryService, settingsSvc *service.SettingsService, subscriptionSvc *service.SubscriptionService, workerPool *worker.Pool) (*Handler, error) {
@@ -37,6 +46,11 @@ func New(cfg *config.Config, presetSvc *service.PresetService, downloadSvc *serv
return nil, fmt.Errorf("load templates: %w", err)
}
sessionSecret, err := loadSessionSecret(settingsSvc)
if err != nil {
return nil, fmt.Errorf("load session secret: %w", err)
}
return &Handler{
cfg: cfg,
templates: tmpl,
@@ -46,6 +60,8 @@ func New(cfg *config.Config, presetSvc *service.PresetService, downloadSvc *serv
settingsSvc: settingsSvc,
subscriptionSvc: subscriptionSvc,
workerPool: workerPool,
loginSem: make(chan struct{}, maxConcurrentLogins),
sessionSecret: sessionSecret,
}, nil
}
@@ -236,7 +252,7 @@ func (h *Handler) renderWithRequest(w http.ResponseWriter, r *http.Request, cont
func (h *Handler) settingsOrDefault() *models.Settings {
settings, err := h.settingsSvc.GetAll()
if err != nil {
log.Printf("settings load failed, using defaults: %v", err)
slog.Error("settings load failed, using defaults", "err", err)
return &models.Settings{RefreshInterval: 5, AutoRefreshDownloads: true}
}
return settings
Minternal/handler/health.go
@@ -3,8 +3,11 @@ package handler
import (
"context"
"encoding/json"
"maps"
"net/http"
"slices"
"strings"
"sync"
"time"
"vidarchive/internal/util"
@@ -14,6 +17,19 @@ import (
// answers promptly, even when a tool is wedged.
const healthProbeTimeout = 3 * time.Second
// toolCacheTTL bounds how stale the reported versions may be. /healthz is
// public and each lookup forks a process, so without a cache anyone who can
// reach the port costs the host three processes per request.
const toolCacheTTL = time.Minute
// toolCache holds the last tool version probe. The mutex is held across the
// probe itself, so a burst of requests collapses into one set of forks.
type toolCache struct {
mu sync.Mutex
at time.Time
results map[string]string
}
type healthResponse struct {
Status string `json:"status"`
Dependencies map[string]string `json:"dependencies"`
@@ -44,15 +60,37 @@ func (h *Handler) Health(w http.ResponseWriter, r *http.Request) {
resp.Dependencies["database"] = "ok"
}
resp.Dependencies["yt-dlp"] = toolVersion(ctx, h.cfg.YTDLPPath, "--version")
resp.Dependencies["ffmpeg"] = toolVersion(ctx, h.cfg.FFmpegPath, "-version")
resp.Dependencies["ffprobe"] = toolVersion(ctx, h.cfg.FFprobePath, "-version")
for tool, version := range h.toolVersions(ctx) {
resp.Dependencies[tool] = version
}
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(status)
_ = json.NewEncoder(w).Encode(resp)
}
// toolVersions returns the external tools' versions, re-probing them at most
// once per toolCacheTTL.
func (h *Handler) toolVersions(ctx context.Context) map[string]string {
h.tools.mu.Lock()
defer h.tools.mu.Unlock()
if h.tools.results != nil && time.Since(h.tools.at) < toolCacheTTL {
return h.tools.results
}
results := map[string]string{
"yt-dlp": toolVersion(ctx, h.cfg.YTDLPPath, "--version"),
"ffmpeg": toolVersion(ctx, h.cfg.FFmpegPath, "-version"),
"ffprobe": toolVersion(ctx, h.cfg.FFprobePath, "-version"),
}
// A timed-out probe says nothing about the tool, so don't cache that verdict.
if !slices.Contains(slices.Collect(maps.Values(results)), "timed out") {
h.tools.results, h.tools.at = results, time.Now()
}
return results
}
// toolVersion returns the first line of the tool's version output. It reports
// "not installed" when the binary is missing and "timed out" when it doesn't
// answer within ctx.
Minternal/handler/helpers.go
@@ -1,7 +1,9 @@
package handler
import (
"log"
"html/template"
"log/slog"
"math"
"net/http"
"strconv"
@@ -31,11 +33,59 @@ func sortFromRequest(w http.ResponseWriter, r *http.Request, cookie, defaultSort
return sortBy
}
// pageSize is how many items a listing page renders. The library page fires one
// thumbnail request per media file, so an unpaginated page of a large archive is
// both a slow render and a request storm.
const pageSize = 50
// Pagination carries what a listing template needs for its prev/next links.
// Query holds the request's other parameters (filter, sort, path) with a
// trailing '&', so a link keeps them. It is template.URL because
// url.Values.Encode already percent-encoded it: as a plain string the template
// escaper would treat the whole query as one value and re-encode its '&' and
// '='.
type Pagination struct {
Page int
Prev int
Next int
HasNext bool
Query template.URL
}
// pageFromRequest reads the 1-based ?page=. Anything unparseable, below 1, or
// large enough that page*pageSize would overflow is page 1: a hand-edited URL
// must not produce a negative offset, which slices a listing out of range.
func pageFromRequest(r *http.Request) int {
p, err := strconv.Atoi(r.URL.Query().Get("page"))
if err != nil || p < 1 || p > math.MaxInt/pageSize {
return 1
}
return p
}
// newPagination describes the page links for a request. hasNext comes from the
// caller because each listing determines it differently.
func newPagination(r *http.Request, page int, hasNext bool) Pagination {
q := r.URL.Query()
q.Del("page")
query := q.Encode()
if query != "" {
query += "&"
}
return Pagination{
Page: page,
Prev: page - 1,
Next: page + 1,
HasNext: hasNext,
Query: template.URL(query),
}
}
// serverError logs the underlying failure and shows the user a generic message.
// Internal error strings can carry filesystem paths and SQL text, so they are
// kept out of the response.
func (h *Handler) serverError(w http.ResponseWriter, r *http.Request, context string, err error) {
log.Printf("%s %s: %s: %v", r.Method, r.URL.Path, context, err)
slog.Error(context, "method", r.Method, "path", r.URL.Path, "err", err)
http.Error(w, "Something went wrong. Please try again.", http.StatusInternalServerError)
}
@@ -44,7 +94,7 @@ func (h *Handler) serverError(w http.ResponseWriter, r *http.Request, context st
// logged rather than shown.
func redirectWithError(w http.ResponseWriter, r *http.Request, path, message string, err error) {
if err != nil {
log.Printf("%s %s: %s: %v", r.Method, r.URL.Path, message, err)
slog.Error(message, "method", r.Method, "path", r.URL.Path, "err", err)
}
flashError(w, message)
http.Redirect(w, r, path, http.StatusSeeOther)
Minternal/handler/library.go
@@ -1,7 +1,7 @@
package handler
import (
"log"
"log/slog"
"net/http"
"net/url"
"strings"
@@ -23,6 +23,14 @@ func (h *Handler) Library(w http.ResponseWriter, r *http.Request) {
h.serverError(w, r, "list library", err)
return
}
total := len(items)
// Only the items are paginated. Folders are few and act as navigation, so
// they stay on every page.
page := pageFromRequest(r)
start := min((page-1)*pageSize, len(items))
end := min(start+pageSize, len(items))
items = items[start:end]
settings := h.settingsOrDefault()
@@ -32,17 +40,19 @@ func (h *Handler) Library(w http.ResponseWriter, r *http.Request) {
AutoRefresh: settings.AutoRefreshLibrary,
RefreshSec: settings.RefreshInterval,
Data: struct {
Items []*models.LibraryItem
Folders []string
Path string
SortBy string
Filter string
Items []*models.LibraryItem
Folders []string
Path string
SortBy string
Filter string
Pagination Pagination
}{
Items: items,
Folders: folders,
Path: path,
SortBy: sortBy,
Filter: filter,
Items: items,
Folders: folders,
Path: path,
SortBy: sortBy,
Filter: filter,
Pagination: newPagination(r, page, end < total),
},
})
}
@@ -56,7 +66,7 @@ func normalizeRelPath(r *http.Request) string {
if decoded, err := url.PathUnescape(relPath); err == nil {
relPath = decoded
} else {
log.Printf("normalizeRelPath: undecodable path %q: %v", relPath, err)
slog.Warn("undecodable path", "path", relPath, "err", err)
}
return relPath
}
@@ -114,15 +124,15 @@ func (h *Handler) libraryDetail(relPath string, w http.ResponseWriter, r *http.R
meta, err := h.librarySvc.GetMetadata(relPath, selectedFilename)
if err != nil {
log.Printf("libraryDetail %q: metadata unavailable: %v", relPath, err)
slog.Warn("metadata unavailable", "path", relPath, "err", err)
}
subtitles, err := h.librarySvc.GetSubtitles(relPath)
if err != nil {
log.Printf("libraryDetail %q: subtitles unavailable: %v", relPath, err)
slog.Warn("subtitles unavailable", "path", relPath, "err", err)
}
comments, heatmap, err := h.librarySvc.GetEngagement(relPath)
if err != nil {
log.Printf("libraryDetail %q: engagement unavailable: %v", relPath, err)
slog.Warn("engagement unavailable", "path", relPath, "err", err)
}
previewComments := comments
if len(previewComments) > commentPreviewLimit {
Minternal/handler/queue.go
@@ -1,9 +1,10 @@
package handler
import (
"log"
"log/slog"
"net/http"
"strconv"
"strings"
"vidarchive/internal/models"
)
@@ -12,11 +13,18 @@ func (h *Handler) Downloads(w http.ResponseWriter, r *http.Request) {
status := r.URL.Query().Get("status")
sortBy := sortFromRequest(w, r, "queue_sort", "date")
downloads, err := h.downloadSvc.GetAll(status, sortBy)
// One row beyond the page tells us whether a next page exists without a
// second COUNT query.
page := pageFromRequest(r)
downloads, err := h.downloadSvc.GetAll(status, sortBy, pageSize+1, (page-1)*pageSize)
if err != nil {
h.serverError(w, r, "list downloads", err)
return
}
hasNext := len(downloads) > pageSize
if hasNext {
downloads = downloads[:pageSize]
}
settings := h.settingsOrDefault()
@@ -26,17 +34,30 @@ func (h *Handler) Downloads(w http.ResponseWriter, r *http.Request) {
AutoRefresh: settings.AutoRefreshDownloads,
RefreshSec: settings.RefreshInterval,
Data: struct {
Items []*models.Download
Status string
SortBy string
Items []*models.Download
Status string
SortBy string
Pagination Pagination
}{
Items: downloads,
Status: status,
SortBy: sortBy,
Items: downloads,
Status: status,
SortBy: sortBy,
Pagination: newPagination(r, page, hasNext),
},
})
}
// backToQueue returns the queue page the action was triggered from, so a
// redirect keeps the filter, sort and page the user was looking at instead of
// dropping them on an unfiltered page 1.
func backToQueue(r *http.Request) string {
ref := localReferer(r)
if strings.HasPrefix(ref, "/queue") {
return ref
}
return "/queue"
}
func (h *Handler) DownloadDetail(w http.ResponseWriter, r *http.Request) {
id, ok := parseID(w, r)
if !ok {
@@ -99,20 +120,72 @@ func (h *Handler) DeleteDownload(w http.ResponseWriter, r *http.Request) {
}
if err := h.downloadSvc.Delete(id); err != nil {
redirectWithError(w, r, "/queue", "Couldn't remove this download.", err)
redirectWithError(w, r, backToQueue(r), "Couldn't remove this download.", err)
return
}
redirectWithSuccess(w, r, backToQueue(r), "Download removed.")
}
func (h *Handler) RetryDownload(w http.ResponseWriter, r *http.Request) {
id, ok := parseID(w, r)
if !ok {
return
}
if err := h.downloadSvc.Retry(id); err != nil {
redirectWithError(w, r, backToQueue(r), "Couldn't retry this download.", err)
return
}
redirectWithSuccess(w, r, backToQueue(r), "Download queued again.")
}
func (h *Handler) CancelDownload(w http.ResponseWriter, r *http.Request) {
id, ok := parseID(w, r)
if !ok {
return
}
if err := h.downloadSvc.Cancel(id); err != nil {
redirectWithError(w, r, backToQueue(r), "Couldn't cancel this download.", err)
return
}
redirectWithSuccess(w, r, "/queue", "Download removed.")
redirectWithSuccess(w, r, backToQueue(r), "Download cancelled.")
}
func (h *Handler) ClearAllDownloads(w http.ResponseWriter, r *http.Request) {
if err := h.downloadSvc.DeleteAll(); err != nil {
redirectWithError(w, r, "/queue", "Couldn't clear the queue.", err)
// clearable maps the queue page's bulk-clear buttons to the status each one
// removes. Only these are accepted, so a hand-made form can't delete rows the
// page never offers to clear (in particular 'downloading').
var clearable = map[string]string{
"completed": "Completed downloads cleared.",
"error": "Failed downloads cleared.",
}
// ClearDownloads removes finished downloads: all of them, or just one status
// when the form names one.
func (h *Handler) ClearDownloads(w http.ResponseWriter, r *http.Request) {
status := r.FormValue("status")
if status == "" {
if err := h.downloadSvc.DeleteAll(); err != nil {
redirectWithError(w, r, backToQueue(r), "Couldn't clear the queue.", err)
return
}
redirectWithSuccess(w, r, backToQueue(r), "Queue cleared.")
return
}
redirectWithSuccess(w, r, "/queue", "Queue cleared.")
message, ok := clearable[status]
if !ok {
http.Error(w, "Invalid status", http.StatusBadRequest)
return
}
if err := h.downloadSvc.DeleteByStatus(status); err != nil {
redirectWithError(w, r, backToQueue(r), "Couldn't clear the queue.", err)
return
}
redirectWithSuccess(w, r, backToQueue(r), message)
}
func (h *Handler) DownloadForm(w http.ResponseWriter, r *http.Request) {
@@ -128,11 +201,12 @@ func (h *Handler) DownloadForm(w http.ResponseWriter, r *http.Request) {
url := r.URL.Query().Get("url")
var formats []*models.FormatInfo
var playlistEntry string
var flash *Flash
if r.URL.Query().Get("list_formats") == "1" && url != "" {
var err error
if formats, err = h.downloadSvc.ListFormats(r.Context(), url); err != nil {
log.Printf("DownloadForm: listing formats for %q failed: %v", url, err)
if formats, playlistEntry, err = h.downloadSvc.ListFormats(r.Context(), url); err != nil {
slog.Error("listing formats failed", "url", url, "err", err)
flash = &Flash{Kind: "error", Message: "Couldn't list formats: " + err.Error()}
}
}
@@ -150,6 +224,7 @@ func (h *Handler) DownloadForm(w http.ResponseWriter, r *http.Request) {
OutputDir string
Formats []*models.FormatInfo
ShowFormats bool
PlaylistEntry string
}{
Presets: presets,
DefaultPreset: defaultPreset,
@@ -159,6 +234,7 @@ func (h *Handler) DownloadForm(w http.ResponseWriter, r *http.Request) {
OutputDir: r.URL.Query().Get("output_dir"),
Formats: formats,
ShowFormats: r.URL.Query().Get("list_formats") == "1",
PlaylistEntry: playlistEntry,
},
})
}
Minternal/handler/settings.go
@@ -2,7 +2,7 @@ package handler
import (
"fmt"
"log"
"log/slog"
"net/http"
"net/url"
"strconv"
@@ -177,7 +177,7 @@ func (h *Handler) UpdateSettings(w http.ResponseWriter, r *http.Request) {
record(h.settingsSvc.SetCookies(r.FormValue("cookies")))
if firstErr != nil {
log.Printf("UpdateSettings: %v", firstErr)
slog.Error("failed to update settings", "err", firstErr)
flashError(w, "Some settings couldn't be saved: "+firstErr.Error())
} else {
flashSuccess(w, "Settings saved.")
Minternal/handler/subscription.go
@@ -3,7 +3,7 @@ package handler
import (
"database/sql"
"fmt"
"log"
"log/slog"
"net/http"
"strconv"
"strings"
@@ -201,7 +201,7 @@ func (h *Handler) RunSubscription(w http.ResponseWriter, r *http.Request) {
// Record the manual run so the subscriptions page shows it. The schedule's
// next run time is deliberately left alone.
if err := h.subscriptionSvc.MarkManualRun(id, time.Now(), "queued"); err != nil {
log.Printf("subscription %d: failed to record manual run: %v", id, err)
slog.Error("failed to record manual run", "subscription_id", id, "err", err)
}
redirectWithSuccess(w, r, "/queue", "Subscription run queued.")
Minternal/repository/download.go
@@ -43,7 +43,9 @@ func (r *DownloadRepository) GetByID(id int64) (*models.Download, error) {
return scanDownload(row)
}
func (r *DownloadRepository) GetAll(status, sortBy string) ([]*models.Download, error) {
// GetAll lists one page of downloads. created_at is indexed, so the ORDER BY
// doesn't sort the whole table to serve a page.
func (r *DownloadRepository) GetAll(status, sortBy string, limit, offset int) ([]*models.Download, error) {
query := `SELECT ` + downloadColumns + ` FROM downloads WHERE 1=1`
var args []interface{}
@@ -59,6 +61,9 @@ func (r *DownloadRepository) GetAll(status, sortBy string) ([]*models.Download,
query += ` ORDER BY created_at DESC`
}
query += ` LIMIT ? OFFSET ?`
args = append(args, limit, offset)
return r.queryDownloads(query, args...)
}
@@ -123,10 +128,39 @@ func (r *DownloadRepository) AppendLogs(id int64, logs string) error {
// It reports whether this call actually claimed it: false means another worker
// already started it, so the caller must not process it again.
func (r *DownloadRepository) MarkStarted(id int64) (bool, error) {
res, err := r.db.Exec(
return r.affected(
`UPDATE downloads SET status = 'downloading', started_at = CURRENT_TIMESTAMP WHERE id = ? AND status = 'queued'`,
id,
)
}
// Requeue moves a finished-but-unsuccessful download back to 'queued' so the
// queue checker picks it up again. Only 'error' and 'cancelled' rows qualify:
// re-queuing a running or completed one would duplicate work. The bool reports
// whether a row actually changed.
func (r *DownloadRepository) Requeue(id int64) (bool, error) {
return r.affected(
`UPDATE downloads SET status = 'queued', error_message = NULL, logs = NULL,
started_at = NULL, completed_at = NULL
WHERE id = ? AND status IN ('error', 'cancelled')`,
id,
)
}
// CancelQueued marks a not-yet-started download as cancelled. A running one is
// stopped by cancelling its context instead (see DownloadService.Cancel), which
// is why the status guard matters: it must not overwrite a row a worker owns.
func (r *DownloadRepository) CancelQueued(id int64) (bool, error) {
return r.affected(
`UPDATE downloads SET status = 'cancelled', completed_at = CURRENT_TIMESTAMP
WHERE id = ? AND status = 'queued'`,
id,
)
}
// affected runs an UPDATE and reports whether it matched a row.
func (r *DownloadRepository) affected(query string, args ...interface{}) (bool, error) {
res, err := r.db.Exec(query, args...)
if err != nil {
return false, err
}
@@ -178,6 +212,13 @@ func (r *DownloadRepository) DeleteAll() error {
return err
}
// DeleteByStatus removes every download in one status, backing the queue page's
// "Clear completed" / "Clear failed" actions.
func (r *DownloadRepository) DeleteByStatus(status string) error {
_, err := r.db.Exec(`DELETE FROM downloads WHERE status = ?`, status)
return err
}
func (r *DownloadRepository) UpdateStatusWhere(oldStatus, newStatus string) error {
_, err := r.db.Exec(
`UPDATE downloads SET status = ? WHERE status = ?`,
Minternal/repository/download_queue_test.go
@@ -64,7 +64,7 @@ func TestGetAllStatusFilter(t *testing.T) {
{"error", 1},
{"cancelled", 0},
} {
got, err := repo.GetAll(tc.status, "date")
got, err := repo.GetAll(tc.status, "date", 100, 0)
if err != nil {
t.Fatalf("get all %q: %v", tc.status, err)
}
@@ -74,7 +74,7 @@ func TestGetAllStatusFilter(t *testing.T) {
}
// Sorting by status must not drop rows.
got, err := repo.GetAll("", "status")
got, err := repo.GetAll("", "status", 100, 0)
if err != nil {
t.Fatalf("get all sorted by status: %v", err)
}
@@ -227,7 +227,7 @@ func TestDeleteAllClearsQueue(t *testing.T) {
if err := repo.DeleteAll(); err != nil {
t.Fatalf("delete all: %v", err)
}
got, err := repo.GetAll("", "date")
got, err := repo.GetAll("", "date", 100, 0)
if err != nil {
t.Fatal(err)
}
Ainternal/server/auth_test.go
@@ -0,0 +1,150 @@
package server
import (
"net/http"
"net/http/httptest"
"net/url"
"strings"
"testing"
)
// postWith is postForm carrying cookies, which /logout needs: it is a guarded
// route, so without a session it never reaches the handler.
func postWith(router http.Handler, path string, cookies []*http.Cookie) *httptest.ResponseRecorder {
req := httptest.NewRequest("POST", path, nil)
for _, c := range cookies {
req.AddCookie(c)
}
w := httptest.NewRecorder()
router.ServeHTTP(w, req)
return w
}
// sessionCookie returns the session cookie set on a response. A cleared cookie
// carries an empty value and does not count as one.
func sessionCookie(cookies []*http.Cookie) *http.Cookie {
for _, c := range cookies {
if c.Name == "session" && c.Value != "" {
return c
}
}
return nil
}
func TestAuthGuardsRoutes(t *testing.T) {
srv, _, cleanup := setupTestServer(t)
defer cleanup()
// Past the test wrapper that signs every request in: these requests must
// arrive without a session.
router := srv.Server.Router()
for _, path := range []string{"/", "/library", "/queue", "/settings", "/subscriptions", "/download"} {
got := getWith(router, path, nil)
if got.Code != http.StatusSeeOther || got.Header().Get("Location") != "/login" {
t.Errorf("%s without a session = %d %q, want 303 /login", path, got.Code, got.Header().Get("Location"))
}
}
// Mutating routes must be guarded too, not just the pages that link to them.
if got := postForm(router, "/queue/clear", nil); got.Header().Get("Location") != "/login" {
t.Errorf("clearing the queue without a session redirected to %q, want /login", got.Header().Get("Location"))
}
login := getWith(router, "/login", nil)
if login.Code != http.StatusOK {
t.Errorf("login page = %d, want 200", login.Code)
}
// Nothing the navigation points at is reachable without a session, so the
// login page shows none of it.
for _, link := range []string{`href="/library"`, `href="/queue"`, `href="/settings"`, "/logout"} {
if strings.Contains(login.Body.String(), link) {
t.Errorf("login page shows %s", link)
}
}
if got := getWith(router, "/healthz", nil); got.Code == http.StatusSeeOther {
t.Error("health check redirected to login")
}
}
func TestLoginRejectsWrongCredentials(t *testing.T) {
srv, _, cleanup := setupTestServer(t)
defer cleanup()
router := srv.Server.Router()
for _, tc := range []struct {
name, user, pass string
}{
{"wrong password", testUsername, "not-the-password"},
{"wrong user", "intruder", testPassword},
{"empty password", testUsername, ""},
// The stored value is a hash, so posting it must not be accepted as if it
// were the password itself.
{"the hash itself", testUsername, testPasswordHash},
} {
w := postForm(router, "/login", url.Values{"username": {tc.user}, "password": {tc.pass}})
if c := sessionCookie(w.Result().Cookies()); c != nil {
t.Errorf("%s issued a session", tc.name)
}
if k, _, ok := flash(w); !ok || k != "error" {
t.Errorf("%s did not flash an error", tc.name)
}
}
}
func TestLoginGrantsAccess(t *testing.T) {
srv, _, cleanup := setupTestServer(t)
defer cleanup()
router := srv.Server.Router()
w := postForm(router, "/login", url.Values{
"username": {testUsername},
"password": {testPassword},
})
session := sessionCookie(w.Result().Cookies())
if session == nil {
t.Fatal("correct credentials issued no session")
}
if !session.HttpOnly {
t.Error("session cookie is not HttpOnly")
}
if got := getWith(router, "/library", []*http.Cookie{session}); got.Code != http.StatusOK {
t.Errorf("library with a session = %d, want 200", got.Code)
}
if got := getWith(router, "/login", []*http.Cookie{session}); got.Code != http.StatusSeeOther {
t.Errorf("login page with a session = %d, want 303", got.Code)
}
// The cookie is signed, so an edited expiry must not extend the session.
forged := &http.Cookie{Name: "session", Value: strings.Replace(session.Value, "|", "9|", 1)}
if got := getWith(router, "/library", []*http.Cookie{forged}); got.Code != http.StatusSeeOther {
t.Errorf("library with a forged session = %d, want 303", got.Code)
}
out := postWith(router, "/logout", []*http.Cookie{session})
if sessionCookie(out.Result().Cookies()) != nil {
t.Error("logout left a session cookie behind")
}
}
func TestLogoutRevokesTheCookie(t *testing.T) {
srv, _, cleanup := setupTestServer(t)
defer cleanup()
router := srv.Server.Router()
in := postForm(router, "/login", url.Values{
"username": {testUsername},
"password": {testPassword},
})
session := sessionCookie(in.Result().Cookies())
if session == nil {
t.Fatal("login issued no session")
}
postWith(router, "/logout", []*http.Cookie{session})
// Clearing the browser's cookie is not enough: a copy taken before the
// sign-out must stop working too.
if got := getWith(router, "/library", []*http.Cookie{session}); got.Code != http.StatusSeeOther {
t.Errorf("a captured cookie still works after logout: %d, want 303", got.Code)
}
}
Ainternal/server/queue_test.go
@@ -0,0 +1,250 @@
package server
import (
"database/sql"
"net/http"
"net/http/httptest"
"net/url"
"strconv"
"strings"
"testing"
)
// insertDownload writes a queue row in a chosen status, which is the only way to
// reach the finished states an HTTP test can't produce on its own.
func insertDownload(t *testing.T, db *sql.DB, status, errMsg string) int64 {
t.Helper()
res, err := db.Exec(
`INSERT INTO downloads (url, status, error_message, format_override, custom_flags)
VALUES (?, ?, ?, '', '')`,
"https://example.com/"+status, status, sql.NullString{String: errMsg, Valid: errMsg != ""},
)
if err != nil {
t.Fatalf("insert %s download: %v", status, err)
}
id, err := res.LastInsertId()
if err != nil {
t.Fatalf("insert id: %v", err)
}
return id
}
func downloadStatus(t *testing.T, db *sql.DB, id int64) (string, sql.NullString) {
t.Helper()
var status string
var errMsg sql.NullString
err := db.QueryRow(`SELECT status, error_message FROM downloads WHERE id = ?`, id).Scan(&status, &errMsg)
if err != nil {
t.Fatalf("read download %d: %v", id, err)
}
return status, errMsg
}
func TestRetryRequeuesFailedDownload(t *testing.T) {
srv, _, db, cleanup := setupTestServerDB(t)
defer cleanup()
router := srv.Router()
failed := insertDownload(t, db, "error", "boom")
w := postForm(router, "/queue/"+strconv.FormatInt(failed, 10)+"/retry", nil)
if w.Code != http.StatusSeeOther {
t.Fatalf("retry = %d, want 303", w.Code)
}
if k, _, ok := flash(w); !ok || k != "success" {
t.Error("retry did not flash success")
}
// A worker may claim the re-queued row on its next tick, so accept either
// state; what matters is that it left the terminal error state.
status, errMsg := downloadStatus(t, db, failed)
if status != "queued" && status != "downloading" {
t.Errorf("status after retry = %q, want queued", status)
}
if errMsg.Valid {
t.Errorf("error message %q survived the retry", errMsg.String)
}
// A completed download is not retryable: re-running it would duplicate work.
done := insertDownload(t, db, "completed", "")
w = postForm(router, "/queue/"+strconv.FormatInt(done, 10)+"/retry", nil)
if k, _, ok := flash(w); !ok || k != "error" {
t.Error("retrying a completed download did not flash an error")
}
if status, _ := downloadStatus(t, db, done); status != "completed" {
t.Errorf("completed download moved to %q", status)
}
}
func TestCancelKeepsTheRow(t *testing.T) {
srv, _, db, cleanup := setupTestServerDB(t)
defer cleanup()
id := insertDownload(t, db, "queued", "")
path := "/queue/" + strconv.FormatInt(id, 10)
detail := getWith(srv.Router(), path, nil).Body.String()
if !strings.Contains(detail, path+"/cancel") {
t.Error("detail page of a queued download offers no cancel button")
}
w := postForm(srv.Router(), path+"/cancel", nil)
if w.Code != http.StatusSeeOther {
t.Fatalf("cancel = %d, want 303", w.Code)
}
if status, _ := downloadStatus(t, db, id); status != "cancelled" {
t.Errorf("status after cancel = %q, want cancelled", status)
}
// The row survives the cancel, and now offers a retry instead.
detail = getWith(srv.Router(), path, nil).Body.String()
if !strings.Contains(detail, path+"/retry") {
t.Error("cancelled download offers no retry button")
}
}
func TestClearByStatus(t *testing.T) {
srv, _, db, cleanup := setupTestServerDB(t)
defer cleanup()
router := srv.Router()
done := insertDownload(t, db, "completed", "")
failed := insertDownload(t, db, "error", "boom")
postForm(router, "/queue/clear", url.Values{"status": {"completed"}})
if err := db.QueryRow(`SELECT status FROM downloads WHERE id = ?`, done).Scan(new(string)); err != sql.ErrNoRows {
t.Errorf("completed download survived the clear: %v", err)
}
if status, _ := downloadStatus(t, db, failed); status != "error" {
t.Errorf("failed download was cleared too, status = %q", status)
}
postForm(router, "/queue/clear", url.Values{"status": {"error"}})
if err := db.QueryRow(`SELECT status FROM downloads WHERE id = ?`, failed).Scan(new(string)); err != sql.ErrNoRows {
t.Errorf("failed download survived the clear: %v", err)
}
// Running downloads are not clearable, so a hand-made form can't kill them.
running := insertDownload(t, db, "downloading", "")
w := postForm(router, "/queue/clear", url.Values{"status": {"downloading"}})
if w.Code != http.StatusBadRequest {
t.Errorf("clearing downloading = %d, want 400", w.Code)
}
if status, _ := downloadStatus(t, db, running); status != "downloading" {
t.Errorf("running download was cleared, status = %q", status)
}
}
func TestQueuePagination(t *testing.T) {
srv, _, db, cleanup := setupTestServerDB(t)
defer cleanup()
router := srv.Router()
// Completed rows, so no worker picks them up mid-test.
for i := 0; i < 55; i++ {
insertDownload(t, db, "completed", "")
}
first := getWith(router, "/queue?status=completed", nil).Body.String()
if got := strings.Count(first, `class="queue-item`); got != 50 {
t.Errorf("page 1 rendered %d items, want 50", got)
}
if !strings.Contains(first, "page=2") {
t.Error("page 1 has no next link")
}
second := getWith(router, "/queue?status=completed&page=2", nil).Body.String()
if got := strings.Count(second, `class="queue-item`); got != 5 {
t.Errorf("page 2 rendered %d items, want 5", got)
}
if !strings.Contains(second, "page=1") {
t.Error("page 2 has no previous link")
}
// The link has to carry the filter, otherwise paging resets it.
if !strings.Contains(second, "status=completed&page=1") {
t.Error("previous link dropped the status filter")
}
if strings.Contains(second, "page=3") {
t.Error("page 2 offers a next page that does not exist")
}
}
func TestLibraryPagination(t *testing.T) {
srv, cfg, cleanup := setupTestServer(t)
defer cleanup()
for i := 0; i < 52; i++ {
name := "item-" + strconv.Itoa(i)
createItem(t, cfg.LibraryDir, name, name, map[string]string{"video.mp4": "dummy"})
}
first := getWith(srv.Router(), "/library", nil).Body.String()
if got := strings.Count(first, `class="media-item`); got != 50 {
t.Errorf("page 1 rendered %d items, want 50", got)
}
second := getWith(srv.Router(), "/library?page=2", nil).Body.String()
if got := strings.Count(second, `class="media-item`); got != 2 {
t.Errorf("page 2 rendered %d items, want 2", got)
}
// A page past the end is empty, not a crash or a wrapped-around first page.
last := getWith(srv.Router(), "/library?page=9", nil)
if last.Code != http.StatusOK {
t.Fatalf("page 9 = %d, want 200", last.Code)
}
if got := strings.Count(last.Body.String(), `class="media-item`); got != 0 {
t.Errorf("page 9 rendered %d items, want 0", got)
}
}
// postFrom is postForm with a Referer, which is how the queue actions know
// which page to send the user back to.
func postFrom(router http.Handler, path, referer string, form url.Values) *httptest.ResponseRecorder {
req := httptest.NewRequest("POST", path, strings.NewReader(form.Encode()))
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
req.Header.Set("Referer", "http://example.test"+referer)
w := httptest.NewRecorder()
router.ServeHTTP(w, req)
return w
}
func TestQueueActionsKeepFilterAndPage(t *testing.T) {
srv, _, db, cleanup := setupTestServerDB(t)
defer cleanup()
router := srv.Router()
from := "/queue?status=error&page=2"
id := insertDownload(t, db, "error", "boom")
w := postFrom(router, "/queue/"+strconv.FormatInt(id, 10)+"/retry", from, nil)
if got := w.Header().Get("Location"); got != from {
t.Errorf("retry redirected to %q, wanted back to %q", got, from)
}
// An off-site Referer must not be honoured as a redirect target.
req := httptest.NewRequest("POST", "/queue/clear", nil)
req.Header.Set("Referer", "https://evil.example/queue?x=1")
rec := httptest.NewRecorder()
router.ServeHTTP(rec, req)
if got := rec.Header().Get("Location"); got != "/queue?x=1" {
t.Errorf("clear redirected to %q, want the local path only", got)
}
}
func TestPaginationRejectsOverflowingPage(t *testing.T) {
srv, cfg, db, cleanup := setupTestServerDB(t)
defer cleanup()
router := srv.Router()
createItem(t, cfg.LibraryDir, "item", "Item", map[string]string{"video.mp4": "dummy"})
insertDownload(t, db, "completed", "")
// A page number large enough to overflow page*pageSize used to slice the
// library listing with a negative index, which panicked into a 500.
for _, page := range []string{"9223372036854775807", "184467440737095518", "99999999999999999999", "-1", "abc"} {
for _, path := range []string{"/library?page=", "/queue?page="} {
if got := getWith(router, path+page, nil); got.Code != http.StatusOK {
t.Errorf("GET %s%s = %d, want 200", path, page, got.Code)
}
}
}
}
Minternal/server/server.go
@@ -5,6 +5,7 @@ import (
"errors"
"fmt"
"io/fs"
"log/slog"
"net/http"
"time"
@@ -37,9 +38,10 @@ func New(cfg *config.Config, h *handler.Handler) *Server {
}
func (s *Server) setupRoutes() {
s.router.Use(middleware.Logger)
s.router.Use(s.requestLogger)
s.router.Use(middleware.Recoverer)
s.router.Use(s.securityHeaders)
s.router.Use(s.handler.RequireAuth)
// Serve embedded static assets (fs.Sub strips the web/static prefix). The
// error is only possible for an invalid constant path, so it can't occur here.
@@ -48,6 +50,10 @@ func (s *Server) setupRoutes() {
s.router.Get("/healthz", s.handler.Health)
s.router.Get("/login", s.handler.LoginForm)
s.router.Post("/login", s.handler.Login)
s.router.Post("/logout", s.handler.Logout)
s.router.Get("/media/item/*", s.handler.ServeMediaItem)
s.router.Get("/", func(w http.ResponseWriter, r *http.Request) {
@@ -61,7 +67,9 @@ func (s *Server) setupRoutes() {
s.router.Get("/queue", s.handler.Downloads)
s.router.Get("/queue/{id}", s.handler.DownloadDetail)
s.router.Post("/queue/{id}/delete", s.handler.DeleteDownload)
s.router.Post("/queue/clear", s.handler.ClearAllDownloads)
s.router.Post("/queue/{id}/retry", s.handler.RetryDownload)
s.router.Post("/queue/{id}/cancel", s.handler.CancelDownload)
s.router.Post("/queue/clear", s.handler.ClearDownloads)
s.router.Get("/download", s.handler.DownloadForm)
s.router.Post("/download", s.handler.CreateDownload)
@@ -82,6 +90,20 @@ func (s *Server) setupRoutes() {
s.router.Post("/theme", s.handler.Theme)
}
// requestLogger replaces chi's middleware.Logger, which writes its own
// preformatted (and ANSI-coloured) line. This one emits the same facts as slog
// attributes, so a request can be filtered and correlated like anything else.
func (s *Server) requestLogger(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
ww := middleware.NewWrapResponseWriter(w, r.ProtoMajor)
start := time.Now()
next.ServeHTTP(ww, r)
slog.Info("request",
"method", r.Method, "path", r.URL.Path, "status", ww.Status(),
"bytes", ww.BytesWritten(), "duration_ms", time.Since(start).Milliseconds())
})
}
func (s *Server) securityHeaders(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("X-Content-Type-Options", "nosniff")
@@ -117,11 +139,7 @@ func (s *Server) Start(ctx context.Context) error {
IdleTimeout: 120 * time.Second,
}
fmt.Printf("Starting server on %s\n", srv.Addr)
if s.cfg.BaseURL != "" {
fmt.Printf("Base URL: %s\n", s.cfg.BaseURL)
fmt.Printf("HTTPS mode: %v\n", s.cfg.IsHTTPS())
}
slog.Info("starting server", "addr", srv.Addr, "base_url", s.cfg.BaseURL, "https", s.cfg.IsHTTPS())
errCh := make(chan error, 1)
go func() {
Minternal/server/server_test.go
@@ -22,17 +22,45 @@ import (
"vidarchive/internal/worker"
)
func setupTestServer(t *testing.T) (*Server, *config.Config, func()) {
// Authentication is mandatory, so the test server is configured with real
// credentials. The hash is bcrypt over testPassword at the minimum cost,
// because every test server logs in once.
const (
testUsername = "tester"
testPassword = "test-password"
testPasswordHash = "$2a$04$6h5pthPxQs2muU4eCZjhReZlMvwf7XP9HUKCEubLMQZsR/84fHs0i"
)
// testServer is a Server whose Router() signs every request in, so the tests
// that are not about authentication don't each have to log in. Tests that do
// exercise the login flow reach past it with srv.Server.Router().
type testServer struct {
*Server
session *http.Cookie
}
func (s testServer) Router() http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if _, err := r.Cookie("session"); err != nil {
r.AddCookie(s.session)
}
s.Server.Router().ServeHTTP(w, r)
})
}
func setupTestServer(t *testing.T) (testServer, *config.Config, func()) {
srv, cfg, _, cleanup := setupTestServerDB(t)
return srv, cfg, cleanup
}
// setupTestServerDB is setupTestServer with the database handle exposed, for
// tests that need to break the database on purpose.
func setupTestServerDB(t *testing.T) (*Server, *config.Config, *sql.DB, func()) {
func setupTestServerDB(t *testing.T) (testServer, *config.Config, *sql.DB, func()) {
t.Helper()
dataDir := t.TempDir()
t.Setenv("VIDARCHIVE_DATA_DIR", dataDir)
t.Setenv("VIDARCHIVE_USERNAME", testUsername)
t.Setenv("VIDARCHIVE_PASSWORD_HASH", testPasswordHash)
cfg := config.New()
if err := os.MkdirAll(cfg.LibraryDir, 0o755); err != nil {
@@ -65,11 +93,21 @@ func setupTestServerDB(t *testing.T) (*Server, *config.Config, *sql.DB, func())
}
srv := New(cfg, h)
login := postForm(srv.Router(), "/login", url.Values{
"username": {testUsername},
"password": {testPassword},
})
session := sessionCookie(login.Result().Cookies())
if session == nil {
t.Fatal("test login issued no session; check testPasswordHash")
}
cleanup := func() {
workerPool.Stop()
db.Close()
}
return srv, cfg, db, cleanup
return testServer{Server: srv, session: session}, cfg, db, cleanup
}
func createItem(t *testing.T, libraryDir, relPath, name string, files map[string]string) {
Minternal/service/download.go
@@ -6,7 +6,7 @@ import (
"database/sql"
"errors"
"fmt"
"log"
"log/slog"
"os"
"path/filepath"
"slices"
@@ -95,8 +95,8 @@ func (s *DownloadService) GetByID(id int64) (*models.Download, error) {
return d, nil
}
func (s *DownloadService) GetAll(status, sortBy string) ([]*models.Download, error) {
downloads, err := s.repo.GetAll(status, sortBy)
func (s *DownloadService) GetAll(status, sortBy string, limit, offset int) ([]*models.Download, error) {
downloads, err := s.repo.GetAll(status, sortBy, limit, offset)
if err != nil {
return nil, err
}
@@ -142,7 +142,9 @@ func (s *DownloadService) registerActive(id int64, cancel context.CancelFunc) fu
}
}
func (s *DownloadService) cancelDownload(id int64) {
// cancelDownload stops the download if it is in flight, reporting whether there
// was one to stop.
func (s *DownloadService) cancelDownload(id int64) bool {
s.activeMu.Lock()
cancel, ok := s.active[id]
delete(s.active, id)
@@ -151,6 +153,7 @@ func (s *DownloadService) cancelDownload(id int64) {
if ok {
cancel()
}
return ok
}
// CancelAll stops every download currently in flight. Used on shutdown and when
@@ -169,6 +172,45 @@ func (s *DownloadService) CancelAll() {
}
}
// Retry puts a failed or cancelled download back in the queue. The worker pool's
// queue checker picks it up on its next tick, so nothing is submitted here.
func (s *DownloadService) Retry(id int64) error {
ok, err := s.repo.Requeue(id)
if err != nil {
return err
}
if !ok {
return fmt.Errorf("download %d is not in a retryable state", id)
}
s.cache.Delete(id)
return nil
}
// Cancel stops a download without deleting its row, so it stays visible and
// retryable. The queued case is marked here; a running one is stopped by
// cancelling its context, and ExecuteDownload records the 'cancelled' status.
//
// The order matters: marking first means a worker racing to claim the row finds
// it no longer 'queued' and skips it, instead of starting work we just cancelled.
func (s *DownloadService) Cancel(id int64) error {
wasQueued, err := s.repo.CancelQueued(id)
if err != nil {
return err
}
// Neither queued nor running: the row is already finished, or it is a stale
// 'downloading' row no worker owns. Say so instead of reporting success.
if !s.cancelDownload(id) && !wasQueued {
return fmt.Errorf("download %d is not running", id)
}
return nil
}
// DeleteByStatus clears one status' worth of rows. Only finished statuses are
// offered in the UI, so nothing it removes needs cancelling first.
func (s *DownloadService) DeleteByStatus(status string) error {
return s.repo.DeleteByStatus(status)
}
func (s *DownloadService) DeleteAll() error {
// Clearing the queue must also stop what is running; otherwise yt-dlp keeps
// going and imports into the library after its row is gone.
@@ -192,7 +234,7 @@ func (s *DownloadService) ResetStalledDownloads() error {
for _, id := range ids {
for _, dir := range s.tempDirsFor(id) {
if err := os.RemoveAll(dir); err != nil {
log.Printf("warning: failed to remove stale temp dir %s: %v", dir, err)
slog.Warn("failed to remove stale temp dir", "dir", dir, "err", err)
}
}
}
@@ -246,14 +288,14 @@ func (s *DownloadService) ExecuteDownload(parent context.Context, d *models.Down
if d.PresetID.Valid {
preset, err = s.presetSvc.GetByID(d.PresetID.Int64)
if err != nil {
log.Printf("download %d: preset %d lookup failed (%v); falling back to default", d.ID, d.PresetID.Int64, err)
slog.Warn("preset lookup failed, falling back to default", "download_id", d.ID, "preset_id", d.PresetID.Int64, "err", err)
preset = nil
}
}
if preset == nil {
var derr error
if preset, derr = s.presetSvc.GetDefault(); derr != nil {
log.Printf("download %d: no default preset available (%v); using built-in defaults", d.ID, derr)
slog.Warn("no default preset available, using built-in defaults", "download_id", d.ID, "err", derr)
preset = &models.Preset{}
}
}
@@ -262,7 +304,7 @@ func (s *DownloadService) ExecuteDownload(parent context.Context, d *models.Down
if d.SubscriptionID.Valid && s.subscriptionSvc != nil {
var serr error
if sub, serr = s.subscriptionSvc.GetByID(d.SubscriptionID.Int64); serr != nil {
log.Printf("download %d: subscription %d lookup failed: %v", d.ID, d.SubscriptionID.Int64, serr)
slog.Error("subscription lookup failed", "download_id", d.ID, "subscription_id", d.SubscriptionID.Int64, "err", serr)
}
}
@@ -455,7 +497,7 @@ func (s *DownloadService) runYTDLP(ctx context.Context, d *models.Download, args
s.cache.AppendLog(d.ID, scanner.Text())
}
if err := scanner.Err(); err != nil {
log.Printf("download %d: error reading yt-dlp output: %v", d.ID, err)
slog.Error("error reading yt-dlp output", "download_id", d.ID, "err", err)
}
close(done)
@@ -497,7 +539,7 @@ func (s *DownloadService) saveRefreshedCookies(path, sent string) {
return
}
if err := s.settingsSvc.SetCookies(string(data)); err != nil {
log.Printf("failed to save refreshed cookies: %v", err)
slog.Error("failed to save refreshed cookies", "err", err)
}
}
@@ -527,7 +569,7 @@ func (s *DownloadService) writeCookiesFile(cookies string) (string, error) {
func (s *DownloadService) finalizeError(d *models.Download, err error) {
s.flushLogs(d.ID)
if markErr := s.repo.MarkError(d.ID, err.Error()); markErr != nil {
log.Printf("download %d: failed to record error: %v", d.ID, markErr)
slog.Error("failed to record download error", "download_id", d.ID, "err", markErr)
}
s.recordSubscriptionStatus(d, "error")
}
@@ -541,8 +583,8 @@ func (s *DownloadService) recordSubscriptionStatus(d *models.Download, status st
return
}
if err := s.subscriptionSvc.SetLastStatus(d.SubscriptionID.Int64, status); err != nil {
log.Printf("download %d: failed to record subscription %d status %q: %v",
d.ID, d.SubscriptionID.Int64, status, err)
slog.Error("failed to record subscription status", "download_id", d.ID,
"subscription_id", d.SubscriptionID.Int64, "status", status, "err", err)
}
}
@@ -568,7 +610,7 @@ func (s *DownloadService) finalizeCancelled(parent context.Context, d *models.Do
}
if err := s.repo.MarkCompleted(d.ID, "cancelled"); err != nil {
log.Printf("download %d: failed to record cancellation: %v", d.ID, err)
slog.Error("failed to record cancellation", "download_id", d.ID, "err", err)
}
s.recordSubscriptionStatus(d, "cancelled")
return ErrCancelled
@@ -581,7 +623,7 @@ func (s *DownloadService) flushLogs(id int64) {
return
}
if err := s.repo.AppendLogs(id, logs); err != nil {
log.Printf("download %d: failed to persist logs: %v", id, err)
slog.Error("failed to persist logs", "download_id", id, "err", err)
}
}
Minternal/service/formats.go
@@ -11,7 +11,10 @@ import (
"vidarchive/internal/util"
)
func (s *DownloadService) ListFormats(ctx context.Context, url string) ([]*models.FormatInfo, error) {
// ListFormats returns the formats offered for url. The second result names the
// playlist entry the formats came from, and is empty for a plain video URL: a
// playlist's entries can differ, so the caller must say which one this is.
func (s *DownloadService) ListFormats(ctx context.Context, url string) ([]*models.FormatInfo, string, error) {
// Use machine-readable JSON (-J) rather than scraping the human "-F" table,
// whose columns/separators shift between yt-dlp versions. stderr is captured
// separately so warnings can't corrupt the JSON on stdout.
@@ -27,7 +30,7 @@ func (s *DownloadService) ListFormats(ctx context.Context, url string) ([]*model
cmd.Stderr = &stderr
output, err := cmd.Output()
if err != nil {
return nil, fmt.Errorf("yt-dlp -J failed: %w\n%s", err, stderr.String())
return nil, "", fmt.Errorf("yt-dlp -J failed: %w\n%s", err, stderr.String())
}
return parseFormatJSON(output)
@@ -53,29 +56,35 @@ type ytFormat struct {
// parseFormatJSON reads yt-dlp's single-JSON dump (-J) and returns the available
// formats. For a single video the formats live at the top level; for a playlist
// URL the dump is restricted to one entry (-I 1), so we fall back to that
// entry's formats and the picker still shows something useful.
func parseFormatJSON(data []byte) ([]*models.FormatInfo, error) {
// URL the dump is restricted to one entry (-I 1), so the formats come from that
// entry and the second result names it, so the UI can say the list describes
// the first item rather than the whole playlist.
func parseFormatJSON(data []byte) ([]*models.FormatInfo, string, error) {
var top struct {
Formats []ytFormat `json:"formats"`
Entries []struct {
Title string `json:"title"`
Formats []ytFormat `json:"formats"`
} `json:"entries"`
}
if err := json.Unmarshal(data, &top); err != nil {
return nil, fmt.Errorf("parse yt-dlp JSON: %w", err)
return nil, "", fmt.Errorf("parse yt-dlp JSON: %w", err)
}
raw := top.Formats
entry := ""
if len(raw) == 0 && len(top.Entries) > 0 {
raw = top.Entries[0].Formats
if entry = top.Entries[0].Title; entry == "" {
entry = "the first entry"
}
}
formats := make([]*models.FormatInfo, 0, len(raw))
for _, f := range raw {
formats = append(formats, f.toFormatInfo())
}
return formats, nil
return formats, entry, nil
}
func (f ytFormat) toFormatInfo() *models.FormatInfo {
Minternal/service/formats_test.go
@@ -13,10 +13,13 @@ func TestParseFormatJSON(t *testing.T) {
{"format_id": "233", "ext": "m4a", "resolution": "audio only", "vcodec": "none", "acodec": "mp4a", "audio_channels": 2, "format_note": "audio"}
]
}`)
formats, err := parseFormatJSON(data)
formats, entry, err := parseFormatJSON(data)
if err != nil {
t.Fatalf("parseFormatJSON: %v", err)
}
if entry != "" {
t.Errorf("single video reported playlist entry %q, want none", entry)
}
if len(formats) != 3 {
t.Fatalf("expected 3 formats, got %d: %+v", len(formats), formats)
}
@@ -44,12 +47,24 @@ func TestParseFormatJSON(t *testing.T) {
func TestParseFormatJSONPlaylistFallback(t *testing.T) {
// A playlist dump exposes formats under the first entry, not at the top level.
data := []byte(`{"_type":"playlist","entries":[{"id":"a","formats":[{"format_id":"18","ext":"mp4"}]}]}`)
formats, err := parseFormatJSON(data)
data := []byte(`{"_type":"playlist","entries":[{"id":"a","title":"First Clip","formats":[{"format_id":"18","ext":"mp4"}]}]}`)
formats, entry, err := parseFormatJSON(data)
if err != nil {
t.Fatalf("parseFormatJSON: %v", err)
}
if len(formats) != 1 || formats[0].ID != "18" {
t.Fatalf("expected 1 format from entry fallback, got %+v", formats)
}
if entry != "First Clip" {
t.Errorf("playlist entry = %q, want First Clip", entry)
}
// An entry without a title still has to be reported as a playlist entry.
_, entry, err = parseFormatJSON([]byte(`{"_type":"playlist","entries":[{"id":"a","formats":[{"format_id":"18"}]}]}`))
if err != nil {
t.Fatalf("parseFormatJSON: %v", err)
}
if entry == "" {
t.Error("untitled playlist entry reported as a single video")
}
}
Minternal/service/import.go
@@ -6,7 +6,7 @@ import (
"errors"
"fmt"
"io"
"log"
"log/slog"
"os"
"path/filepath"
"sort"
@@ -82,7 +82,7 @@ func (s *DownloadService) importDownloadedItems(ctx context.Context, d *models.D
if ctx.Err() != nil {
return imported, ctx.Err()
}
log.Printf("warning: failed to import item %s: %v", itemDir, err)
slog.Warn("import item failed", "dir", itemDir, "err", err)
continue
}
imported++
@@ -233,7 +233,7 @@ func readInfoJSON(infoJSONPath string) infoJSON {
return info
}
if err := json.Unmarshal(data, &info); err != nil {
log.Printf("ignoring malformed %s: %v", infoJSONPath, err)
slog.Warn("ignoring malformed info.json", "path", infoJSONPath, "err", err)
return infoJSON{}
}
return info
Minternal/service/library.go
@@ -4,7 +4,7 @@ import (
"bytes"
"encoding/json"
"fmt"
"log"
"log/slog"
"os"
"path/filepath"
"sort"
@@ -40,22 +40,24 @@ type LibraryService struct {
ffmpegPath string
ffprobePath string
// thumbLocks holds a per-media-file mutex serializing extraction so two
// callers never write the same temp file at once. Entries are deliberately
// never pruned: dropping one would reintroduce the race it prevents.
thumbLocks sync.Map
// callers never write the same temp file at once. Entries are reference
// counted and removed once nobody holds them. Evicting a lock that is still
// held would let a second caller take a fresh one and reintroduce the race.
thumbMu sync.Mutex
thumbLocks map[string]*refLock
thumbSem chan struct{}
// thumbFailed records media filepaths whose extraction already failed this
// run, so we trust ffmpeg's verdict and don't re-run it on every request.
thumbFailed sync.Map
// Bounded: an evicted path just means one more ffmpeg attempt.
thumbFailed *lru[bool]
// scanCache memoizes scanned items for a short TTL so the listing page (which
// fans out one thumbnail request per media file) and quick auto-refreshes
// don't re-parse each item's marker + info.json on every request. Only item
// scans are cached — the directory listing itself is always read fresh, so
// newly added/removed items and subfolders appear immediately.
scanMu sync.Mutex
scanCache map[string]scanCacheEntry
scanCache *lru[scanCacheEntry]
scanTTL time.Duration
}
@@ -67,13 +69,20 @@ type scanCacheEntry struct {
// scanCacheTTL is how long a scanned item is reused before being re-read.
const scanCacheTTL = 10 * time.Second
// maxCachedPaths bounds each per-path cache. Without it both grow with the
// number of distinct files touched over the process lifetime, which is fine for
// a personal archive and not for a large one.
const maxCachedPaths = 1024
func NewLibraryService(libraryDir, ffmpegPath, ffprobePath string) *LibraryService {
return &LibraryService{
libraryDir: libraryDir,
ffmpegPath: ffmpegPath,
ffprobePath: ffprobePath,
thumbLocks: make(map[string]*refLock),
thumbSem: make(chan struct{}, maxConcurrentThumbnails),
scanCache: make(map[string]scanCacheEntry),
thumbFailed: newLRU[bool](maxCachedPaths),
scanCache: newLRU[scanCacheEntry](maxCachedPaths),
scanTTL: scanCacheTTL,
}
}
@@ -82,9 +91,7 @@ func (s *LibraryService) getCachedScan(relPath string) (*models.LibraryItem, boo
if s.scanTTL <= 0 {
return nil, false
}
s.scanMu.Lock()
defer s.scanMu.Unlock()
e, ok := s.scanCache[relPath]
e, ok := s.scanCache.Get(relPath)
if !ok || time.Since(e.at) > s.scanTTL {
return nil, false
}
@@ -95,15 +102,41 @@ func (s *LibraryService) putCachedScan(relPath string, item *models.LibraryItem)
if s.scanTTL <= 0 {
return
}
s.scanMu.Lock()
s.scanCache[relPath] = scanCacheEntry{item: item, at: time.Now()}
s.scanMu.Unlock()
s.scanCache.Put(relPath, scanCacheEntry{item: item, at: time.Now()})
}
func (s *LibraryService) evictCachedScan(relPath string) {
s.scanMu.Lock()
delete(s.scanCache, relPath)
s.scanMu.Unlock()
s.scanCache.Delete(relPath)
}
// refLock is a mutex plus the number of callers holding or waiting for it.
type refLock struct {
mu sync.Mutex
refs int
}
// lockThumbFile locks the mutex guarding path and returns its release func.
func (s *LibraryService) lockThumbFile(path string) func() {
s.thumbMu.Lock()
l, ok := s.thumbLocks[path]
if !ok {
l = &refLock{}
s.thumbLocks[path] = l
}
l.refs++
s.thumbMu.Unlock()
l.mu.Lock()
return func() {
l.mu.Unlock()
s.thumbMu.Lock()
l.refs--
if l.refs == 0 {
delete(s.thumbLocks, path)
}
s.thumbMu.Unlock()
}
}
// scannedItem returns a cached scan if fresh, otherwise scans and caches it.
@@ -160,7 +193,7 @@ func (s *LibraryService) GetAll(path, sortBy, filter string) ([]*models.LibraryI
dir, err := s.resolveItemDir(path)
if err != nil {
log.Printf("GetAll: invalid path %q: %v", path, err)
slog.Warn("library listing: invalid path", "path", path, "err", err)
return nil, nil, nil
}
entries, err := os.ReadDir(dir)
@@ -189,7 +222,7 @@ func (s *LibraryService) GetAll(path, sortBy, filter string) ([]*models.LibraryI
if _, err := os.Stat(markerPath); err == nil {
item, err := s.scannedItem(itemDir, relPath)
if err != nil {
log.Printf("warning: failed to scan item %s: %v", relPath, err)
slog.Warn("failed to scan item", "path", relPath, "err", err)
continue
}
if filter != "" && !strings.Contains(strings.ToLower(item.Name), strings.ToLower(filter)) && !strings.Contains(strings.ToLower(item.RelPath), strings.ToLower(filter)) {
@@ -261,7 +294,7 @@ func (s *LibraryService) scanItem(itemDir, relPath string) (*models.LibraryItem,
data, err := os.ReadFile(infoJSONPath)
if err == nil {
if err := json.Unmarshal(data, &info); err != nil {
log.Printf("scanItem: ignoring malformed %s: %v", infoJSONPath, err)
slog.Warn("ignoring malformed info.json", "path", infoJSONPath, "err", err)
}
}
}
@@ -342,7 +375,7 @@ func (s *LibraryService) scanItem(itemDir, relPath string) (*models.LibraryItem,
if dirty {
if err := s.writeMetadata(itemDir, metadata); err != nil {
log.Printf("scanItem: failed to persist derived metadata for %s: %v", itemDir, err)
slog.Warn("failed to persist derived metadata", "dir", itemDir, "err", err)
}
}
@@ -369,7 +402,7 @@ func (s *LibraryService) readMetadata(itemDir string) (models.ItemMetadata, erro
data, err := os.ReadFile(markerPath)
if err == nil {
if _, err := toml.Decode(string(data), &metadata); err != nil {
log.Printf("warning: failed to parse %s: %v", markerPath, err)
slog.Warn("failed to parse item marker", "path", markerPath, "err", err)
}
}
return metadata, nil
@@ -461,7 +494,7 @@ func (s *LibraryService) listItemFiles(itemDir string) ([]models.MediaFile, stri
sort.Strings(infoJSONFiles)
infoJSONPath = infoJSONFiles[0]
if len(infoJSONFiles) > 1 {
log.Printf("warning: multiple info.json files in %s, using %s", itemDir, infoJSONPath)
slog.Warn("multiple info.json files", "dir", itemDir, "using", infoJSONPath)
}
}
@@ -606,7 +639,7 @@ func (s *LibraryService) eachItemDir(baseDir, logLabel string, fn func(itemDir s
}
meta, err := s.readMetadata(itemDir)
if err != nil {
log.Printf("%s: skipping %s: %v", logLabel, itemDir, err)
slog.Warn("skipping item", "op", logLabel, "dir", itemDir, "err", err)
continue
}
if !fn(itemDir, meta) {
@@ -632,7 +665,7 @@ func (s *LibraryService) PruneToIDSet(baseDir string, keep map[string]bool) (int
s.evictCachedScan(filepath.ToSlash(rel))
}
if err := os.RemoveAll(itemDir); err != nil {
log.Printf("warning: prune failed to remove %s: %v", itemDir, err)
slog.Warn("prune failed to remove item", "dir", itemDir, "err", err)
return true
}
removed++
Ainternal/service/lru.go
@@ -0,0 +1,75 @@
package service
import (
"container/list"
"sync"
)
// lru is a fixed-size map keyed by path, dropping the least recently used entry
// when it is full.
//
// Only use it for values that are safe to lose: an eviction means the work is
// redone, never that correctness changes.
type lru[V any] struct {
mu sync.Mutex
maxSize int
order *list.List // front is most recently used
entries map[string]*list.Element
}
type lruEntry[V any] struct {
key string
val V
}
func newLRU[V any](maxSize int) *lru[V] {
// A size below 1 would evict each entry as it is stored, turning the cache
// into a silent miss on every lookup.
return &lru[V]{
maxSize: max(maxSize, 1),
order: list.New(),
entries: make(map[string]*list.Element),
}
}
func (c *lru[V]) Get(key string) (V, bool) {
c.mu.Lock()
defer c.mu.Unlock()
el, ok := c.entries[key]
if !ok {
var zero V
return zero, false
}
c.order.MoveToFront(el)
return el.Value.(*lruEntry[V]).val, true
}
func (c *lru[V]) Put(key string, val V) {
c.mu.Lock()
defer c.mu.Unlock()
if el, ok := c.entries[key]; ok {
el.Value.(*lruEntry[V]).val = val
c.order.MoveToFront(el)
return
}
c.entries[key] = c.order.PushFront(&lruEntry[V]{key: key, val: val})
if c.order.Len() > c.maxSize {
oldest := c.order.Back()
c.order.Remove(oldest)
delete(c.entries, oldest.Value.(*lruEntry[V]).key)
}
}
func (c *lru[V]) Delete(key string) {
c.mu.Lock()
defer c.mu.Unlock()
if el, ok := c.entries[key]; ok {
c.order.Remove(el)
delete(c.entries, key)
}
}
Ainternal/service/lru_test.go
@@ -0,0 +1,73 @@
package service
import (
"strconv"
"sync"
"testing"
)
func TestLRUEvictsLeastRecentlyUsed(t *testing.T) {
c := newLRU[int](2)
c.Put("a", 1)
c.Put("b", 2)
// Touching "a" makes "b" the eviction candidate.
if _, ok := c.Get("a"); !ok {
t.Fatal("a missing right after Put")
}
c.Put("c", 3)
if _, ok := c.Get("b"); ok {
t.Error("b survived, want it evicted as least recently used")
}
for _, key := range []string{"a", "c"} {
if _, ok := c.Get(key); !ok {
t.Errorf("%s evicted, want it kept", key)
}
}
// Overwriting an existing key must not grow the cache past its size.
c.Put("a", 9)
if v, _ := c.Get("a"); v != 9 {
t.Errorf("a = %d, want 9", v)
}
if got := c.order.Len(); got != 2 {
t.Errorf("cache holds %d entries, want 2", got)
}
c.Delete("a")
if _, ok := c.Get("a"); ok {
t.Error("a readable after Delete")
}
}
// The per-file thumbnail locks are reference counted rather than cached. The map
// must not grow with every file ever touched, and must not drop a lock that is
// still held.
func TestThumbLocksReleasedWhenUnused(t *testing.T) {
s := NewLibraryService(t.TempDir(), "ffmpeg", "ffprobe")
var wg sync.WaitGroup
shared := 0
for i := 0; i < 50; i++ {
wg.Add(1)
go func(i int) {
defer wg.Done()
// Half contend on one path, half take a path of their own.
if i%2 == 0 {
defer s.lockThumbFile("shared")()
shared++
return
}
defer s.lockThumbFile("file-" + strconv.Itoa(i))()
}(i)
}
wg.Wait()
if shared != 25 {
t.Errorf("shared counter = %d, want 25: the lock did not serialize callers", shared)
}
if got := len(s.thumbLocks); got != 0 {
t.Errorf("%d locks left behind, want 0", got)
}
}
Minternal/service/settings.go
@@ -44,3 +44,14 @@ func (s *SettingsService) SetCookies(cookies string) error {
func (s *SettingsService) GetCookies() (string, error) {
return s.repo.Get("cookies")
}
// GetSessionSecret returns the secret that signs login cookies, or "" when none
// has been generated yet. It lives in the database so sessions survive a
// restart, and is rotated on sign-out so cookies issued earlier stop verifying.
func (s *SettingsService) GetSessionSecret() (string, error) {
return s.repo.Get("session_secret")
}
func (s *SettingsService) SetSessionSecret(secret string) error {
return s.repo.Set("session_secret", secret)
}
Minternal/service/subscription.go
@@ -2,7 +2,7 @@ package service
import (
"fmt"
"log"
"log/slog"
"os"
"path/filepath"
"time"
@@ -90,7 +90,7 @@ func (s *SubscriptionService) Delete(id int64) error {
return err
}
if err := os.Remove(s.ArchivePath(id)); err != nil && !os.IsNotExist(err) {
log.Printf("subscription %d: failed to remove archive file: %v", id, err)
slog.Warn("failed to remove archive file", "subscription_id", id, "err", err)
}
return nil
}
Minternal/service/subscription_run.go
@@ -4,7 +4,7 @@ import (
"context"
"encoding/json"
"fmt"
"log"
"log/slog"
"os"
"path/filepath"
"slices"
@@ -58,7 +58,7 @@ func (s *DownloadService) refreshAndAddNew(ctx context.Context, d *models.Downlo
}
if existing, ok := s.librarySvc.FindByVideoID(baseLibraryDir, info.ID); ok {
if err := s.applyMetadata(existing, info, infoJSONPath); err != nil {
log.Printf("warning: failed to refresh metadata for %s: %v", itemDir, err)
slog.Warn("metadata refresh failed", "dir", itemDir, "err", err)
}
continue
}
@@ -103,7 +103,7 @@ func (s *DownloadService) downloadFresh(ctx context.Context, d *models.Download,
// nothing at all was imported.
imported, err := s.importDownloadedItems(ctx, d, tempDir, "", ytdlpFlags)
if err != nil {
log.Printf("warning: failed to import new metadata-mode items: %v", err)
slog.Warn("failed to import new metadata-mode items", "err", err)
}
if imported > 0 {
return nil
@@ -223,27 +223,27 @@ func (s *DownloadService) applyMetadata(existing string, info infoJSON, sourceIn
func (s *DownloadService) pruneSubscription(ctx context.Context, d *models.Download, sub *models.Subscription) {
baseLibraryDir, err := s.resolveBaseLibraryDir(d)
if err != nil {
log.Printf("subscription %d prune skipped: %v", sub.ID, err)
slog.Warn("prune skipped", "subscription_id", sub.ID, "err", err)
return
}
keep, err := s.enumeratePlaylistIDs(ctx, sub.URL)
if err != nil {
log.Printf("subscription %d prune skipped: enumeration failed: %v", sub.ID, err)
slog.Warn("prune skipped, enumeration failed", "subscription_id", sub.ID, "err", err)
return
}
if len(keep) == 0 {
log.Printf("subscription %d prune skipped: source returned no entries", sub.ID)
slog.Warn("prune skipped, source returned no entries", "subscription_id", sub.ID)
return
}
removed, err := s.librarySvc.PruneToIDSet(baseLibraryDir, keep)
if err != nil {
log.Printf("subscription %d prune error: %v", sub.ID, err)
slog.Error("prune failed", "subscription_id", sub.ID, "err", err)
return
}
if removed > 0 {
log.Printf("subscription %d pruned %d item(s) removed upstream", sub.ID, removed)
slog.Info("pruned items removed upstream", "subscription_id", sub.ID, "removed", removed)
}
}
Minternal/service/thumbnail.go
@@ -3,11 +3,10 @@ package service
import (
"context"
"fmt"
"log"
"log/slog"
"os"
"path/filepath"
"strings"
"sync"
"time"
"vidarchive/internal/models"
@@ -51,10 +50,8 @@ func (s *LibraryService) ThumbnailForFile(relPath, filename string) (string, boo
// the work and the rest reuse the result. A prior in-process failure short-
// circuits to avoid re-running ffmpeg on every request (see thumbFailed).
func (s *LibraryService) ensureThumbnailForFile(mf models.MediaFile) (string, bool) {
actual, _ := s.thumbLocks.LoadOrStore(mf.Filepath, &sync.Mutex{})
lock := actual.(*sync.Mutex)
lock.Lock()
defer lock.Unlock()
release := s.lockThumbFile(mf.Filepath)
defer release()
if path, ok := s.findExistingThumbnail(mf.Filepath); ok {
return path, true
@@ -63,7 +60,7 @@ func (s *LibraryService) ensureThumbnailForFile(mf models.MediaFile) (string, bo
// Trust a prior failure for this run rather than re-running ffmpeg every
// request; a restart clears thumbFailed and retries. Checked after the disk
// so a thumbnail that appears later (e.g. added manually) still wins.
if _, failed := s.thumbFailed.Load(mf.Filepath); failed {
if _, failed := s.thumbFailed.Get(mf.Filepath); failed {
return "", false
}
@@ -72,8 +69,8 @@ func (s *LibraryService) ensureThumbnailForFile(mf models.MediaFile) (string, bo
path, err := s.extractThumbnail(mf)
if err != nil {
log.Printf("thumbnail extraction failed for %s: %v", mf.Filepath, err)
s.thumbFailed.Store(mf.Filepath, struct{}{})
slog.Warn("thumbnail extraction failed", "path", mf.Filepath, "err", err)
s.thumbFailed.Put(mf.Filepath, true)
return "", false
}
return path, true
Minternal/service/ytdlp_live_test.go
@@ -63,10 +63,13 @@ func liveEnv(t *testing.T) (*execEnv, string, string) {
func TestLiveListFormats(t *testing.T) {
e, videoURL, _ := liveEnv(t)
formats, err := e.svc.ListFormats(t.Context(), videoURL)
formats, entry, err := e.svc.ListFormats(t.Context(), videoURL)
if err != nil {
t.Fatalf("ListFormats: %v", err)
}
if entry != "" {
t.Errorf("single video URL reported playlist entry %q", entry)
}
if len(formats) == 0 {
t.Fatal("ListFormats returned no formats")
}
Minternal/worker/pool.go
@@ -3,7 +3,7 @@ package worker
import (
"context"
"errors"
"log"
"log/slog"
"sync"
"time"
@@ -35,7 +35,7 @@ func (p *Pool) Start() {
// Reset any downloads that were in progress during a previous run, and clear
// the temp dirs they left behind so a re-run doesn't import a duplicate.
if err := p.downloadSvc.ResetStalledDownloads(); err != nil {
log.Printf("Warning: failed to reset stalled downloads: %v", err)
slog.Error("failed to reset stalled downloads", "err", err)
}
for i := 0; i < p.workers; i++ {
@@ -65,32 +65,32 @@ func (p *Pool) Submit(d *models.Download) {
select {
case p.queue <- d:
log.Printf("Download %d queued", d.ID)
slog.Debug("download handed to a worker", "download_id", d.ID)
default:
log.Printf("Download %d left in database queue (worker buffer full)", d.ID)
slog.Info("download left in database queue, worker buffer full", "download_id", d.ID)
}
}
func (p *Pool) worker(id int) {
defer p.wg.Done()
log.Printf("Worker %d started", id)
slog.Debug("worker started", "worker", id)
for {
select {
case d := <-p.queue:
log.Printf("Worker %d processing download %d", id, d.ID)
slog.Info("processing download", "worker", id, "download_id", d.ID)
processed, err := p.downloadSvc.ExecuteDownload(p.ctx, d)
switch {
case errors.Is(err, service.ErrCancelled):
log.Printf("Worker %d download %d cancelled", id, d.ID)
slog.Info("download cancelled", "worker", id, "download_id", d.ID)
case err != nil:
log.Printf("Worker %d download %d failed: %v", id, d.ID, err)
slog.Error("download failed", "worker", id, "download_id", d.ID, "err", err)
case processed:
log.Printf("Worker %d download %d completed", id, d.ID)
slog.Info("download completed", "worker", id, "download_id", d.ID)
default:
log.Printf("Worker %d download %d already claimed by another worker, skipping", id, d.ID)
slog.Debug("download already claimed by another worker", "worker", id, "download_id", d.ID)
}
case <-p.ctx.Done():
log.Printf("Worker %d stopped", id)
slog.Debug("worker stopped", "worker", id)
return
}
}
@@ -117,7 +117,7 @@ func (p *Pool) checkQueue() {
// on a later tick; duplicates are harmless (MarkStarted claims atomically).
downloads, err := p.downloadSvc.GetQueued(cap(p.queue))
if err != nil {
log.Printf("Queue check error: %v", err)
slog.Error("queue check failed", "err", err)
return
}
Minternal/worker/scheduler.go
@@ -3,7 +3,7 @@ package worker
import (
"context"
"database/sql"
"log"
"log/slog"
"time"
"vidarchive/internal/models"
@@ -70,7 +70,7 @@ func (s *Scheduler) loop() {
func (s *Scheduler) backfillNextRuns() {
subs, err := s.subscriptionSvc.GetAll()
if err != nil {
log.Printf("scheduler: backfill failed to list subscriptions: %v", err)
slog.Error("scheduler backfill failed to list subscriptions", "err", err)
return
}
now := time.Now()
@@ -80,12 +80,12 @@ func (s *Scheduler) backfillNextRuns() {
}
next, err := s.subscriptionSvc.ComputeNextRun(sub, now)
if err != nil {
log.Printf("scheduler: subscription %d has invalid schedule: %v", sub.ID, err)
slog.Warn("scheduler: invalid schedule", "subscription_id", sub.ID, "err", err)
continue
}
sub.NextRunAt = sql.NullTime{Time: next, Valid: true}
if err := s.subscriptionSvc.Update(sub); err != nil {
log.Printf("scheduler: failed to set next run for subscription %d: %v", sub.ID, err)
slog.Error("scheduler: failed to set next run", "subscription_id", sub.ID, "err", err)
}
}
}
@@ -94,7 +94,7 @@ func (s *Scheduler) checkDue() {
now := time.Now()
due, err := s.subscriptionSvc.GetDue(now)
if err != nil {
log.Printf("scheduler: failed to query due subscriptions: %v", err)
slog.Error("scheduler: failed to query due subscriptions", "err", err)
return
}
for _, sub := range due {
@@ -106,7 +106,7 @@ func (s *Scheduler) run(sub *models.Subscription, now time.Time) {
next, err := s.subscriptionSvc.ComputeNextRun(sub, now)
if err != nil {
// Don't keep retrying a broken schedule every tick; push it out a day.
log.Printf("scheduler: subscription %d has invalid schedule, deferring: %v", sub.ID, err)
slog.Warn("scheduler: invalid schedule, deferring", "subscription_id", sub.ID, "err", err)
next = now.Add(24 * time.Hour)
}
@@ -114,31 +114,31 @@ func (s *Scheduler) run(sub *models.Subscription, now time.Time) {
// longer than the interval would otherwise stack duplicate downloads. Advance
// next_run_at so we don't re-evaluate it every tick, preserving the last run.
if active, aErr := s.downloadSvc.HasActiveForSubscription(sub.ID); aErr != nil {
log.Printf("scheduler: subscription %d active-run check failed: %v", sub.ID, aErr)
slog.Error("scheduler: active-run check failed", "subscription_id", sub.ID, "err", aErr)
} else if active {
lastRun := now
if sub.LastRunAt.Valid {
lastRun = sub.LastRunAt.Time
}
log.Printf("scheduler: subscription %d still has an active run, skipping until %s", sub.ID, next.Format(time.RFC3339))
slog.Info("scheduler: active run in progress, skipping", "subscription_id", sub.ID, "next_run", next.Format(time.RFC3339))
if mErr := s.subscriptionSvc.MarkRun(sub.ID, lastRun, next, "skipped"); mErr != nil {
log.Printf("scheduler: failed to mark subscription %d run: %v", sub.ID, mErr)
slog.Error("scheduler: failed to mark run", "subscription_id", sub.ID, "err", mErr)
}
return
}
d, err := s.downloadSvc.CreateForSubscription(sub)
if err != nil {
log.Printf("scheduler: failed to queue subscription %d: %v", sub.ID, err)
slog.Error("scheduler: failed to queue subscription", "subscription_id", sub.ID, "err", err)
if mErr := s.subscriptionSvc.MarkRun(sub.ID, now, next, "error"); mErr != nil {
log.Printf("scheduler: failed to mark subscription %d run: %v", sub.ID, mErr)
slog.Error("scheduler: failed to mark run", "subscription_id", sub.ID, "err", mErr)
}
return
}
s.pool.Submit(d)
if err := s.subscriptionSvc.MarkRun(sub.ID, now, next, "queued"); err != nil {
log.Printf("scheduler: failed to mark subscription %d run: %v", sub.ID, err)
slog.Error("scheduler: failed to mark run", "subscription_id", sub.ID, "err", err)
}
log.Printf("scheduler: queued subscription %d (download %d), next run %s", sub.ID, d.ID, next.Format(time.RFC3339))
slog.Info("scheduler: queued subscription", "subscription_id", sub.ID, "download_id", d.ID, "next_run", next.Format(time.RFC3339))
}
Mweb/static/style.css
@@ -51,7 +51,15 @@ body > header h1 {
nav {
display: flex;
align-items: center;
gap: 1rem;
/* Take the rest of the header row, so the sign-out button can sit at its far
right while the links stay next to the brand. */
flex: 1;
}
.nav-signout {
margin-left: auto;
}
nav a {
@@ -1127,6 +1135,34 @@ input[name="cron_expr"] {
padding: 3rem;
}
/* Login */
.login-page {
max-width: 24rem;
margin: 0 auto;
}
/* The login page is one narrow column, so the flash above the form is kept to
the same width instead of spanning the whole page. */
.login-main {
display: flex;
flex-direction: column;
justify-content: center;
}
.login-main > * {
max-width: 24rem;
width: 100%;
}
/* Pagination */
.pagination {
display: flex;
align-items: center;
justify-content: space-between;
gap: 1rem;
margin: 1.5rem 0;
}
/* Utility */
.inline {
display: inline;
Mweb/templates/download_form.html
@@ -44,6 +44,11 @@
{{if .Data.Formats}}
<article>
<h4>Available Formats</h4>
{{if .Data.PlaylistEntry}}
<p class="format-note">This URL is a playlist. The formats below are the
ones offered for <strong>{{.Data.PlaylistEntry}}</strong>; other entries
may offer different formats.</p>
{{end}}
<table>
<thead>
<tr>
Mweb/templates/layout.html
@@ -21,16 +21,23 @@
</svg>
<span>VidArchive</span>
</h1>
{{/* The login page shows no navigation: every link behind it needs the
session the visitor does not have yet. */}}
{{if ne .ActiveTab "login"}}
<nav>
<a href="/library" class="{{if eq .ActiveTab "library"}}active{{end}}">Library</a>
<a href="/queue" class="{{if eq .ActiveTab "queue"}}active{{end}}">Queue</a>
<a href="/download" class="{{if eq .ActiveTab "download"}}active{{end}}">Download</a>
<a href="/subscriptions" class="{{if eq .ActiveTab "subscriptions"}}active{{end}}">Subscriptions</a>
<a href="/settings" class="{{if eq .ActiveTab "settings"}}active{{end}}">Settings</a>
<form method="post" action="/logout" class="nav-signout">
<button type="submit" class="small secondary">Sign out</button>
</form>
</nav>
{{end}}
</header>
<main>
<main{{if eq .ActiveTab "login"}} class="login-main"{{end}}>
{{if .Flash}}
<div class="flash flash-{{.Flash.Kind}}" role="status" aria-live="polite">{{.Flash.Message}}</div>
{{end}}
@@ -43,3 +50,14 @@
</body>
</html>
{{end}}
{{/* Shared prev/next links for the paginated listing pages. */}}
{{define "pagination"}}
{{if or (gt .Page 1) .HasNext}}
<nav class="pagination">
{{if gt .Page 1}}<a href="?{{.Query}}page={{.Prev}}">← Previous</a>{{end}}
<span>Page {{.Page}}</span>
{{if .HasNext}}<a href="?{{.Query}}page={{.Next}}">Next →</a>{{end}}
</nav>
{{end}}
{{end}}
Mweb/templates/library.html
@@ -86,4 +86,6 @@
<p>No media found. <a href="/download">Start a download</a> or add existing files.</p>
</div>
{{end}}
{{template "pagination" .Data.Pagination}}
{{end}}
Aweb/templates/login.html
@@ -0,0 +1,14 @@
{{define "login"}}
<div class="login-page">
<h2>Sign in</h2>
<form method="post" action="/login">
<label for="username">Username</label>
<input type="text" id="username" name="username" autocomplete="username" required autofocus>
<label for="password">Password</label>
<input type="password" id="password" name="password" autocomplete="current-password" required>
<button type="submit">Sign in</button>
</form>
</div>
{{end}}
Mweb/templates/queue.html
@@ -5,6 +5,14 @@
<form method="post" action="/queue/clear" class="inline">
<button type="submit" class="secondary">Clear All</button>
</form>
<form method="post" action="/queue/clear" class="inline">
<input type="hidden" name="status" value="completed">
<button type="submit" class="secondary">Clear Completed</button>
</form>
<form method="post" action="/queue/clear" class="inline">
<input type="hidden" name="status" value="error">
<button type="submit" class="secondary">Clear Failed</button>
</form>
<form method="get" action="/queue" class="inline filter-form">
<select name="status">
<option value="all" {{if eq .Data.Status "all"}}selected{{end}}>All</option>
@@ -32,6 +40,16 @@
<div class="queue-status">
<span class="status-badge {{.Status}}">{{.Status}}</span>
</div>
{{if or (eq .Status "error") (eq .Status "cancelled")}}
<form method="post" action="/queue/{{.ID}}/retry" class="inline queue-action">
<button type="submit" class="small">Retry</button>
</form>
{{end}}
{{if or (eq .Status "queued") (eq .Status "downloading")}}
<form method="post" action="/queue/{{.ID}}/cancel" class="inline queue-action">
<button type="submit" class="small secondary">Cancel</button>
</form>
{{end}}
<form method="post" action="/queue/{{.ID}}/delete" class="inline queue-action">
<button type="submit" class="small danger">Remove</button>
</form>
@@ -52,4 +70,6 @@
</div>
{{end}}
</div>
{{template "pagination" .Data.Pagination}}
{{end}}
Mweb/templates/queue_detail.html
@@ -4,6 +4,16 @@
<h2>Queue Details</h2>
<div class="download-status-header">
<span class="status-badge {{.Data.Status}}">{{.Data.Status}}</span>
{{if or (eq .Data.Status "error") (eq .Data.Status "cancelled")}}
<form method="post" action="/queue/{{.Data.ID}}/retry" class="inline">
<button type="submit" class="small">Retry</button>
</form>
{{end}}
{{if or (eq .Data.Status "queued") (eq .Data.Status "downloading")}}
<form method="post" action="/queue/{{.Data.ID}}/cancel" class="inline">
<button type="submit" class="small secondary">Cancel</button>
</form>
{{end}}
</div>
</div>