add lint, format and vuln CI checks; fix what they found
Mirror the Hearthforge pipeline: tidy, gofumpt, golangci-lint, govulncheck, and -race on the test step. - localReferer accepted "/\host", which browsers read as a host. Open redirect; now rejected, with cases added to the existing test. - Five exec.Command calls bypassed util.KillableCommand and had no timeout, so a stuck yt-dlp or ffmpeg held a worker forever. ListFormats takes the caller's context; the ffmpeg/ffprobe paths get bounded ones. - go 1.26.2 -> 1.26.8, clearing 10 stdlib advisories including two html/template XSS on the render path. - gofumpt across the tree. .golangci.yml records why each remaining gosec, noctx and ST1005 hit is excluded. Raise memory_limit to 4g for the race detector. Also add a library screenshot to the README.
A.golangci.yml
@@ -0,0 +1,62 @@
version: "2"
linters:
default: none
enable:
- errcheck
- govet
- staticcheck
- unused
- ineffassign
- gosec
- bodyclose
- noctx
settings:
errcheck:
# Best-effort cleanup and response writes. A failure here has no useful
# handler; the caller already logs or returns the main error.
exclude-functions:
- (io.Closer).Close
- (*os.File).Close
- (*database/sql.Rows).Close
- (*database/sql.Tx).Rollback
- os.Remove
- os.RemoveAll
- (net/http.ResponseWriter).Write
- (*encoding/json.Encoder).Encode
- io.Copy
- io.WriteString
- fmt.Fprintf
gosec:
excludes:
- G104 # errcheck covers unchecked errors
- G124 # cookies set HttpOnly and SameSite; Secure would break plain-HTTP self-hosting
- G203 # the only template.HTML is rendered template output, not user input
- G204 # running yt-dlp, ffmpeg and ffprobe with request args is the core of this program
- G301 # library and temp directories are meant to be world-readable
- G304 # paths come from LibraryService.resolveItemDir, which rejects escapes
- G702 # taint analysis: same as G204
- G703 # taint analysis: same as G304
- G706 # log injection: log lines are for the operator
- G710 # localReferer keeps only a local path, see its test
exclusions:
rules:
- path: _test\.go
linters: [gosec, errcheck, noctx, bodyclose]
# The repository layer is deliberately context-free: SQLite is local and
# every query is short. noctx still guards the outbound HTTP calls.
- path: internal/(repository|database)/
linters: [noctx]
# These error strings are the flash message shown to the user, so they are
# written as sentences on purpose.
- path: internal/handler/settings\.go
text: "ST1005"
formatters:
enable:
- gofumpt
settings:
gofumpt:
module-path: vidarchive
extra:
group-params: true
M.hearthforge-ci.toml
@@ -8,7 +8,8 @@ clone_project_to = "/ci/build/project"
shell_setup = "set -euo pipefail"
timeout = 1800
cpu_limit = 2.0
memory_limit = "2g"
# The race detector needs several times the normal heap.
memory_limit = "4g"
# Go's build and module caches. Both live outside clone_project_to.
cache = [
@@ -35,16 +36,34 @@ apt-get update -qq && apt-get install -y -qq --no-install-recommends ffmpeg podm
curl -fsSL https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp_linux -o /usr/local/bin/yt-dlp
chmod +x /usr/local/bin/yt-dlp
yt-dlp --version
go install mvdan.cc/gofumpt@latest
go install github.com/golangci/golangci-lint/v2/cmd/golangci-lint@latest
go install golang.org/x/vuln/cmd/govulncheck@latest
"""
# gofmt has no failure exit code, so an empty report is the pass condition.
# The checkout has no .git, so compare copies instead of using git diff.
[[steps]]
name = "vet"
run_sh = "cd project && gofmt -l . | tee /ci/build/gofmt.txt && test ! -s /ci/build/gofmt.txt && go vet ./..."
name = "tidy"
run_sh = "cd project && cp go.mod /tmp/go.mod && cp go.sum /tmp/go.sum && go mod tidy && diff /tmp/go.mod go.mod && diff /tmp/go.sum go.sum"
# gofumpt has no failure exit code, so an empty report is the pass condition.
[[steps]]
name = "format"
run_sh = "cd project && test -z \"$(gofumpt -l -extra .)\" || (gofumpt -l -extra . && exit 1)"
[[steps]]
name = "lint"
run_sh = "cd project && golangci-lint run ./..."
# Advisories appear without any code change, so a hit must not block a run.
[[steps]]
name = "vulncheck"
warn_on_fail = true
run_sh = "cd project && govulncheck ./..."
[[steps]]
name = "test"
run_sh = "cd project && go test ./..."
run_sh = "cd project && go test -race -count=1 ./..."
# The live tests hit YouTube, which may block CI IPs. A failure must stay
# visible but must not fail the run.
MREADME.md
@@ -9,6 +9,8 @@ metadata, subtitles and comments, and re-runs saved subscriptions on a schedule.
Go standard library plus chi, SQLite (pure-Go driver), and server-rendered
templates. No JavaScript, no build step for the front end.
## Requirements
- `yt-dlp` on `PATH` (or set `VIDARCHIVE_YTDLP_PATH`)
Mcmd/vidarchive/main.go
@@ -37,16 +37,16 @@ func main() {
checkDependencies(cfg)
if err := os.MkdirAll(cfg.DataDir, 0755); err != nil {
if err := os.MkdirAll(cfg.DataDir, 0o755); err != nil {
log.Fatalf("Failed to create data dir: %v", err)
}
if err := os.MkdirAll(cfg.LibraryDir, 0755); err != nil {
if err := os.MkdirAll(cfg.LibraryDir, 0o755); err != nil {
log.Fatalf("Failed to create library dir: %v", err)
}
if err := os.MkdirAll(cfg.TempDir, 0755); err != nil {
if err := os.MkdirAll(cfg.TempDir, 0o755); err != nil {
log.Fatalf("Failed to create temp dir: %v", err)
}
if err := os.MkdirAll(filepath.Join(cfg.DataDir, "archives"), 0755); err != nil {
if err := os.MkdirAll(filepath.Join(cfg.DataDir, "archives"), 0o755); err != nil {
log.Fatalf("Failed to create archives dir: %v", err)
}
Mgo.mod
@@ -1,6 +1,6 @@
module vidarchive
go 1.26.2
go 1.26.8
require (
github.com/BurntSushi/toml v1.6.0
Minternal/database/database.go
@@ -12,7 +12,7 @@ import (
)
func New(cfg *config.Config) (*sql.DB, error) {
if err := os.MkdirAll(filepath.Dir(cfg.DBPath), 0755); err != nil {
if err := os.MkdirAll(filepath.Dir(cfg.DBPath), 0o755); err != nil {
return nil, fmt.Errorf("create db dir: %w", err)
}
Minternal/handler/handler_test.go
@@ -64,6 +64,9 @@ func TestLocalRefererKeepsPathOnly(t *testing.T) {
{"https://vidarchive.local/library?path=music", "/library?path=music"},
{"/queue?sort=status", "/queue?sort=status"},
{"not a url", "/"},
// Protocol-relative paths: a browser reads these as a host, not a path.
{"https://evil.example//evil.example/phish", "/"},
{`https://evil.example/\evil.example/phish`, "/"},
}
for _, tc := range tests {
Minternal/handler/health.go
@@ -56,7 +56,7 @@ func (h *Handler) Health(w http.ResponseWriter, r *http.Request) {
// toolVersion returns the first line of the tool's version output. It reports
// "not installed" when the binary is missing and "timed out" when it doesn't
// answer within ctx.
func toolVersion(ctx context.Context, path string, versionArg string) string {
func toolVersion(ctx context.Context, path, versionArg string) string {
out, err := util.KillableCommand(ctx, path, versionArg).Output()
if err != nil {
if ctx.Err() != nil {
Minternal/handler/queue.go
@@ -131,7 +131,7 @@ func (h *Handler) DownloadForm(w http.ResponseWriter, r *http.Request) {
var flash *Flash
if r.URL.Query().Get("list_formats") == "1" && url != "" {
var err error
if formats, err = h.downloadSvc.ListFormats(url); err != nil {
if formats, err = h.downloadSvc.ListFormats(r.Context(), url); err != nil {
log.Printf("DownloadForm: listing formats for %q failed: %v", url, err)
flash = &Flash{Kind: "error", Message: "Couldn't list formats: " + err.Error()}
}
Minternal/handler/settings.go
@@ -209,6 +209,12 @@ func localReferer(r *http.Request) string {
if err != nil || !strings.HasPrefix(ref.Path, "/") {
return "/"
}
// "//host" and "/\host" are protocol-relative URLs to browsers, so a path
// starting with them would redirect off-site. Only a single leading slash
// followed by a normal path segment stays local.
if strings.HasPrefix(ref.Path, "//") || strings.HasPrefix(ref.Path, `/\`) {
return "/"
}
if ref.RawQuery == "" {
return ref.Path
}
Minternal/repository/settings.go
@@ -3,6 +3,7 @@ package repository
import (
"database/sql"
"strconv"
"vidarchive/internal/models"
)
Minternal/server/health_test.go
@@ -16,7 +16,7 @@ import (
func fakeTool(t *testing.T, envVar, name, body string) {
t.Helper()
path := filepath.Join(t.TempDir(), name)
if err := os.WriteFile(path, []byte("#!/bin/sh\n"+body+"\n"), 0755); err != nil {
if err := os.WriteFile(path, []byte("#!/bin/sh\n"+body+"\n"), 0o755); err != nil {
t.Fatal(err)
}
t.Setenv(envVar, path)
Minternal/server/server_test.go
@@ -35,10 +35,10 @@ func setupTestServerDB(t *testing.T) (*Server, *config.Config, *sql.DB, func())
t.Setenv("VIDARCHIVE_DATA_DIR", dataDir)
cfg := config.New()
if err := os.MkdirAll(cfg.LibraryDir, 0755); err != nil {
if err := os.MkdirAll(cfg.LibraryDir, 0o755); err != nil {
t.Fatalf("create library dir: %v", err)
}
if err := os.MkdirAll(cfg.TempDir, 0755); err != nil {
if err := os.MkdirAll(cfg.TempDir, 0o755); err != nil {
t.Fatalf("create temp dir: %v", err)
}
@@ -75,16 +75,16 @@ func setupTestServerDB(t *testing.T) (*Server, *config.Config, *sql.DB, func())
func createItem(t *testing.T, libraryDir, relPath, name string, files map[string]string) {
t.Helper()
itemDir := filepath.Join(libraryDir, relPath)
if err := os.MkdirAll(itemDir, 0755); err != nil {
if err := os.MkdirAll(itemDir, 0o755); err != nil {
t.Fatalf("create item dir: %v", err)
}
marker := filepath.Join(itemDir, ".vidarchive-item.toml")
if err := os.WriteFile(marker, []byte("name = \""+name+"\"\nduration = -1\n"), 0644); err != nil {
if err := os.WriteFile(marker, []byte("name = \""+name+"\"\nduration = -1\n"), 0o644); err != nil {
t.Fatalf("write marker: %v", err)
}
for filename, content := range files {
path := filepath.Join(itemDir, filename)
if err := os.WriteFile(path, []byte(content), 0644); err != nil {
if err := os.WriteFile(path, []byte(content), 0o644); err != nil {
t.Fatalf("write file %s: %v", filename, err)
}
}
@@ -250,10 +250,10 @@ func TestSubtitleServedByPathSegment(t *testing.T) {
})
vtt := "WEBVTT\n\n00:00:00.000 --> 00:00:01.000\nhi\n"
subDir := filepath.Join(cfg.LibraryDir, item, "subtitles")
if err := os.MkdirAll(subDir, 0755); err != nil {
if err := os.MkdirAll(subDir, 0o755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(subDir, "eng.vtt"), []byte(vtt), 0644); err != nil {
if err := os.WriteFile(filepath.Join(subDir, "eng.vtt"), []byte(vtt), 0o644); err != nil {
t.Fatal(err)
}
@@ -287,11 +287,11 @@ func TestPathTraversalBlocked(t *testing.T) {
defer cleanup()
outside := filepath.Join(cfg.DataDir, "secret")
if err := os.MkdirAll(outside, 0755); err != nil {
if err := os.MkdirAll(outside, 0o755); err != nil {
t.Fatalf("create outside dir: %v", err)
}
marker := filepath.Join(outside, ".vidarchive-item.toml")
if err := os.WriteFile(marker, []byte("name = \"secret\"\nduration = -1\n"), 0644); err != nil {
if err := os.WriteFile(marker, []byte("name = \"secret\"\nduration = -1\n"), 0o644); err != nil {
t.Fatalf("write marker: %v", err)
}
Minternal/service/download.go
@@ -281,7 +281,7 @@ func (s *DownloadService) ExecuteDownload(parent context.Context, d *models.Down
s.recordSubscriptionStatus(d, "downloading")
tempDownloadDir := s.tempDirFor(d.ID)
if err := os.MkdirAll(tempDownloadDir, 0755); err != nil {
if err := os.MkdirAll(tempDownloadDir, 0o755); err != nil {
// MarkStarted already moved the row to "downloading"; returning without
// finalizing would strand it there until the next restart.
err = fmt.Errorf("create temp download dir: %w", err)
@@ -314,7 +314,7 @@ func (s *DownloadService) ExecuteDownload(parent context.Context, d *models.Down
case "skip":
// Let yt-dlp skip entries already recorded — no re-download.
archive := s.subscriptionSvc.ArchivePath(sub.ID)
if err := os.MkdirAll(filepath.Dir(archive), 0755); err == nil {
if err := os.MkdirAll(filepath.Dir(archive), 0o755); err == nil {
args = append(args, "--download-archive", archive)
}
case "metadata":
@@ -505,7 +505,7 @@ func (s *DownloadService) saveRefreshedCookies(path, sent string) {
// same volume the rest of the run uses). On any failure the partial file is
// removed — a truncated cookies file must not be handed to yt-dlp.
func (s *DownloadService) writeCookiesFile(cookies string) (string, error) {
if err := os.MkdirAll(s.cfg.TempDir, 0755); err != nil {
if err := os.MkdirAll(s.cfg.TempDir, 0o755); err != nil {
return "", err
}
tmpFile, err := os.CreateTemp(s.cfg.TempDir, "cookies-*.txt")
Minternal/service/execute_download_test.go
@@ -40,7 +40,7 @@ func newExecEnv(t *testing.T) *execEnv {
FFprobePath: "ffprobe",
}
for _, dir := range []string{cfg.LibraryDir, cfg.TempDir} {
if err := os.MkdirAll(dir, 0755); err != nil {
if err := os.MkdirAll(dir, 0o755); err != nil {
t.Fatal(err)
}
}
@@ -93,7 +93,7 @@ echo 3.0`)
func writeScript(t *testing.T, path, body string) string {
t.Helper()
if err := os.WriteFile(path, []byte("#!/bin/sh\n"+body+"\n"), 0755); err != nil {
if err := os.WriteFile(path, []byte("#!/bin/sh\n"+body+"\n"), 0o755); err != nil {
t.Fatal(err)
}
return path
@@ -426,7 +426,7 @@ while [ ! -f "$SCRATCH/proceed" ]; do sleep 0.05; done`)
waitForFile(t, filepath.Join(e.scratch, "probing"))
e.svc.cancelDownload(d.ID)
if err := os.WriteFile(filepath.Join(e.scratch, "proceed"), nil, 0644); err != nil {
if err := os.WriteFile(filepath.Join(e.scratch, "proceed"), nil, 0o644); err != nil {
t.Fatal(err)
}
@@ -483,7 +483,7 @@ while [ ! -f "$SCRATCH/proceed" ]; do sleep 0.05; done`)
waitForFile(t, filepath.Join(e.scratch, "probing"))
e.svc.cancelDownload(d.ID)
if err := os.WriteFile(filepath.Join(e.scratch, "proceed"), nil, 0644); err != nil {
if err := os.WriteFile(filepath.Join(e.scratch, "proceed"), nil, 0o644); err != nil {
t.Fatal(err)
}
@@ -517,10 +517,10 @@ func TestResetStalledDownloadsClearsTempDirs(t *testing.T) {
t.Fatalf("tempDirsFor returned %d dirs, want the main and second-pass dirs", len(dirs))
}
for _, dir := range dirs {
if err := os.MkdirAll(dir, 0755); err != nil {
if err := os.MkdirAll(dir, 0o755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(dir, "partial.mp4.part"), []byte("x"), 0644); err != nil {
if err := os.WriteFile(filepath.Join(dir, "partial.mp4.part"), []byte("x"), 0o644); err != nil {
t.Fatal(err)
}
}
@@ -528,7 +528,7 @@ func TestResetStalledDownloadsClearsTempDirs(t *testing.T) {
// A download that is not stalled must keep whatever it owns.
other := e.queue(t, &models.Download{})
keep := e.svc.tempDirFor(other.ID)
if err := os.MkdirAll(keep, 0755); err != nil {
if err := os.MkdirAll(keep, 0o755); err != nil {
t.Fatal(err)
}
@@ -679,7 +679,7 @@ func TestExecuteDownloadTempDirFailureIsError(t *testing.T) {
// A regular file where the temp tree needs a directory makes MkdirAll fail
// with ENOTDIR.
blocker := filepath.Join(e.scratch, "blocker")
if err := os.WriteFile(blocker, nil, 0644); err != nil {
if err := os.WriteFile(blocker, nil, 0o644); err != nil {
t.Fatal(err)
}
e.cfg.TempDir = filepath.Join(blocker, "temp")
Minternal/service/formats.go
@@ -2,16 +2,16 @@ package service
import (
"bytes"
"context"
"encoding/json"
"fmt"
"os/exec"
"strconv"
"vidarchive/internal/models"
"vidarchive/internal/util"
)
func (s *DownloadService) ListFormats(url string) ([]*models.FormatInfo, error) {
func (s *DownloadService) ListFormats(ctx context.Context, url string) ([]*models.FormatInfo, error) {
// Use machine-readable JSON (-J) rather than scraping the human "-F" table,
// whose columns/separators shift between yt-dlp versions. stderr is captured
// separately so warnings can't corrupt the JSON on stdout.
@@ -22,7 +22,7 @@ func (s *DownloadService) ListFormats(url string) ([]*models.FormatInfo, error)
defer cleanup()
args = append(args, url)
cmd := exec.Command(s.cfg.YTDLPPath, args...)
cmd := util.KillableCommand(ctx, s.cfg.YTDLPPath, args...)
var stderr bytes.Buffer
cmd.Stderr = &stderr
output, err := cmd.Output()
Minternal/service/import.go
@@ -55,7 +55,7 @@ func (s *DownloadService) importDownloadedItems(ctx context.Context, d *models.D
if err != nil {
return 0, err
}
if err := os.MkdirAll(baseLibraryDir, 0755); err != nil {
if err := os.MkdirAll(baseLibraryDir, 0o755); err != nil {
return 0, err
}
@@ -158,7 +158,7 @@ func (s *DownloadService) importItemDir(ctx context.Context, url, itemDir, baseL
if err != nil {
return err
}
if err := os.MkdirAll(targetDir, 0755); err != nil {
if err := os.MkdirAll(targetDir, 0o755); err != nil {
return err
}
@@ -186,7 +186,7 @@ func (s *DownloadService) importItemDir(ctx context.Context, url, itemDir, baseL
if len(subtitleFiles) > 0 {
subtitlesDir := filepath.Join(targetDir, subtitlesDirName)
if err := os.MkdirAll(subtitlesDir, 0755); err != nil {
if err := os.MkdirAll(subtitlesDir, 0o755); err != nil {
return err
}
for _, sf := range subtitleFiles {
Minternal/service/import_test.go
@@ -56,14 +56,14 @@ func TestUniqueDir(t *testing.T) {
if filepath.Base(first) != "item" {
t.Errorf("first uniqueDir = %q, want .../item", first)
}
if err := os.MkdirAll(first, 0755); err != nil {
if err := os.MkdirAll(first, 0o755); err != nil {
t.Fatal(err)
}
second := mustUnique("item")
if filepath.Base(second) != "item-1" {
t.Errorf("second uniqueDir = %q, want .../item-1", second)
}
if err := os.MkdirAll(second, 0755); err != nil {
if err := os.MkdirAll(second, 0o755); err != nil {
t.Fatal(err)
}
third := mustUnique("item")
@@ -112,11 +112,11 @@ func TestDeriveItemName(t *testing.T) {
// info.json title wins and is sanitized.
infoPath := filepath.Join(itemDir, "info.json")
if err := os.WriteFile(infoPath, []byte(`{"title":"Cool: Video"}`), 0644); err != nil {
if err := os.WriteFile(infoPath, []byte(`{"title":"Cool: Video"}`), 0o644); err != nil {
t.Fatal(err)
}
bigName := "big.mp4"
if err := os.WriteFile(filepath.Join(itemDir, bigName), []byte("xxxxxxxxxx"), 0644); err != nil {
if err := os.WriteFile(filepath.Join(itemDir, bigName), []byte("xxxxxxxxxx"), 0o644); err != nil {
t.Fatal(err)
}
media := []os.DirEntry{dirEntry(t, itemDir, bigName)}
@@ -126,7 +126,7 @@ func TestDeriveItemName(t *testing.T) {
// Without info.json, falls back to the largest media file's stem.
small := "small.mp4"
if err := os.WriteFile(filepath.Join(itemDir, small), []byte("x"), 0644); err != nil {
if err := os.WriteFile(filepath.Join(itemDir, small), []byte("x"), 0o644); err != nil {
t.Fatal(err)
}
media = []os.DirEntry{dirEntry(t, itemDir, small), dirEntry(t, itemDir, bigName)}
@@ -161,10 +161,10 @@ func TestImportItemDir(t *testing.T) {
src := t.TempDir()
makeTestVideo(t, filepath.Join(src, "raw.mp4"))
if err := os.WriteFile(filepath.Join(src, "info.json"), []byte(`{"title":"My Clip"}`), 0644); err != nil {
if err := os.WriteFile(filepath.Join(src, "info.json"), []byte(`{"title":"My Clip"}`), 0o644); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(src, "raw.en.srt"), []byte("1\n00:00:00,000 --> 00:00:01,000\nhi\n"), 0644); err != nil {
if err := os.WriteFile(filepath.Join(src, "raw.en.srt"), []byte("1\n00:00:00,000 --> 00:00:01,000\nhi\n"), 0o644); err != nil {
t.Fatal(err)
}
@@ -210,7 +210,7 @@ func TestImportDownloadedItemsRejectsOutputTraversal(t *testing.T) {
// A temp download dir with one item subdir.
tempDir := t.TempDir()
itemDir := filepath.Join(tempDir, "item-00001")
if err := os.MkdirAll(itemDir, 0755); err != nil {
if err := os.MkdirAll(itemDir, 0o755); err != nil {
t.Fatal(err)
}
@@ -237,7 +237,7 @@ func TestImportDownloadedItemsStopsOnCancel(t *testing.T) {
tempDir := t.TempDir()
for _, name := range []string{"item-00001", "item-00002"} {
itemDir := filepath.Join(tempDir, name)
if err := os.MkdirAll(itemDir, 0755); err != nil {
if err := os.MkdirAll(itemDir, 0o755); err != nil {
t.Fatal(err)
}
makeTestVideo(t, filepath.Join(itemDir, "raw.mp4"))
@@ -276,12 +276,12 @@ func TestImportDownloadedItemsStopsBetweenItems(t *testing.T) {
tempDir := t.TempDir()
for i, name := range []string{"item-00001", "item-00002", "item-00003"} {
itemDir := filepath.Join(tempDir, name)
if err := os.MkdirAll(itemDir, 0755); err != nil {
if err := os.MkdirAll(itemDir, 0o755); err != nil {
t.Fatal(err)
}
makeTestVideo(t, filepath.Join(itemDir, "raw.mp4"))
info := fmt.Sprintf(`{"id":"v%d","title":"Clip %d"}`, i+1, i+1)
if err := os.WriteFile(filepath.Join(itemDir, "clip.info.json"), []byte(info), 0644); err != nil {
if err := os.WriteFile(filepath.Join(itemDir, "clip.info.json"), []byte(info), 0o644); err != nil {
t.Fatal(err)
}
}
@@ -342,20 +342,20 @@ func TestImportItemDirOverwriteReplacesExisting(t *testing.T) {
// First import establishes the item.
first := t.TempDir()
makeTestVideo(t, filepath.Join(first, "raw.mp4"))
if err := os.WriteFile(filepath.Join(first, "clip.info.json"), []byte(`{"id":"vid1","title":"Old Title"}`), 0644); err != nil {
if err := os.WriteFile(filepath.Join(first, "clip.info.json"), []byte(`{"id":"vid1","title":"Old Title"}`), 0o644); err != nil {
t.Fatal(err)
}
if err := svc.importItemDir(context.Background(), "https://example.com/v", first, libDir, "overwrite", ""); err != nil {
t.Fatalf("first import: %v", err)
}
if err := os.WriteFile(filepath.Join(libDir, "Old Title", "stale.txt"), []byte("gone"), 0644); err != nil {
if err := os.WriteFile(filepath.Join(libDir, "Old Title", "stale.txt"), []byte("gone"), 0o644); err != nil {
t.Fatal(err)
}
// Second import of the same video id, now retitled upstream.
second := t.TempDir()
makeTestVideo(t, filepath.Join(second, "raw.mp4"))
if err := os.WriteFile(filepath.Join(second, "clip.info.json"), []byte(`{"id":"vid1","title":"New Title"}`), 0644); err != nil {
if err := os.WriteFile(filepath.Join(second, "clip.info.json"), []byte(`{"id":"vid1","title":"New Title"}`), 0o644); err != nil {
t.Fatal(err)
}
if err := svc.importItemDir(context.Background(), "https://example.com/v", second, libDir, "overwrite", ""); err != nil {
@@ -392,7 +392,7 @@ func TestImportItemDirWithoutOverwriteKeepsBoth(t *testing.T) {
for i := 0; i < 2; i++ {
src := t.TempDir()
makeTestVideo(t, filepath.Join(src, "raw.mp4"))
if err := os.WriteFile(filepath.Join(src, "clip.info.json"), []byte(`{"id":"vid1","title":"Same"}`), 0644); err != nil {
if err := os.WriteFile(filepath.Join(src, "clip.info.json"), []byte(`{"id":"vid1","title":"Same"}`), 0o644); err != nil {
t.Fatal(err)
}
if err := svc.importItemDir(context.Background(), "https://example.com/v", src, libDir, "", ""); err != nil {
@@ -427,7 +427,7 @@ func TestMoveFileCrossDeviceFallback(t *testing.T) {
src := filepath.Join(srcDir, "clip.mp4")
content := []byte("not really a video, but the bytes must survive")
if err := os.WriteFile(src, content, 0640); err != nil {
if err := os.WriteFile(src, content, 0o640); err != nil {
t.Fatal(err)
}
@@ -452,7 +452,7 @@ func TestMoveFileCrossDeviceFallback(t *testing.T) {
if err != nil {
t.Fatal(err)
}
if info.Mode().Perm() != 0640 {
if info.Mode().Perm() != 0o640 {
t.Errorf("destination mode = %v, want 0640", info.Mode().Perm())
}
if _, err := os.Stat(src); !os.IsNotExist(err) {
Minternal/service/library.go
@@ -17,8 +17,10 @@ import (
"vidarchive/internal/models"
)
const itemMarkerName = ".vidarchive-item.toml"
const subtitlesDirName = "subtitles"
const (
itemMarkerName = ".vidarchive-item.toml"
subtitlesDirName = "subtitles"
)
var mediaExts = map[string]struct{}{
".mp4": {}, ".webm": {}, ".mkv": {}, ".avi": {}, ".mov": {},
Minternal/service/library_test.go
@@ -23,17 +23,17 @@ func newLibrary(t *testing.T) (*LibraryService, string) {
func writeItem(t *testing.T, libraryDir, relPath, markerBody string, files map[string]string) string {
t.Helper()
itemDir := filepath.Join(libraryDir, filepath.FromSlash(relPath))
if err := os.MkdirAll(itemDir, 0755); err != nil {
if err := os.MkdirAll(itemDir, 0o755); err != nil {
t.Fatalf("mkdir item: %v", err)
}
if markerBody == "" {
markerBody = "duration = -1\n"
}
if err := os.WriteFile(filepath.Join(itemDir, itemMarkerName), []byte(markerBody), 0644); err != nil {
if err := os.WriteFile(filepath.Join(itemDir, itemMarkerName), []byte(markerBody), 0o644); err != nil {
t.Fatalf("write marker: %v", err)
}
for name, content := range files {
if err := os.WriteFile(filepath.Join(itemDir, name), []byte(content), 0644); err != nil {
if err := os.WriteFile(filepath.Join(itemDir, name), []byte(content), 0o644); err != nil {
t.Fatalf("write file %s: %v", name, err)
}
}
@@ -113,7 +113,7 @@ func TestPrimaryMediaFile(t *testing.T) {
dir := t.TempDir()
write := func(name string, n int) string {
p := filepath.Join(dir, name)
if err := os.WriteFile(p, make([]byte, n), 0644); err != nil {
if err := os.WriteFile(p, make([]byte, n), 0o644); err != nil {
t.Fatal(err)
}
return p
@@ -225,7 +225,7 @@ func TestGetAllSeparatesFoldersAndItems(t *testing.T) {
// A plain folder (no marker) containing a nested item
writeItem(t, dir, "folder/nested", "name = \"Nested\"\nduration = -1\n", map[string]string{"b.mp4": "v"})
// The reserved subtitles dir at root must be ignored
if err := os.MkdirAll(filepath.Join(dir, subtitlesDirName), 0755); err != nil {
if err := os.MkdirAll(filepath.Join(dir, subtitlesDirName), 0o755); err != nil {
t.Fatal(err)
}
Minternal/service/media_probe.go
@@ -1,15 +1,17 @@
package service
import (
"context"
"encoding/json"
"fmt"
"math"
"os"
"os/exec"
"strconv"
"strings"
"time"
"vidarchive/internal/models"
"vidarchive/internal/util"
)
// GetMetadata probes media details for the named file within an item. An empty
@@ -40,13 +42,21 @@ func (s *LibraryService) GetMetadata(relPath, filename string) (*MediaMetadata,
return s.probeMedia(target.Filepath)
}
const probeTimeout = 30 * time.Second
// probeStreams runs ffprobe once and returns the container format and every
// stream in it. It is the single ffprobe entry point: callers that only care
// about one stream kind filter the result themselves, rather than each
// re-declaring the same command and JSON shape.
func (s *LibraryService) probeStreams(path string) (ffprobeOutput, error) {
// Reading a local file's headers is quick. A longer run means ffprobe is
// stuck on a truncated or unreadable file, so cut it off rather than block
// the request that asked for it.
ctx, cancel := context.WithTimeout(context.Background(), probeTimeout)
defer cancel()
var probe ffprobeOutput
output, err := exec.Command(s.ffprobePath,
output, err := util.KillableCommand(ctx, s.ffprobePath,
"-v", "error",
"-show_format",
"-show_streams",
Minternal/service/preset.go
@@ -4,6 +4,7 @@ import (
"fmt"
"strconv"
"strings"
"vidarchive/internal/models"
"vidarchive/internal/repository"
)
Minternal/service/subscription_run.go
@@ -78,7 +78,7 @@ func (s *DownloadService) refreshAndAddNew(ctx context.Context, d *models.Downlo
// to add entries that don't exist in the library yet.
func (s *DownloadService) downloadFresh(ctx context.Context, d *models.Download, preset *models.Preset, urls []string, ytdlpFlags string) error {
tempDir := s.tempNewDirFor(d.ID)
if err := os.MkdirAll(tempDir, 0755); err != nil {
if err := os.MkdirAll(tempDir, 0o755); err != nil {
return err
}
defer os.RemoveAll(tempDir)
@@ -161,7 +161,7 @@ func restoreFileAtomically(path string, data []byte) error {
}
tmpPath := tmp.Name()
defer os.Remove(tmpPath)
if err := tmp.Chmod(0644); err != nil {
if err := tmp.Chmod(0o644); err != nil {
tmp.Close()
return err
}
@@ -231,7 +231,7 @@ func (s *DownloadService) applyMetadata(existing string, info infoJSON, sourceIn
return fmt.Errorf("create refreshed info JSON: %w", err)
}
stagedInfo = tmp.Name()
if err := tmp.Chmod(0644); err != nil {
if err := tmp.Chmod(0o644); err != nil {
tmp.Close()
return fmt.Errorf("set refreshed info JSON permissions: %w", err)
}
Minternal/service/subscription_run_test.go
@@ -16,7 +16,7 @@ video_id = "old"
"info.json": `{"id":"old","title":"Old","comments":[{"id":"c1","text":"archived"}],"heatmap":[{"start_time":0,"end_time":1,"value":1}],"filesize":123}`,
})
source := filepath.Join(e.scratch, "refreshed.info.json")
if err := os.WriteFile(source, []byte(`{"id":"new","title":"New","description":"updated","comments":[]}`), 0644); err != nil {
if err := os.WriteFile(source, []byte(`{"id":"new","title":"New","description":"updated","comments":[]}`), 0o644); err != nil {
t.Fatal(err)
}
@@ -41,7 +41,7 @@ video_id = "old"
if string(got["title"]) != `"New"` {
t.Errorf("title = %s, want New", got["title"])
}
if mode := fileMode(t, filepath.Join(existing, "info.json")); mode != 0644 {
if mode := fileMode(t, filepath.Join(existing, "info.json")); mode != 0o644 {
t.Errorf("info.json mode = %o, want 0644", mode)
}
}
@@ -59,11 +59,11 @@ video_id = "old"
// A directory at the destination makes the final rename fail after the
// marker has been written, exercising the rollback path.
infoPath := filepath.Join(existing, "info.json")
if err := os.Mkdir(infoPath, 0755); err != nil {
if err := os.Mkdir(infoPath, 0o755); err != nil {
t.Fatal(err)
}
source := filepath.Join(e.scratch, "rollback.info.json")
if err := os.WriteFile(source, []byte(`{"id":"new","title":"New"}`), 0644); err != nil {
if err := os.WriteFile(source, []byte(`{"id":"new","title":"New"}`), 0o644); err != nil {
t.Fatal(err)
}
Minternal/service/subscription_test.go
@@ -91,10 +91,10 @@ func TestSubscriptionDeleteRemovesArchive(t *testing.T) {
}
archive := svc.ArchivePath(sub.ID)
if err := os.MkdirAll(filepath.Dir(archive), 0755); err != nil {
if err := os.MkdirAll(filepath.Dir(archive), 0o755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(archive, []byte("youtube abc123\n"), 0644); err != nil {
if err := os.WriteFile(archive, []byte("youtube abc123\n"), 0o644); err != nil {
t.Fatal(err)
}
Minternal/service/subtitles.go
@@ -1,11 +1,12 @@
package service
import (
"context"
"fmt"
"os"
"os/exec"
"path/filepath"
"strings"
"time"
"vidarchive/internal/models"
"vidarchive/internal/util"
@@ -62,7 +63,7 @@ func (s *LibraryService) GetSubtitles(relPath string) ([]models.SubtitleTrack, e
if err != nil || len(streams) == 0 {
continue
}
if err := os.MkdirAll(cacheDir, 0755); err != nil {
if err := os.MkdirAll(cacheDir, 0o755); err != nil {
return nil, err
}
var tracks []models.SubtitleTrack
@@ -135,8 +136,15 @@ func selectSubtitleStreams(all []ffprobeStream) []subtitleStream {
return streams
}
// A subtitle track is small, but ffmpeg still walks the whole container to
// find it, so allow more than a probe and less than a download.
const subtitleExtractTimeout = 5 * time.Minute
func (s *LibraryService) extractSubtitleToVTT(inputPath, outputPath string, streamIndex int) error {
cmd := exec.Command(s.ffmpegPath,
ctx, cancel := context.WithTimeout(context.Background(), subtitleExtractTimeout)
defer cancel()
cmd := util.KillableCommand(ctx, s.ffmpegPath,
"-i", inputPath,
"-map", fmt.Sprintf("0:s:%d", streamIndex),
"-f", "webvtt",
Minternal/service/thumbnail.go
@@ -1,14 +1,15 @@
package service
import (
"context"
"fmt"
"log"
"os"
"os/exec"
"path/filepath"
"strings"
"sync"
"sync/atomic"
"time"
"vidarchive/internal/models"
"vidarchive/internal/util"
@@ -155,8 +156,10 @@ func (s *LibraryService) extractThumbnail(mf models.MediaFile) (string, error) {
attemptErrs = append(attemptErrs, fmt.Sprintf("attachment-dump: %v", err))
} else {
defer os.Remove(rawPath)
attempts = append(attempts, thumbAttempt{"attachment-webp", webpPath,
[]string{"-i", rawPath, "-c:v", "libwebp"}})
attempts = append(attempts, thumbAttempt{
"attachment-webp", webpPath,
[]string{"-i", rawPath, "-c:v", "libwebp"},
})
}
}
@@ -194,6 +197,10 @@ func (s *LibraryService) extractThumbnail(mf models.MediaFile) (string, error) {
return "", fmt.Errorf("all thumbnail extraction attempts failed:\n%s", strings.Join(attemptErrs, "\n"))
}
// Seeking one frame out of a file is fast. A longer run means ffmpeg is stuck,
// and a stuck thumbnail must not hold a worker or a request forever.
const thumbnailTimeout = 2 * time.Minute
// dumpAttachment extracts the raw bytes of attachment stream idx (e.g. the
// cover.jpg/cover.webp yt-dlp embeds into MKV with --embed-thumbnail) into a
// temp file and returns its path. The caller removes the file.
@@ -208,7 +215,9 @@ func (s *LibraryService) dumpAttachment(path string, idx int) (string, error) {
fmt.Sprintf("-dump_attachment:t:%d", idx), rawPath,
"-i", path, "-y", "-t", "0", "-f", "null", "-",
}
if out, err := exec.Command(s.ffmpegPath, dumpArgs...).CombinedOutput(); err != nil {
ctx, cancel := context.WithTimeout(context.Background(), thumbnailTimeout)
defer cancel()
if out, err := util.KillableCommand(ctx, s.ffmpegPath, dumpArgs...).CombinedOutput(); err != nil {
os.Remove(rawPath)
return "", fmt.Errorf("%v\n%s", err, out)
}
@@ -222,7 +231,9 @@ func (s *LibraryService) tryWriteThumbnail(outputPath string, args []string) (st
outExt := filepath.Ext(outputPath)
tmpPath := strings.TrimSuffix(outputPath, outExt) + ".tmp" + outExt
os.Remove(tmpPath)
cmd := exec.Command(s.ffmpegPath, append(args, tmpPath)...)
ctx, cancel := context.WithTimeout(context.Background(), thumbnailTimeout)
defer cancel()
cmd := util.KillableCommand(ctx, s.ffmpegPath, append(args, tmpPath)...)
if output, err := cmd.CombinedOutput(); err != nil {
os.Remove(tmpPath)
return "", fmt.Errorf("ffmpeg failed: %v\n%s", err, string(output))
Minternal/service/ytdlp_live_test.go
@@ -63,7 +63,7 @@ func liveEnv(t *testing.T) (*execEnv, string, string) {
func TestLiveListFormats(t *testing.T) {
e, videoURL, _ := liveEnv(t)
formats, err := e.svc.ListFormats(videoURL)
formats, err := e.svc.ListFormats(t.Context(), videoURL)
if err != nil {
t.Fatalf("ListFormats: %v", err)
}
Minternal/worker/pool_test.go
@@ -28,7 +28,7 @@ func newTestPool(t *testing.T) (pool *Pool, repo *repository.DownloadRepository,
FFprobePath: "ffprobe",
}
for _, dir := range []string{cfg.LibraryDir, cfg.TempDir} {
if err := os.MkdirAll(dir, 0755); err != nil {
if err := os.MkdirAll(dir, 0o755); err != nil {
t.Fatal(err)
}
}
@@ -37,7 +37,7 @@ func newTestPool(t *testing.T) (pool *Pool, repo *repository.DownloadRepository,
// interrupts a download rather than waiting one out.
cfg.YTDLPPath = filepath.Join(root, "yt-dlp")
script := "#!/bin/sh\ntouch " + filepath.Join(root, "started") + "\nsleep 300\n"
if err := os.WriteFile(cfg.YTDLPPath, []byte(script), 0755); err != nil {
if err := os.WriteFile(cfg.YTDLPPath, []byte(script), 0o755); err != nil {
t.Fatal(err)
}
Minternal/worker/scheduler_test.go
@@ -38,7 +38,7 @@ func newTestScheduler(t *testing.T) *schedEnv {
FFprobePath: "ffprobe",
}
for _, dir := range []string{cfg.LibraryDir, cfg.TempDir} {
if err := os.MkdirAll(dir, 0755); err != nil {
if err := os.MkdirAll(dir, 0o755); err != nil {
t.Fatal(err)
}
}