various fixes
Aassets.go
@@ -0,0 +1,18 @@
// Package vidarchive embeds the web assets (templates and static files) into the
// binary so it runs correctly regardless of the working directory or where the
// binary is installed — no reliance on locating the source tree at runtime.
package vidarchive
import "embed"
// TemplatesFS holds the HTML templates under web/templates.
//
//go:embed web/templates/*.html
var TemplatesFS embed.FS
// StaticFS holds the static assets (CSS, icons) under web/static. The all: prefix
// ensures files that would otherwise be skipped (none currently, but future
// dotfiles) are still embedded.
//
//go:embed all:web/static
var StaticFS embed.FS
Minternal/handler/handler.go
@@ -7,13 +7,13 @@ import (
"net/http"
"net/url"
"path/filepath"
"runtime"
"strconv"
"strings"
"github.com/gabriel-vasile/mimetype"
"github.com/go-chi/chi/v5"
"vidarchive"
"vidarchive/internal/config"
"vidarchive/internal/models"
"vidarchive/internal/service"
@@ -74,23 +74,10 @@ func loadTemplates() (*template.Template, error) {
},
})
if repoRoot == "" {
return nil, fmt.Errorf("failed to determine template path")
}
return tmpl.ParseGlob(filepath.Join(repoRoot, "web", "templates", "*.html"))
return tmpl.ParseFS(vidarchive.TemplatesFS, "web/templates/*.html")
}
var (
presetSvcGlobal *service.PresetService
repoRoot string
)
func init() {
_, file, _, ok := runtime.Caller(0)
if ok {
repoRoot = filepath.Join(filepath.Dir(file), "..", "..")
}
}
var presetSvcGlobal *service.PresetService
func formatDuration(seconds int) string {
if seconds <= 0 {
@@ -225,8 +212,10 @@ func (h *Handler) Library(w http.ResponseWriter, r *http.Request) {
}
func normalizeRelPath(r *http.Request) string {
// chi gives the raw, still-encoded wildcard. Decode it as a URL path, where
// '+' is a literal plus (only query strings treat '+' as space) — so an item
// directory named "a+b" round-trips correctly.
relPath := chi.URLParam(r, "*")
relPath = strings.ReplaceAll(relPath, "+", " ")
relPath = strings.Trim(relPath, "/")
relPath, _ = url.PathUnescape(relPath)
return relPath
@@ -323,7 +312,13 @@ func (h *Handler) serveThumbnail(relPath string, w http.ResponseWriter, r *http.
icon = "audio-icon.svg"
}
}
http.ServeFile(w, r, filepath.Join(repoRoot, "web", "static", "icons", icon))
data, err := vidarchive.StaticFS.ReadFile("web/static/icons/" + icon)
if err != nil {
http.Error(w, "icon not found", http.StatusInternalServerError)
return
}
w.Header().Set("Content-Type", "image/svg+xml")
w.Write(data)
}
// isAudioFile reports whether the named file (or, if unnamed, the first media
Minternal/models/models.go
@@ -84,7 +84,6 @@ type SubtitleTrack struct {
type ItemMetadata struct {
Name string `toml:"name"`
SourceURL string `toml:"source_url"`
Duration int `toml:"duration"`
Description string `toml:"description"`
FileDurations map[string]int `toml:"file_durations"`
}
Minternal/repository/download.go
@@ -98,11 +98,6 @@ func (r *DownloadRepository) UpdateStatus(id int64, status string) error {
return err
}
func (r *DownloadRepository) UpdateLogs(id int64, logs string) error {
_, err := r.db.Exec(`UPDATE downloads SET logs = ? WHERE id = ?`, logs, id)
return err
}
func (r *DownloadRepository) AppendLogs(id int64, logs string) error {
_, err := r.db.Exec(
`UPDATE downloads SET logs = COALESCE(logs, '') || ? WHERE id = ?`,
@@ -111,12 +106,22 @@ func (r *DownloadRepository) AppendLogs(id int64, logs string) error {
return err
}
func (r *DownloadRepository) MarkStarted(id int64) error {
_, err := r.db.Exec(
`UPDATE downloads SET status = 'downloading', started_at = CURRENT_TIMESTAMP WHERE id = ?`,
// MarkStarted atomically transitions a download from 'queued' to 'downloading'.
// It reports whether this call actually claimed it: false means another worker
// already started it, so the caller must not process it again.
func (r *DownloadRepository) MarkStarted(id int64) (bool, error) {
res, err := r.db.Exec(
`UPDATE downloads SET status = 'downloading', started_at = CURRENT_TIMESTAMP WHERE id = ? AND status = 'queued'`,
id,
)
return err
if err != nil {
return false, err
}
n, err := res.RowsAffected()
if err != nil {
return false, err
}
return n > 0, nil
}
func (r *DownloadRepository) MarkCompleted(id int64, status string) error {
Ainternal/repository/download_test.go
@@ -0,0 +1,37 @@
package repository
import (
"testing"
"vidarchive/internal/models"
)
// MarkStarted must claim a download exactly once: the worker pool's Submit and
// its 2s queue checker can both enqueue the same queued download, and only one
// worker may actually process it.
func TestMarkStartedClaimsOnce(t *testing.T) {
db := setupTestDB(t)
defer db.Close()
repo := NewDownloadRepository(db)
d := &models.Download{URL: "https://example.com/v", Status: "queued"}
if err := repo.Create(d); err != nil {
t.Fatalf("create: %v", err)
}
claimed, err := repo.MarkStarted(d.ID)
if err != nil {
t.Fatalf("first MarkStarted: %v", err)
}
if !claimed {
t.Fatal("first MarkStarted should claim the queued download")
}
claimed, err = repo.MarkStarted(d.ID)
if err != nil {
t.Fatalf("second MarkStarted: %v", err)
}
if claimed {
t.Error("second MarkStarted should not re-claim a download that is already downloading")
}
}
Minternal/server/server.go
@@ -2,16 +2,15 @@ package server
import (
"fmt"
"io/fs"
"net/http"
"path/filepath"
"runtime"
"github.com/go-chi/chi/v5"
"github.com/go-chi/chi/v5/middleware"
"vidarchive"
"vidarchive/internal/config"
"vidarchive/internal/handler"
"vidarchive/internal/service"
)
type Server struct {
@@ -31,19 +30,14 @@ func New(cfg *config.Config, h *handler.Handler) *Server {
}
func (s *Server) setupRoutes() {
_, file, _, ok := runtime.Caller(0)
staticRoot := "web/static"
if ok {
root := filepath.Join(filepath.Dir(file), "..", "..")
staticRoot = filepath.Join(root, "web", "static")
}
s.router.Use(middleware.Logger)
s.router.Use(middleware.Recoverer)
s.router.Use(s.securityHeaders)
s.router.Use(itemCacheMiddleware)
s.router.Handle("/static/*", http.StripPrefix("/static/", http.FileServer(http.Dir(staticRoot))))
// Serve embedded static assets (fs.Sub strips the web/static prefix). The
// error is only possible for an invalid constant path, so it can't occur here.
staticFS, _ := fs.Sub(vidarchive.StaticFS, "web/static")
s.router.Handle("/static/*", http.StripPrefix("/static/", http.FileServer(http.FS(staticFS))))
s.router.Get("/media/item/*", s.handler.ServeMediaItem)
@@ -73,14 +67,6 @@ func (s *Server) setupRoutes() {
s.router.Get("/api/presets/{id}/flags", s.handler.GetPresetFlags)
}
// itemCacheMiddleware attaches a fresh per-request library item cache so a
// single request that looks up the same item multiple times only scans it once.
func itemCacheMiddleware(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
next.ServeHTTP(w, r.WithContext(service.WithItemCache(r.Context())))
})
}
func (s *Server) securityHeaders(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("X-Content-Type-Options", "nosniff")
Minternal/server/server_test.go
@@ -89,11 +89,30 @@ func TestNestedLibraryItem(t *testing.T) {
router := srv.Router()
tests := []string{
"/library/item/test/My%20Item%20%5Bid%5D",
"/library/item/test/My+Item+%5Bid%5D",
req := httptest.NewRequest("GET", "/library/item/test/My%20Item%20%5Bid%5D", nil)
w := httptest.NewRecorder()
router.ServeHTTP(w, req)
if w.Code != http.StatusOK {
body, _ := io.ReadAll(w.Body)
t.Errorf("expected 200, got %d: %s", w.Code, string(body))
}
for _, path := range tests {
}
// A directory whose name contains a literal '+' must round-trip: in a URL path
// '+' is a literal plus (not a space), reachable raw or percent-encoded.
func TestLiteralPlusInPathSegment(t *testing.T) {
srv, cfg, cleanup := setupTestServer(t)
defer cleanup()
createItem(t, cfg.LibraryDir, "C++ Tutorial", "C++ Tutorial", map[string]string{
"C++ Tutorial.mp4": "dummy video",
})
router := srv.Router()
for _, path := range []string{
"/library/item/C++%20Tutorial",
"/library/item/C%2B%2B%20Tutorial",
} {
req := httptest.NewRequest("GET", path, nil)
w := httptest.NewRecorder()
router.ServeHTTP(w, req)
Minternal/service/download.go
@@ -9,7 +9,9 @@ import (
"os"
"os/exec"
"path/filepath"
"regexp"
"sort"
"strconv"
"strings"
"sync"
"syscall"
@@ -66,18 +68,14 @@ func (s *DownloadService) Create(url string, presetID *int64, formatOverride, cu
}
func (s *DownloadService) GetByID(id int64) (*models.Download, error) {
if live, ok := s.cache.Get(id); ok {
d, err := s.repo.GetByID(id)
if err != nil {
return nil, err
}
logs := live.Logs.String()
if logs != "" {
d.Logs = sql.NullString{String: logs, Valid: true}
}
return d, nil
d, err := s.repo.GetByID(id)
if err != nil {
return nil, err
}
if logs := s.cache.Snapshot(id); logs != "" {
d.Logs = sql.NullString{String: logs, Valid: true}
}
return s.repo.GetByID(id)
return d, nil
}
func (s *DownloadService) GetAll(status, sortBy string) ([]*models.Download, error) {
@@ -87,11 +85,8 @@ func (s *DownloadService) GetAll(status, sortBy string) ([]*models.Download, err
}
for _, d := range downloads {
if live, ok := s.cache.Get(d.ID); ok {
logs := live.Logs.String()
if logs != "" {
d.Logs = sql.NullString{String: logs, Valid: true}
}
if logs := s.cache.Snapshot(d.ID); logs != "" {
d.Logs = sql.NullString{String: logs, Valid: true}
}
}
@@ -139,16 +134,25 @@ func (s *DownloadService) ListFormats(url string) ([]*models.FormatInfo, error)
return parseFormatList(string(output)), nil
}
func (s *DownloadService) ExecuteDownload(d *models.Download) error {
if err := s.repo.MarkStarted(d.ID); err != nil {
return err
// ExecuteDownload runs the download for d. The bool reports whether this call
// actually processed it: false means another worker already claimed it (Submit
// and the queue checker can both enqueue the same row within the 2s poll window),
// so the caller should not log it as completed.
func (s *DownloadService) ExecuteDownload(d *models.Download) (bool, error) {
// Atomically claim the download. If it's no longer queued, another worker
// already took it — bail rather than download it twice.
claimed, err := s.repo.MarkStarted(d.ID)
if err != nil {
return false, err
}
if !claimed {
return false, nil
}
s.cache.Set(d.ID, &LiveDownload{LastUpdate: time.Now()})
defer s.cache.Delete(d.ID)
var preset *models.Preset
var err error
if d.PresetID.Valid {
preset, err = s.presetSvc.GetByID(d.PresetID.Int64)
@@ -165,7 +169,7 @@ func (s *DownloadService) ExecuteDownload(d *models.Download) error {
tempDownloadDir := filepath.Join(s.cfg.TempDir, fmt.Sprintf("%d", d.ID))
if err := os.MkdirAll(tempDownloadDir, 0755); err != nil {
return fmt.Errorf("create temp download dir: %w", err)
return false, fmt.Errorf("create temp download dir: %w", err)
}
args := s.presetSvc.BuildArgs(preset, d.FormatOverride, d.CustomFlags)
@@ -191,13 +195,13 @@ func (s *DownloadService) ExecuteDownload(d *models.Download) error {
stdout, err := cmd.StdoutPipe()
if err != nil {
s.finalizeError(d.ID, err)
return err
return false, err
}
cmd.Stderr = cmd.Stdout
if err := cmd.Start(); err != nil {
s.finalizeError(d.ID, err)
return err
return false, err
}
s.processMu.Lock()
@@ -220,7 +224,7 @@ func (s *DownloadService) ExecuteDownload(d *models.Download) error {
case <-ticker.C:
logs := s.cache.FlushLogs(d.ID)
if logs != "" {
s.repo.UpdateLogs(d.ID, logs)
s.repo.AppendLogs(d.ID, logs)
}
case <-done:
return
@@ -237,29 +241,29 @@ func (s *DownloadService) ExecuteDownload(d *models.Download) error {
logs := s.cache.FlushLogs(d.ID)
if logs != "" {
s.repo.UpdateLogs(d.ID, logs)
s.repo.AppendLogs(d.ID, logs)
}
if err := cmd.Wait(); err != nil {
s.finalizeError(d.ID, err)
return err
return false, err
}
if err := s.repo.MarkCompleted(d.ID, "completed"); err != nil {
return err
return false, err
}
if err := s.importDownloadedItems(d, tempDownloadDir); err != nil {
log.Printf("Download %d completed but import failed: %v", d.ID, err)
}
return nil
return true, nil
}
func (s *DownloadService) finalizeError(id int64, err error) {
logs := s.cache.FlushLogs(id)
if logs != "" {
s.repo.UpdateLogs(id, logs)
s.repo.AppendLogs(id, logs)
}
s.repo.MarkError(id, err.Error())
}
@@ -365,6 +369,16 @@ func (s *DownloadService) importItemDir(url, itemDir, baseLibraryDir string) err
}
}
// Probe each media file's duration once, here in the worker (off the request
// path), and cache it in the marker so the library never has to probe while
// serving pages. Files we can't probe simply get no duration.
fileDurations := make(map[string]int)
for _, entry := range mediaFiles {
if d, ok := probeDuration(filepath.Join(targetDir, entry.Name())); ok {
fileDurations[entry.Name()] = d
}
}
if len(subtitleFiles) > 0 {
subtitlesDir := filepath.Join(targetDir, subtitlesDirName)
if err := os.MkdirAll(subtitlesDir, 0755); err != nil {
@@ -378,9 +392,9 @@ func (s *DownloadService) importItemDir(url, itemDir, baseLibraryDir string) err
}
metadata := models.ItemMetadata{
Name: name,
SourceURL: url,
Duration: -1,
Name: name,
SourceURL: url,
FileDurations: fileDurations,
}
markerPath := filepath.Join(targetDir, itemMarkerName)
@@ -456,6 +470,25 @@ func sanitizeDirName(name string) string {
return name
}
// resolutionRe matches a yt-dlp resolution column like "1920x1080".
var resolutionRe = regexp.MustCompile(`^\d+x\d+$`)
// probeDuration returns the duration of a media file in whole seconds. The bool
// is false when ffprobe is unavailable or the file has no usable duration.
func probeDuration(path string) (int, bool) {
out, err := exec.Command("ffprobe", "-v", "error",
"-show_entries", "format=duration",
"-of", "default=nw=1:nk=1", path).Output()
if err != nil {
return 0, false
}
f, err := strconv.ParseFloat(strings.TrimSpace(string(out)), 64)
if err != nil || f <= 0 {
return 0, false
}
return int(f + 0.5), true
}
func parseFormatList(output string) []*models.FormatInfo {
lines := strings.Split(output, "\n")
var formats []*models.FormatInfo
@@ -484,7 +517,9 @@ func parseFormatList(output string) []*models.FormatInfo {
}
for i, part := range parts {
if strings.Contains(part, "x") && !strings.Contains(part, "http") {
// A resolution token is strictly <digits>x<digits> (e.g. 1920x1080);
// matching on a literal "x" anywhere misclassified notes/codecs.
if resolutionRe.MatchString(part) {
format.Resolution = part
if i+1 < len(parts) {
format.FPS = parts[i+1]
Minternal/service/download_test.go
@@ -99,7 +99,7 @@ func TestParseFormatList(t *testing.T) {
ID EXT RESOLUTION FPS
18 mp4 640x360 30
137 mp4 1920x1080 60
140 m4a audio only
233 m4a audio_only_xtra 128k
`
formats := parseFormatList(output)
if len(formats) != 3 {
@@ -111,6 +111,11 @@ ID EXT RESOLUTION FPS
if formats[1].Resolution != "1920x1080" {
t.Errorf("format[1] resolution = %q, want 1920x1080", formats[1].Resolution)
}
// A non-resolution token containing the letter 'x' must NOT be parsed as a
// resolution (the old "contains x" heuristic misclassified it).
if formats[2].Resolution != "" {
t.Errorf("format[2] resolution = %q, want empty (token has 'x' but isn't WxH)", formats[2].Resolution)
}
}
// TestImportItemDir exercises the full import: media + info.json + subtitles get
@@ -149,11 +154,15 @@ func TestImportItemDir(t *testing.T) {
t.Errorf("subtitle not moved into subtitles dir: %v", err)
}
// The marker should record the source URL.
// The marker should record the source URL and a probed per-file duration
// (the test video is 3s), so the library never has to probe at serve time.
markerData, _ := os.ReadFile(filepath.Join(targetDir, itemMarkerName))
if want := "https://example.com/v"; !strings.Contains(string(markerData), want) {
t.Errorf("marker missing source url %q: %s", want, markerData)
}
if !strings.Contains(string(markerData), "file_durations") || !strings.Contains(string(markerData), "raw.mp4") {
t.Errorf("marker missing probed per-file duration: %s", markerData)
}
}
func TestImportDownloadedItemsRejectsOutputTraversal(t *testing.T) {
Minternal/service/library.go
@@ -15,6 +15,7 @@ import (
"strings"
"sync"
"sync/atomic"
"time"
"github.com/BurntSushi/toml"
@@ -60,40 +61,73 @@ type LibraryService struct {
extractInFlight int32
extractMaxConcurrent int32
extractAttempts int32
// scanCache memoizes scanned items for a short TTL so the listing page (which
// fans out one thumbnail request per media file) and quick auto-refreshes
// don't re-parse each item's marker + info.json on every request. Only item
// scans are cached — the directory listing itself is always read fresh, so
// newly added/removed items and subfolders appear immediately.
scanMu sync.Mutex
scanCache map[string]scanCacheEntry
scanTTL time.Duration
}
type scanCacheEntry struct {
item *models.LibraryItem
at time.Time
}
// scanCacheTTL is how long a scanned item is reused before being re-read.
const scanCacheTTL = 10 * time.Second
func NewLibraryService(libraryDir string) *LibraryService {
return &LibraryService{
libraryDir: libraryDir,
thumbSem: make(chan struct{}, maxConcurrentThumbnails),
scanCache: make(map[string]scanCacheEntry),
scanTTL: scanCacheTTL,
}
}
// itemCache memoizes scanned items within the scope of a single request, so
// repeated lookups of the same item (e.g. the detail page calling GetByRelPath,
// GetMetadata and GetSubtitles) don't each rescan the item directory.
type itemCache struct {
mu sync.Mutex
items map[string]*models.LibraryItem
func (s *LibraryService) getCachedScan(relPath string) (*models.LibraryItem, bool) {
if s.scanTTL <= 0 {
return nil, false
}
s.scanMu.Lock()
defer s.scanMu.Unlock()
e, ok := s.scanCache[relPath]
if !ok || time.Since(e.at) > s.scanTTL {
return nil, false
}
return e.item, true
}
type itemCacheKeyType struct{}
var itemCacheKey itemCacheKeyType
func (s *LibraryService) putCachedScan(relPath string, item *models.LibraryItem) {
if s.scanTTL <= 0 {
return
}
s.scanMu.Lock()
s.scanCache[relPath] = scanCacheEntry{item: item, at: time.Now()}
s.scanMu.Unlock()
}
// WithItemCache returns a context carrying a fresh per-request item cache.
// Lookups made with the returned context reuse already-scanned items; a context
// without a cache (e.g. context.Background()) always rescans.
func WithItemCache(ctx context.Context) context.Context {
return context.WithValue(ctx, itemCacheKey, &itemCache{items: make(map[string]*models.LibraryItem)})
func (s *LibraryService) evictCachedScan(relPath string) {
s.scanMu.Lock()
delete(s.scanCache, relPath)
s.scanMu.Unlock()
}
func cacheFromCtx(ctx context.Context) *itemCache {
if ctx == nil {
return nil
// scannedItem returns a cached scan if fresh, otherwise scans and caches it.
func (s *LibraryService) scannedItem(itemDir, relPath string) (*models.LibraryItem, error) {
if item, ok := s.getCachedScan(relPath); ok {
return item, nil
}
c, _ := ctx.Value(itemCacheKey).(*itemCache)
return c
item, err := s.scanItem(itemDir, relPath)
if err != nil {
return nil, err
}
s.putCachedScan(relPath, item)
return item, nil
}
func (s *LibraryService) resolveItemDir(relPath string) (string, error) {
@@ -113,7 +147,9 @@ func (s *LibraryService) resolveItemDir(relPath string) (string, error) {
if !strings.HasPrefix(cleanDir, base+string(filepath.Separator)) && cleanDir != base {
return "", fmt.Errorf("invalid path")
}
return itemDir, nil
// Return the cleaned/symlink-resolved path we just validated, so callers do
// I/O on exactly the path that passed the boundary check.
return cleanDir, nil
}
func (s *LibraryService) GetAll(path, sortBy, filter string) ([]*models.LibraryItem, []string, error) {
@@ -149,7 +185,7 @@ func (s *LibraryService) GetAll(path, sortBy, filter string) ([]*models.LibraryI
markerPath := filepath.Join(itemDir, itemMarkerName)
if _, err := os.Stat(markerPath); err == nil {
item, err := s.scanItem(itemDir, relPath)
item, err := s.scannedItem(itemDir, relPath)
if err != nil {
log.Printf("warning: failed to scan item %s: %v", relPath, err)
continue
@@ -171,13 +207,15 @@ func (s *LibraryService) GetAll(path, sortBy, filter string) ([]*models.LibraryI
case "date":
fallthrough
default:
sort.Slice(items, func(i, j int) bool {
ii, _ := os.Stat(filepath.Join(s.libraryDir, items[i].RelPath))
jj, _ := os.Stat(filepath.Join(s.libraryDir, items[j].RelPath))
if ii == nil || jj == nil {
return false
// Stat each item once up front rather than twice per comparison.
modTime := make(map[string]int64, len(items))
for _, it := range items {
if info, err := os.Stat(it.DirPath); err == nil {
modTime[it.RelPath] = info.ModTime().UnixNano()
}
return ii.ModTime().After(jj.ModTime())
}
sort.Slice(items, func(i, j int) bool {
return modTime[items[i].RelPath] > modTime[items[j].RelPath]
})
}
@@ -186,17 +224,14 @@ func (s *LibraryService) GetAll(path, sortBy, filter string) ([]*models.LibraryI
return items, folders, nil
}
// GetByRelPath returns the item at relPath, reusing a recent cached scan when
// available (see scanCache). The context is accepted for cancellation/threading
// but item caching is keyed on relPath at the service level.
func (s *LibraryService) GetByRelPath(ctx context.Context, relPath string) (*models.LibraryItem, error) {
relPath = strings.Trim(relPath, "/")
cache := cacheFromCtx(ctx)
if cache != nil {
cache.mu.Lock()
item, ok := cache.items[relPath]
cache.mu.Unlock()
if ok {
return item, nil
}
if item, ok := s.getCachedScan(relPath); ok {
return item, nil
}
itemDir, err := s.resolveItemDir(relPath)
@@ -211,12 +246,7 @@ func (s *LibraryService) GetByRelPath(ctx context.Context, relPath string) (*mod
if err != nil {
return nil, err
}
if cache != nil {
cache.mu.Lock()
cache.items[relPath] = item
cache.mu.Unlock()
}
s.putCachedScan(relPath, item)
return item, nil
}
@@ -275,30 +305,32 @@ func (s *LibraryService) scanItem(itemDir, relPath string) (*models.LibraryItem,
metadata.FileDurations = make(map[string]int)
}
if metadata.Duration < 0 {
if duration, ok := infoDuration(info); ok && duration > 0 {
metadata.Duration = duration
dirty = true
}
}
// Per-file durations come from the marker's file_durations map (populated at
// import time). For a single-file item we also seed it from info.json's
// duration, which covers the common case without a probe. We deliberately do
// not run ffprobe here — keeping it off the scan/listing path is the point of
// the marker cache. Files without a known duration simply show no badge.
for i := range mediaFiles {
mf := &mediaFiles[i]
if mf.Duration < 0 {
if d, ok := metadata.FileDurations[mf.Filename]; ok && d > 0 {
if d, ok := metadata.FileDurations[mf.Filename]; ok {
mf.Duration = d
continue
}
if len(mediaFiles) == 1 {
if d, ok := infoDuration(info); ok && d > 0 {
mf.Duration = d
} else if metadata.Duration > 0 {
mf.Duration = metadata.Duration
metadata.FileDurations[mf.Filename] = mf.Duration
metadata.FileDurations[mf.Filename] = d
dirty = true
}
}
}
if metadata.Duration < 0 {
if d := commonDuration(mediaFiles); d >= 0 {
metadata.Duration = d
dirty = true
// The item-level duration is the sum of known per-file durations, used only
// for the "duration" sort — there is no single "overall" duration shown.
total := 0
for _, mf := range mediaFiles {
if mf.Duration > 0 {
total += mf.Duration
}
}
@@ -307,7 +339,7 @@ func (s *LibraryService) scanItem(itemDir, relPath string) (*models.LibraryItem,
RelPath: relPath,
DirPath: itemDir,
SourceURL: metadata.SourceURL,
Duration: metadata.Duration,
Duration: total,
Description: metadata.Description,
MediaFiles: mediaFiles,
}
@@ -328,38 +360,19 @@ func (s *LibraryService) readOrCreateMetadata(itemDir string) (models.ItemMetada
log.Printf("warning: failed to parse %s: %v", markerPath, err)
}
}
if metadata.Duration == 0 && !hasField(data, "duration") {
metadata.Duration = -1
}
return metadata, nil
}
// hasField reports whether the TOML data declares the given top-level key. It
// matches the key only at the start of a line (after optional whitespace) so a
// value that happens to contain "<key> =" — e.g. a description mentioning
// "duration = 5" — does not produce a false positive.
func hasField(data []byte, key string) bool {
for _, line := range strings.Split(string(data), "\n") {
line = strings.TrimSpace(line)
if !strings.HasPrefix(line, key) {
continue
}
if strings.HasPrefix(strings.TrimSpace(line[len(key):]), "=") {
return true
}
}
return false
}
func (s *LibraryService) writeMetadata(itemDir string, metadata models.ItemMetadata) error {
markerPath := filepath.Join(itemDir, itemMarkerName)
// Write to a temp file and rename so a crash mid-encode can't leave a
// truncated marker behind.
tmpPath := markerPath + ".tmp"
f, err := os.Create(tmpPath)
// Write to a uniquely-named temp file and rename so a crash mid-encode can't
// leave a truncated marker, and two concurrent writers never collide on a
// shared temp path.
f, err := os.CreateTemp(itemDir, ".vidarchive-item-*.tmp")
if err != nil {
return err
}
tmpPath := f.Name()
if err := toml.NewEncoder(f).Encode(metadata); err != nil {
f.Close()
os.Remove(tmpPath)
@@ -470,22 +483,6 @@ func mediaFileStem(mf models.MediaFile) string {
return strings.TrimSuffix(filepath.Base(mf.Filename), filepath.Ext(mf.Filename))
}
func commonDuration(files []models.MediaFile) int {
if len(files) == 0 {
return -1
}
d := files[0].Duration
if d < 0 {
return -1
}
for _, f := range files[1:] {
if f.Duration != d {
return -1
}
}
return d
}
// primaryMediaFile picks the representative file for an item: the largest video
// file, or — if there are none — the largest file overall. Returns nil for an
// item with no media files. Used for the item-level thumbnail and as the default
@@ -808,6 +805,9 @@ func (s *LibraryService) Delete(relPath string) error {
return fmt.Errorf("refusing to delete library root")
}
}
// Evict the cached scan so the deletion is reflected immediately rather than
// lingering until the TTL expires.
s.evictCachedScan(strings.Trim(relPath, "/"))
return os.RemoveAll(itemDir)
}
Minternal/service/library_test.go
@@ -134,34 +134,6 @@ func TestInfoDuration(t *testing.T) {
}
}
func TestCommonDuration(t *testing.T) {
mk := func(ds ...int) []models.MediaFile {
var fs []models.MediaFile
for _, d := range ds {
fs = append(fs, models.MediaFile{Duration: d})
}
return fs
}
tests := []struct {
name string
files []models.MediaFile
want int
}{
{"empty", nil, -1},
{"single", mk(100), 100},
{"all equal", mk(100, 100, 100), 100},
{"mismatch", mk(100, 90), -1},
{"unknown first", mk(-1, 100), -1},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
if got := commonDuration(tc.files); got != tc.want {
t.Errorf("commonDuration = %d, want %d", got, tc.want)
}
})
}
}
func TestUrlEncodePath(t *testing.T) {
tests := []struct{ in, want string }{
{"a/b c", "a/b%20c"},
@@ -209,27 +181,6 @@ func TestInfoString(t *testing.T) {
}
}
func TestHasField(t *testing.T) {
tests := []struct {
data string
key string
want bool
}{
{"duration = 10\n", "duration", true},
{"duration=10\n", "duration", true},
{" duration = 10\n", "duration", true},
{"name = \"x\"\nduration = 5\n", "duration", true},
{"description = \"mentions duration = 5 inline\"\n", "duration", false},
{"name = \"x\"\n", "duration", false},
{"", "duration", false},
}
for _, tc := range tests {
if got := hasField([]byte(tc.data), tc.key); got != tc.want {
t.Errorf("hasField(%q, %q) = %v, want %v", tc.data, tc.key, got, tc.want)
}
}
}
func TestPrimaryMediaFile(t *testing.T) {
dir := t.TempDir()
write := func(name string, n int) string {
@@ -440,9 +391,35 @@ func TestScanItemDerivesMetadataFromInfoJSON(t *testing.T) {
}
}
func TestScanItemUsesFileDurations(t *testing.T) {
svc, dir := newLibrary(t)
marker := "name = \"M\"\n\n[file_durations]\n \"a.mp4\" = 30\n \"b.mp4\" = 12\n"
writeItem(t, dir, "multi", marker, map[string]string{"a.mp4": "v", "b.mp4": "v"})
item, err := svc.GetByRelPath(context.Background(), "multi")
if err != nil {
t.Fatalf("GetByRelPath: %v", err)
}
got := map[string]int{}
for _, mf := range item.MediaFiles {
got[mf.Filename] = mf.Duration
}
if got["a.mp4"] != 30 || got["b.mp4"] != 12 {
t.Errorf("per-file durations = %v, want a=30 b=12", got)
}
// The item duration is the sum of known per-file durations (used for sorting);
// there is no single "overall" duration any more.
if item.Duration != 42 {
t.Errorf("item Duration = %d, want 42 (sum of file durations)", item.Duration)
}
}
func TestScanItemDoesNotRewriteConvergedMarker(t *testing.T) {
svc, dir := newLibrary(t)
marker := "name = \"Done\"\nsource_url = \"u\"\nduration = 10\ndescription = \"d\"\n"
// Disable the scan cache so every read actually rescans — this exercises
// scanItem's own idempotency rather than just a cache hit.
svc.scanTTL = 0
marker := "name = \"Done\"\nsource_url = \"u\"\ndescription = \"d\"\n"
itemDir := writeItem(t, dir, "y", marker, map[string]string{"y.mp4": "v"})
markerPath := filepath.Join(itemDir, itemMarkerName)
@@ -473,33 +450,41 @@ func TestScanItemDoesNotRewriteConvergedMarker(t *testing.T) {
}
}
// --- per-request item cache ---
// --- scan cache ---
func TestItemCacheDedupesScans(t *testing.T) {
func TestScanCacheReusesWithinTTL(t *testing.T) {
svc, dir := newLibrary(t)
writeItem(t, dir, "x", "name = \"X\"\nsource_url = \"u\"\nduration = 5\n", map[string]string{"x.mp4": "v"})
writeItem(t, dir, "x", "name = \"X\"\nsource_url = \"u\"\n", map[string]string{"x.mp4": "v"})
// Within one cached context, repeated lookups return the same scanned item.
ctx := WithItemCache(context.Background())
a, err := svc.GetByRelPath(ctx, "x")
// Within the TTL, repeated lookups return the same cached scan (no rescan).
a, err := svc.GetByRelPath(context.Background(), "x")
if err != nil {
t.Fatalf("first lookup: %v", err)
}
b, err := svc.GetByRelPath(ctx, "x")
b, err := svc.GetByRelPath(context.Background(), "x")
if err != nil {
t.Fatalf("second lookup: %v", err)
}
if a != b {
t.Error("cached context should return the same item pointer (no rescan)")
t.Error("expected the cached scan to be reused (same pointer) within the TTL")
}
// A context without a cache rescans, yielding a distinct item.
c, err := svc.GetByRelPath(context.Background(), "x")
if err != nil {
t.Fatalf("uncached lookup: %v", err)
// Deleting the item evicts its cached scan immediately.
if err := svc.Delete("x"); err != nil {
t.Fatalf("Delete: %v", err)
}
if c == a {
t.Error("uncached context should rescan, not reuse the cached pointer")
if _, ok := svc.getCachedScan("x"); ok {
t.Error("Delete should evict the cached scan")
}
// With caching disabled, each lookup rescans into a distinct item.
svc2, dir2 := newLibrary(t)
svc2.scanTTL = 0
writeItem(t, dir2, "y", "name = \"Y\"\n", map[string]string{"y.mp4": "v"})
c, _ := svc2.GetByRelPath(context.Background(), "y")
d, _ := svc2.GetByRelPath(context.Background(), "y")
if c == nil || d == nil || c == d {
t.Error("with TTL disabled each lookup should rescan, not reuse a pointer")
}
}
Minternal/service/progress_cache.go
@@ -9,6 +9,7 @@ import (
type LiveDownload struct {
Logs strings.Builder
LastUpdate time.Time
flushed int // length of Logs already persisted to the DB
}
type ProgressCache struct {
@@ -28,13 +29,6 @@ func (c *ProgressCache) Set(id int64, d *LiveDownload) {
c.data[id] = d
}
func (c *ProgressCache) Get(id int64) (*LiveDownload, bool) {
c.mu.RLock()
defer c.mu.RUnlock()
d, ok := c.data[id]
return d, ok
}
func (c *ProgressCache) AppendLog(id int64, line string) {
c.mu.Lock()
defer c.mu.Unlock()
@@ -45,7 +39,10 @@ func (c *ProgressCache) AppendLog(id int64, line string) {
}
}
func (c *ProgressCache) GetLogs(id int64) string {
// Snapshot returns the full in-memory log buffer for a live download, read under
// the lock so it never races the writer in AppendLog (strings.Builder is not
// safe for concurrent read/write). Empty string if the download isn't live.
func (c *ProgressCache) Snapshot(id int64) string {
c.mu.RLock()
defer c.mu.RUnlock()
if d, ok := c.data[id]; ok {
@@ -60,12 +57,21 @@ func (c *ProgressCache) Delete(id int64) {
delete(c.data, id)
}
// FlushLogs returns only the log content appended since the last flush and
// advances the flushed mark, so the caller can append (rather than rewrite the
// whole buffer) to the DB. Empty string when there's nothing new.
func (c *ProgressCache) FlushLogs(id int64) string {
c.mu.Lock()
defer c.mu.Unlock()
if d, ok := c.data[id]; ok {
logs := d.Logs.String()
return logs
d, ok := c.data[id]
if !ok {
return ""
}
return ""
full := d.Logs.String()
if d.flushed >= len(full) {
return ""
}
tail := full[d.flushed:]
d.flushed = len(full)
return tail
}
Ainternal/service/progress_cache_test.go
@@ -0,0 +1,60 @@
package service
import (
"sync"
"testing"
)
func TestProgressCacheFlushAndSnapshot(t *testing.T) {
c := NewProgressCache()
c.Set(1, &LiveDownload{})
c.AppendLog(1, "line1")
c.AppendLog(1, "line2")
if got := c.Snapshot(1); got != "line1\nline2\n" {
t.Errorf("Snapshot = %q, want the full buffer", got)
}
// First flush returns everything appended so far.
if first := c.FlushLogs(1); first != "line1\nline2\n" {
t.Errorf("first flush = %q, want full buffer", first)
}
// With nothing new, a flush returns empty (no DB rewrite of the whole log).
if again := c.FlushLogs(1); again != "" {
t.Errorf("flush with no new logs = %q, want empty", again)
}
// A later flush returns only the newly appended tail.
c.AppendLog(1, "line3")
if tail := c.FlushLogs(1); tail != "line3\n" {
t.Errorf("incremental flush = %q, want only the new tail", tail)
}
if got := c.Snapshot(999); got != "" {
t.Errorf("Snapshot of unknown id = %q, want empty", got)
}
}
// TestProgressCacheConcurrentSnapshot exercises the data-race fix: a reader
// (Snapshot) and the writer (AppendLog) touch the same strings.Builder, so they
// must be serialized by the cache lock. Run with -race to catch a regression.
func TestProgressCacheConcurrentSnapshot(t *testing.T) {
c := NewProgressCache()
c.Set(1, &LiveDownload{})
var wg sync.WaitGroup
wg.Add(2)
go func() {
defer wg.Done()
for i := 0; i < 2000; i++ {
c.AppendLog(1, "log line")
}
}()
go func() {
defer wg.Done()
for i := 0; i < 2000; i++ {
_ = c.Snapshot(1)
}
}()
wg.Wait()
}
Minternal/worker/pool.go
@@ -64,10 +64,14 @@ func (p *Pool) worker(id int) {
continue
}
log.Printf("Worker %d processing download %d", id, d.ID)
if err := p.downloadSvc.ExecuteDownload(d); err != nil {
processed, err := p.downloadSvc.ExecuteDownload(d)
switch {
case err != nil:
log.Printf("Worker %d download %d failed: %v", id, d.ID, err)
} else {
case processed:
log.Printf("Worker %d download %d completed", id, d.ID)
default:
log.Printf("Worker %d download %d already claimed by another worker, skipping", id, d.ID)
}
case <-p.ctx.Done():
log.Printf("Worker %d stopped", id)
Mweb/static/style.css
@@ -135,6 +135,30 @@ footer {
margin-top: 1.5rem;
}
.file-selector {
display: flex;
align-items: center;
gap: 0.75rem;
flex-wrap: wrap;
margin-bottom: 1rem;
}
.file-selector label {
margin: 0;
}
.file-selector select {
flex: 1;
min-width: 0;
margin: 0;
}
.file-selector button {
margin: 0;
width: auto;
flex: none;
}
/* Buttons */
button.small,
[role="button"].small {
@@ -233,7 +257,21 @@ button:disabled {
position: relative;
}
.media-thumbnail .media-duration {
/* Each thumbnail sits in a tile so it can carry its own per-file duration
badge; the tile fills the container (single file) or a grid cell (multi). */
.thumb-tile {
position: relative;
display: flex;
align-items: center;
justify-content: center;
overflow: hidden;
width: 100%;
height: 100%;
min-width: 0;
min-height: 0;
}
.thumb-tile .media-duration {
position: absolute;
bottom: 0.4rem;
right: 0.4rem;
@@ -241,7 +279,7 @@ button:disabled {
color: #fff;
padding: 0.15rem 0.4rem;
border-radius: var(--pico-border-radius);
font-size: 0.75rem;
font-size: 0.7rem;
font-weight: 500;
line-height: 1;
}
@@ -255,7 +293,7 @@ button:disabled {
font-size: 3rem;
}
.media-thumbnail img {
.thumb-tile img {
width: 100%;
height: 100%;
/* contain: show the whole thumbnail (letterboxed) and never overflow the
@@ -263,7 +301,7 @@ button:disabled {
object-fit: contain;
}
.media-thumbnail img[src$=".svg"] {
.thumb-tile img[src$=".svg"] {
width: 48px;
height: 48px;
opacity: 0.5;
@@ -289,20 +327,10 @@ button:disabled {
}
/* For 3 files, let the first thumbnail span the full top row. */
.media-thumbnail.count-3 img:first-of-type {
.media-thumbnail.count-3 .thumb-tile:first-of-type {
grid-column: 1 / -1;
}
.media-thumbnail.count-2 img,
.media-thumbnail.count-3 img,
.media-thumbnail.count-many img {
width: 100%;
height: 100%;
object-fit: contain;
min-width: 0;
min-height: 0;
}
/* "+N" overlay on the last tile when there are more than 4 files. */
.media-thumbnail .thumb-more {
position: absolute;
Mweb/templates/library.html
@@ -49,14 +49,23 @@
<div class="media-thumbnail {{if le $count 1}}count-1{{else if eq $count 2}}count-2{{else if eq $count 3}}count-3{{else}}count-many{{end}}">
{{if le $count 1}}
{{if gt $count 0}}
<img src="/media/item/{{urlEncodePath .RelPath}}/thumbnail?file={{(index .MediaFiles 0).Filename}}" alt="" loading="lazy">
{{$mf := index .MediaFiles 0}}
<div class="thumb-tile">
<img src="/media/item/{{urlEncodePath .RelPath}}/thumbnail?file={{$mf.Filename}}" alt="" loading="lazy">
{{if gt $mf.Duration 0}}<span class="media-duration">{{formatDuration $mf.Duration}}</span>{{end}}
</div>
{{else}}
<img src="/static/icons/video-icon.svg" alt="" loading="lazy">
<div class="thumb-tile">
<img src="/static/icons/video-icon.svg" alt="" loading="lazy">
</div>
{{end}}
{{else}}
{{range $i, $mf := .MediaFiles}}
{{if lt $i 4}}
<img src="/media/item/{{urlEncodePath $item.RelPath}}/thumbnail?file={{$mf.Filename}}" alt="" loading="lazy">
<div class="thumb-tile">
<img src="/media/item/{{urlEncodePath $item.RelPath}}/thumbnail?file={{$mf.Filename}}" alt="" loading="lazy">
{{if gt $mf.Duration 0}}<span class="media-duration">{{formatDuration $mf.Duration}}</span>{{end}}
</div>
{{end}}
{{end}}
{{if gt $count 4}}
@@ -64,9 +73,6 @@
{{end}}
<span class="thumb-count">{{$count}} files</span>
{{end}}
{{if ge .Duration 0}}
<span class="media-duration">{{formatDuration .Duration}}</span>
{{end}}
</div>
<div class="media-info">
<div class="media-title">{{.Name}}</div>
Mweb/templates/library_detail.html
@@ -16,11 +16,12 @@
{{if gt (len .Data.Item.MediaFiles) 1}}
<form method="get" action="/library/item/{{urlEncodePath .Data.Item.RelPath}}" class="file-selector">
<label for="file">Playing</label>
<select id="file" name="file" onchange="this.form.submit()">
<select id="file" name="file">
{{range .Data.Item.MediaFiles}}
<option value="{{.Filename}}" {{if eq .Filename $.Data.SelectedFilename}}selected{{end}}>{{.Filename}}</option>
{{end}}
</select>
<button type="submit" class="secondary">Switch</button>
</form>
{{end}}