vidarchive

vidarchive/internal/service/download_test.go
download_test.go
⎇
Raw
1package service
2
3import (
4 "context"
5 "database/sql"
6 "errors"
7 "fmt"
8 "os"
9 "path/filepath"
10 "strings"
11 "syscall"
12 "testing"
13 "time"
14
15 "vidarchive/internal/config"
16 "vidarchive/internal/models"
17)
18
19func TestSanitizeDirName(t *testing.T) {
20 tests := []struct{ in, want string }{
21 {"normal title", "normal title"},
22 {"a/b:c*d?e\"f<g>h|i", "a-b-c-d-e-f-g-h-i"},
23 {" trimmed ", "trimmed"},
24 {"", "untitled"},
25 {"///", "---"},
26 }
27 for _, tc := range tests {
28 if got := sanitizeDirName(tc.in); got != tc.want {
29 t.Errorf("sanitizeDirName(%q) = %q, want %q", tc.in, got, tc.want)
30 }
31 }
32}
33
34func TestUniqueDir(t *testing.T) {
35 base := t.TempDir()
36 svc := &DownloadService{}
37
38 first := svc.uniqueDir(base, "item")
39 if filepath.Base(first) != "item" {
40 t.Errorf("first uniqueDir = %q, want .../item", first)
41 }
42 if err := os.MkdirAll(first, 0755); err != nil {
43 t.Fatal(err)
44 }
45 second := svc.uniqueDir(base, "item")
46 if filepath.Base(second) != "item-1" {
47 t.Errorf("second uniqueDir = %q, want .../item-1", second)
48 }
49 if err := os.MkdirAll(second, 0755); err != nil {
50 t.Fatal(err)
51 }
52 third := svc.uniqueDir(base, "item")
53 if filepath.Base(third) != "item-2" {
54 t.Errorf("third uniqueDir = %q, want .../item-2", third)
55 }
56}
57
58func TestDeriveItemName(t *testing.T) {
59 svc := &DownloadService{}
60 itemDir := t.TempDir()
61
62 // info.json title wins and is sanitized.
63 infoPath := filepath.Join(itemDir, "info.json")
64 if err := os.WriteFile(infoPath, []byte(`{"title":"Cool: Video"}`), 0644); err != nil {
65 t.Fatal(err)
66 }
67 bigName := "big.mp4"
68 if err := os.WriteFile(filepath.Join(itemDir, bigName), []byte("xxxxxxxxxx"), 0644); err != nil {
69 t.Fatal(err)
70 }
71 media := []os.DirEntry{dirEntry(t, itemDir, bigName)}
72 if got := svc.deriveItemName(itemDir, readInfoJSON(infoPath), media); got != "Cool- Video" {
73 t.Errorf("deriveItemName with info.json = %q, want %q", got, "Cool- Video")
74 }
75
76 // Without info.json, falls back to the largest media file's stem.
77 small := "small.mp4"
78 if err := os.WriteFile(filepath.Join(itemDir, small), []byte("x"), 0644); err != nil {
79 t.Fatal(err)
80 }
81 media = []os.DirEntry{dirEntry(t, itemDir, small), dirEntry(t, itemDir, bigName)}
82 if got := svc.deriveItemName(itemDir, infoJSON{}, media); got != "big" {
83 t.Errorf("deriveItemName fallback = %q, want %q (largest file stem)", got, "big")
84 }
85}
86
87func dirEntry(t *testing.T, dir, name string) os.DirEntry {
88 t.Helper()
89 entries, err := os.ReadDir(dir)
90 if err != nil {
91 t.Fatal(err)
92 }
93 for _, e := range entries {
94 if e.Name() == name {
95 return e
96 }
97 }
98 t.Fatalf("dir entry %q not found", name)
99 return nil
100}
101
102func TestParseFormatJSON(t *testing.T) {
103 data := []byte(`{
104 "id": "vid",
105 "formats": [
106 {"format_id": "18", "ext": "mp4", "resolution": "640x360", "fps": 30, "vcodec": "avc1", "acodec": "mp4a", "format_note": "360p"},
107 {"format_id": "137", "ext": "mp4", "width": 1920, "height": 1080, "fps": 60, "vcodec": "avc1", "acodec": "none", "filesize": 1048576, "format_note": "1080p"},
108 {"format_id": "233", "ext": "m4a", "resolution": "audio only", "vcodec": "none", "acodec": "mp4a", "audio_channels": 2, "format_note": "audio"}
109 ]
110 }`)
111 formats, err := parseFormatJSON(data)
112 if err != nil {
113 t.Fatalf("parseFormatJSON: %v", err)
114 }
115 if len(formats) != 3 {
116 t.Fatalf("expected 3 formats, got %d: %+v", len(formats), formats)
117 }
118 if formats[0].ID != "18" || formats[0].Ext != "mp4" || formats[0].Resolution != "640x360" {
119 t.Errorf("format[0] = %+v", formats[0])
120 }
121 if formats[0].FPS != "30" {
122 t.Errorf("format[0] fps = %q, want 30", formats[0].FPS)
123 }
124 // Resolution is derived from width/height when no resolution string is present.
125 if formats[1].Resolution != "1920x1080" {
126 t.Errorf("format[1] resolution = %q, want 1920x1080", formats[1].Resolution)
127 }
128 if formats[1].FileSize != "1.0 MiB" {
129 t.Errorf("format[1] filesize = %q, want 1.0 MiB", formats[1].FileSize)
130 }
131 // Audio-only format: codec falls back to acodec and channels are populated.
132 if formats[2].Codec != "mp4a" {
133 t.Errorf("format[2] codec = %q, want mp4a", formats[2].Codec)
134 }
135 if formats[2].Channels != "2" {
136 t.Errorf("format[2] channels = %q, want 2", formats[2].Channels)
137 }
138}
139
140func TestParseFormatJSONPlaylistFallback(t *testing.T) {
141 // A playlist dump exposes formats under the first entry, not at the top level.
142 data := []byte(`{"_type":"playlist","entries":[{"id":"a","formats":[{"format_id":"18","ext":"mp4"}]}]}`)
143 formats, err := parseFormatJSON(data)
144 if err != nil {
145 t.Fatalf("parseFormatJSON: %v", err)
146 }
147 if len(formats) != 1 || formats[0].ID != "18" {
148 t.Fatalf("expected 1 format from entry fallback, got %+v", formats)
149 }
150}
151
152// TestImportItemDir exercises the full import: media + info.json + subtitles get
153// sorted into a named item directory with a marker. Uses ffmpeg to produce real
154// media so the mimetype-based classification in importItemDir matches.
155func TestImportItemDir(t *testing.T) {
156 requireFFmpeg(t)
157
158 libDir := t.TempDir()
159 svc := &DownloadService{cfg: &config.Config{LibraryDir: libDir, FFprobePath: "ffprobe"}}
160
161 src := t.TempDir()
162 makeTestVideo(t, filepath.Join(src, "raw.mp4"))
163 if err := os.WriteFile(filepath.Join(src, "info.json"), []byte(`{"title":"My Clip"}`), 0644); err != nil {
164 t.Fatal(err)
165 }
166 if err := os.WriteFile(filepath.Join(src, "raw.en.srt"), []byte("1\n00:00:00,000 --> 00:00:01,000\nhi\n"), 0644); err != nil {
167 t.Fatal(err)
168 }
169
170 if err := svc.importItemDir(context.Background(), "https://example.com/v", src, libDir, "", "-f bestvideo+bestaudio"); err != nil {
171 t.Fatalf("importItemDir: %v", err)
172 }
173
174 targetDir := filepath.Join(libDir, "My Clip")
175 if _, err := os.Stat(filepath.Join(targetDir, itemMarkerName)); err != nil {
176 t.Errorf("marker not created: %v", err)
177 }
178 if _, err := os.Stat(filepath.Join(targetDir, "info.json")); err != nil {
179 t.Errorf("info.json not moved: %v", err)
180 }
181 if _, err := os.Stat(filepath.Join(targetDir, "raw.mp4")); err != nil {
182 t.Errorf("media not moved: %v", err)
183 }
184 if _, err := os.Stat(filepath.Join(targetDir, subtitlesDirName, "raw.en.srt")); err != nil {
185 t.Errorf("subtitle not moved into subtitles dir: %v", err)
186 }
187
188 // The marker should record the source URL and a probed per-file duration
189 // (the test video is 3s), so the library never has to probe at serve time.
190 markerData, _ := os.ReadFile(filepath.Join(targetDir, itemMarkerName))
191 if want := "https://example.com/v"; !strings.Contains(string(markerData), want) {
192 t.Errorf("marker missing source url %q: %s", want, markerData)
193 }
194 if !strings.Contains(string(markerData), "file_durations") || !strings.Contains(string(markerData), "raw.mp4") {
195 t.Errorf("marker missing probed per-file duration: %s", markerData)
196 }
197 if want := "-f bestvideo+bestaudio"; !strings.Contains(string(markerData), want) {
198 t.Errorf("marker missing yt-dlp flags %q: %s", want, markerData)
199 }
200}
201
202func TestCheckReservedFlags(t *testing.T) {
203 cases := []struct {
204 name string
205 flags string
206 subscription bool
207 wantErr bool
208 }{
209 {"empty", "", false, false},
210 {"harmless", "--no-playlist --write-thumbnail", false, false},
211 {"output short", "-o foo.mp4", false, true},
212 {"output long", "--output foo.mp4", false, true},
213 {"output equals form", "--output=foo.mp4", false, true},
214 {"paths short", "-P /tmp", false, true},
215 {"cookies", "--cookies x.txt", false, true},
216 {"cookies inverse", "--no-cookies", false, true},
217 // Subscription-only reserved flags pass for normal downloads...
218 {"skip-download non-sub", "--skip-download", false, false},
219 {"write-info-json non-sub", "--write-info-json", false, false},
220 // ...but are rejected for subscription runs (and their inverses).
221 {"skip-download sub", "--skip-download", true, true},
222 {"no-skip-download sub", "--no-skip-download", true, true},
223 {"write-info-json sub", "--write-info-json", true, true},
224 {"no-write-info-json sub", "--no-write-info-json", true, true},
225 {"download-archive sub", "--download-archive a.txt", true, true},
226 // The "--flag=value" spelling names the same option as "--flag value",
227 // for the subscription-only table as well as the base one.
228 {"download-archive equals form sub", "--download-archive=a.txt", true, true},
229 // Base reserved flags still apply to subscriptions.
230 {"output sub", "-o x", true, true},
231 }
232 for _, tc := range cases {
233 t.Run(tc.name, func(t *testing.T) {
234 err := checkReservedFlags(tc.flags, tc.subscription)
235 if tc.wantErr != (err != nil) {
236 t.Errorf("checkReservedFlags(%q, %v) error = %v, wantErr %v", tc.flags, tc.subscription, err, tc.wantErr)
237 }
238 })
239 }
240}
241
242func TestImportDownloadedItemsRejectsOutputTraversal(t *testing.T) {
243 libDir := t.TempDir()
244 svc := &DownloadService{
245 cfg: &config.Config{LibraryDir: libDir},
246 librarySvc: NewLibraryService(libDir, "ffmpeg", "ffprobe"),
247 }
248
249 // A temp download dir with one item subdir.
250 tempDir := t.TempDir()
251 itemDir := filepath.Join(tempDir, "item-00001")
252 if err := os.MkdirAll(itemDir, 0755); err != nil {
253 t.Fatal(err)
254 }
255
256 d := &models.Download{
257 URL: "u",
258 OutputDir: sql.NullString{String: "../escape", Valid: true},
259 }
260 if _, err := svc.importDownloadedItems(context.Background(), d, tempDir, "", ""); err == nil {
261 t.Error("expected path-traversal output dir to be rejected")
262 }
263}
264
265// A download deleted while its items are being imported must stop, not keep
266// filling the library with items whose queue row no longer exists.
267func TestImportDownloadedItemsStopsOnCancel(t *testing.T) {
268 requireFFmpeg(t)
269
270 libDir := t.TempDir()
271 svc := &DownloadService{
272 cfg: &config.Config{LibraryDir: libDir, FFprobePath: "ffprobe"},
273 librarySvc: NewLibraryService(libDir, "ffmpeg", "ffprobe"),
274 }
275
276 tempDir := t.TempDir()
277 for _, name := range []string{"item-00001", "item-00002"} {
278 itemDir := filepath.Join(tempDir, name)
279 if err := os.MkdirAll(itemDir, 0755); err != nil {
280 t.Fatal(err)
281 }
282 makeTestVideo(t, filepath.Join(itemDir, "raw.mp4"))
283 }
284
285 ctx, cancel := context.WithCancel(context.Background())
286 cancel()
287
288 imported, err := svc.importDownloadedItems(ctx, &models.Download{URL: "u"}, tempDir, "", "")
289 if !errors.Is(err, context.Canceled) {
290 t.Errorf("error = %v, want context.Canceled", err)
291 }
292 if imported != 0 {
293 t.Errorf("imported = %d, want 0", imported)
294 }
295 entries, err := os.ReadDir(libDir)
296 if err != nil {
297 t.Fatal(err)
298 }
299 if len(entries) != 0 {
300 t.Errorf("library got %d entries, want 0", len(entries))
301 }
302}
303
304// A cancel part-way through a playlist must stop at an item boundary: the items
305// already imported stay, and no half-written directory is left behind.
306func TestImportDownloadedItemsStopsBetweenItems(t *testing.T) {
307 requireFFmpeg(t)
308
309 libDir := t.TempDir()
310 svc := &DownloadService{
311 cfg: &config.Config{LibraryDir: libDir, FFprobePath: "ffprobe"},
312 librarySvc: NewLibraryService(libDir, "ffmpeg", "ffprobe"),
313 }
314
315 tempDir := t.TempDir()
316 for i, name := range []string{"item-00001", "item-00002", "item-00003"} {
317 itemDir := filepath.Join(tempDir, name)
318 if err := os.MkdirAll(itemDir, 0755); err != nil {
319 t.Fatal(err)
320 }
321 makeTestVideo(t, filepath.Join(itemDir, "raw.mp4"))
322 info := fmt.Sprintf(`{"id":"v%d","title":"Clip %d"}`, i+1, i+1)
323 if err := os.WriteFile(filepath.Join(itemDir, "clip.info.json"), []byte(info), 0644); err != nil {
324 t.Fatal(err)
325 }
326 }
327
328 // Cancel as soon as the first item lands, so the second boundary check trips.
329 ctx, cancel := context.WithCancel(context.Background())
330 stop := make(chan struct{})
331 go func() {
332 defer close(stop)
333 for {
334 if _, err := os.Stat(filepath.Join(libDir, "Clip 1", itemMarkerName)); err == nil {
335 cancel()
336 return
337 }
338 time.Sleep(2 * time.Millisecond)
339 }
340 }()
341
342 imported, err := svc.importDownloadedItems(ctx, &models.Download{URL: "u"}, tempDir, "", "")
343 <-stop
344 cancel()
345
346 if !errors.Is(err, context.Canceled) {
347 t.Fatalf("error = %v, want context.Canceled", err)
348 }
349 if imported < 1 || imported > 2 {
350 t.Errorf("imported = %d, want the cancel to land after 1 or 2 items", imported)
351 }
352
353 // Whatever was imported must be complete: an item dir without a marker means
354 // the import stopped half-way through writing it.
355 entries, err := os.ReadDir(libDir)
356 if err != nil {
357 t.Fatal(err)
358 }
359 if len(entries) != imported {
360 t.Errorf("library has %d dirs but reported %d imported", len(entries), imported)
361 }
362 for _, entry := range entries {
363 marker := filepath.Join(libDir, entry.Name(), itemMarkerName)
364 if _, err := os.Stat(marker); err != nil {
365 t.Errorf("item %q left without a marker: %v", entry.Name(), err)
366 }
367 }
368}
369
370// Overwrite mode replaces the existing copy of a video in place instead of
371// adding a second directory for it.
372func TestImportItemDirOverwriteReplacesExisting(t *testing.T) {
373 requireFFmpeg(t)
374
375 libDir := t.TempDir()
376 svc := &DownloadService{
377 cfg: &config.Config{LibraryDir: libDir, FFprobePath: "ffprobe"},
378 librarySvc: NewLibraryService(libDir, "ffmpeg", "ffprobe"),
379 }
380
381 // First import establishes the item.
382 first := t.TempDir()
383 makeTestVideo(t, filepath.Join(first, "raw.mp4"))
384 if err := os.WriteFile(filepath.Join(first, "clip.info.json"), []byte(`{"id":"vid1","title":"Old Title"}`), 0644); err != nil {
385 t.Fatal(err)
386 }
387 if err := svc.importItemDir(context.Background(), "https://example.com/v", first, libDir, "overwrite", ""); err != nil {
388 t.Fatalf("first import: %v", err)
389 }
390 if err := os.WriteFile(filepath.Join(libDir, "Old Title", "stale.txt"), []byte("gone"), 0644); err != nil {
391 t.Fatal(err)
392 }
393
394 // Second import of the same video id, now retitled upstream.
395 second := t.TempDir()
396 makeTestVideo(t, filepath.Join(second, "raw.mp4"))
397 if err := os.WriteFile(filepath.Join(second, "clip.info.json"), []byte(`{"id":"vid1","title":"New Title"}`), 0644); err != nil {
398 t.Fatal(err)
399 }
400 if err := svc.importItemDir(context.Background(), "https://example.com/v", second, libDir, "overwrite", ""); err != nil {
401 t.Fatalf("second import: %v", err)
402 }
403
404 entries, err := os.ReadDir(libDir)
405 if err != nil {
406 t.Fatal(err)
407 }
408 if len(entries) != 1 || entries[0].Name() != "New Title" {
409 var names []string
410 for _, e := range entries {
411 names = append(names, e.Name())
412 }
413 t.Fatalf("library = %v, want exactly [New Title]", names)
414 }
415 if _, err := os.Stat(filepath.Join(libDir, "New Title", "stale.txt")); err == nil {
416 t.Error("the replaced item kept a file from the old copy")
417 }
418}
419
420// Without overwrite mode the same video is imported again beside the first, so
421// the guard above is really the thing preventing duplicates.
422func TestImportItemDirWithoutOverwriteKeepsBoth(t *testing.T) {
423 requireFFmpeg(t)
424
425 libDir := t.TempDir()
426 svc := &DownloadService{
427 cfg: &config.Config{LibraryDir: libDir, FFprobePath: "ffprobe"},
428 librarySvc: NewLibraryService(libDir, "ffmpeg", "ffprobe"),
429 }
430
431 for i := 0; i < 2; i++ {
432 src := t.TempDir()
433 makeTestVideo(t, filepath.Join(src, "raw.mp4"))
434 if err := os.WriteFile(filepath.Join(src, "clip.info.json"), []byte(`{"id":"vid1","title":"Same"}`), 0644); err != nil {
435 t.Fatal(err)
436 }
437 if err := svc.importItemDir(context.Background(), "https://example.com/v", src, libDir, "", ""); err != nil {
438 t.Fatalf("import %d: %v", i, err)
439 }
440 }
441
442 entries, err := os.ReadDir(libDir)
443 if err != nil {
444 t.Fatal(err)
445 }
446 if len(entries) != 2 {
447 t.Errorf("library has %d dirs, want 2 (Same and Same-1)", len(entries))
448 }
449}
450
451// moveFile falls back to copy+delete when the source and destination are on
452// different filesystems, which is the normal case for a temp dir on tmpfs and a
453// library on disk. /dev/shm is a second tmpfs mount, so a rename across it
454// really does return EXDEV.
455func TestMoveFileCrossDeviceFallback(t *testing.T) {
456 dstDir, err := os.MkdirTemp("/dev/shm", "vidarchive-move-")
457 if err != nil {
458 t.Skipf("no second filesystem available for a cross-device move: %v", err)
459 }
460 defer os.RemoveAll(dstDir)
461
462 srcDir := t.TempDir()
463 if sameDevice(t, srcDir, dstDir) {
464 t.Skip("temp dir and /dev/shm are on the same filesystem")
465 }
466
467 src := filepath.Join(srcDir, "clip.mp4")
468 content := []byte("not really a video, but the bytes must survive")
469 if err := os.WriteFile(src, content, 0640); err != nil {
470 t.Fatal(err)
471 }
472
473 // Confirm the fallback is actually what gets exercised.
474 dst := filepath.Join(dstDir, "clip.mp4")
475 if err := os.Rename(src, dst); !errors.Is(err, syscall.EXDEV) {
476 t.Skipf("rename across the two dirs did not report EXDEV: %v", err)
477 }
478
479 if err := moveFile(src, dst); err != nil {
480 t.Fatalf("moveFile: %v", err)
481 }
482
483 got, err := os.ReadFile(dst)
484 if err != nil {
485 t.Fatalf("read destination: %v", err)
486 }
487 if string(got) != string(content) {
488 t.Errorf("destination content = %q, want %q", got, content)
489 }
490 info, err := os.Stat(dst)
491 if err != nil {
492 t.Fatal(err)
493 }
494 if info.Mode().Perm() != 0640 {
495 t.Errorf("destination mode = %v, want 0640", info.Mode().Perm())
496 }
497 if _, err := os.Stat(src); !os.IsNotExist(err) {
498 t.Error("source still exists after a successful move")
499 }
500}
501
502// A move that cannot even read the source must not leave a destination behind.
503func TestMoveFileMissingSourceLeavesNoDestination(t *testing.T) {
504 dir := t.TempDir()
505 dst := filepath.Join(dir, "dst.mp4")
506
507 if err := moveFile(filepath.Join(dir, "missing.mp4"), dst); err == nil {
508 t.Fatal("expected an error moving a file that does not exist")
509 }
510 if _, err := os.Stat(dst); !os.IsNotExist(err) {
511 t.Error("destination was created for a failed move")
512 }
513}
514
515func sameDevice(t *testing.T, a, b string) bool {
516 t.Helper()
517 var sa, sb syscall.Stat_t
518 if err := syscall.Stat(a, &sa); err != nil {
519 t.Fatal(err)
520 }
521 if err := syscall.Stat(b, &sb); err != nil {
522 t.Fatal(err)
523 }
524 return sa.Dev == sb.Dev
525}
526