vidarchive

vidarchive/internal/service/import_test.go
import_test.go
⎇
Raw
1package service
2
3import (
4 "context"
5 "database/sql"
6 "errors"
7 "fmt"
8 "os"
9 "path/filepath"
10 "strings"
11 "syscall"
12 "testing"
13 "time"
14 "unicode/utf8"
15
16 "vidarchive/internal/config"
17 "vidarchive/internal/models"
18)
19
20func TestSanitizeDirName(t *testing.T) {
21 tests := []struct{ in, want string }{
22 {"normal title", "normal title"},
23 {"a/b:c*d?e\"f<g>h|i", "a-b-c-d-e-f-g-h-i"},
24 {" trimmed ", "trimmed"},
25 {"", "untitled"},
26 {"///", "---"},
27 // A dot-only name would resolve to the target directory or its parent, so
28 // a title like this must not become a directory name.
29 {".", "untitled"},
30 {"..", "untitled"},
31 {" .. ", "untitled"},
32 {"...", "untitled"},
33 {".hidden", ".hidden"},
34 }
35 for _, tc := range tests {
36 if got := sanitizeDirName(tc.in); got != tc.want {
37 t.Errorf("sanitizeDirName(%q) = %q, want %q", tc.in, got, tc.want)
38 }
39 }
40}
41
42func TestUniqueDir(t *testing.T) {
43 base := t.TempDir()
44 svc := &DownloadService{}
45
46 mustUnique := func(name string) string {
47 t.Helper()
48 dir, err := svc.uniqueDir(base, name)
49 if err != nil {
50 t.Fatalf("uniqueDir(%q): %v", name, err)
51 }
52 return dir
53 }
54
55 first := mustUnique("item")
56 if filepath.Base(first) != "item" {
57 t.Errorf("first uniqueDir = %q, want .../item", first)
58 }
59 if err := os.MkdirAll(first, 0o755); err != nil {
60 t.Fatal(err)
61 }
62 second := mustUnique("item")
63 if filepath.Base(second) != "item-1" {
64 t.Errorf("second uniqueDir = %q, want .../item-1", second)
65 }
66 if err := os.MkdirAll(second, 0o755); err != nil {
67 t.Fatal(err)
68 }
69 third := mustUnique("item")
70 if filepath.Base(third) != "item-2" {
71 t.Errorf("third uniqueDir = %q, want .../item-2", third)
72 }
73}
74
75// A name the filesystem rejects makes every os.Stat fail with something other
76// than ENOENT. uniqueDir must report that instead of looping forever looking
77// for a free suffix.
78func TestUniqueDirRejectsUnusableName(t *testing.T) {
79 svc := &DownloadService{}
80
81 done := make(chan struct{})
82 go func() {
83 defer close(done)
84 if _, err := svc.uniqueDir(t.TempDir(), strings.Repeat("a", 300)); err == nil {
85 t.Error("uniqueDir accepted an over-long name, want an error")
86 }
87 }()
88
89 select {
90 case <-done:
91 case <-time.After(5 * time.Second):
92 t.Fatal("uniqueDir did not return: it is looping on a non-ENOENT stat error")
93 }
94}
95
96// Titles come from remote metadata, where nothing bounds their length.
97func TestSanitizeDirNameBoundsLength(t *testing.T) {
98 for _, in := range []string{strings.Repeat("a", 500), strings.Repeat("ミク", 300)} {
99 got := sanitizeDirName(in)
100 if len(got) > maxDirNameBytes {
101 t.Errorf("sanitizeDirName(%d bytes) = %d bytes, want <= %d", len(in), len(got), maxDirNameBytes)
102 }
103 if !utf8.ValidString(got) {
104 t.Errorf("sanitizeDirName(%d bytes) split a rune: %q", len(in), got)
105 }
106 }
107}
108
109func TestDeriveItemName(t *testing.T) {
110 svc := &DownloadService{}
111 itemDir := t.TempDir()
112
113 // info.json title wins and is sanitized.
114 infoPath := filepath.Join(itemDir, "info.json")
115 if err := os.WriteFile(infoPath, []byte(`{"title":"Cool: Video"}`), 0o644); err != nil {
116 t.Fatal(err)
117 }
118 bigName := "big.mp4"
119 if err := os.WriteFile(filepath.Join(itemDir, bigName), []byte("xxxxxxxxxx"), 0o644); err != nil {
120 t.Fatal(err)
121 }
122 media := []os.DirEntry{dirEntry(t, itemDir, bigName)}
123 if got := svc.deriveItemName(itemDir, readInfoJSON(infoPath), media); got != "Cool- Video" {
124 t.Errorf("deriveItemName with info.json = %q, want %q", got, "Cool- Video")
125 }
126
127 // Without info.json, falls back to the largest media file's stem.
128 small := "small.mp4"
129 if err := os.WriteFile(filepath.Join(itemDir, small), []byte("x"), 0o644); err != nil {
130 t.Fatal(err)
131 }
132 media = []os.DirEntry{dirEntry(t, itemDir, small), dirEntry(t, itemDir, bigName)}
133 if got := svc.deriveItemName(itemDir, infoJSON{}, media); got != "big" {
134 t.Errorf("deriveItemName fallback = %q, want %q (largest file stem)", got, "big")
135 }
136}
137
138func dirEntry(t *testing.T, dir, name string) os.DirEntry {
139 t.Helper()
140 entries, err := os.ReadDir(dir)
141 if err != nil {
142 t.Fatal(err)
143 }
144 for _, e := range entries {
145 if e.Name() == name {
146 return e
147 }
148 }
149 t.Fatalf("dir entry %q not found", name)
150 return nil
151}
152
153// TestImportItemDir exercises the full import: media + info.json + subtitles get
154// sorted into a named item directory with a marker. Uses ffmpeg to produce real
155// media so the mimetype-based classification in importItemDir matches.
156func TestImportItemDir(t *testing.T) {
157 requireFFmpeg(t)
158
159 libDir := t.TempDir()
160 svc := &DownloadService{cfg: &config.Config{LibraryDir: libDir, FFprobePath: "ffprobe"}}
161
162 src := t.TempDir()
163 makeTestVideo(t, filepath.Join(src, "raw.mp4"))
164 if err := os.WriteFile(filepath.Join(src, "info.json"), []byte(`{"title":"My Clip"}`), 0o644); err != nil {
165 t.Fatal(err)
166 }
167 if err := os.WriteFile(filepath.Join(src, "raw.en.srt"), []byte("1\n00:00:00,000 --> 00:00:01,000\nhi\n"), 0o644); err != nil {
168 t.Fatal(err)
169 }
170
171 if err := svc.importItemDir(context.Background(), "https://example.com/v", src, libDir, "", "-f bestvideo+bestaudio"); err != nil {
172 t.Fatalf("importItemDir: %v", err)
173 }
174
175 targetDir := filepath.Join(libDir, "My Clip")
176 if _, err := os.Stat(filepath.Join(targetDir, itemMarkerName)); err != nil {
177 t.Errorf("marker not created: %v", err)
178 }
179 if _, err := os.Stat(filepath.Join(targetDir, "info.json")); err != nil {
180 t.Errorf("info.json not moved: %v", err)
181 }
182 if _, err := os.Stat(filepath.Join(targetDir, "raw.mp4")); err != nil {
183 t.Errorf("media not moved: %v", err)
184 }
185 if _, err := os.Stat(filepath.Join(targetDir, subtitlesDirName, "raw.en.srt")); err != nil {
186 t.Errorf("subtitle not moved into subtitles dir: %v", err)
187 }
188
189 // The marker should record the source URL and a probed per-file duration
190 // (the test video is 3s), so the library never has to probe at serve time.
191 markerData, _ := os.ReadFile(filepath.Join(targetDir, itemMarkerName))
192 if want := "https://example.com/v"; !strings.Contains(string(markerData), want) {
193 t.Errorf("marker missing source url %q: %s", want, markerData)
194 }
195 if !strings.Contains(string(markerData), "file_durations") || !strings.Contains(string(markerData), "raw.mp4") {
196 t.Errorf("marker missing probed per-file duration: %s", markerData)
197 }
198 if want := "-f bestvideo+bestaudio"; !strings.Contains(string(markerData), want) {
199 t.Errorf("marker missing yt-dlp flags %q: %s", want, markerData)
200 }
201}
202
203func TestImportDownloadedItemsRejectsOutputTraversal(t *testing.T) {
204 libDir := t.TempDir()
205 svc := &DownloadService{
206 cfg: &config.Config{LibraryDir: libDir},
207 librarySvc: NewLibraryService(libDir, "ffmpeg", "ffprobe"),
208 }
209
210 tempDir := t.TempDir()
211 itemDir := filepath.Join(tempDir, "item-00001")
212 if err := os.MkdirAll(itemDir, 0o755); err != nil {
213 t.Fatal(err)
214 }
215
216 d := &models.Download{
217 URL: "u",
218 OutputDir: sql.NullString{String: "../escape", Valid: true},
219 }
220 if _, err := svc.importDownloadedItems(context.Background(), d, tempDir, "", ""); err == nil {
221 t.Error("expected path-traversal output dir to be rejected")
222 }
223}
224
225// A download deleted while its items are being imported must stop, not keep
226// filling the library with items whose queue row no longer exists.
227func TestImportDownloadedItemsStopsOnCancel(t *testing.T) {
228 requireFFmpeg(t)
229
230 libDir := t.TempDir()
231 svc := &DownloadService{
232 cfg: &config.Config{LibraryDir: libDir, FFprobePath: "ffprobe"},
233 librarySvc: NewLibraryService(libDir, "ffmpeg", "ffprobe"),
234 }
235
236 tempDir := t.TempDir()
237 for _, name := range []string{"item-00001", "item-00002"} {
238 itemDir := filepath.Join(tempDir, name)
239 if err := os.MkdirAll(itemDir, 0o755); err != nil {
240 t.Fatal(err)
241 }
242 makeTestVideo(t, filepath.Join(itemDir, "raw.mp4"))
243 }
244
245 ctx, cancel := context.WithCancel(context.Background())
246 cancel()
247
248 imported, err := svc.importDownloadedItems(ctx, &models.Download{URL: "u"}, tempDir, "", "")
249 if !errors.Is(err, context.Canceled) {
250 t.Errorf("error = %v, want context.Canceled", err)
251 }
252 if imported != 0 {
253 t.Errorf("imported = %d, want 0", imported)
254 }
255 entries, err := os.ReadDir(libDir)
256 if err != nil {
257 t.Fatal(err)
258 }
259 if len(entries) != 0 {
260 t.Errorf("library got %d entries, want 0", len(entries))
261 }
262}
263
264// A cancel part-way through a playlist must stop at an item boundary: the items
265// already imported stay, and no half-written directory is left behind.
266func TestImportDownloadedItemsStopsBetweenItems(t *testing.T) {
267 requireFFmpeg(t)
268
269 libDir := t.TempDir()
270 svc := &DownloadService{
271 cfg: &config.Config{LibraryDir: libDir, FFprobePath: "ffprobe"},
272 librarySvc: NewLibraryService(libDir, "ffmpeg", "ffprobe"),
273 }
274
275 tempDir := t.TempDir()
276 for i, name := range []string{"item-00001", "item-00002", "item-00003"} {
277 itemDir := filepath.Join(tempDir, name)
278 if err := os.MkdirAll(itemDir, 0o755); err != nil {
279 t.Fatal(err)
280 }
281 makeTestVideo(t, filepath.Join(itemDir, "raw.mp4"))
282 info := fmt.Sprintf(`{"id":"v%d","title":"Clip %d"}`, i+1, i+1)
283 if err := os.WriteFile(filepath.Join(itemDir, "clip.info.json"), []byte(info), 0o644); err != nil {
284 t.Fatal(err)
285 }
286 }
287
288 // Cancel as soon as the first item lands, so the second boundary check trips.
289 ctx, cancel := context.WithCancel(context.Background())
290 stop := make(chan struct{})
291 go func() {
292 defer close(stop)
293 for {
294 if _, err := os.Stat(filepath.Join(libDir, "Clip 1", itemMarkerName)); err == nil {
295 cancel()
296 return
297 }
298 time.Sleep(2 * time.Millisecond)
299 }
300 }()
301
302 imported, err := svc.importDownloadedItems(ctx, &models.Download{URL: "u"}, tempDir, "", "")
303 <-stop
304 cancel()
305
306 if !errors.Is(err, context.Canceled) {
307 t.Fatalf("error = %v, want context.Canceled", err)
308 }
309 if imported < 1 || imported > 2 {
310 t.Errorf("imported = %d, want the cancel to land after 1 or 2 items", imported)
311 }
312
313 // Whatever was imported must be complete: an item dir without a marker means
314 // the import stopped half-way through writing it.
315 entries, err := os.ReadDir(libDir)
316 if err != nil {
317 t.Fatal(err)
318 }
319 if len(entries) != imported {
320 t.Errorf("library has %d dirs but reported %d imported", len(entries), imported)
321 }
322 for _, entry := range entries {
323 marker := filepath.Join(libDir, entry.Name(), itemMarkerName)
324 if _, err := os.Stat(marker); err != nil {
325 t.Errorf("item %q left without a marker: %v", entry.Name(), err)
326 }
327 }
328}
329
330// Overwrite mode replaces the existing copy of a video in place instead of
331// adding a second directory for it.
332func TestImportItemDirOverwriteReplacesExisting(t *testing.T) {
333 requireFFmpeg(t)
334
335 libDir := t.TempDir()
336 svc := &DownloadService{
337 cfg: &config.Config{LibraryDir: libDir, FFprobePath: "ffprobe"},
338 librarySvc: NewLibraryService(libDir, "ffmpeg", "ffprobe"),
339 }
340
341 first := t.TempDir()
342 makeTestVideo(t, filepath.Join(first, "raw.mp4"))
343 if err := os.WriteFile(filepath.Join(first, "clip.info.json"), []byte(`{"id":"vid1","title":"Old Title"}`), 0o644); err != nil {
344 t.Fatal(err)
345 }
346 if err := svc.importItemDir(context.Background(), "https://example.com/v", first, libDir, "overwrite", ""); err != nil {
347 t.Fatalf("first import: %v", err)
348 }
349 if err := os.WriteFile(filepath.Join(libDir, "Old Title", "stale.txt"), []byte("gone"), 0o644); err != nil {
350 t.Fatal(err)
351 }
352
353 // Second import of the same video id, now retitled upstream.
354 second := t.TempDir()
355 makeTestVideo(t, filepath.Join(second, "raw.mp4"))
356 if err := os.WriteFile(filepath.Join(second, "clip.info.json"), []byte(`{"id":"vid1","title":"New Title"}`), 0o644); err != nil {
357 t.Fatal(err)
358 }
359 if err := svc.importItemDir(context.Background(), "https://example.com/v", second, libDir, "overwrite", ""); err != nil {
360 t.Fatalf("second import: %v", err)
361 }
362
363 entries, err := os.ReadDir(libDir)
364 if err != nil {
365 t.Fatal(err)
366 }
367 if len(entries) != 1 || entries[0].Name() != "New Title" {
368 var names []string
369 for _, e := range entries {
370 names = append(names, e.Name())
371 }
372 t.Fatalf("library = %v, want exactly [New Title]", names)
373 }
374 if _, err := os.Stat(filepath.Join(libDir, "New Title", "stale.txt")); err == nil {
375 t.Error("the replaced item kept a file from the old copy")
376 }
377}
378
379// Without overwrite mode the same video is imported again beside the first, so
380// the guard above is really the thing preventing duplicates.
381func TestImportItemDirWithoutOverwriteKeepsBoth(t *testing.T) {
382 requireFFmpeg(t)
383
384 libDir := t.TempDir()
385 svc := &DownloadService{
386 cfg: &config.Config{LibraryDir: libDir, FFprobePath: "ffprobe"},
387 librarySvc: NewLibraryService(libDir, "ffmpeg", "ffprobe"),
388 }
389
390 for i := 0; i < 2; i++ {
391 src := t.TempDir()
392 makeTestVideo(t, filepath.Join(src, "raw.mp4"))
393 if err := os.WriteFile(filepath.Join(src, "clip.info.json"), []byte(`{"id":"vid1","title":"Same"}`), 0o644); err != nil {
394 t.Fatal(err)
395 }
396 if err := svc.importItemDir(context.Background(), "https://example.com/v", src, libDir, "", ""); err != nil {
397 t.Fatalf("import %d: %v", i, err)
398 }
399 }
400
401 entries, err := os.ReadDir(libDir)
402 if err != nil {
403 t.Fatal(err)
404 }
405 if len(entries) != 2 {
406 t.Errorf("library has %d dirs, want 2 (Same and Same-1)", len(entries))
407 }
408}
409
410// moveFile falls back to copy+delete when the source and destination are on
411// different filesystems, which is the normal case for a temp dir on tmpfs and a
412// library on disk. /dev/shm is a second tmpfs mount, so a rename across it
413// really does return EXDEV.
414func TestMoveFileCrossDeviceFallback(t *testing.T) {
415 dstDir, err := os.MkdirTemp("/dev/shm", "vidarchive-move-")
416 if err != nil {
417 t.Skipf("no second filesystem available for a cross-device move: %v", err)
418 }
419 defer os.RemoveAll(dstDir)
420
421 srcDir := t.TempDir()
422 if sameDevice(t, srcDir, dstDir) {
423 t.Skip("temp dir and /dev/shm are on the same filesystem")
424 }
425
426 src := filepath.Join(srcDir, "clip.mp4")
427 content := []byte("not really a video, but the bytes must survive")
428 if err := os.WriteFile(src, content, 0o640); err != nil {
429 t.Fatal(err)
430 }
431
432 // Confirm the fallback is actually what gets exercised.
433 dst := filepath.Join(dstDir, "clip.mp4")
434 if err := os.Rename(src, dst); !errors.Is(err, syscall.EXDEV) {
435 t.Skipf("rename across the two dirs did not report EXDEV: %v", err)
436 }
437
438 if err := moveFile(src, dst); err != nil {
439 t.Fatalf("moveFile: %v", err)
440 }
441
442 got, err := os.ReadFile(dst)
443 if err != nil {
444 t.Fatalf("read destination: %v", err)
445 }
446 if string(got) != string(content) {
447 t.Errorf("destination content = %q, want %q", got, content)
448 }
449 info, err := os.Stat(dst)
450 if err != nil {
451 t.Fatal(err)
452 }
453 if info.Mode().Perm() != 0o640 {
454 t.Errorf("destination mode = %v, want 0640", info.Mode().Perm())
455 }
456 if _, err := os.Stat(src); !os.IsNotExist(err) {
457 t.Error("source still exists after a successful move")
458 }
459}
460
461// A move that cannot even read the source must not leave a destination behind.
462func TestMoveFileMissingSourceLeavesNoDestination(t *testing.T) {
463 dir := t.TempDir()
464 dst := filepath.Join(dir, "dst.mp4")
465
466 if err := moveFile(filepath.Join(dir, "missing.mp4"), dst); err == nil {
467 t.Fatal("expected an error moving a file that does not exist")
468 }
469 if _, err := os.Stat(dst); !os.IsNotExist(err) {
470 t.Error("destination was created for a failed move")
471 }
472}
473
474func sameDevice(t *testing.T, a, b string) bool {
475 t.Helper()
476 var sa, sb syscall.Stat_t
477 if err := syscall.Stat(a, &sa); err != nil {
478 t.Fatal(err)
479 }
480 if err := syscall.Stat(b, &sb); err != nil {
481 t.Fatal(err)
482 }
483 return sa.Dev == sb.Dev
484}
485