vidarchive

vidarchive/internal/service/import_test.go
import_test.go
⎇
Raw
1package service
2
3import (
4 "context"
5 "database/sql"
6 "errors"
7 "fmt"
8 "os"
9 "path/filepath"
10 "strings"
11 "syscall"
12 "testing"
13 "time"
14 "unicode/utf8"
15
16 "vidarchive/internal/config"
17 "vidarchive/internal/models"
18)
19
20func TestSanitizeDirName(t *testing.T) {
21 tests := []struct{ in, want string }{
22 {"normal title", "normal title"},
23 {"a/b:c*d?e\"f<g>h|i", "a-b-c-d-e-f-g-h-i"},
24 {" trimmed ", "trimmed"},
25 {"", "untitled"},
26 {"///", "---"},
27 // A dot-only name would resolve to the target directory or its parent, so
28 // a title like this must not become a directory name.
29 {".", "untitled"},
30 {"..", "untitled"},
31 {" .. ", "untitled"},
32 {"...", "untitled"},
33 {".hidden", ".hidden"},
34 }
35 for _, tc := range tests {
36 if got := sanitizeDirName(tc.in); got != tc.want {
37 t.Errorf("sanitizeDirName(%q) = %q, want %q", tc.in, got, tc.want)
38 }
39 }
40}
41
42func TestUniqueDir(t *testing.T) {
43 base := t.TempDir()
44 svc := &DownloadService{}
45
46 mustUnique := func(name string) string {
47 t.Helper()
48 dir, err := svc.uniqueDir(base, name)
49 if err != nil {
50 t.Fatalf("uniqueDir(%q): %v", name, err)
51 }
52 return dir
53 }
54
55 first := mustUnique("item")
56 if filepath.Base(first) != "item" {
57 t.Errorf("first uniqueDir = %q, want .../item", first)
58 }
59 if err := os.MkdirAll(first, 0o755); err != nil {
60 t.Fatal(err)
61 }
62 second := mustUnique("item")
63 if filepath.Base(second) != "item-1" {
64 t.Errorf("second uniqueDir = %q, want .../item-1", second)
65 }
66 if err := os.MkdirAll(second, 0o755); err != nil {
67 t.Fatal(err)
68 }
69 third := mustUnique("item")
70 if filepath.Base(third) != "item-2" {
71 t.Errorf("third uniqueDir = %q, want .../item-2", third)
72 }
73}
74
75// A name the filesystem rejects makes every os.Stat fail with something other
76// than ENOENT. uniqueDir must report that instead of looping forever looking
77// for a free suffix.
78func TestUniqueDirRejectsUnusableName(t *testing.T) {
79 svc := &DownloadService{}
80
81 done := make(chan struct{})
82 go func() {
83 defer close(done)
84 if _, err := svc.uniqueDir(t.TempDir(), strings.Repeat("a", 300)); err == nil {
85 t.Error("uniqueDir accepted an over-long name, want an error")
86 }
87 }()
88
89 select {
90 case <-done:
91 case <-time.After(5 * time.Second):
92 t.Fatal("uniqueDir did not return: it is looping on a non-ENOENT stat error")
93 }
94}
95
96// Titles come from remote metadata, where nothing bounds their length.
97func TestSanitizeDirNameBoundsLength(t *testing.T) {
98 for _, in := range []string{strings.Repeat("a", 500), strings.Repeat("ミク", 300)} {
99 got := sanitizeDirName(in)
100 if len(got) > maxDirNameBytes {
101 t.Errorf("sanitizeDirName(%d bytes) = %d bytes, want <= %d", len(in), len(got), maxDirNameBytes)
102 }
103 if !utf8.ValidString(got) {
104 t.Errorf("sanitizeDirName(%d bytes) split a rune: %q", len(in), got)
105 }
106 }
107}
108
109func TestDeriveItemName(t *testing.T) {
110 svc := &DownloadService{}
111 itemDir := t.TempDir()
112
113 // info.json title wins and is sanitized.
114 infoPath := filepath.Join(itemDir, "info.json")
115 if err := os.WriteFile(infoPath, []byte(`{"title":"Cool: Video"}`), 0o644); err != nil {
116 t.Fatal(err)
117 }
118 bigName := "big.mp4"
119 if err := os.WriteFile(filepath.Join(itemDir, bigName), []byte("xxxxxxxxxx"), 0o644); err != nil {
120 t.Fatal(err)
121 }
122 media := []os.DirEntry{dirEntry(t, itemDir, bigName)}
123 if got := svc.deriveItemName(itemDir, readInfoJSON(infoPath), media); got != "Cool- Video" {
124 t.Errorf("deriveItemName with info.json = %q, want %q", got, "Cool- Video")
125 }
126
127 // Without info.json, falls back to the largest media file's stem.
128 small := "small.mp4"
129 if err := os.WriteFile(filepath.Join(itemDir, small), []byte("x"), 0o644); err != nil {
130 t.Fatal(err)
131 }
132 media = []os.DirEntry{dirEntry(t, itemDir, small), dirEntry(t, itemDir, bigName)}
133 if got := svc.deriveItemName(itemDir, infoJSON{}, media); got != "big" {
134 t.Errorf("deriveItemName fallback = %q, want %q (largest file stem)", got, "big")
135 }
136}
137
138func dirEntry(t *testing.T, dir, name string) os.DirEntry {
139 t.Helper()
140 entries, err := os.ReadDir(dir)
141 if err != nil {
142 t.Fatal(err)
143 }
144 for _, e := range entries {
145 if e.Name() == name {
146 return e
147 }
148 }
149 t.Fatalf("dir entry %q not found", name)
150 return nil
151}
152
153// TestImportItemDir exercises the full import: media + info.json + subtitles get
154// sorted into a named item directory with a marker. Uses ffmpeg to produce real
155// media so the mimetype-based classification in importItemDir matches.
156func TestImportItemDir(t *testing.T) {
157 requireFFmpeg(t)
158
159 libDir := t.TempDir()
160 svc := &DownloadService{cfg: &config.Config{LibraryDir: libDir, FFprobePath: "ffprobe"}}
161
162 src := t.TempDir()
163 makeTestVideo(t, filepath.Join(src, "raw.mp4"))
164 if err := os.WriteFile(filepath.Join(src, "info.json"), []byte(`{"title":"My Clip"}`), 0o644); err != nil {
165 t.Fatal(err)
166 }
167 if err := os.WriteFile(filepath.Join(src, "raw.en.srt"), []byte("1\n00:00:00,000 --> 00:00:01,000\nhi\n"), 0o644); err != nil {
168 t.Fatal(err)
169 }
170
171 if err := svc.importItemDir(context.Background(), "https://example.com/v", src, libDir, "", "-f bestvideo+bestaudio"); err != nil {
172 t.Fatalf("importItemDir: %v", err)
173 }
174
175 targetDir := filepath.Join(libDir, "My Clip")
176 if _, err := os.Stat(filepath.Join(targetDir, itemMarkerName)); err != nil {
177 t.Errorf("marker not created: %v", err)
178 }
179 if _, err := os.Stat(filepath.Join(targetDir, "info.json")); err != nil {
180 t.Errorf("info.json not moved: %v", err)
181 }
182 if _, err := os.Stat(filepath.Join(targetDir, "raw.mp4")); err != nil {
183 t.Errorf("media not moved: %v", err)
184 }
185 if _, err := os.Stat(filepath.Join(targetDir, subtitlesDirName, "raw.en.srt")); err != nil {
186 t.Errorf("subtitle not moved into subtitles dir: %v", err)
187 }
188
189 // The marker should record the source URL and a probed per-file duration
190 // (the test video is 3s), so the library never has to probe at serve time.
191 markerData, _ := os.ReadFile(filepath.Join(targetDir, itemMarkerName))
192 if want := "https://example.com/v"; !strings.Contains(string(markerData), want) {
193 t.Errorf("marker missing source url %q: %s", want, markerData)
194 }
195 if !strings.Contains(string(markerData), "file_durations") || !strings.Contains(string(markerData), "raw.mp4") {
196 t.Errorf("marker missing probed per-file duration: %s", markerData)
197 }
198 if want := "-f bestvideo+bestaudio"; !strings.Contains(string(markerData), want) {
199 t.Errorf("marker missing yt-dlp flags %q: %s", want, markerData)
200 }
201}
202
203func TestImportDownloadedItemsRejectsOutputTraversal(t *testing.T) {
204 libDir := t.TempDir()
205 svc := &DownloadService{
206 cfg: &config.Config{LibraryDir: libDir},
207 librarySvc: NewLibraryService(libDir, "ffmpeg", "ffprobe"),
208 }
209
210 // A temp download dir with one item subdir.
211 tempDir := t.TempDir()
212 itemDir := filepath.Join(tempDir, "item-00001")
213 if err := os.MkdirAll(itemDir, 0o755); err != nil {
214 t.Fatal(err)
215 }
216
217 d := &models.Download{
218 URL: "u",
219 OutputDir: sql.NullString{String: "../escape", Valid: true},
220 }
221 if _, err := svc.importDownloadedItems(context.Background(), d, tempDir, "", ""); err == nil {
222 t.Error("expected path-traversal output dir to be rejected")
223 }
224}
225
226// A download deleted while its items are being imported must stop, not keep
227// filling the library with items whose queue row no longer exists.
228func TestImportDownloadedItemsStopsOnCancel(t *testing.T) {
229 requireFFmpeg(t)
230
231 libDir := t.TempDir()
232 svc := &DownloadService{
233 cfg: &config.Config{LibraryDir: libDir, FFprobePath: "ffprobe"},
234 librarySvc: NewLibraryService(libDir, "ffmpeg", "ffprobe"),
235 }
236
237 tempDir := t.TempDir()
238 for _, name := range []string{"item-00001", "item-00002"} {
239 itemDir := filepath.Join(tempDir, name)
240 if err := os.MkdirAll(itemDir, 0o755); err != nil {
241 t.Fatal(err)
242 }
243 makeTestVideo(t, filepath.Join(itemDir, "raw.mp4"))
244 }
245
246 ctx, cancel := context.WithCancel(context.Background())
247 cancel()
248
249 imported, err := svc.importDownloadedItems(ctx, &models.Download{URL: "u"}, tempDir, "", "")
250 if !errors.Is(err, context.Canceled) {
251 t.Errorf("error = %v, want context.Canceled", err)
252 }
253 if imported != 0 {
254 t.Errorf("imported = %d, want 0", imported)
255 }
256 entries, err := os.ReadDir(libDir)
257 if err != nil {
258 t.Fatal(err)
259 }
260 if len(entries) != 0 {
261 t.Errorf("library got %d entries, want 0", len(entries))
262 }
263}
264
265// A cancel part-way through a playlist must stop at an item boundary: the items
266// already imported stay, and no half-written directory is left behind.
267func TestImportDownloadedItemsStopsBetweenItems(t *testing.T) {
268 requireFFmpeg(t)
269
270 libDir := t.TempDir()
271 svc := &DownloadService{
272 cfg: &config.Config{LibraryDir: libDir, FFprobePath: "ffprobe"},
273 librarySvc: NewLibraryService(libDir, "ffmpeg", "ffprobe"),
274 }
275
276 tempDir := t.TempDir()
277 for i, name := range []string{"item-00001", "item-00002", "item-00003"} {
278 itemDir := filepath.Join(tempDir, name)
279 if err := os.MkdirAll(itemDir, 0o755); err != nil {
280 t.Fatal(err)
281 }
282 makeTestVideo(t, filepath.Join(itemDir, "raw.mp4"))
283 info := fmt.Sprintf(`{"id":"v%d","title":"Clip %d"}`, i+1, i+1)
284 if err := os.WriteFile(filepath.Join(itemDir, "clip.info.json"), []byte(info), 0o644); err != nil {
285 t.Fatal(err)
286 }
287 }
288
289 // Cancel as soon as the first item lands, so the second boundary check trips.
290 ctx, cancel := context.WithCancel(context.Background())
291 stop := make(chan struct{})
292 go func() {
293 defer close(stop)
294 for {
295 if _, err := os.Stat(filepath.Join(libDir, "Clip 1", itemMarkerName)); err == nil {
296 cancel()
297 return
298 }
299 time.Sleep(2 * time.Millisecond)
300 }
301 }()
302
303 imported, err := svc.importDownloadedItems(ctx, &models.Download{URL: "u"}, tempDir, "", "")
304 <-stop
305 cancel()
306
307 if !errors.Is(err, context.Canceled) {
308 t.Fatalf("error = %v, want context.Canceled", err)
309 }
310 if imported < 1 || imported > 2 {
311 t.Errorf("imported = %d, want the cancel to land after 1 or 2 items", imported)
312 }
313
314 // Whatever was imported must be complete: an item dir without a marker means
315 // the import stopped half-way through writing it.
316 entries, err := os.ReadDir(libDir)
317 if err != nil {
318 t.Fatal(err)
319 }
320 if len(entries) != imported {
321 t.Errorf("library has %d dirs but reported %d imported", len(entries), imported)
322 }
323 for _, entry := range entries {
324 marker := filepath.Join(libDir, entry.Name(), itemMarkerName)
325 if _, err := os.Stat(marker); err != nil {
326 t.Errorf("item %q left without a marker: %v", entry.Name(), err)
327 }
328 }
329}
330
331// Overwrite mode replaces the existing copy of a video in place instead of
332// adding a second directory for it.
333func TestImportItemDirOverwriteReplacesExisting(t *testing.T) {
334 requireFFmpeg(t)
335
336 libDir := t.TempDir()
337 svc := &DownloadService{
338 cfg: &config.Config{LibraryDir: libDir, FFprobePath: "ffprobe"},
339 librarySvc: NewLibraryService(libDir, "ffmpeg", "ffprobe"),
340 }
341
342 // First import establishes the item.
343 first := t.TempDir()
344 makeTestVideo(t, filepath.Join(first, "raw.mp4"))
345 if err := os.WriteFile(filepath.Join(first, "clip.info.json"), []byte(`{"id":"vid1","title":"Old Title"}`), 0o644); err != nil {
346 t.Fatal(err)
347 }
348 if err := svc.importItemDir(context.Background(), "https://example.com/v", first, libDir, "overwrite", ""); err != nil {
349 t.Fatalf("first import: %v", err)
350 }
351 if err := os.WriteFile(filepath.Join(libDir, "Old Title", "stale.txt"), []byte("gone"), 0o644); err != nil {
352 t.Fatal(err)
353 }
354
355 // Second import of the same video id, now retitled upstream.
356 second := t.TempDir()
357 makeTestVideo(t, filepath.Join(second, "raw.mp4"))
358 if err := os.WriteFile(filepath.Join(second, "clip.info.json"), []byte(`{"id":"vid1","title":"New Title"}`), 0o644); err != nil {
359 t.Fatal(err)
360 }
361 if err := svc.importItemDir(context.Background(), "https://example.com/v", second, libDir, "overwrite", ""); err != nil {
362 t.Fatalf("second import: %v", err)
363 }
364
365 entries, err := os.ReadDir(libDir)
366 if err != nil {
367 t.Fatal(err)
368 }
369 if len(entries) != 1 || entries[0].Name() != "New Title" {
370 var names []string
371 for _, e := range entries {
372 names = append(names, e.Name())
373 }
374 t.Fatalf("library = %v, want exactly [New Title]", names)
375 }
376 if _, err := os.Stat(filepath.Join(libDir, "New Title", "stale.txt")); err == nil {
377 t.Error("the replaced item kept a file from the old copy")
378 }
379}
380
381// Without overwrite mode the same video is imported again beside the first, so
382// the guard above is really the thing preventing duplicates.
383func TestImportItemDirWithoutOverwriteKeepsBoth(t *testing.T) {
384 requireFFmpeg(t)
385
386 libDir := t.TempDir()
387 svc := &DownloadService{
388 cfg: &config.Config{LibraryDir: libDir, FFprobePath: "ffprobe"},
389 librarySvc: NewLibraryService(libDir, "ffmpeg", "ffprobe"),
390 }
391
392 for i := 0; i < 2; i++ {
393 src := t.TempDir()
394 makeTestVideo(t, filepath.Join(src, "raw.mp4"))
395 if err := os.WriteFile(filepath.Join(src, "clip.info.json"), []byte(`{"id":"vid1","title":"Same"}`), 0o644); err != nil {
396 t.Fatal(err)
397 }
398 if err := svc.importItemDir(context.Background(), "https://example.com/v", src, libDir, "", ""); err != nil {
399 t.Fatalf("import %d: %v", i, err)
400 }
401 }
402
403 entries, err := os.ReadDir(libDir)
404 if err != nil {
405 t.Fatal(err)
406 }
407 if len(entries) != 2 {
408 t.Errorf("library has %d dirs, want 2 (Same and Same-1)", len(entries))
409 }
410}
411
412// moveFile falls back to copy+delete when the source and destination are on
413// different filesystems, which is the normal case for a temp dir on tmpfs and a
414// library on disk. /dev/shm is a second tmpfs mount, so a rename across it
415// really does return EXDEV.
416func TestMoveFileCrossDeviceFallback(t *testing.T) {
417 dstDir, err := os.MkdirTemp("/dev/shm", "vidarchive-move-")
418 if err != nil {
419 t.Skipf("no second filesystem available for a cross-device move: %v", err)
420 }
421 defer os.RemoveAll(dstDir)
422
423 srcDir := t.TempDir()
424 if sameDevice(t, srcDir, dstDir) {
425 t.Skip("temp dir and /dev/shm are on the same filesystem")
426 }
427
428 src := filepath.Join(srcDir, "clip.mp4")
429 content := []byte("not really a video, but the bytes must survive")
430 if err := os.WriteFile(src, content, 0o640); err != nil {
431 t.Fatal(err)
432 }
433
434 // Confirm the fallback is actually what gets exercised.
435 dst := filepath.Join(dstDir, "clip.mp4")
436 if err := os.Rename(src, dst); !errors.Is(err, syscall.EXDEV) {
437 t.Skipf("rename across the two dirs did not report EXDEV: %v", err)
438 }
439
440 if err := moveFile(src, dst); err != nil {
441 t.Fatalf("moveFile: %v", err)
442 }
443
444 got, err := os.ReadFile(dst)
445 if err != nil {
446 t.Fatalf("read destination: %v", err)
447 }
448 if string(got) != string(content) {
449 t.Errorf("destination content = %q, want %q", got, content)
450 }
451 info, err := os.Stat(dst)
452 if err != nil {
453 t.Fatal(err)
454 }
455 if info.Mode().Perm() != 0o640 {
456 t.Errorf("destination mode = %v, want 0640", info.Mode().Perm())
457 }
458 if _, err := os.Stat(src); !os.IsNotExist(err) {
459 t.Error("source still exists after a successful move")
460 }
461}
462
463// A move that cannot even read the source must not leave a destination behind.
464func TestMoveFileMissingSourceLeavesNoDestination(t *testing.T) {
465 dir := t.TempDir()
466 dst := filepath.Join(dir, "dst.mp4")
467
468 if err := moveFile(filepath.Join(dir, "missing.mp4"), dst); err == nil {
469 t.Fatal("expected an error moving a file that does not exist")
470 }
471 if _, err := os.Stat(dst); !os.IsNotExist(err) {
472 t.Error("destination was created for a failed move")
473 }
474}
475
476func sameDevice(t *testing.T, a, b string) bool {
477 t.Helper()
478 var sa, sb syscall.Stat_t
479 if err := syscall.Stat(a, &sa); err != nil {
480 t.Fatal(err)
481 }
482 if err := syscall.Stat(b, &sb); err != nil {
483 t.Fatal(err)
484 }
485 return sa.Dev == sb.Dev
486}
487