vidarchive

vidarchive/internal/service/import_test.go
import_test.go
⎇
Raw
1package service
2
3import (
4 "context"
5 "database/sql"
6 "errors"
7 "fmt"
8 "os"
9 "path/filepath"
10 "strings"
11 "syscall"
12 "testing"
13 "time"
14
15 "vidarchive/internal/config"
16 "vidarchive/internal/models"
17)
18
19func TestSanitizeDirName(t *testing.T) {
20 tests := []struct{ in, want string }{
21 {"normal title", "normal title"},
22 {"a/b:c*d?e\"f<g>h|i", "a-b-c-d-e-f-g-h-i"},
23 {" trimmed ", "trimmed"},
24 {"", "untitled"},
25 {"///", "---"},
26 // A dot-only name would resolve to the target directory or its parent, so
27 // a title like this must not become a directory name.
28 {".", "untitled"},
29 {"..", "untitled"},
30 {" .. ", "untitled"},
31 {"...", "untitled"},
32 {".hidden", ".hidden"},
33 }
34 for _, tc := range tests {
35 if got := sanitizeDirName(tc.in); got != tc.want {
36 t.Errorf("sanitizeDirName(%q) = %q, want %q", tc.in, got, tc.want)
37 }
38 }
39}
40
41func TestUniqueDir(t *testing.T) {
42 base := t.TempDir()
43 svc := &DownloadService{}
44
45 first := svc.uniqueDir(base, "item")
46 if filepath.Base(first) != "item" {
47 t.Errorf("first uniqueDir = %q, want .../item", first)
48 }
49 if err := os.MkdirAll(first, 0755); err != nil {
50 t.Fatal(err)
51 }
52 second := svc.uniqueDir(base, "item")
53 if filepath.Base(second) != "item-1" {
54 t.Errorf("second uniqueDir = %q, want .../item-1", second)
55 }
56 if err := os.MkdirAll(second, 0755); err != nil {
57 t.Fatal(err)
58 }
59 third := svc.uniqueDir(base, "item")
60 if filepath.Base(third) != "item-2" {
61 t.Errorf("third uniqueDir = %q, want .../item-2", third)
62 }
63}
64
65func TestDeriveItemName(t *testing.T) {
66 svc := &DownloadService{}
67 itemDir := t.TempDir()
68
69 // info.json title wins and is sanitized.
70 infoPath := filepath.Join(itemDir, "info.json")
71 if err := os.WriteFile(infoPath, []byte(`{"title":"Cool: Video"}`), 0644); err != nil {
72 t.Fatal(err)
73 }
74 bigName := "big.mp4"
75 if err := os.WriteFile(filepath.Join(itemDir, bigName), []byte("xxxxxxxxxx"), 0644); err != nil {
76 t.Fatal(err)
77 }
78 media := []os.DirEntry{dirEntry(t, itemDir, bigName)}
79 if got := svc.deriveItemName(itemDir, readInfoJSON(infoPath), media); got != "Cool- Video" {
80 t.Errorf("deriveItemName with info.json = %q, want %q", got, "Cool- Video")
81 }
82
83 // Without info.json, falls back to the largest media file's stem.
84 small := "small.mp4"
85 if err := os.WriteFile(filepath.Join(itemDir, small), []byte("x"), 0644); err != nil {
86 t.Fatal(err)
87 }
88 media = []os.DirEntry{dirEntry(t, itemDir, small), dirEntry(t, itemDir, bigName)}
89 if got := svc.deriveItemName(itemDir, infoJSON{}, media); got != "big" {
90 t.Errorf("deriveItemName fallback = %q, want %q (largest file stem)", got, "big")
91 }
92}
93
94func dirEntry(t *testing.T, dir, name string) os.DirEntry {
95 t.Helper()
96 entries, err := os.ReadDir(dir)
97 if err != nil {
98 t.Fatal(err)
99 }
100 for _, e := range entries {
101 if e.Name() == name {
102 return e
103 }
104 }
105 t.Fatalf("dir entry %q not found", name)
106 return nil
107}
108
109// TestImportItemDir exercises the full import: media + info.json + subtitles get
110// sorted into a named item directory with a marker. Uses ffmpeg to produce real
111// media so the mimetype-based classification in importItemDir matches.
112func TestImportItemDir(t *testing.T) {
113 requireFFmpeg(t)
114
115 libDir := t.TempDir()
116 svc := &DownloadService{cfg: &config.Config{LibraryDir: libDir, FFprobePath: "ffprobe"}}
117
118 src := t.TempDir()
119 makeTestVideo(t, filepath.Join(src, "raw.mp4"))
120 if err := os.WriteFile(filepath.Join(src, "info.json"), []byte(`{"title":"My Clip"}`), 0644); err != nil {
121 t.Fatal(err)
122 }
123 if err := os.WriteFile(filepath.Join(src, "raw.en.srt"), []byte("1\n00:00:00,000 --> 00:00:01,000\nhi\n"), 0644); err != nil {
124 t.Fatal(err)
125 }
126
127 if err := svc.importItemDir(context.Background(), "https://example.com/v", src, libDir, "", "-f bestvideo+bestaudio"); err != nil {
128 t.Fatalf("importItemDir: %v", err)
129 }
130
131 targetDir := filepath.Join(libDir, "My Clip")
132 if _, err := os.Stat(filepath.Join(targetDir, itemMarkerName)); err != nil {
133 t.Errorf("marker not created: %v", err)
134 }
135 if _, err := os.Stat(filepath.Join(targetDir, "info.json")); err != nil {
136 t.Errorf("info.json not moved: %v", err)
137 }
138 if _, err := os.Stat(filepath.Join(targetDir, "raw.mp4")); err != nil {
139 t.Errorf("media not moved: %v", err)
140 }
141 if _, err := os.Stat(filepath.Join(targetDir, subtitlesDirName, "raw.en.srt")); err != nil {
142 t.Errorf("subtitle not moved into subtitles dir: %v", err)
143 }
144
145 // The marker should record the source URL and a probed per-file duration
146 // (the test video is 3s), so the library never has to probe at serve time.
147 markerData, _ := os.ReadFile(filepath.Join(targetDir, itemMarkerName))
148 if want := "https://example.com/v"; !strings.Contains(string(markerData), want) {
149 t.Errorf("marker missing source url %q: %s", want, markerData)
150 }
151 if !strings.Contains(string(markerData), "file_durations") || !strings.Contains(string(markerData), "raw.mp4") {
152 t.Errorf("marker missing probed per-file duration: %s", markerData)
153 }
154 if want := "-f bestvideo+bestaudio"; !strings.Contains(string(markerData), want) {
155 t.Errorf("marker missing yt-dlp flags %q: %s", want, markerData)
156 }
157}
158
159func TestImportDownloadedItemsRejectsOutputTraversal(t *testing.T) {
160 libDir := t.TempDir()
161 svc := &DownloadService{
162 cfg: &config.Config{LibraryDir: libDir},
163 librarySvc: NewLibraryService(libDir, "ffmpeg", "ffprobe"),
164 }
165
166 // A temp download dir with one item subdir.
167 tempDir := t.TempDir()
168 itemDir := filepath.Join(tempDir, "item-00001")
169 if err := os.MkdirAll(itemDir, 0755); err != nil {
170 t.Fatal(err)
171 }
172
173 d := &models.Download{
174 URL: "u",
175 OutputDir: sql.NullString{String: "../escape", Valid: true},
176 }
177 if _, err := svc.importDownloadedItems(context.Background(), d, tempDir, "", ""); err == nil {
178 t.Error("expected path-traversal output dir to be rejected")
179 }
180}
181
182// A download deleted while its items are being imported must stop, not keep
183// filling the library with items whose queue row no longer exists.
184func TestImportDownloadedItemsStopsOnCancel(t *testing.T) {
185 requireFFmpeg(t)
186
187 libDir := t.TempDir()
188 svc := &DownloadService{
189 cfg: &config.Config{LibraryDir: libDir, FFprobePath: "ffprobe"},
190 librarySvc: NewLibraryService(libDir, "ffmpeg", "ffprobe"),
191 }
192
193 tempDir := t.TempDir()
194 for _, name := range []string{"item-00001", "item-00002"} {
195 itemDir := filepath.Join(tempDir, name)
196 if err := os.MkdirAll(itemDir, 0755); err != nil {
197 t.Fatal(err)
198 }
199 makeTestVideo(t, filepath.Join(itemDir, "raw.mp4"))
200 }
201
202 ctx, cancel := context.WithCancel(context.Background())
203 cancel()
204
205 imported, err := svc.importDownloadedItems(ctx, &models.Download{URL: "u"}, tempDir, "", "")
206 if !errors.Is(err, context.Canceled) {
207 t.Errorf("error = %v, want context.Canceled", err)
208 }
209 if imported != 0 {
210 t.Errorf("imported = %d, want 0", imported)
211 }
212 entries, err := os.ReadDir(libDir)
213 if err != nil {
214 t.Fatal(err)
215 }
216 if len(entries) != 0 {
217 t.Errorf("library got %d entries, want 0", len(entries))
218 }
219}
220
221// A cancel part-way through a playlist must stop at an item boundary: the items
222// already imported stay, and no half-written directory is left behind.
223func TestImportDownloadedItemsStopsBetweenItems(t *testing.T) {
224 requireFFmpeg(t)
225
226 libDir := t.TempDir()
227 svc := &DownloadService{
228 cfg: &config.Config{LibraryDir: libDir, FFprobePath: "ffprobe"},
229 librarySvc: NewLibraryService(libDir, "ffmpeg", "ffprobe"),
230 }
231
232 tempDir := t.TempDir()
233 for i, name := range []string{"item-00001", "item-00002", "item-00003"} {
234 itemDir := filepath.Join(tempDir, name)
235 if err := os.MkdirAll(itemDir, 0755); err != nil {
236 t.Fatal(err)
237 }
238 makeTestVideo(t, filepath.Join(itemDir, "raw.mp4"))
239 info := fmt.Sprintf(`{"id":"v%d","title":"Clip %d"}`, i+1, i+1)
240 if err := os.WriteFile(filepath.Join(itemDir, "clip.info.json"), []byte(info), 0644); err != nil {
241 t.Fatal(err)
242 }
243 }
244
245 // Cancel as soon as the first item lands, so the second boundary check trips.
246 ctx, cancel := context.WithCancel(context.Background())
247 stop := make(chan struct{})
248 go func() {
249 defer close(stop)
250 for {
251 if _, err := os.Stat(filepath.Join(libDir, "Clip 1", itemMarkerName)); err == nil {
252 cancel()
253 return
254 }
255 time.Sleep(2 * time.Millisecond)
256 }
257 }()
258
259 imported, err := svc.importDownloadedItems(ctx, &models.Download{URL: "u"}, tempDir, "", "")
260 <-stop
261 cancel()
262
263 if !errors.Is(err, context.Canceled) {
264 t.Fatalf("error = %v, want context.Canceled", err)
265 }
266 if imported < 1 || imported > 2 {
267 t.Errorf("imported = %d, want the cancel to land after 1 or 2 items", imported)
268 }
269
270 // Whatever was imported must be complete: an item dir without a marker means
271 // the import stopped half-way through writing it.
272 entries, err := os.ReadDir(libDir)
273 if err != nil {
274 t.Fatal(err)
275 }
276 if len(entries) != imported {
277 t.Errorf("library has %d dirs but reported %d imported", len(entries), imported)
278 }
279 for _, entry := range entries {
280 marker := filepath.Join(libDir, entry.Name(), itemMarkerName)
281 if _, err := os.Stat(marker); err != nil {
282 t.Errorf("item %q left without a marker: %v", entry.Name(), err)
283 }
284 }
285}
286
287// Overwrite mode replaces the existing copy of a video in place instead of
288// adding a second directory for it.
289func TestImportItemDirOverwriteReplacesExisting(t *testing.T) {
290 requireFFmpeg(t)
291
292 libDir := t.TempDir()
293 svc := &DownloadService{
294 cfg: &config.Config{LibraryDir: libDir, FFprobePath: "ffprobe"},
295 librarySvc: NewLibraryService(libDir, "ffmpeg", "ffprobe"),
296 }
297
298 // First import establishes the item.
299 first := t.TempDir()
300 makeTestVideo(t, filepath.Join(first, "raw.mp4"))
301 if err := os.WriteFile(filepath.Join(first, "clip.info.json"), []byte(`{"id":"vid1","title":"Old Title"}`), 0644); err != nil {
302 t.Fatal(err)
303 }
304 if err := svc.importItemDir(context.Background(), "https://example.com/v", first, libDir, "overwrite", ""); err != nil {
305 t.Fatalf("first import: %v", err)
306 }
307 if err := os.WriteFile(filepath.Join(libDir, "Old Title", "stale.txt"), []byte("gone"), 0644); err != nil {
308 t.Fatal(err)
309 }
310
311 // Second import of the same video id, now retitled upstream.
312 second := t.TempDir()
313 makeTestVideo(t, filepath.Join(second, "raw.mp4"))
314 if err := os.WriteFile(filepath.Join(second, "clip.info.json"), []byte(`{"id":"vid1","title":"New Title"}`), 0644); err != nil {
315 t.Fatal(err)
316 }
317 if err := svc.importItemDir(context.Background(), "https://example.com/v", second, libDir, "overwrite", ""); err != nil {
318 t.Fatalf("second import: %v", err)
319 }
320
321 entries, err := os.ReadDir(libDir)
322 if err != nil {
323 t.Fatal(err)
324 }
325 if len(entries) != 1 || entries[0].Name() != "New Title" {
326 var names []string
327 for _, e := range entries {
328 names = append(names, e.Name())
329 }
330 t.Fatalf("library = %v, want exactly [New Title]", names)
331 }
332 if _, err := os.Stat(filepath.Join(libDir, "New Title", "stale.txt")); err == nil {
333 t.Error("the replaced item kept a file from the old copy")
334 }
335}
336
337// Without overwrite mode the same video is imported again beside the first, so
338// the guard above is really the thing preventing duplicates.
339func TestImportItemDirWithoutOverwriteKeepsBoth(t *testing.T) {
340 requireFFmpeg(t)
341
342 libDir := t.TempDir()
343 svc := &DownloadService{
344 cfg: &config.Config{LibraryDir: libDir, FFprobePath: "ffprobe"},
345 librarySvc: NewLibraryService(libDir, "ffmpeg", "ffprobe"),
346 }
347
348 for i := 0; i < 2; i++ {
349 src := t.TempDir()
350 makeTestVideo(t, filepath.Join(src, "raw.mp4"))
351 if err := os.WriteFile(filepath.Join(src, "clip.info.json"), []byte(`{"id":"vid1","title":"Same"}`), 0644); err != nil {
352 t.Fatal(err)
353 }
354 if err := svc.importItemDir(context.Background(), "https://example.com/v", src, libDir, "", ""); err != nil {
355 t.Fatalf("import %d: %v", i, err)
356 }
357 }
358
359 entries, err := os.ReadDir(libDir)
360 if err != nil {
361 t.Fatal(err)
362 }
363 if len(entries) != 2 {
364 t.Errorf("library has %d dirs, want 2 (Same and Same-1)", len(entries))
365 }
366}
367
368// moveFile falls back to copy+delete when the source and destination are on
369// different filesystems, which is the normal case for a temp dir on tmpfs and a
370// library on disk. /dev/shm is a second tmpfs mount, so a rename across it
371// really does return EXDEV.
372func TestMoveFileCrossDeviceFallback(t *testing.T) {
373 dstDir, err := os.MkdirTemp("/dev/shm", "vidarchive-move-")
374 if err != nil {
375 t.Skipf("no second filesystem available for a cross-device move: %v", err)
376 }
377 defer os.RemoveAll(dstDir)
378
379 srcDir := t.TempDir()
380 if sameDevice(t, srcDir, dstDir) {
381 t.Skip("temp dir and /dev/shm are on the same filesystem")
382 }
383
384 src := filepath.Join(srcDir, "clip.mp4")
385 content := []byte("not really a video, but the bytes must survive")
386 if err := os.WriteFile(src, content, 0640); err != nil {
387 t.Fatal(err)
388 }
389
390 // Confirm the fallback is actually what gets exercised.
391 dst := filepath.Join(dstDir, "clip.mp4")
392 if err := os.Rename(src, dst); !errors.Is(err, syscall.EXDEV) {
393 t.Skipf("rename across the two dirs did not report EXDEV: %v", err)
394 }
395
396 if err := moveFile(src, dst); err != nil {
397 t.Fatalf("moveFile: %v", err)
398 }
399
400 got, err := os.ReadFile(dst)
401 if err != nil {
402 t.Fatalf("read destination: %v", err)
403 }
404 if string(got) != string(content) {
405 t.Errorf("destination content = %q, want %q", got, content)
406 }
407 info, err := os.Stat(dst)
408 if err != nil {
409 t.Fatal(err)
410 }
411 if info.Mode().Perm() != 0640 {
412 t.Errorf("destination mode = %v, want 0640", info.Mode().Perm())
413 }
414 if _, err := os.Stat(src); !os.IsNotExist(err) {
415 t.Error("source still exists after a successful move")
416 }
417}
418
419// A move that cannot even read the source must not leave a destination behind.
420func TestMoveFileMissingSourceLeavesNoDestination(t *testing.T) {
421 dir := t.TempDir()
422 dst := filepath.Join(dir, "dst.mp4")
423
424 if err := moveFile(filepath.Join(dir, "missing.mp4"), dst); err == nil {
425 t.Fatal("expected an error moving a file that does not exist")
426 }
427 if _, err := os.Stat(dst); !os.IsNotExist(err) {
428 t.Error("destination was created for a failed move")
429 }
430}
431
432func sameDevice(t *testing.T, a, b string) bool {
433 t.Helper()
434 var sa, sb syscall.Stat_t
435 if err := syscall.Stat(a, &sa); err != nil {
436 t.Fatal(err)
437 }
438 if err := syscall.Stat(b, &sb); err != nil {
439 t.Fatal(err)
440 }
441 return sa.Dev == sb.Dev
442}
443