vidarchive

vidarchive/internal/service/import.go
import.go
⎇
Raw
1package service
2
3import (
4 "context"
5 "encoding/json"
6 "errors"
7 "fmt"
8 "io"
9 "log/slog"
10 "os"
11 "path/filepath"
12 "sort"
13 "strconv"
14 "strings"
15 "syscall"
16 "unicode/utf8"
17
18 "github.com/gabriel-vasile/mimetype"
19
20 "vidarchive/internal/models"
21 "vidarchive/internal/util"
22)
23
24// resolveBaseLibraryDir returns the absolute library directory a download writes
25// into, applying the optional per-download OutputDir while rejecting any path
26// that escapes the library root.
27func (s *DownloadService) resolveBaseLibraryDir(d *models.Download) (string, error) {
28 // Both branches go through the library service's guard, so every caller gets
29 // a symlink-resolved path. Cache keys are derived from that resolved root.
30 if !d.OutputDir.Valid || d.OutputDir.String == "" {
31 return s.librarySvc.ResolveWithinLibrary("")
32 }
33 // Reuse the library service's guard so both entry points enforce the boundary
34 // the same way — it resolves symlinks, which a plain prefix check does not.
35 dir, err := s.librarySvc.ResolveWithinLibrary(d.OutputDir.String)
36 if err != nil {
37 return "", fmt.Errorf("invalid output directory: %w", err)
38 }
39 return dir, nil
40}
41
42// importDownloadedItems moves each downloaded item from the temp dir into the
43// library and returns the number of items successfully imported. Per-item
44// failures are logged and skipped (a playlist with a few bad entries still
45// imports the rest); a non-nil error means the import couldn't even start.
46//
47// A cancel stops the import between items and returns ctx.Err() with the count
48// imported so far. Importing a long playlist takes real time (a move plus an
49// ffprobe per file), so a deleted download must not keep filling the library.
50func (s *DownloadService) importDownloadedItems(ctx context.Context, d *models.Download, tempDownloadDir, mode, ytdlpFlags string) (int, error) {
51 entries, err := os.ReadDir(tempDownloadDir)
52 if err != nil {
53 return 0, err
54 }
55
56 baseLibraryDir, err := s.resolveBaseLibraryDir(d)
57 if err != nil {
58 return 0, err
59 }
60 if err := os.MkdirAll(baseLibraryDir, 0o755); err != nil {
61 return 0, err
62 }
63
64 var itemDirs []string
65 for _, entry := range entries {
66 if !entry.IsDir() {
67 continue
68 }
69 name := entry.Name()
70 if strings.HasPrefix(name, "item-") {
71 itemDirs = append(itemDirs, filepath.Join(tempDownloadDir, name))
72 }
73 }
74 sort.Strings(itemDirs)
75
76 imported := 0
77 for _, itemDir := range itemDirs {
78 if err := ctx.Err(); err != nil {
79 return imported, err
80 }
81 if err := s.importItemDir(ctx, d.URL, itemDir, baseLibraryDir, mode, ytdlpFlags); err != nil {
82 // A cancelled item isn't a bad item: stop instead of logging a warning
83 // for it and every one that follows.
84 if ctx.Err() != nil {
85 return imported, ctx.Err()
86 }
87 slog.Warn("import item failed", "dir", itemDir, "err", err)
88 continue
89 }
90 imported++
91 }
92
93 // The temp dir (and any leftovers from failed imports) is removed by the
94 // caller's deferred cleanup, so partial state never leaks even on a crash.
95 return imported, nil
96}
97
98func (s *DownloadService) importItemDir(ctx context.Context, url, itemDir, baseLibraryDir, mode, ytdlpFlags string) error {
99 entries, err := os.ReadDir(itemDir)
100 if err != nil {
101 return err
102 }
103
104 var mediaFiles []os.DirEntry
105 var infoJSONPath string
106 var subtitleFiles []string
107
108 for _, entry := range entries {
109 if entry.IsDir() {
110 continue
111 }
112 name := entry.Name()
113 path := filepath.Join(itemDir, name)
114 ext := strings.ToLower(filepath.Ext(name))
115
116 if isInfoJSON(name) {
117 infoJSONPath = path
118 continue
119 }
120 if ext == ".vtt" || ext == ".srt" || ext == ".ass" || ext == ".ssa" {
121 subtitleFiles = append(subtitleFiles, path)
122 continue
123 }
124
125 mtype, err := mimetype.DetectFile(path)
126 if err == nil && mtype != nil && (strings.HasPrefix(mtype.String(), "audio/") || strings.HasPrefix(mtype.String(), "video/")) {
127 mediaFiles = append(mediaFiles, entry)
128 }
129 }
130
131 if len(mediaFiles) == 0 {
132 return fmt.Errorf("no media files found in %s", itemDir)
133 }
134
135 info := readInfoJSON(infoJSONPath)
136 name := s.deriveItemName(itemDir, info, mediaFiles)
137 videoID := info.ID
138
139 // Last point at which nothing has been written to the library yet: give up
140 // here on a cancel rather than part-way through, which would leave a folder
141 // with some of its files and no marker — or, in overwrite mode, delete the
142 // existing item and not replace it.
143 if err := ctx.Err(); err != nil {
144 return err
145 }
146
147 // Overwrite mode: replace the existing copy of this video in place rather than
148 // creating a duplicate folder. Removing the old dir lets uniqueDir reuse its
149 // name (or land on the new title if it changed upstream).
150 if mode == "overwrite" && videoID != "" {
151 if existing, ok := s.librarySvc.FindByVideoID(baseLibraryDir, videoID); ok {
152 s.librarySvc.evictCachedDir(existing)
153 os.RemoveAll(existing)
154 }
155 }
156
157 targetDir, err := s.uniqueDir(baseLibraryDir, name)
158 if err != nil {
159 return err
160 }
161 if err := os.MkdirAll(targetDir, 0o755); err != nil {
162 return err
163 }
164
165 if infoJSONPath != "" {
166 if err := moveFile(infoJSONPath, filepath.Join(targetDir, "info.json")); err != nil {
167 return err
168 }
169 }
170
171 for _, entry := range mediaFiles {
172 if err := moveFile(filepath.Join(itemDir, entry.Name()), filepath.Join(targetDir, entry.Name())); err != nil {
173 return err
174 }
175 }
176
177 // Probe each media file's duration once, here in the worker (off the request
178 // path), and cache it in the marker so the library never has to probe while
179 // serving pages. Files we can't probe simply get no duration.
180 fileDurations := make(map[string]int)
181 for _, entry := range mediaFiles {
182 if d, ok := probeDuration(ctx, s.cfg.FFprobePath, filepath.Join(targetDir, entry.Name())); ok {
183 fileDurations[entry.Name()] = d
184 }
185 }
186
187 if len(subtitleFiles) > 0 {
188 subtitlesDir := filepath.Join(targetDir, subtitlesDirName)
189 if err := os.MkdirAll(subtitlesDir, 0o755); err != nil {
190 return err
191 }
192 for _, sf := range subtitleFiles {
193 if err := moveFile(sf, filepath.Join(subtitlesDir, filepath.Base(sf))); err != nil {
194 return err
195 }
196 }
197 }
198
199 metadata := models.ItemMetadata{
200 Name: name,
201 SourceURL: url,
202 VideoID: videoID,
203 YtdlpFlags: ytdlpFlags,
204 FileDurations: fileDurations,
205 }
206
207 return s.librarySvc.writeMetadata(targetDir, metadata)
208}
209
210// infoJSON is the subset of yt-dlp's info.json VidArchive reads. ID is the
211// stable item identity; within a single subscription's own directory it is
212// enough to match items, so the extractor is not needed.
213type infoJSON struct {
214 ID string `json:"id"`
215 // Type is yt-dlp's "_type": "playlist" marks a playlist-level sidecar rather
216 // than an item.
217 Type string `json:"_type"`
218 Title string `json:"title"`
219 Description string `json:"description"`
220 WebpageURL string `json:"webpage_url"`
221}
222
223// readInfoJSON parses an info.json. A missing, unreadable or malformed file
224// yields a zero-value struct: every caller treats absent fields as "unknown"
225// and falls back, so there is nothing to distinguish.
226func readInfoJSON(infoJSONPath string) infoJSON {
227 var info infoJSON
228 if infoJSONPath == "" {
229 return info
230 }
231 data, err := os.ReadFile(infoJSONPath)
232 if err != nil {
233 return info
234 }
235 if err := json.Unmarshal(data, &info); err != nil {
236 slog.Warn("ignoring malformed info.json", "path", infoJSONPath, "err", err)
237 return infoJSON{}
238 }
239 return info
240}
241
242// isInfoJSON reports whether a file name is yt-dlp's metadata sidecar. yt-dlp
243// writes "<title>.info.json" next to the media, but a bare "info.json" is what
244// an already-imported item holds.
245func isInfoJSON(name string) bool {
246 return name == "info.json" || strings.HasSuffix(name, ".info.json")
247}
248
249// findInfoJSON returns the path to an info.json directly inside itemDir, or "".
250func findInfoJSON(itemDir string) string {
251 entries, err := os.ReadDir(itemDir)
252 if err != nil {
253 return ""
254 }
255 for _, entry := range entries {
256 if entry.IsDir() {
257 continue
258 }
259 name := entry.Name()
260 if isInfoJSON(name) {
261 return filepath.Join(itemDir, name)
262 }
263 }
264 return ""
265}
266
267// deriveItemName names the imported item after its title, falling back to the
268// largest media file's base name when there is no usable info.json.
269func (s *DownloadService) deriveItemName(itemDir string, info infoJSON, mediaFiles []os.DirEntry) string {
270 if info.Title != "" {
271 return sanitizeDirName(info.Title)
272 }
273
274 var largest os.DirEntry
275 var maxSize int64
276 for _, f := range mediaFiles {
277 st, err := os.Stat(filepath.Join(itemDir, f.Name()))
278 if err == nil && (largest == nil || st.Size() > maxSize) {
279 largest, maxSize = f, st.Size()
280 }
281 }
282 if largest == nil {
283 largest = mediaFiles[0]
284 }
285 base := strings.TrimSuffix(largest.Name(), filepath.Ext(largest.Name()))
286 return sanitizeDirName(base)
287}
288
289// uniqueDir returns a directory under base that does not exist yet, appending
290// "-1", "-2", ... until it finds one. A stat error other than "does not exist"
291// is returned rather than treated as "taken": every candidate would fail the
292// same way, so the loop would never end.
293func (s *DownloadService) uniqueDir(base, name string) (string, error) {
294 dir := filepath.Join(base, name)
295 for i := 0; ; i++ {
296 candidate := dir
297 if i > 0 {
298 candidate = fmt.Sprintf("%s-%d", dir, i)
299 }
300 _, err := os.Stat(candidate)
301 if os.IsNotExist(err) {
302 return candidate, nil
303 }
304 if err != nil {
305 return "", fmt.Errorf("choose item directory: %w", err)
306 }
307 }
308}
309
310// moveFile moves src to dst, falling back to copy-and-delete when the two are on
311// different filesystems. The temp and library directories are independently
312// configurable, so they can legitimately live on separate mounts — where a plain
313// rename fails with EXDEV.
314func moveFile(src, dst string) error {
315 if err := os.Rename(src, dst); err == nil {
316 return nil
317 } else if !errors.Is(err, syscall.EXDEV) {
318 return err
319 }
320
321 in, err := os.Open(src)
322 if err != nil {
323 return err
324 }
325 defer in.Close()
326
327 info, err := in.Stat()
328 if err != nil {
329 return err
330 }
331
332 out, err := os.OpenFile(dst, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, info.Mode())
333 if err != nil {
334 return err
335 }
336
337 if _, err := io.Copy(out, in); err != nil {
338 out.Close()
339 os.Remove(dst)
340 return err
341 }
342 // Close explicitly: a deferred close would hide a flush error on the copy.
343 if err := out.Close(); err != nil {
344 os.Remove(dst)
345 return err
346 }
347
348 return os.Remove(src)
349}
350
351func sanitizeDirName(name string) string {
352 name = strings.TrimSpace(name)
353 replacer := strings.NewReplacer(
354 "/", "-",
355 "\\", "-",
356 ":", "-",
357 "*", "-",
358 "?", "-",
359 "\"", "-",
360 "<", "-",
361 ">", "-",
362 "|", "-",
363 )
364 name = replacer.Replace(name)
365 name = strings.TrimSpace(name)
366 // A name made only of dots resolves to the parent ("..") or to the target
367 // directory itself ("."), so joining it would place the item outside the
368 // library. Titles come from remote metadata, so refuse them here.
369 if strings.Trim(name, ".") == "" {
370 name = "untitled"
371 }
372 return truncateDirName(name)
373}
374
375// maxDirNameBytes leaves room under the 255-byte filesystem component limit for
376// uniqueDir's "-N" suffix.
377const maxDirNameBytes = 240
378
379// truncateDirName shortens name to maxDirNameBytes without splitting a rune.
380// Over the limit, every filesystem call on the name fails with ENAMETOOLONG.
381func truncateDirName(name string) string {
382 if len(name) <= maxDirNameBytes {
383 return name
384 }
385 cut := maxDirNameBytes
386 for cut > 0 && !utf8.RuneStart(name[cut]) {
387 cut--
388 }
389 truncated := strings.TrimSpace(name[:cut])
390 // Trimming can reintroduce the empty/all-dots case the caller already ruled
391 // out.
392 if strings.Trim(truncated, ".") == "" {
393 return "untitled"
394 }
395 return truncated
396}
397
398// probeDuration returns the duration of a media file in whole seconds. The bool
399// is false when ffprobe is unavailable or the file has no usable duration.
400func probeDuration(ctx context.Context, ffprobePath, path string) (int, bool) {
401 out, err := util.KillableCommand(ctx, ffprobePath, "-v", "error",
402 "-show_entries", "format=duration",
403 "-of", "default=nw=1:nk=1", path).Output()
404 if err != nil {
405 return 0, false
406 }
407 f, err := strconv.ParseFloat(strings.TrimSpace(string(out)), 64)
408 if err != nil || f <= 0 {
409 return 0, false
410 }
411 return int(f + 0.5), true
412}
413