vidarchive

vidarchive/internal/service/import.go
import.go
⎇
Raw
1package service
2
3import (
4 "context"
5 "encoding/json"
6 "errors"
7 "fmt"
8 "io"
9 "log"
10 "os"
11 "os/exec"
12 "path/filepath"
13 "sort"
14 "strconv"
15 "strings"
16 "syscall"
17
18 "github.com/gabriel-vasile/mimetype"
19
20 "vidarchive/internal/models"
21)
22
23// resolveBaseLibraryDir returns the absolute library directory a download writes
24// into, applying the optional per-download OutputDir while rejecting any path
25// that escapes the library root.
26func (s *DownloadService) resolveBaseLibraryDir(d *models.Download) (string, error) {
27 if !d.OutputDir.Valid || d.OutputDir.String == "" {
28 return s.cfg.LibraryDir, nil
29 }
30 // Reuse the library service's guard so both entry points enforce the boundary
31 // the same way — it resolves symlinks, which a plain prefix check does not.
32 dir, err := s.librarySvc.ResolveWithinLibrary(d.OutputDir.String)
33 if err != nil {
34 return "", fmt.Errorf("invalid output directory: %w", err)
35 }
36 return dir, nil
37}
38
39// importDownloadedItems moves each downloaded item from the temp dir into the
40// library and returns the number of items successfully imported. Per-item
41// failures are logged and skipped (a playlist with a few bad entries still
42// imports the rest); a non-nil error means the import couldn't even start.
43//
44// A cancel stops the import between items and returns ctx.Err() with the count
45// imported so far. Importing a long playlist takes real time (a move plus an
46// ffprobe per file), so a deleted download must not keep filling the library.
47func (s *DownloadService) importDownloadedItems(ctx context.Context, d *models.Download, tempDownloadDir, mode, ytdlpFlags string) (int, error) {
48 entries, err := os.ReadDir(tempDownloadDir)
49 if err != nil {
50 return 0, err
51 }
52
53 baseLibraryDir, err := s.resolveBaseLibraryDir(d)
54 if err != nil {
55 return 0, err
56 }
57 if err := os.MkdirAll(baseLibraryDir, 0755); err != nil {
58 return 0, err
59 }
60
61 var itemDirs []string
62 for _, entry := range entries {
63 if !entry.IsDir() {
64 continue
65 }
66 name := entry.Name()
67 if strings.HasPrefix(name, "item-") {
68 itemDirs = append(itemDirs, filepath.Join(tempDownloadDir, name))
69 }
70 }
71 sort.Strings(itemDirs)
72
73 imported := 0
74 for _, itemDir := range itemDirs {
75 if err := ctx.Err(); err != nil {
76 return imported, err
77 }
78 if err := s.importItemDir(ctx, d.URL, itemDir, baseLibraryDir, mode, ytdlpFlags); err != nil {
79 // A cancelled item isn't a bad item: stop instead of logging a warning
80 // for it and every one that follows.
81 if ctx.Err() != nil {
82 return imported, ctx.Err()
83 }
84 log.Printf("warning: failed to import item %s: %v", itemDir, err)
85 continue
86 }
87 imported++
88 }
89
90 // The temp dir (and any leftovers from failed imports) is removed by the
91 // caller's deferred cleanup, so partial state never leaks even on a crash.
92 return imported, nil
93}
94
95func (s *DownloadService) importItemDir(ctx context.Context, url, itemDir, baseLibraryDir, mode, ytdlpFlags string) error {
96 entries, err := os.ReadDir(itemDir)
97 if err != nil {
98 return err
99 }
100
101 var mediaFiles []os.DirEntry
102 var infoJSONPath string
103 var subtitleFiles []string
104
105 for _, entry := range entries {
106 if entry.IsDir() {
107 continue
108 }
109 name := entry.Name()
110 path := filepath.Join(itemDir, name)
111 ext := strings.ToLower(filepath.Ext(name))
112
113 if isInfoJSON(name) {
114 infoJSONPath = path
115 continue
116 }
117 if ext == ".vtt" || ext == ".srt" || ext == ".ass" || ext == ".ssa" {
118 subtitleFiles = append(subtitleFiles, path)
119 continue
120 }
121
122 mtype, err := mimetype.DetectFile(path)
123 if err == nil && mtype != nil && (strings.HasPrefix(mtype.String(), "audio/") || strings.HasPrefix(mtype.String(), "video/")) {
124 mediaFiles = append(mediaFiles, entry)
125 }
126 }
127
128 if len(mediaFiles) == 0 {
129 return fmt.Errorf("no media files found in %s", itemDir)
130 }
131
132 info := readInfoJSON(infoJSONPath)
133 name := s.deriveItemName(itemDir, info, mediaFiles)
134 videoID := info.ID
135
136 // Last point at which nothing has been written to the library yet: give up
137 // here on a cancel rather than part-way through, which would leave a folder
138 // with some of its files and no marker — or, in overwrite mode, delete the
139 // existing item and not replace it.
140 if err := ctx.Err(); err != nil {
141 return err
142 }
143
144 // Overwrite mode: replace the existing copy of this video in place rather than
145 // creating a duplicate folder. Removing the old dir lets uniqueDir reuse its
146 // name (or land on the new title if it changed upstream).
147 if mode == "overwrite" && videoID != "" {
148 if existing, ok := s.librarySvc.FindByVideoID(baseLibraryDir, videoID); ok {
149 if rel, err := filepath.Rel(s.cfg.LibraryDir, existing); err == nil {
150 s.librarySvc.evictCachedScan(filepath.ToSlash(rel))
151 }
152 os.RemoveAll(existing)
153 }
154 }
155
156 targetDir := s.uniqueDir(baseLibraryDir, name)
157 if err := os.MkdirAll(targetDir, 0755); err != nil {
158 return err
159 }
160
161 if infoJSONPath != "" {
162 if err := moveFile(infoJSONPath, filepath.Join(targetDir, "info.json")); err != nil {
163 return err
164 }
165 }
166
167 for _, entry := range mediaFiles {
168 if err := moveFile(filepath.Join(itemDir, entry.Name()), filepath.Join(targetDir, entry.Name())); err != nil {
169 return err
170 }
171 }
172
173 // Probe each media file's duration once, here in the worker (off the request
174 // path), and cache it in the marker so the library never has to probe while
175 // serving pages. Files we can't probe simply get no duration.
176 fileDurations := make(map[string]int)
177 for _, entry := range mediaFiles {
178 if d, ok := probeDuration(ctx, s.cfg.FFprobePath, filepath.Join(targetDir, entry.Name())); ok {
179 fileDurations[entry.Name()] = d
180 }
181 }
182
183 if len(subtitleFiles) > 0 {
184 subtitlesDir := filepath.Join(targetDir, subtitlesDirName)
185 if err := os.MkdirAll(subtitlesDir, 0755); err != nil {
186 return err
187 }
188 for _, sf := range subtitleFiles {
189 if err := moveFile(sf, filepath.Join(subtitlesDir, filepath.Base(sf))); err != nil {
190 return err
191 }
192 }
193 }
194
195 metadata := models.ItemMetadata{
196 Name: name,
197 SourceURL: url,
198 VideoID: videoID,
199 YtdlpFlags: ytdlpFlags,
200 FileDurations: fileDurations,
201 }
202
203 return s.librarySvc.writeMetadata(targetDir, metadata)
204}
205
206// infoJSON is the subset of yt-dlp's info.json VidArchive reads. ID is the
207// stable item identity; within a single subscription's own directory it is
208// enough to match items, so the extractor is not needed.
209type infoJSON struct {
210 ID string `json:"id"`
211 // Type is yt-dlp's "_type": "playlist" marks a playlist-level sidecar rather
212 // than an item.
213 Type string `json:"_type"`
214 Title string `json:"title"`
215 Description string `json:"description"`
216 WebpageURL string `json:"webpage_url"`
217}
218
219// readInfoJSON parses an info.json. A missing, unreadable or malformed file
220// yields a zero-value struct: every caller treats absent fields as "unknown"
221// and falls back, so there is nothing to distinguish.
222func readInfoJSON(infoJSONPath string) infoJSON {
223 var info infoJSON
224 if infoJSONPath == "" {
225 return info
226 }
227 data, err := os.ReadFile(infoJSONPath)
228 if err != nil {
229 return info
230 }
231 if err := json.Unmarshal(data, &info); err != nil {
232 log.Printf("ignoring malformed %s: %v", infoJSONPath, err)
233 return infoJSON{}
234 }
235 return info
236}
237
238// isInfoJSON reports whether a file name is yt-dlp's metadata sidecar. yt-dlp
239// writes "<title>.info.json" next to the media, but a bare "info.json" is what
240// an already-imported item holds.
241func isInfoJSON(name string) bool {
242 return name == "info.json" || strings.HasSuffix(name, ".info.json")
243}
244
245// findInfoJSON returns the path to an info.json directly inside itemDir, or "".
246func findInfoJSON(itemDir string) string {
247 entries, err := os.ReadDir(itemDir)
248 if err != nil {
249 return ""
250 }
251 for _, entry := range entries {
252 if entry.IsDir() {
253 continue
254 }
255 name := entry.Name()
256 if isInfoJSON(name) {
257 return filepath.Join(itemDir, name)
258 }
259 }
260 return ""
261}
262
263// deriveItemName names the imported item after its title, falling back to the
264// largest media file's base name when there is no usable info.json.
265func (s *DownloadService) deriveItemName(itemDir string, info infoJSON, mediaFiles []os.DirEntry) string {
266 if info.Title != "" {
267 return sanitizeDirName(info.Title)
268 }
269
270 var largest os.DirEntry
271 var maxSize int64
272 for _, f := range mediaFiles {
273 st, err := os.Stat(filepath.Join(itemDir, f.Name()))
274 if err == nil && (largest == nil || st.Size() > maxSize) {
275 largest, maxSize = f, st.Size()
276 }
277 }
278 if largest == nil {
279 largest = mediaFiles[0]
280 }
281 base := strings.TrimSuffix(largest.Name(), filepath.Ext(largest.Name()))
282 return sanitizeDirName(base)
283}
284
285func (s *DownloadService) uniqueDir(base, name string) string {
286 dir := filepath.Join(base, name)
287 if _, err := os.Stat(dir); os.IsNotExist(err) {
288 return dir
289 }
290 for i := 1; ; i++ {
291 candidate := fmt.Sprintf("%s-%d", dir, i)
292 if _, err := os.Stat(candidate); os.IsNotExist(err) {
293 return candidate
294 }
295 }
296}
297
298// moveFile moves src to dst, falling back to copy-and-delete when the two are on
299// different filesystems. The temp and library directories are independently
300// configurable, so they can legitimately live on separate mounts — where a plain
301// rename fails with EXDEV.
302func moveFile(src, dst string) error {
303 if err := os.Rename(src, dst); err == nil {
304 return nil
305 } else if !errors.Is(err, syscall.EXDEV) {
306 return err
307 }
308
309 in, err := os.Open(src)
310 if err != nil {
311 return err
312 }
313 defer in.Close()
314
315 info, err := in.Stat()
316 if err != nil {
317 return err
318 }
319
320 out, err := os.OpenFile(dst, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, info.Mode())
321 if err != nil {
322 return err
323 }
324
325 if _, err := io.Copy(out, in); err != nil {
326 out.Close()
327 os.Remove(dst)
328 return err
329 }
330 // Close explicitly: a deferred close would hide a flush error on the copy.
331 if err := out.Close(); err != nil {
332 os.Remove(dst)
333 return err
334 }
335
336 return os.Remove(src)
337}
338
339func sanitizeDirName(name string) string {
340 name = strings.TrimSpace(name)
341 replacer := strings.NewReplacer(
342 "/", "-",
343 "\\", "-",
344 ":", "-",
345 "*", "-",
346 "?", "-",
347 "\"", "-",
348 "<", "-",
349 ">", "-",
350 "|", "-",
351 )
352 name = replacer.Replace(name)
353 name = strings.TrimSpace(name)
354 // A name made only of dots resolves to the parent ("..") or to the target
355 // directory itself ("."), so joining it would place the item outside the
356 // library. Titles come from remote metadata, so refuse them here.
357 if strings.Trim(name, ".") == "" {
358 name = "untitled"
359 }
360 return name
361}
362
363// probeDuration returns the duration of a media file in whole seconds. The bool
364// is false when ffprobe is unavailable or the file has no usable duration.
365func probeDuration(ctx context.Context, ffprobePath, path string) (int, bool) {
366 out, err := exec.CommandContext(ctx, ffprobePath, "-v", "error",
367 "-show_entries", "format=duration",
368 "-of", "default=nw=1:nk=1", path).Output()
369 if err != nil {
370 return 0, false
371 }
372 f, err := strconv.ParseFloat(strings.TrimSpace(string(out)), 64)
373 if err != nil || f <= 0 {
374 return 0, false
375 }
376 return int(f + 0.5), true
377}
378