vidarchive

vidarchive/internal/service/import.go
import.go
⎇
Raw
1package service
2
3import (
4 "context"
5 "encoding/json"
6 "errors"
7 "fmt"
8 "io"
9 "log/slog"
10 "os"
11 "path/filepath"
12 "sort"
13 "strconv"
14 "strings"
15 "syscall"
16 "unicode/utf8"
17
18 "github.com/gabriel-vasile/mimetype"
19
20 "vidarchive/internal/models"
21 "vidarchive/internal/util"
22)
23
24// resolveBaseLibraryDir applies the optional per-download OutputDir. Both
25// branches go through the library guard: it resolves symlinks, which a plain
26// prefix check does not, and the cache keys are derived from the resolved root.
27func (s *DownloadService) resolveBaseLibraryDir(d *models.Download) (string, error) {
28 if !d.OutputDir.Valid || d.OutputDir.String == "" {
29 return s.librarySvc.ResolveWithinLibrary("")
30 }
31 dir, err := s.librarySvc.ResolveWithinLibrary(d.OutputDir.String)
32 if err != nil {
33 return "", fmt.Errorf("invalid output directory: %w", err)
34 }
35 return dir, nil
36}
37
38// importDownloadedItems returns the number of items imported. Per-item failures
39// are logged and skipped, so a playlist with a few bad entries still imports the
40// rest; a non-nil error means the import couldn't even start. A cancel stops
41// between items and returns ctx.Err() with the count imported so far.
42func (s *DownloadService) importDownloadedItems(ctx context.Context, d *models.Download, tempDownloadDir, mode, ytdlpFlags string) (int, error) {
43 entries, err := os.ReadDir(tempDownloadDir)
44 if err != nil {
45 return 0, err
46 }
47
48 baseLibraryDir, err := s.resolveBaseLibraryDir(d)
49 if err != nil {
50 return 0, err
51 }
52 if err := os.MkdirAll(baseLibraryDir, 0o755); err != nil {
53 return 0, err
54 }
55
56 var itemDirs []string
57 for _, entry := range entries {
58 if !entry.IsDir() {
59 continue
60 }
61 name := entry.Name()
62 if strings.HasPrefix(name, "item-") {
63 itemDirs = append(itemDirs, filepath.Join(tempDownloadDir, name))
64 }
65 }
66 sort.Strings(itemDirs)
67
68 imported := 0
69 for _, itemDir := range itemDirs {
70 if err := ctx.Err(); err != nil {
71 return imported, err
72 }
73 if err := s.importItemDir(ctx, d.URL, itemDir, baseLibraryDir, mode, ytdlpFlags); err != nil {
74 // A cancelled item is not a bad item, and every one after it would warn
75 // too.
76 if ctx.Err() != nil {
77 return imported, ctx.Err()
78 }
79 slog.Warn("import item failed", "dir", itemDir, "err", err)
80 continue
81 }
82 imported++
83 }
84
85 // Leftovers from failed imports go with the caller's deferred temp-dir cleanup.
86 return imported, nil
87}
88
89func (s *DownloadService) importItemDir(ctx context.Context, url, itemDir, baseLibraryDir, mode, ytdlpFlags string) error {
90 entries, err := os.ReadDir(itemDir)
91 if err != nil {
92 return err
93 }
94
95 var mediaFiles []os.DirEntry
96 var infoJSONPath string
97 var subtitleFiles []string
98
99 for _, entry := range entries {
100 if entry.IsDir() {
101 continue
102 }
103 name := entry.Name()
104 path := filepath.Join(itemDir, name)
105 ext := strings.ToLower(filepath.Ext(name))
106
107 if isInfoJSON(name) {
108 infoJSONPath = path
109 continue
110 }
111 if ext == ".vtt" || ext == ".srt" || ext == ".ass" || ext == ".ssa" {
112 subtitleFiles = append(subtitleFiles, path)
113 continue
114 }
115
116 mtype, err := mimetype.DetectFile(path)
117 if err == nil && mtype != nil && (strings.HasPrefix(mtype.String(), "audio/") || strings.HasPrefix(mtype.String(), "video/")) {
118 mediaFiles = append(mediaFiles, entry)
119 }
120 }
121
122 if len(mediaFiles) == 0 {
123 return fmt.Errorf("no media files found in %s", itemDir)
124 }
125
126 info := readInfoJSON(infoJSONPath)
127 name := s.deriveItemName(itemDir, info, mediaFiles)
128 videoID := info.ID
129
130 // Last point at which nothing has been written to the library yet. Cancelling
131 // later leaves a folder with no marker, or in overwrite mode deletes the
132 // existing item without replacing it.
133 if err := ctx.Err(); err != nil {
134 return err
135 }
136
137 // Removing the old dir avoids a duplicate folder and lets uniqueDir reuse its
138 // name, or take the new title if it changed upstream.
139 if mode == "overwrite" && videoID != "" {
140 if existing, ok := s.librarySvc.FindByVideoID(baseLibraryDir, videoID); ok {
141 s.librarySvc.evictCachedDir(existing)
142 os.RemoveAll(existing)
143 }
144 }
145
146 targetDir, err := s.uniqueDir(baseLibraryDir, name)
147 if err != nil {
148 return err
149 }
150 if err := os.MkdirAll(targetDir, 0o755); err != nil {
151 return err
152 }
153
154 if infoJSONPath != "" {
155 if err := moveFile(infoJSONPath, filepath.Join(targetDir, "info.json")); err != nil {
156 return err
157 }
158 }
159
160 for _, entry := range mediaFiles {
161 if err := moveFile(filepath.Join(itemDir, entry.Name()), filepath.Join(targetDir, entry.Name())); err != nil {
162 return err
163 }
164 }
165
166 // Probing once here, in the worker, is what keeps ffprobe off the request
167 // path. An unprobeable file simply gets no duration.
168 fileDurations := make(map[string]int)
169 for _, entry := range mediaFiles {
170 if d, ok := probeDuration(ctx, s.cfg.FFprobePath, filepath.Join(targetDir, entry.Name())); ok {
171 fileDurations[entry.Name()] = d
172 }
173 }
174
175 if len(subtitleFiles) > 0 {
176 subtitlesDir := filepath.Join(targetDir, subtitlesDirName)
177 if err := os.MkdirAll(subtitlesDir, 0o755); err != nil {
178 return err
179 }
180 for _, sf := range subtitleFiles {
181 if err := moveFile(sf, filepath.Join(subtitlesDir, filepath.Base(sf))); err != nil {
182 return err
183 }
184 }
185 }
186
187 metadata := models.ItemMetadata{
188 Name: name,
189 SourceURL: url,
190 VideoID: videoID,
191 YtdlpFlags: ytdlpFlags,
192 FileDurations: fileDurations,
193 }
194
195 return s.librarySvc.writeMetadata(targetDir, metadata)
196}
197
198// infoJSON is the subset of yt-dlp's info.json VidArchive reads. ID alone
199// identifies an item within one subscription's directory, so the extractor is
200// not needed.
201type infoJSON struct {
202 ID string `json:"id"`
203 // Type is yt-dlp's "_type": "playlist" marks a playlist-level sidecar rather
204 // than an item.
205 Type string `json:"_type"`
206 Title string `json:"title"`
207 Description string `json:"description"`
208 WebpageURL string `json:"webpage_url"`
209}
210
211// readInfoJSON yields a zero-value struct for a missing, unreadable or malformed
212// file: every caller already falls back on an absent field.
213func readInfoJSON(infoJSONPath string) infoJSON {
214 var info infoJSON
215 if infoJSONPath == "" {
216 return info
217 }
218 data, err := os.ReadFile(infoJSONPath)
219 if err != nil {
220 return info
221 }
222 if err := json.Unmarshal(data, &info); err != nil {
223 slog.Warn("ignoring malformed info.json", "path", infoJSONPath, "err", err)
224 return infoJSON{}
225 }
226 return info
227}
228
229// isInfoJSON accepts both forms: yt-dlp writes "<title>.info.json", while an
230// already-imported item holds a bare "info.json".
231func isInfoJSON(name string) bool {
232 return name == "info.json" || strings.HasSuffix(name, ".info.json")
233}
234
235func findInfoJSON(itemDir string) string {
236 entries, err := os.ReadDir(itemDir)
237 if err != nil {
238 return ""
239 }
240 for _, entry := range entries {
241 if entry.IsDir() {
242 continue
243 }
244 name := entry.Name()
245 if isInfoJSON(name) {
246 return filepath.Join(itemDir, name)
247 }
248 }
249 return ""
250}
251
252// deriveItemName falls back to the largest media file's base name when there is
253// no usable info.json title.
254func (s *DownloadService) deriveItemName(itemDir string, info infoJSON, mediaFiles []os.DirEntry) string {
255 if info.Title != "" {
256 return sanitizeDirName(info.Title)
257 }
258
259 var largest os.DirEntry
260 var maxSize int64
261 for _, f := range mediaFiles {
262 st, err := os.Stat(filepath.Join(itemDir, f.Name()))
263 if err == nil && (largest == nil || st.Size() > maxSize) {
264 largest, maxSize = f, st.Size()
265 }
266 }
267 if largest == nil {
268 largest = mediaFiles[0]
269 }
270 base := strings.TrimSuffix(largest.Name(), filepath.Ext(largest.Name()))
271 return sanitizeDirName(base)
272}
273
274// uniqueDir appends "-1", "-2", ... until the name is free. A stat error other
275// than "does not exist" is returned rather than treated as taken: every
276// candidate would fail the same way and the loop would never end.
277func (s *DownloadService) uniqueDir(base, name string) (string, error) {
278 dir := filepath.Join(base, name)
279 for i := 0; ; i++ {
280 candidate := dir
281 if i > 0 {
282 candidate = fmt.Sprintf("%s-%d", dir, i)
283 }
284 _, err := os.Stat(candidate)
285 if os.IsNotExist(err) {
286 return candidate, nil
287 }
288 if err != nil {
289 return "", fmt.Errorf("choose item directory: %w", err)
290 }
291 }
292}
293
294// moveFile falls back to copy-and-delete on EXDEV: the temp and library
295// directories are configured separately and can live on different mounts.
296func moveFile(src, dst string) error {
297 if err := os.Rename(src, dst); err == nil {
298 return nil
299 } else if !errors.Is(err, syscall.EXDEV) {
300 return err
301 }
302
303 in, err := os.Open(src)
304 if err != nil {
305 return err
306 }
307 defer in.Close()
308
309 info, err := in.Stat()
310 if err != nil {
311 return err
312 }
313
314 out, err := os.OpenFile(dst, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, info.Mode())
315 if err != nil {
316 return err
317 }
318
319 if _, err := io.Copy(out, in); err != nil {
320 out.Close()
321 os.Remove(dst)
322 return err
323 }
324 // Close explicitly: a deferred close would hide a flush error on the copy.
325 if err := out.Close(); err != nil {
326 os.Remove(dst)
327 return err
328 }
329
330 return os.Remove(src)
331}
332
333func sanitizeDirName(name string) string {
334 name = strings.TrimSpace(name)
335 replacer := strings.NewReplacer(
336 "/", "-",
337 "\\", "-",
338 ":", "-",
339 "*", "-",
340 "?", "-",
341 "\"", "-",
342 "<", "-",
343 ">", "-",
344 "|", "-",
345 )
346 name = replacer.Replace(name)
347 name = strings.TrimSpace(name)
348 // Titles come from remote metadata, and a name of only dots resolves to the
349 // parent or to the target dir itself, placing the item outside the library.
350 if strings.Trim(name, ".") == "" {
351 name = "untitled"
352 }
353 return truncateDirName(name)
354}
355
356// maxDirNameBytes leaves room under the 255-byte filesystem component limit for
357// uniqueDir's "-N" suffix.
358const maxDirNameBytes = 240
359
360// truncateDirName cuts on a rune boundary. Over the limit, every filesystem
361// call on the name fails with ENAMETOOLONG.
362func truncateDirName(name string) string {
363 if len(name) <= maxDirNameBytes {
364 return name
365 }
366 cut := maxDirNameBytes
367 for cut > 0 && !utf8.RuneStart(name[cut]) {
368 cut--
369 }
370 truncated := strings.TrimSpace(name[:cut])
371 // Trimming can reintroduce the empty/all-dots case the caller already ruled
372 // out.
373 if strings.Trim(truncated, ".") == "" {
374 return "untitled"
375 }
376 return truncated
377}
378
379// probeDuration returns the duration of a media file in whole seconds. The bool
380// is false when ffprobe is unavailable or the file has no usable duration.
381func probeDuration(ctx context.Context, ffprobePath, path string) (int, bool) {
382 out, err := util.KillableCommand(ctx, ffprobePath, "-v", "error",
383 "-show_entries", "format=duration",
384 "-of", "default=nw=1:nk=1", path).Output()
385 if err != nil {
386 return 0, false
387 }
388 f, err := strconv.ParseFloat(strings.TrimSpace(string(out)), 64)
389 if err != nil || f <= 0 {
390 return 0, false
391 }
392 return int(f + 0.5), true
393}
394