vidarchive

vidarchive/internal/tools/tools.go
tools.go
⎇
Raw
1// Package tools installs and updates the external binaries VidArchive drives.
2//
3// It only ever touches binaries it owns. When the operator set the matching
4// VIDARCHIVE_*_PATH, the tool is external and every method here skips it.
5package tools
6
7import (
8 "archive/zip"
9 "context"
10 "crypto/sha256"
11 "encoding/hex"
12 "errors"
13 "fmt"
14 "io"
15 "log/slog"
16 "net/http"
17 "os"
18 "path"
19 "path/filepath"
20 "regexp"
21 "runtime"
22 "strings"
23 "sync"
24 "time"
25
26 "vidarchive/internal/config"
27 "vidarchive/internal/util"
28)
29
30const (
31 ytdlpRepo = "yt-dlp/yt-dlp"
32 denoRepo = "denoland/deno"
33)
34
35// maxAssetSize bounds both the download and the unpacked binary, so a corrupt
36// or hostile release cannot fill the data volume.
37const maxAssetSize = 512 << 20
38
39// Candidate release assets per architecture, best first: the installer keeps
40// the first one that runs here. armv7 is absent because yt-dlp only ships it
41// zipped.
42var (
43 ytdlpAssets = map[string][]string{
44 "amd64": {"yt-dlp_linux", "yt-dlp_musllinux"},
45 "arm64": {"yt-dlp_linux_aarch64", "yt-dlp_musllinux_aarch64"},
46 }
47 // Deno publishes one build per architecture, glibc only.
48 denoAssets = map[string][]string{
49 "amd64": {"deno-x86_64-unknown-linux-gnu.zip"},
50 "arm64": {"deno-aarch64-unknown-linux-gnu.zip"},
51 }
52)
53
54// trialRunTimeout: the standalone yt-dlp build unpacks itself on every start,
55// which is slow the first time.
56const trialRunTimeout = time.Minute
57
58type Manager struct {
59 cfg *config.Config
60 // mu serializes installs and updates: they rewrite the same files, and two
61 // at once would race on the rename.
62 mu sync.Mutex
63 // baseURL is the GitHub release host. Tests point it at a local server.
64 baseURL string
65}
66
67func New(cfg *config.Config) *Manager {
68 return &Manager{cfg: cfg, baseURL: "https://github.com"}
69}
70
71// Installed reports whether path holds something that could be executed. The
72// permission check matters: a truncated or 0644 file left by a volume restore
73// would otherwise block the install forever and fail every download instead.
74func Installed(path string) bool {
75 info, err := os.Stat(path)
76 return err == nil && info.Mode().IsRegular() && info.Mode().Perm()&0o111 != 0
77}
78
79// Ensure installs every managed tool that is missing. It is safe to call on
80// every start, because updating an existing tool is a separate operation.
81// wantJSRuntime comes from the settings toggle, because deno is a large
82// download nobody should pay for unless they use it.
83func (m *Manager) Ensure(ctx context.Context, wantJSRuntime bool) error {
84 // TryLock, not Lock: a blocked caller would sit on the mutex without any way
85 // to honour its context, and there is nothing to do once another install of
86 // the same files is already running.
87 if !m.mu.TryLock() {
88 slog.Info("tool install skipped, another one is already running")
89 return nil
90 }
91 defer m.mu.Unlock()
92
93 if err := os.MkdirAll(m.cfg.BinDir, 0o755); err != nil {
94 return fmt.Errorf("create bin dir: %w", err)
95 }
96 cleanTemps(m.cfg.BinDir)
97
98 var errs []error
99 if m.cfg.YTDLPManaged && !Installed(m.cfg.YTDLPPath) {
100 slog.Info("installing yt-dlp", "path", m.cfg.YTDLPPath)
101 if err := m.installYTDLP(ctx); err != nil {
102 errs = append(errs, fmt.Errorf("install yt-dlp: %w", err))
103 } else {
104 slog.Info("yt-dlp installed", "path", m.cfg.YTDLPPath)
105 }
106 }
107 if wantJSRuntime && m.cfg.DenoManaged && !Installed(m.cfg.DenoPath) {
108 slog.Info("installing deno JS runtime", "path", m.cfg.DenoPath)
109 if err := m.installDeno(ctx); err != nil {
110 errs = append(errs, fmt.Errorf("install deno: %w", err))
111 } else {
112 slog.Info("deno installed", "path", m.cfg.DenoPath)
113 }
114 }
115 return errors.Join(errs...)
116}
117
118// Update brings the managed tools up to date and returns a one-line summary for
119// the UI. Both yt-dlp and deno ship an updater that checks the version, verifies
120// the download and replaces the binary atomically, so none of that is
121// reimplemented here. A tool that is not installed yet is installed instead.
122func (m *Manager) Update(ctx context.Context) (string, error) {
123 // An update takes minutes and the page has no JavaScript, so a second click
124 // is likely. Queueing it behind the mutex would spend its whole context
125 // waiting, then kill the child and report a failure for an update that
126 // worked.
127 if !m.mu.TryLock() {
128 return "An install or update is already running.", nil
129 }
130 defer m.mu.Unlock()
131
132 if err := os.MkdirAll(m.cfg.BinDir, 0o755); err != nil {
133 return "", fmt.Errorf("create bin dir: %w", err)
134 }
135
136 var parts []string
137 var errs []error
138
139 if m.cfg.CanUpdateYTDLP() {
140 summary, err := m.updateOne(ctx, "yt-dlp", m.cfg.YTDLPPath, m.cfg.YTDLPManaged, m.installYTDLP, "-U")
141 if err != nil {
142 errs = append(errs, err)
143 } else {
144 parts = append(parts, summary)
145 }
146 }
147 // deno is only updated when it is already there. Installing it is the
148 // settings toggle's job, not the update button's.
149 if m.cfg.CanUpdateDeno() && Installed(m.cfg.DenoPath) {
150 summary, err := m.updateOne(ctx, "deno", m.cfg.DenoPath, m.cfg.DenoManaged, m.installDeno, "upgrade")
151 if err != nil {
152 errs = append(errs, err)
153 } else {
154 parts = append(parts, summary)
155 }
156 }
157
158 if len(parts) == 0 && len(errs) == 0 {
159 return "Nothing to update: no tool is under VidArchive's control.", nil
160 }
161 return strings.Join(parts, " "), errors.Join(errs...)
162}
163
164func (m *Manager) updateOne(ctx context.Context, name, path string, managed bool, install func(context.Context) error, updateArg string) (string, error) {
165 if !Installed(path) {
166 // Writing a fresh binary to a path the operator chose is not this
167 // program's business.
168 if !managed {
169 return "", fmt.Errorf("%s is not installed at %s; install it there first", name, path)
170 }
171 if err := install(ctx); err != nil {
172 return "", fmt.Errorf("install %s: %w", name, err)
173 }
174 return name + ": installed.", nil
175 }
176
177 out, err := util.KillableCommand(ctx, path, updateArg).CombinedOutput()
178 text := strings.TrimSpace(string(out))
179 if err != nil {
180 slog.Error("tool update failed", "tool", name, "err", err, "output", text)
181 return "", fmt.Errorf("update %s: %w: %s", name, err, lastLine(text))
182 }
183 slog.Info("tool update finished", "tool", name, "output", text)
184 return name + ": " + lastLine(text), nil
185}
186
187// ansiColor matches the SGR escapes deno writes even with NO_COLOR set. They
188// would reach the browser as literal control characters in the flash message.
189var ansiColor = regexp.MustCompile("\x1b\\[[0-9;]*m")
190
191// lastLine is what the updaters put their verdict on ("up to date", "Updated to
192// ..."). The lines before it are progress noise, which the log already has.
193func lastLine(s string) string {
194 lines := strings.Split(s, "\n")
195 for i := len(lines) - 1; i >= 0; i-- {
196 if t := strings.TrimSpace(lines[i]); t != "" {
197 return ansiColor.ReplaceAllString(t, "")
198 }
199 }
200 return "done"
201}
202
203func (m *Manager) installYTDLP(ctx context.Context) error {
204 candidates := ytdlpAssets[runtime.GOARCH]
205 if len(candidates) == 0 {
206 return fmt.Errorf("no yt-dlp release build for %s/%s; set VIDARCHIVE_YTDLP_PATH", runtime.GOOS, runtime.GOARCH)
207 }
208
209 // yt-dlp publishes SHA2-256SUMS in every release, so a missing or
210 // unparseable sums file means something is wrong and the install stops.
211 sums, err := m.get(ctx, ytdlpRepo, "SHA2-256SUMS")
212 if err != nil {
213 return err
214 }
215
216 var errs []error
217 for _, asset := range candidates {
218 want, ok := parseChecksum(sums, asset)
219 if !ok {
220 errs = append(errs, fmt.Errorf("no checksum for %s in SHA2-256SUMS", asset))
221 continue
222 }
223 err := m.tryCandidate(ctx, ytdlpRepo, asset, want, m.cfg.YTDLPPath, placeBinary)
224 if err == nil {
225 return nil
226 }
227 errs = append(errs, err)
228 }
229 return errors.Join(errs...)
230}
231
232func (m *Manager) installDeno(ctx context.Context) error {
233 candidates := denoAssets[runtime.GOARCH]
234 if len(candidates) == 0 {
235 return fmt.Errorf("no deno release build for %s/%s; set VIDARCHIVE_DENO_PATH", runtime.GOOS, runtime.GOARCH)
236 }
237
238 unzip := func(src, dest string) error { return extractBinary(src, "deno", dest) }
239
240 var errs []error
241 for _, asset := range candidates {
242 // Fail closed. Deno publishes a sums file for every Linux asset, so a
243 // missing one means something between here and GitHub is wrong. The
244 // binary is executed and left on PATH, so it is never taken unverified.
245 sums, err := m.get(ctx, denoRepo, asset+".sha256sum")
246 if err != nil {
247 errs = append(errs, fmt.Errorf("checksum file for %s: %w", asset, err))
248 continue
249 }
250 want, ok := parseChecksum(sums, asset)
251 if !ok {
252 errs = append(errs, fmt.Errorf("no checksum for %s in %s.sha256sum", asset, asset))
253 continue
254 }
255
256 err = m.tryCandidate(ctx, denoRepo, asset, want, m.cfg.DenoPath, unzip)
257 if err == nil {
258 return nil
259 }
260 errs = append(errs, err)
261 }
262 return errors.Join(errs...)
263}
264
265// tryCandidate downloads one release asset, unpacks it and runs it once before
266// moving it into place. That trial run replaces libc and architecture
267// detection: a build for the wrong platform fails to exec, and the caller moves
268// on to the next candidate.
269func (m *Manager) tryCandidate(ctx context.Context, repo, asset, wantSum, dest string, unpack func(src, dest string) error) error {
270 tmp, err := m.fetchAsset(ctx, repo, asset, wantSum)
271 if err != nil {
272 return err
273 }
274 defer os.Remove(tmp)
275
276 staged := dest + ".tmp"
277 defer os.Remove(staged)
278
279 if err := unpack(tmp, staged); err != nil {
280 return fmt.Errorf("%s: %w", asset, err)
281 }
282 if err := trialRun(ctx, staged); err != nil {
283 return fmt.Errorf("%s: %w", asset, err)
284 }
285 // Rename is atomic, and on Linux it leaves a currently running copy of the
286 // old binary untouched, so a download in flight is not disturbed.
287 return os.Rename(staged, dest)
288}
289
290// trialRun checks that the downloaded binary starts on this system. Both tools
291// answer --version without touching the network.
292func trialRun(ctx context.Context, path string) error {
293 ctx, cancel := context.WithTimeout(ctx, trialRunTimeout)
294 defer cancel()
295
296 out, err := util.KillableCommand(ctx, path, "--version").CombinedOutput()
297 if err != nil {
298 return fmt.Errorf("does not run on this system: %w: %s", err, lastLine(string(out)))
299 }
300 return nil
301}
302
303// assetURL uses the "latest" alias for both the asset and its checksum file, so
304// the two always come from the same release without asking the GitHub API.
305func (m *Manager) assetURL(repo, asset string) string {
306 return fmt.Sprintf("%s/%s/releases/latest/download/%s", m.baseURL, repo, asset)
307}
308
309func (m *Manager) get(ctx context.Context, repo, asset string) ([]byte, error) {
310 resp, err := m.open(ctx, repo, asset)
311 if err != nil {
312 return nil, err
313 }
314 defer resp.Body.Close()
315 return io.ReadAll(io.LimitReader(resp.Body, 1<<20))
316}
317
318func (m *Manager) open(ctx context.Context, repo, asset string) (*http.Response, error) {
319 url := m.assetURL(repo, asset)
320 req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
321 if err != nil {
322 return nil, err
323 }
324 resp, err := http.DefaultClient.Do(req)
325 if err != nil {
326 return nil, fmt.Errorf("fetch %s: %w", url, err)
327 }
328 if resp.StatusCode != http.StatusOK {
329 resp.Body.Close()
330 return nil, fmt.Errorf("fetch %s: %s", url, resp.Status)
331 }
332 return resp, nil
333}
334
335// fetchAsset downloads a release asset into BinDir and verifies its SHA-256
336// while writing. The caller removes the returned file.
337func (m *Manager) fetchAsset(ctx context.Context, repo, asset, wantSum string) (string, error) {
338 resp, err := m.open(ctx, repo, asset)
339 if err != nil {
340 return "", err
341 }
342 defer resp.Body.Close()
343
344 f, err := os.CreateTemp(m.cfg.BinDir, "download-*")
345 if err != nil {
346 return "", err
347 }
348 tmp := f.Name()
349
350 sum := sha256.New()
351 n, err := io.Copy(io.MultiWriter(f, sum), io.LimitReader(resp.Body, maxAssetSize+1))
352 closeErr := f.Close()
353 switch {
354 case err != nil:
355 os.Remove(tmp)
356 return "", fmt.Errorf("download %s: %w", asset, err)
357 case closeErr != nil:
358 os.Remove(tmp)
359 return "", closeErr
360 case n > maxAssetSize:
361 os.Remove(tmp)
362 return "", fmt.Errorf("download %s: larger than %d bytes", asset, int64(maxAssetSize))
363 }
364
365 if got := hex.EncodeToString(sum.Sum(nil)); got != wantSum {
366 os.Remove(tmp)
367 return "", fmt.Errorf("checksum mismatch for %s: got %s, want %s", asset, got, wantSum)
368 }
369 return tmp, nil
370}
371
372// parseChecksum reads sha256sum output: one "<hex> <filename>" line per file.
373// Both repositories publish that format.
374func parseChecksum(data []byte, asset string) (string, bool) {
375 for _, line := range strings.Split(string(data), "\n") {
376 fields := strings.Fields(line)
377 if len(fields) >= 2 && path.Base(fields[1]) == asset {
378 return strings.ToLower(fields[0]), true
379 }
380 }
381 return "", false
382}
383
384// cleanTemps removes downloads a hard kill interrupted. fetchAsset cannot, and
385// a partial file can be as large as maxAssetSize. The age check spares a
386// download another process may still be writing.
387func cleanTemps(dir string) {
388 matches, _ := filepath.Glob(filepath.Join(dir, "download-*"))
389 for _, match := range matches {
390 info, err := os.Stat(match)
391 if err != nil || time.Since(info.ModTime()) < time.Hour {
392 continue
393 }
394 if err := os.Remove(match); err != nil {
395 slog.Warn("failed to remove an orphaned download", "path", match, "err", err)
396 }
397 }
398}
399
400// placeBinary is the unpack step for an asset that is already the binary.
401func placeBinary(src, dest string) error {
402 if err := os.Chmod(src, 0o755); err != nil {
403 return err
404 }
405 return os.Rename(src, dest)
406}
407
408func extractBinary(archive, entry, dest string) error {
409 zr, err := zip.OpenReader(archive)
410 if err != nil {
411 return fmt.Errorf("open %s: %w", filepath.Base(archive), err)
412 }
413 defer zr.Close()
414
415 for _, f := range zr.File {
416 if path.Base(f.Name) != entry {
417 continue
418 }
419 rc, err := f.Open()
420 if err != nil {
421 return err
422 }
423 defer rc.Close()
424
425 out, err := os.OpenFile(dest, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o755)
426 if err != nil {
427 return err
428 }
429 n, err := io.Copy(out, io.LimitReader(rc, maxAssetSize+1))
430 if cErr := out.Close(); err == nil {
431 err = cErr
432 }
433 if err == nil && n > maxAssetSize {
434 err = fmt.Errorf("%s in archive is larger than %d bytes", entry, int64(maxAssetSize))
435 }
436 if err != nil {
437 os.Remove(dest)
438 return err
439 }
440 // Not left to the OpenFile mode: umask reduces it, and it is ignored
441 // outright when a leftover file from a crashed install already exists.
442 return os.Chmod(dest, 0o755)
443 }
444 return fmt.Errorf("%s not found in %s", entry, filepath.Base(archive))
445}
446