// Package tools installs and updates the external binaries VidArchive drives. // // It only ever touches binaries it owns. When the operator set the matching // VIDARCHIVE_*_PATH, the tool is external and every method here skips it. package tools import ( "archive/zip" "context" "crypto/sha256" "encoding/hex" "errors" "fmt" "io" "log/slog" "net/http" "os" "path" "path/filepath" "regexp" "runtime" "strings" "sync" "time" "vidarchive/internal/config" "vidarchive/internal/util" ) const ( ytdlpRepo = "yt-dlp/yt-dlp" denoRepo = "denoland/deno" ) // maxAssetSize bounds both the download and the unpacked binary, so a corrupt // or hostile release cannot fill the data volume. const maxAssetSize = 512 << 20 // Candidate release assets per architecture, best first: the installer keeps // the first one that runs here. armv7 is absent because yt-dlp only ships it // zipped. var ( ytdlpAssets = map[string][]string{ "amd64": {"yt-dlp_linux", "yt-dlp_musllinux"}, "arm64": {"yt-dlp_linux_aarch64", "yt-dlp_musllinux_aarch64"}, } // Deno publishes one build per architecture, glibc only. denoAssets = map[string][]string{ "amd64": {"deno-x86_64-unknown-linux-gnu.zip"}, "arm64": {"deno-aarch64-unknown-linux-gnu.zip"}, } ) // trialRunTimeout: the standalone yt-dlp build unpacks itself on every start, // which is slow the first time. const trialRunTimeout = time.Minute type Manager struct { cfg *config.Config // mu serializes installs and updates: they rewrite the same files, and two // at once would race on the rename. mu sync.Mutex // baseURL is the GitHub release host. Tests point it at a local server. baseURL string } func New(cfg *config.Config) *Manager { return &Manager{cfg: cfg, baseURL: "https://github.com"} } // Installed reports whether path holds something that could be executed. The // permission check matters: a truncated or 0644 file left by a volume restore // would otherwise block the install forever and fail every download instead. func Installed(path string) bool { info, err := os.Stat(path) return err == nil && info.Mode().IsRegular() && info.Mode().Perm()&0o111 != 0 } // Ensure installs every managed tool that is missing. It is safe to call on // every start, because updating an existing tool is a separate operation. // wantJSRuntime comes from the settings toggle, because deno is a large // download nobody should pay for unless they use it. func (m *Manager) Ensure(ctx context.Context, wantJSRuntime bool) error { // TryLock, not Lock: a blocked caller would sit on the mutex without any way // to honour its context, and there is nothing to do once another install of // the same files is already running. if !m.mu.TryLock() { slog.Info("tool install skipped, another one is already running") return nil } defer m.mu.Unlock() if err := os.MkdirAll(m.cfg.BinDir, 0o755); err != nil { return fmt.Errorf("create bin dir: %w", err) } cleanTemps(m.cfg.BinDir) var errs []error if m.cfg.YTDLPManaged && !Installed(m.cfg.YTDLPPath) { slog.Info("installing yt-dlp", "path", m.cfg.YTDLPPath) if err := m.installYTDLP(ctx); err != nil { errs = append(errs, fmt.Errorf("install yt-dlp: %w", err)) } else { slog.Info("yt-dlp installed", "path", m.cfg.YTDLPPath) } } if wantJSRuntime && m.cfg.DenoManaged && !Installed(m.cfg.DenoPath) { slog.Info("installing deno JS runtime", "path", m.cfg.DenoPath) if err := m.installDeno(ctx); err != nil { errs = append(errs, fmt.Errorf("install deno: %w", err)) } else { slog.Info("deno installed", "path", m.cfg.DenoPath) } } return errors.Join(errs...) } // Update brings the managed tools up to date and returns a one-line summary for // the UI. Both yt-dlp and deno ship an updater that checks the version, verifies // the download and replaces the binary atomically, so none of that is // reimplemented here. A tool that is not installed yet is installed instead. func (m *Manager) Update(ctx context.Context) (string, error) { // An update takes minutes and the page has no JavaScript, so a second click // is likely. Queueing it behind the mutex would spend its whole context // waiting, then kill the child and report a failure for an update that // worked. if !m.mu.TryLock() { return "An install or update is already running.", nil } defer m.mu.Unlock() if err := os.MkdirAll(m.cfg.BinDir, 0o755); err != nil { return "", fmt.Errorf("create bin dir: %w", err) } var parts []string var errs []error if m.cfg.CanUpdateYTDLP() { summary, err := m.updateOne(ctx, "yt-dlp", m.cfg.YTDLPPath, m.cfg.YTDLPManaged, m.installYTDLP, "-U") if err != nil { errs = append(errs, err) } else { parts = append(parts, summary) } } // deno is only updated when it is already there. Installing it is the // settings toggle's job, not the update button's. if m.cfg.CanUpdateDeno() && Installed(m.cfg.DenoPath) { summary, err := m.updateOne(ctx, "deno", m.cfg.DenoPath, m.cfg.DenoManaged, m.installDeno, "upgrade") if err != nil { errs = append(errs, err) } else { parts = append(parts, summary) } } if len(parts) == 0 && len(errs) == 0 { return "Nothing to update: no tool is under VidArchive's control.", nil } return strings.Join(parts, " "), errors.Join(errs...) } func (m *Manager) updateOne(ctx context.Context, name, path string, managed bool, install func(context.Context) error, updateArg string) (string, error) { if !Installed(path) { // Writing a fresh binary to a path the operator chose is not this // program's business. if !managed { return "", fmt.Errorf("%s is not installed at %s; install it there first", name, path) } if err := install(ctx); err != nil { return "", fmt.Errorf("install %s: %w", name, err) } return name + ": installed.", nil } out, err := util.KillableCommand(ctx, path, updateArg).CombinedOutput() text := strings.TrimSpace(string(out)) if err != nil { slog.Error("tool update failed", "tool", name, "err", err, "output", text) return "", fmt.Errorf("update %s: %w: %s", name, err, lastLine(text)) } slog.Info("tool update finished", "tool", name, "output", text) return name + ": " + lastLine(text), nil } // ansiColor matches the SGR escapes deno writes even with NO_COLOR set. They // would reach the browser as literal control characters in the flash message. var ansiColor = regexp.MustCompile("\x1b\\[[0-9;]*m") // lastLine is what the updaters put their verdict on ("up to date", "Updated to // ..."). The lines before it are progress noise, which the log already has. func lastLine(s string) string { lines := strings.Split(s, "\n") for i := len(lines) - 1; i >= 0; i-- { if t := strings.TrimSpace(lines[i]); t != "" { return ansiColor.ReplaceAllString(t, "") } } return "done" } func (m *Manager) installYTDLP(ctx context.Context) error { candidates := ytdlpAssets[runtime.GOARCH] if len(candidates) == 0 { return fmt.Errorf("no yt-dlp release build for %s/%s; set VIDARCHIVE_YTDLP_PATH", runtime.GOOS, runtime.GOARCH) } // yt-dlp publishes SHA2-256SUMS in every release, so a missing or // unparseable sums file means something is wrong and the install stops. sums, err := m.get(ctx, ytdlpRepo, "SHA2-256SUMS") if err != nil { return err } var errs []error for _, asset := range candidates { want, ok := parseChecksum(sums, asset) if !ok { errs = append(errs, fmt.Errorf("no checksum for %s in SHA2-256SUMS", asset)) continue } err := m.tryCandidate(ctx, ytdlpRepo, asset, want, m.cfg.YTDLPPath, placeBinary) if err == nil { return nil } errs = append(errs, err) } return errors.Join(errs...) } func (m *Manager) installDeno(ctx context.Context) error { candidates := denoAssets[runtime.GOARCH] if len(candidates) == 0 { return fmt.Errorf("no deno release build for %s/%s; set VIDARCHIVE_DENO_PATH", runtime.GOOS, runtime.GOARCH) } unzip := func(src, dest string) error { return extractBinary(src, "deno", dest) } var errs []error for _, asset := range candidates { // Fail closed. Deno publishes a sums file for every Linux asset, so a // missing one means something between here and GitHub is wrong. The // binary is executed and left on PATH, so it is never taken unverified. sums, err := m.get(ctx, denoRepo, asset+".sha256sum") if err != nil { errs = append(errs, fmt.Errorf("checksum file for %s: %w", asset, err)) continue } want, ok := parseChecksum(sums, asset) if !ok { errs = append(errs, fmt.Errorf("no checksum for %s in %s.sha256sum", asset, asset)) continue } err = m.tryCandidate(ctx, denoRepo, asset, want, m.cfg.DenoPath, unzip) if err == nil { return nil } errs = append(errs, err) } return errors.Join(errs...) } // tryCandidate downloads one release asset, unpacks it and runs it once before // moving it into place. That trial run replaces libc and architecture // detection: a build for the wrong platform fails to exec, and the caller moves // on to the next candidate. func (m *Manager) tryCandidate(ctx context.Context, repo, asset, wantSum, dest string, unpack func(src, dest string) error) error { tmp, err := m.fetchAsset(ctx, repo, asset, wantSum) if err != nil { return err } defer os.Remove(tmp) staged := dest + ".tmp" defer os.Remove(staged) if err := unpack(tmp, staged); err != nil { return fmt.Errorf("%s: %w", asset, err) } if err := trialRun(ctx, staged); err != nil { return fmt.Errorf("%s: %w", asset, err) } // Rename is atomic, and on Linux it leaves a currently running copy of the // old binary untouched, so a download in flight is not disturbed. return os.Rename(staged, dest) } // trialRun checks that the downloaded binary starts on this system. Both tools // answer --version without touching the network. func trialRun(ctx context.Context, path string) error { ctx, cancel := context.WithTimeout(ctx, trialRunTimeout) defer cancel() out, err := util.KillableCommand(ctx, path, "--version").CombinedOutput() if err != nil { return fmt.Errorf("does not run on this system: %w: %s", err, lastLine(string(out))) } return nil } // assetURL uses the "latest" alias for both the asset and its checksum file, so // the two always come from the same release without asking the GitHub API. func (m *Manager) assetURL(repo, asset string) string { return fmt.Sprintf("%s/%s/releases/latest/download/%s", m.baseURL, repo, asset) } func (m *Manager) get(ctx context.Context, repo, asset string) ([]byte, error) { resp, err := m.open(ctx, repo, asset) if err != nil { return nil, err } defer resp.Body.Close() return io.ReadAll(io.LimitReader(resp.Body, 1<<20)) } func (m *Manager) open(ctx context.Context, repo, asset string) (*http.Response, error) { url := m.assetURL(repo, asset) req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) if err != nil { return nil, err } resp, err := http.DefaultClient.Do(req) if err != nil { return nil, fmt.Errorf("fetch %s: %w", url, err) } if resp.StatusCode != http.StatusOK { resp.Body.Close() return nil, fmt.Errorf("fetch %s: %s", url, resp.Status) } return resp, nil } // fetchAsset downloads a release asset into BinDir and verifies its SHA-256 // while writing. The caller removes the returned file. func (m *Manager) fetchAsset(ctx context.Context, repo, asset, wantSum string) (string, error) { resp, err := m.open(ctx, repo, asset) if err != nil { return "", err } defer resp.Body.Close() f, err := os.CreateTemp(m.cfg.BinDir, "download-*") if err != nil { return "", err } tmp := f.Name() sum := sha256.New() n, err := io.Copy(io.MultiWriter(f, sum), io.LimitReader(resp.Body, maxAssetSize+1)) closeErr := f.Close() switch { case err != nil: os.Remove(tmp) return "", fmt.Errorf("download %s: %w", asset, err) case closeErr != nil: os.Remove(tmp) return "", closeErr case n > maxAssetSize: os.Remove(tmp) return "", fmt.Errorf("download %s: larger than %d bytes", asset, int64(maxAssetSize)) } if got := hex.EncodeToString(sum.Sum(nil)); got != wantSum { os.Remove(tmp) return "", fmt.Errorf("checksum mismatch for %s: got %s, want %s", asset, got, wantSum) } return tmp, nil } // parseChecksum reads sha256sum output: one " " line per file. // Both repositories publish that format. func parseChecksum(data []byte, asset string) (string, bool) { for _, line := range strings.Split(string(data), "\n") { fields := strings.Fields(line) if len(fields) >= 2 && path.Base(fields[1]) == asset { return strings.ToLower(fields[0]), true } } return "", false } // cleanTemps removes downloads a hard kill interrupted. fetchAsset cannot, and // a partial file can be as large as maxAssetSize. The age check spares a // download another process may still be writing. func cleanTemps(dir string) { matches, _ := filepath.Glob(filepath.Join(dir, "download-*")) for _, match := range matches { info, err := os.Stat(match) if err != nil || time.Since(info.ModTime()) < time.Hour { continue } if err := os.Remove(match); err != nil { slog.Warn("failed to remove an orphaned download", "path", match, "err", err) } } } // placeBinary is the unpack step for an asset that is already the binary. func placeBinary(src, dest string) error { if err := os.Chmod(src, 0o755); err != nil { return err } return os.Rename(src, dest) } func extractBinary(archive, entry, dest string) error { zr, err := zip.OpenReader(archive) if err != nil { return fmt.Errorf("open %s: %w", filepath.Base(archive), err) } defer zr.Close() for _, f := range zr.File { if path.Base(f.Name) != entry { continue } rc, err := f.Open() if err != nil { return err } defer rc.Close() out, err := os.OpenFile(dest, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o755) if err != nil { return err } n, err := io.Copy(out, io.LimitReader(rc, maxAssetSize+1)) if cErr := out.Close(); err == nil { err = cErr } if err == nil && n > maxAssetSize { err = fmt.Errorf("%s in archive is larger than %d bytes", entry, int64(maxAssetSize)) } if err != nil { os.Remove(dest) return err } // Not left to the OpenFile mode: umask reduces it, and it is ignored // outright when a leftover file from a crashed install already exists. return os.Chmod(dest, 0o755) } return fmt.Errorf("%s not found in %s", entry, filepath.Base(archive)) }