ci: export the commit with git archive instead of cloning in the container
Running `git clone` inside the container made every CI image depend on
git. The dependency was invisible in the config and only surfaced as a
failed run ("sh: git: not found" on Alpine). HearthForge has git, so it
now streams `git archive` straight into the container. No .git reaches
the container any more.
A work tree piped through tar would have worked too, but its `./` entry
stamps the host uid and 0700 onto the destination. Verified on podman:
/ci/build/project went from `drwxr-xr-x 0 0` to `drwx------ 1000 1000`,
which locks out images that do not run as HearthForge's uid.
Fixed while reviewing:
- `clear` removed and recreated the directory in two execs. With
clone_project_to == work_dir that deleted the container's WorkingDir
and every later exec failed to chdir. Now one exec.
- validateCiConfig missed a cache above clone_project_to, which merged
two commits into one tree. It also accepted a relative path, which
named two different directories.
- The spawn uses gitEnv, honours the abort signal, and is reaped in a
finally. --end-of-options guards the sha.Mpublic/assets/hearthforge-ci-template.toml
@@ -3,6 +3,9 @@
image = "docker.io/debian:stable"
work_dir = "/ci/build"
# Must be absolute. The image needs no git. HearthForge exports the commit
# and uploads it. No .git reaches the container, so `git describe` needs a
# step that fetches history itself.
clone_project_to = "/ci/build/project"
# shell = ["/bin/sh", "-c"]
shell_setup = "set -euo pipefail"
@@ -16,8 +19,8 @@ shell_setup = "set -euo pipefail"
# { path = "/root/.cargo", max_size = "8g" },
# "/root/.npm",
# ]
# A cache path must sit outside clone_project_to. The volume is mounted
# before the clone runs, and git refuses to clone into a non-empty directory.
# A cache path must not overlap clone_project_to, in either direction. The
# checkout is extracted over that directory. A `clear` step deletes it.
[on]
push = ["main"] # trigger on push to these branches; use ["*"] for all
Msrc/services/ci.ts
@@ -4,7 +4,7 @@ import path from "node:path";
import config from "../config.ts";
import { CI_MAX_LOG_BYTES, paths } from "../constants.ts";
import { db } from "../db/index.ts";
import { repoPath } from "./git.ts";
import { gitEnv, repoPath } from "./git.ts";
// --- Types ---
@@ -79,8 +79,6 @@ export interface TriggerOpts {
variableOverrides?: Record<string, string>;
}
const CONTAINER_REPO_PATH = "/hearthforge-repo.git";
// In-memory map of running tasks for cancellation
const runningTasks = new Map<
number,
@@ -259,20 +257,31 @@ export function validateCiConfig(cfg: CiConfig): string | null {
}
}
if (!cfg.clone_project_to || !cfg.cache) return null;
if (!cfg.clone_project_to) return null;
// The archive endpoint resolves `path` against /, while the execs that
// create and clear the directory resolve against work_dir. A relative
// value would name two different directories.
if (!path.isAbsolute(cfg.clone_project_to)) {
return `clone_project_to ("${cfg.clone_project_to}") must be an absolute path.`;
}
const clone = path.resolve(cfg.clone_project_to);
for (const entry of cfg.cache) {
for (const entry of cfg.cache ?? []) {
const cachePath = path.resolve(entry.path);
// Either nesting direction breaks. A cache below the checkout is
// overwritten by the extract and deleted by `clear`. A cache above it
// carries the previous run's tree back in. The extract then merges two
// commits instead of replacing one.
if (
cachePath === clone ||
cachePath.startsWith(`${clone}${path.sep}`)
cachePath.startsWith(`${clone}${path.sep}`) ||
clone.startsWith(`${cachePath}${path.sep}`)
) {
return (
`cache path "${entry.path}" is inside clone_project_to ("${cfg.clone_project_to}"). ` +
"The cache volume is mounted before the clone runs, which leaves the " +
"directory non-empty, and git refuses to clone into it. Move the cache " +
"outside the clone directory."
`cache path "${entry.path}" overlaps clone_project_to ("${cfg.clone_project_to}"). ` +
"The checkout is extracted over that directory and a `clear` step " +
"deletes it. Move the cache outside the clone directory."
);
}
}
@@ -709,64 +718,90 @@ async function execInContainer(
}
/**
* Copy the bare repo into the container at CONTAINER_REPO_PATH.
* Upload the commit's tree into the container at `destPath`.
*
* The checkout deliberately does not happen inside the container. That would
* force every CI image to carry a git binary. The missing dependency would
* only surface as a failed run.
*
* Not a bind mount: the Docker daemon resolves bind sources in the host
* filesystem, so when HearthForge itself runs in a container it finds nothing
* at our DATA_DIR path and silently mounts an empty directory instead.
* Not a bind mount either. The Docker daemon resolves bind sources in the
* host filesystem. When HearthForge itself runs in a container, it finds
* nothing at our DATA_DIR path and silently mounts an empty directory.
*
* `git archive` writes uid 0 and mode 0644/0755 into the tar headers, and
* emits no entry for the archive root. The destination directory therefore
* keeps the ownership and mode the container gave it. Piping a work tree
* through `tar` instead would stamp the host's uid and 0700 onto it, which
* locks out any image whose default user is not HearthForge's uid.
*
* No `.git` reaches the container. A step that needs history must fetch it.
*/
async function uploadRepo(
async function uploadCheckout(
containerId: string,
repoAbsPath: string,
commitSha: string,
destPath: string,
signal: AbortSignal,
): Promise<void> {
const mk = await execInContainer(containerId, [
"mkdir",
"-p",
CONTAINER_REPO_PATH,
]);
const mk = await execInContainer(containerId, ["mkdir", "-p", destPath]);
if (mk.exitCode !== 0) {
throw new Error(
`Failed to create ${CONTAINER_REPO_PATH} in the container: ${mk.log.trim()}`,
`Failed to create ${destPath} in the container: ${mk.log.trim()}`,
);
}
const tar = Bun.spawn(["tar", "-cf", "-", "-C", repoAbsPath, "."], {
stdout: "pipe",
stderr: "pipe",
});
const resp = await dockerFetch(
`/containers/${containerId}/archive?path=${encodeURIComponent(CONTAINER_REPO_PATH)}`,
{
method: "PUT",
headers: { "Content-Type": "application/x-tar" },
body: tar.stdout,
},
// --end-of-options: commit_sha is unvalidated pkt-line text from the push.
const archive = Bun.spawn(
[
"git",
"-C",
repoAbsPath,
"archive",
"--format=tar",
"--end-of-options",
commitSha,
],
{ stdout: "pipe", stderr: "pipe", env: gitEnv, signal },
);
// The PUT stopped reading, so tar would block writing into a full pipe.
if (!resp.ok) tar.kill();
try {
const resp = await dockerFetch(
`/containers/${containerId}/archive?path=${encodeURIComponent(destPath)}`,
{
method: "PUT",
headers: { "Content-Type": "application/x-tar" },
body: archive.stdout,
signal,
},
);
// stderr must be drained in the same turn as the wait. A tar that fills
// the pipe buffer blocks on the write, and awaiting `exited` first would
// then hang the run for good: this path has no timeout and no signal.
const [tarExit, tarErr] = await Promise.all([
tar.exited,
new Response(tar.stderr).text(),
]);
// The PUT stopped reading, so git would block writing into a full pipe.
if (!resp.ok) archive.kill();
if (!resp.ok) {
const detail = (await resp.text().catch(() => "")).trim();
throw new Error(
`Failed to upload the repository: HTTP ${resp.status}${detail ? ` ${detail}` : ""}`,
);
}
await resp.body?.cancel();
if (tarExit !== 0) {
const err = tarErr.trim();
throw new Error(
`Failed to read the repository: tar exited ${tarExit}${err ? ` ${err}` : ""}`,
);
// stderr must be drained in the same turn as the wait. A git that fills
// the pipe buffer blocks on the write, and awaiting `exited` first would
// then hang the run for good.
const [exitCode, err] = await Promise.all([
archive.exited,
new Response(archive.stderr).text(),
]);
if (!resp.ok) {
const detail = (await resp.text().catch(() => "")).trim();
throw new Error(
`Failed to upload the checkout: HTTP ${resp.status}${detail ? ` ${detail}` : ""}`,
);
}
await resp.body?.cancel();
if (exitCode !== 0) {
throw new Error(
`Failed to read ${commitSha}: git archive exited ${exitCode}${err.trim() ? ` ${err.trim()}` : ""}`,
);
}
} finally {
// Reached on a thrown dockerFetch too, where nothing has reaped git.
archive.kill();
await archive.exited;
}
}
@@ -1164,26 +1199,14 @@ async function executeRun(runId: number, signal: AbortSignal): Promise<void> {
if (signal.aborted) throw new Error("Cancelled");
}
// Clone project if requested
if (cfg.clone_project_to && run.commit_sha) {
await uploadRepo(containerId, repoPath(repo.name));
const clone = await execInContainer(
await uploadCheckout(
containerId,
[
"sh",
"-c",
// safe.directory: the upload carries the host's uids, and
// git refuses a repo it does not appear to own.
`git -c safe.directory=${CONTAINER_REPO_PATH} clone ${CONTAINER_REPO_PATH} ${cfg.clone_project_to} && git -C ${cfg.clone_project_to} checkout --detach ${run.commit_sha}`,
],
cfg.work_dir,
envArray,
repoPath(repo.name),
run.commit_sha,
cfg.clone_project_to,
signal,
);
if (clone.exitCode !== 0) {
throw new Error(
`Failed to clone the repository into ${cfg.clone_project_to}: ${clone.log.trim()}`,
);
}
}
// Execute steps
@@ -1231,18 +1254,35 @@ async function executeRun(runId: number, signal: AbortSignal): Promise<void> {
}
// Handle clear option
// Drop the directory and re-extract. `git clean` would need git
// in the image, and this also removes files git never tracked.
if (step.clear && cfg.clone_project_to && run.commit_sha) {
const cleared = await execInContainer(
containerId,
[
let clearError: string | null = null;
try {
// One exec, not two. `rm -rf` can delete the container's
// WorkingDir, and every later exec then fails to chdir
// before its command starts. This exec chdirs first.
const reset = await execInContainer(containerId, [
...shell,
'rm -rf "$1" && mkdir -p "$1"',
"sh",
"-c",
`git -C ${cfg.clone_project_to} reset --hard ${run.commit_sha} && git -C ${cfg.clone_project_to} clean -fdx`,
],
cfg.work_dir,
envArray,
);
if (cleared.exitCode !== 0) {
cfg.clone_project_to,
]);
if (reset.exitCode !== 0) {
throw new Error(reset.log.trim());
}
await uploadCheckout(
containerId,
repoPath(repo.name),
run.commit_sha,
cfg.clone_project_to,
signal,
);
} catch (err) {
clearError =
err instanceof Error ? err.message : String(err);
}
if (clearError !== null) {
// Not thrown: the outer handler only records a message
// when no step has failed yet, so after an earlier failure
// it would vanish. The step row always survives.
@@ -1252,7 +1292,7 @@ async function executeRun(runId: number, signal: AbortSignal): Promise<void> {
status: "failure",
started_at: now(),
finished_at: now(),
log: `Failed to reset ${cfg.clone_project_to}: ${cleared.log.trim()}\n`,
log: `Failed to reset ${cfg.clone_project_to}: ${clearError}\n`,
})
.where("id", "=", stepId)
.execute();
@@ -1704,6 +1744,7 @@ export async function purgeRepoCaches(repoName: string): Promise<void> {
export async function cancelStaleRuns(): Promise<void> {
const now = new Date().toISOString();
const stale = await db
.selectFrom("ci_runs")
.select("id")
Msrc/services/git.ts
@@ -11,7 +11,7 @@ import {
REF_CACHE_TTL_MS,
} from "../constants.ts";
const gitEnv = {
export const gitEnv = {
...process.env,
LC_ALL: "C",
LANG: "C",
Msrc/views/ci/CiHistory.tsx
@@ -124,7 +124,10 @@ function CiHelp({ repo }: { repo: RepositoryRow }) {
"warn_on_fail",
"step warns instead of failing; the run reports warning",
],
["clear", "reset the clone before the step"],
[
"clear",
"re-extract a clean checkout before the step",
],
["timeout", "per-step timeout in seconds"],
].map(([k, v]) => (
<>
Mtests/e2e.ci.test.ts
@@ -40,8 +40,35 @@ interface ExecResp {
delayMs?: number;
}
/** Parse the 512-byte headers of an uncompressed tar. */
function tarHeaders(tar: Uint8Array): Array<{ name: string; uid: number }> {
const dec = new TextDecoder();
const out: Array<{ name: string; uid: number }> = [];
for (let off = 0; off + 512 <= tar.length; ) {
const name = dec.decode(tar.subarray(off, off + 100)).replace(/\0.*$/, "");
if (name === "") break; // end-of-archive padding
const uid = Number.parseInt(
dec.decode(tar.subarray(off + 108, off + 116)).replace(/\0.*$/, "").trim() ||
"0",
8,
);
const size = Number.parseInt(
dec.decode(tar.subarray(off + 124, off + 136)).replace(/\0.*$/, "").trim() ||
"0",
8,
);
out.push({ name, uid });
off += 512 + Math.ceil(size / 512) * 512;
}
return out;
}
function tarEntryNames(tar: Uint8Array): string[] {
return tarHeaders(tar).map((h) => h.name);
}
// Repo archive uploads (PUT /containers/*/archive)
const uploads: Array<{ path: string; bytes: number }> = [];
const uploads: Array<{ path: string; bytes: number; body: Uint8Array }> = [];
// Images passed to POST /images/create
const pulls: string[] = [];
// Volumes created, and the ones deleted, so cache pruning can be asserted
@@ -59,6 +86,8 @@ const execMap = new Map<string, ExecResp>();
// Queue consumed in order when execs are created — allows tests to pre-program
// specific step responses
const execQueue: ExecResp[] = [];
/** Every command run inside a container, in order. */
const execCmds: string[][] = [];
let execCounter = 0;
function queueExec(resp: ExecResp) {
@@ -68,6 +97,7 @@ function queueExec(resp: ExecResp) {
function resetMock() {
execMap.clear();
execQueue.length = 0;
execCmds.length = 0;
execCounter = 0;
uploads.length = 0;
pulls.length = 0;
@@ -152,6 +182,8 @@ function startMockDocker() {
) {
execCounter++;
const execId = `mock-exec-${execCounter}`;
const cmd = ((await req.json()) as { Cmd?: string[] }).Cmd;
execCmds.push(cmd ?? []);
execMap.set(
execId,
execQueue.shift() ?? { output: "", exitCode: 0 },
@@ -173,7 +205,7 @@ function startMockDocker() {
const resp = execMap.get(id) ?? { output: "", exitCode: 0 };
return Response.json({ ExitCode: resp.exitCode });
}
// Archive upload (used to copy the bare repo into the container)
// Archive upload (the checkout, and [[copy]] sources)
if (
req.method === "PUT" &&
/\/containers\/[^/]+\/archive/.test(p)
@@ -182,6 +214,7 @@ function startMockDocker() {
uploads.push({
path: qs.get("path") ?? "",
bytes: body.length,
body,
});
return new Response(null, { status: 200 });
}
@@ -999,21 +1032,28 @@ run_sh = "echo hi"
});
}
test("bare repo is uploaded instead of bind-mounted", async () => {
test("the checkout is uploaded, not bind-mounted", async () => {
const runId = await trigger();
expect(await waitForRun(runId)).toBe("success");
expect(uploads.map((u) => u.path)).toContain("/hearthforge-repo.git");
expect(uploads.map((u) => u.path)).toContain("/ci/build/project");
expect(uploads[0]!.bytes).toBeGreaterThan(0);
const binds = JSON.stringify(lastCreateBody?.HostConfig?.Binds ?? []);
expect(binds).not.toContain("hearthforge-repo.git");
expect(binds).not.toContain(DATA_DIR);
});
test("a failing clone fails the run before any step runs", async () => {
test("the container never runs git", async () => {
const runId = await trigger();
expect(await waitForRun(runId)).toBe("success");
const ran = execCmds.flat().join(" ");
expect(ran).not.toContain("git");
});
test("a failing checkout fails the run before any step runs", async () => {
queueExec({ output: "", exitCode: 0 }); // mkdir work_dir
queueExec({ output: "", exitCode: 0 }); // mkdir repo path
queueExec({ output: "fatal: not empty\n", exitCode: 128 }); // clone
queueExec({ output: "mkdir: read-only\n", exitCode: 1 }); // mkdir dest
const runId = await trigger();
expect(await waitForRun(runId)).toBe("failure");
@@ -1033,7 +1073,7 @@ run_sh = "echo hi"
.where("name", "=", "pipeline setup")
.executeTakeFirst();
expect(setup?.status).toBe("failure");
expect(setup?.log).toContain("fatal: not empty");
expect(setup?.log).toContain("mkdir: read-only");
});
test("a cache path inside the clone directory is rejected", async () => {
@@ -1067,7 +1107,87 @@ run_sh = "echo hi"
.where("run_id", "=", runId)
.where("name", "=", "pipeline setup")
.executeTakeFirst();
expect(setup?.log).toContain("is inside clone_project_to");
expect(setup?.log).toContain("overlaps clone_project_to");
});
test("a cache path above the clone directory is rejected", async () => {
const badSha = seedCiToml(
"ci-repo",
`
image = "debian:latest"
clone_project_to = "/ci/build/project"
cache = ["/ci/build"]
[on]
manual = true
[[steps]]
name = "hello"
run_sh = "echo hi"
`,
);
const runId = await triggerRun("ci-repo", {
triggerSource: "manual",
commitSha: badSha,
commitBranch: "main",
triggeredBy: adminUserId,
});
expect(await waitForRun(runId)).toBe("failure");
expect(uploads).toHaveLength(0);
});
test("a relative clone_project_to is rejected", async () => {
const badSha = seedCiToml(
"ci-repo",
`
image = "debian:latest"
work_dir = "/ci/build"
clone_project_to = "project"
[on]
manual = true
[[steps]]
name = "hello"
run_sh = "echo hi"
`,
);
const runId = await triggerRun("ci-repo", {
triggerSource: "manual",
commitSha: badSha,
commitBranch: "main",
triggeredBy: adminUserId,
});
expect(await waitForRun(runId)).toBe("failure");
const setup = await db
.selectFrom("ci_steps")
.select("log")
.where("run_id", "=", runId)
.where("name", "=", "pipeline setup")
.executeTakeFirst();
expect(setup?.log).toContain("must be an absolute path");
});
test("the upload carries the requested commit", async () => {
const runId = await trigger();
expect(await waitForRun(runId)).toBe("success");
const upload = uploads.find((u) => u.path === "/ci/build/project");
expect(upload).toBeDefined();
// The archive must hold the CI config at the triggered commit, and no
// .git. A dropped commit argument would still produce a valid tar.
const names = tarEntryNames(upload!.body);
expect(names).toContain(".hearthforge-ci.toml");
expect(names.some((n) => n.startsWith(".git/"))).toBe(false);
// git archive writes uid 0 and no entry for the archive root, so the
// destination keeps the mode the container gave it.
for (const h of tarHeaders(upload!.body)) {
expect(h.uid).toBe(0);
expect(h.name).not.toBe("./");
}
});
});
@@ -1340,10 +1460,9 @@ run_sh = "echo hi"
test("a clear failure lands on the step, not the console", async () => {
const sha = seedCiToml("ci-repo", CLEAR_TOML);
queueExec({ output: "", exitCode: 0 }); // mkdir work_dir
queueExec({ output: "", exitCode: 0 }); // mkdir repo path
queueExec({ output: "", exitCode: 0 }); // clone
queueExec({ output: "", exitCode: 0 }); // mkdir clone_project_to
queueExec({ output: "boom\n", exitCode: 1 }); // first, fails
queueExec({ output: "index.lock exists\n", exitCode: 1 }); // clear
queueExec({ output: "rm: device busy\n", exitCode: 1 }); // clear
const runId = await triggerRun("ci-repo", {
triggerSource: "manual",
@@ -1361,7 +1480,66 @@ run_sh = "echo hi"
.executeTakeFirst();
expect(second?.status).toBe("failure");
expect(second?.log).toContain("Failed to reset");
expect(second?.log).toContain("index.lock");
expect(second?.log).toContain("device busy");
});
test("a clear step re-extracts the checkout", async () => {
const sha = seedCiToml("ci-repo", CLEAR_TOML);
queueExec({ output: "", exitCode: 0 }); // mkdir work_dir
queueExec({ output: "", exitCode: 0 }); // mkdir clone_project_to
queueExec({ output: "ok\n", exitCode: 0 }); // first
queueExec({ output: "", exitCode: 0 }); // clear: rm -rf
queueExec({ output: "", exitCode: 0 }); // mkdir clone_project_to again
queueExec({ output: "hi\n", exitCode: 0 }); // second
const runId = await triggerRun("ci-repo", {
triggerSource: "manual",
commitSha: sha,
commitBranch: "main",
triggeredBy: adminUserId,
});
expect(await waitForRun(runId)).toBe("success");
const toProject = uploads.filter((u) => u.path === "/ci/build/project");
expect(toProject.length).toBe(2);
expect(execCmds.flat().join(" ")).not.toContain("git");
});
test("clear removes and recreates the directory in one exec", async () => {
// `rm -rf` can delete the container's WorkingDir. A second exec would
// then fail to chdir before its command starts, with exit 127 and an
// opaque OCI message. Splitting these is the regression.
const sha = seedCiToml(
"ci-repo",
`
image = "debian:latest"
work_dir = "/ci/build"
clone_project_to = "/ci/build"
[on]
manual = true
[[steps]]
name = "first"
run_sh = "true"
[[steps]]
name = "second"
clear = true
run_sh = "echo hi"
`,
);
const runId = await triggerRun("ci-repo", {
triggerSource: "manual",
commitSha: sha,
commitBranch: "main",
triggeredBy: adminUserId,
});
expect(await waitForRun(runId)).toBe("success");
const removals = execCmds.filter((c) => c.join(" ").includes("rm -rf"));
expect(removals).toHaveLength(1);
expect(removals[0]!.join(" ")).toContain("mkdir -p");
});
});