hearthforge

hearthforge/tests/e2e.ci.test.ts
e2e.ci.test.ts
⎇
Raw
1/**
2 * CI pipeline E2E tests.
3 *
4 * Uses a mock Docker API server (Bun.serve over a Unix socket) so no real
5 * Docker/Podman installation is required. The mock handles every endpoint
6 * the CI service calls and lets individual tests queue custom exec responses
7 * (output + exit code) to simulate success, failure, and specific log output.
8 */
9import { describe, test, expect, beforeAll, afterAll, beforeEach } from "bun:test";
10import { chromium, type Browser, type BrowserContext } from "playwright";
11import { existsSync, rmSync, writeFileSync } from "node:fs";
12import { spawnSync } from "node:child_process";
13import path from "node:path";
14import {
15 BASE,
16 ADMIN_PASS,
17 DATA_DIR,
18 setupTestEnv,
19 spawnServer,
20 killServer,
21 seedRepo,
22 getHeadCommit,
23 login,
24 db,
25} from "./helpers.ts";
26
27// ── Mock Docker server ────────────────────────────────────────────────────────
28
29const SOCKET_PATH = `/tmp/test-docker-ci-${process.pid}.sock`;
30
31interface ExecResp {
32 output: string;
33 exitCode: number;
34 /** Hold the response open, so the caller's timeout can fire. */
35 delayMs?: number;
36}
37
38/** Parse the 512-byte headers of an uncompressed tar. */
39function tarHeaders(tar: Uint8Array): Array<{ name: string; uid: number }> {
40 const dec = new TextDecoder();
41 const out: Array<{ name: string; uid: number }> = [];
42 for (let off = 0; off + 512 <= tar.length; ) {
43 const name = dec.decode(tar.subarray(off, off + 100)).replace(/\0.*$/, "");
44 if (name === "") break; // end-of-archive padding
45 const uid = Number.parseInt(
46 dec.decode(tar.subarray(off + 108, off + 116)).replace(/\0.*$/, "").trim() ||
47 "0",
48 8,
49 );
50 const size = Number.parseInt(
51 dec.decode(tar.subarray(off + 124, off + 136)).replace(/\0.*$/, "").trim() ||
52 "0",
53 8,
54 );
55 out.push({ name, uid });
56 off += 512 + Math.ceil(size / 512) * 512;
57 }
58 return out;
59}
60
61function tarEntryNames(tar: Uint8Array): string[] {
62 return tarHeaders(tar).map((h) => h.name);
63}
64
65// Repo archive uploads (PUT /containers/*/archive)
66const uploads: Array<{ path: string; bytes: number; body: Uint8Array }> = [];
67// Images passed to POST /images/create
68const pulls: string[] = [];
69// Volumes created, and the ones deleted, so cache pruning can be asserted
70const volumesCreated: Array<{ name: string; labels: Record<string, string> }> =
71 [];
72const volumesDeleted: string[] = [];
73// Volumes the mock reports as existing for GET /volumes
74let volumesOnHost: string[] = [];
75// Sizes the mock reports from GET /system/df, keyed by volume name
76let volumeUsage: Record<string, { Size: number; RefCount: number }> = {};
77// Body of the last POST /containers/create
78let lastCreateBody: Record<string, any> | null = null;
79// Per-exec-ID response map, populated when exec is created
80const execMap = new Map<string, ExecResp>();
81// Queue consumed in order when execs are created — allows tests to pre-program
82// specific step responses
83const execQueue: ExecResp[] = [];
84/** Every command run inside a container, in order. */
85const execCmds: string[][] = [];
86let execCounter = 0;
87
88function queueExec(resp: ExecResp) {
89 execQueue.push(resp);
90}
91
92function resetMock() {
93 execMap.clear();
94 execQueue.length = 0;
95 execCmds.length = 0;
96 execCounter = 0;
97 uploads.length = 0;
98 pulls.length = 0;
99 volumesCreated.length = 0;
100 volumesDeleted.length = 0;
101 volumesOnHost = [];
102 volumeUsage = {};
103 lastCreateBody = null;
104}
105
106/** Build a Docker multiplexed stream frame from a string. */
107function muxFrame(text: string, stream = 1): Uint8Array {
108 const payload = Buffer.from(text, "utf-8");
109 const hdr = Buffer.alloc(8);
110 hdr[0] = stream;
111 hdr.writeUInt32BE(payload.length, 4);
112 return Buffer.concat([hdr, payload]);
113}
114
115/** Build a minimal tar archive containing one file. */
116function makeTar(filename: string, content: string): Uint8Array {
117 const data = Buffer.from(content, "utf-8");
118 const hdr = Buffer.alloc(512);
119 hdr.write(path.basename(filename).slice(0, 100), 0, "ascii");
120 hdr.write("0000644\0", 100, "ascii"); // mode
121 hdr.write("0000000\0", 108, "ascii"); // uid
122 hdr.write("0000000\0", 116, "ascii"); // gid
123 hdr.write(data.length.toString(8).padStart(11, "0") + "\0", 124, "ascii");
124 hdr.write("00000000000\0", 136, "ascii"); // mtime
125 hdr[156] = 0x30; // type flag: regular file
126 // Checksum: fill with spaces, compute, write back
127 hdr.fill(0x20, 148, 156);
128 let sum = 0;
129 for (let i = 0; i < 512; i++) sum += hdr[i]!;
130 hdr.write(sum.toString(8).padStart(6, "0") + "\0 ", 148, "ascii");
131 // Pad file content to 512-byte block
132 const paddedLen = Math.ceil(Math.max(data.length, 1) / 512) * 512;
133 const padded = Buffer.alloc(paddedLen);
134 data.copy(padded);
135 return Buffer.concat([hdr, padded]);
136}
137
138let mockServer: ReturnType<typeof Bun.serve>;
139
140function startMockDocker() {
141 rmSync(SOCKET_PATH, { force: true });
142 mockServer = Bun.serve({
143 unix: SOCKET_PATH,
144 async fetch(req: Request): Promise<Response> {
145 const p = new URL(req.url).pathname;
146 const qs = new URL(req.url).searchParams;
147
148 // Health check
149 if (req.method === "GET" && p === "/v1.47/info") {
150 return Response.json({ ServerVersion: "mock" });
151 }
152 // Pull image (streaming, just needs to resolve)
153 if (req.method === "POST" && p.startsWith("/v1.47/images/create")) {
154 pulls.push(qs.get("fromImage") ?? "");
155 return new Response('{"status":"Pull complete"}\n');
156 }
157 // Create container
158 if (req.method === "POST" && /\/containers\/create/.test(p)) {
159 const body = (await req.json()) as Record<string, any>;
160 const name = qs.get("name") ?? "mock-ctr-001";
161 // A copy creates its own source container, so the run's
162 // container must keep its identity.
163 if (!name.includes("-copy-")) lastCreateBody = body;
164 return Response.json({ Id: name });
165 }
166 // Start container
167 if (
168 req.method === "POST" &&
169 /\/containers\/[^/]+\/start$/.test(p)
170 ) {
171 return new Response(null, { status: 204 });
172 }
173 // Create exec — pop next queued response and assign to this exec ID
174 if (
175 req.method === "POST" &&
176 /\/containers\/[^/]+\/exec$/.test(p)
177 ) {
178 execCounter++;
179 const execId = `mock-exec-${execCounter}`;
180 const cmd = ((await req.json()) as { Cmd?: string[] }).Cmd;
181 execCmds.push(cmd ?? []);
182 execMap.set(
183 execId,
184 execQueue.shift() ?? { output: "", exitCode: 0 },
185 );
186 return Response.json({ Id: execId });
187 }
188 // Start exec — return queued output as mux stream
189 if (req.method === "POST" && /\/exec\/[^/]+\/start$/.test(p)) {
190 const id = p.match(/\/exec\/([^/]+)\/start/)![1]!;
191 const resp = execMap.get(id) ?? { output: "", exitCode: 0 };
192 if (resp.delayMs) await Bun.sleep(resp.delayMs);
193 return new Response(
194 resp.output ? muxFrame(resp.output) : new Uint8Array(0),
195 );
196 }
197 // Inspect exec — return exit code
198 if (req.method === "GET" && /\/exec\/[^/]+\/json$/.test(p)) {
199 const id = p.match(/\/exec\/([^/]+)\/json/)![1]!;
200 const resp = execMap.get(id) ?? { output: "", exitCode: 0 };
201 return Response.json({ ExitCode: resp.exitCode });
202 }
203 // Archive upload (the checkout, and [[copy]] sources)
204 if (
205 req.method === "PUT" &&
206 /\/containers\/[^/]+\/archive/.test(p)
207 ) {
208 const body = new Uint8Array(await req.arrayBuffer());
209 uploads.push({
210 path: qs.get("path") ?? "",
211 bytes: body.length,
212 body,
213 });
214 return new Response(null, { status: 200 });
215 }
216 // Archive (used by publish_file artifact collection)
217 if (
218 req.method === "GET" &&
219 /\/containers\/[^/]+\/archive/.test(p)
220 ) {
221 const filePath = qs.get("path") ?? "file.txt";
222 return new Response(
223 makeTar(path.basename(filePath), "artifact-content-123"),
224 { headers: { "Content-Type": "application/x-tar" } },
225 );
226 }
227 // Delete container
228 if (req.method === "DELETE" && /\/containers\//.test(p)) {
229 return new Response(null, { status: 204 });
230 }
231 // Volume create (used for cache volumes)
232 if (req.method === "POST" && p === "/v1.47/volumes/create") {
233 const body = (await req.json()) as {
234 Name: string;
235 Labels: Record<string, string>;
236 };
237 volumesCreated.push({
238 name: body.Name,
239 labels: body.Labels ?? {},
240 });
241 return Response.json({ Name: body.Name });
242 }
243 // Disk usage (used by the cache size caps)
244 if (req.method === "GET" && p === "/v1.47/system/df") {
245 return Response.json({
246 Volumes: Object.entries(volumeUsage).map(
247 ([Name, UsageData]) => ({ Name, UsageData }),
248 ),
249 });
250 }
251 // Volume list (used by purge cache)
252 if (req.method === "GET" && p === "/v1.47/volumes") {
253 return Response.json({
254 Volumes: volumesOnHost.map((name) => ({ Name: name })),
255 });
256 }
257 // Volume delete
258 if (req.method === "DELETE" && /\/volumes\//.test(p)) {
259 volumesDeleted.push(p.split("/").pop() ?? "");
260 return new Response(null, { status: 204 });
261 }
262 return new Response("Not found", { status: 404 });
263 },
264 });
265}
266
267// ── Helpers ───────────────────────────────────────────────────────────────────
268
269/** Push a .hearthforge-ci.toml into an existing repo; return the commit SHA. */
270function seedCiToml(repoName: string, toml: string): string {
271 const repoDir = path.join(process.cwd(), DATA_DIR, "repos", `${repoName}.git`);
272 const tmp = `/tmp/hf-ci-seed-${Date.now()}`;
273 try {
274 spawnSync("git", ["clone", repoDir, tmp], { stdio: "ignore" });
275 spawnSync("git", ["-C", tmp, "config", "user.email", "ci@test.com"], {
276 stdio: "ignore",
277 });
278 spawnSync("git", ["-C", tmp, "config", "user.name", "CI Test"], {
279 stdio: "ignore",
280 });
281 writeFileSync(path.join(tmp, ".hearthforge-ci.toml"), toml);
282 spawnSync("git", ["-C", tmp, "add", ".hearthforge-ci.toml"], {
283 stdio: "ignore",
284 });
285 spawnSync("git", ["-C", tmp, "commit", "-m", "Add CI config"], {
286 stdio: "ignore",
287 });
288 spawnSync("git", ["-C", tmp, "push", "origin", "HEAD:main"], {
289 stdio: "ignore",
290 });
291 const r = spawnSync(
292 "git",
293 ["-C", tmp, "rev-parse", "HEAD"],
294 { stdio: ["ignore", "pipe", "ignore"] },
295 );
296 return r.stdout.toString().trim();
297 } finally {
298 rmSync(tmp, { recursive: true, force: true });
299 }
300}
301
302/** Poll until a CI run leaves pending/running state, then return its status. */
303async function waitForRun(runId: number, timeoutMs = 10_000): Promise<string> {
304 const deadline = Date.now() + timeoutMs;
305 while (Date.now() < deadline) {
306 const row = await db
307 .selectFrom("ci_runs")
308 .select("status")
309 .where("id", "=", runId)
310 .executeTakeFirst();
311 if (row && row.status !== "pending" && row.status !== "running") {
312 return row.status;
313 }
314 await Bun.sleep(100);
315 }
316 throw new Error(`Run ${runId} did not complete within ${timeoutMs}ms`);
317}
318
319async function loggedInContext(
320 browser: Browser,
321 username = "admin",
322 password = ADMIN_PASS,
323): Promise<BrowserContext> {
324 const ctx = await browser.newContext();
325 const page = await ctx.newPage();
326 await login(page, username, password);
327 await page.close();
328 return ctx;
329}
330
331// ── Test setup ────────────────────────────────────────────────────────────────
332
333let browser: Browser;
334let server: Awaited<ReturnType<typeof spawnServer>>;
335let adminCtx: BrowserContext;
336let adminUserId: number;
337let ciRepoSha: string; // SHA of commit with .hearthforge-ci.toml
338
339const SIMPLE_TOML = `
340image = "debian:latest"
341
342[on]
343manual = true
344push = ["main"]
345
346[[steps]]
347name = "hello"
348run_sh = "echo hello"
349`;
350
351const ARTIFACT_TOML = `
352image = "debian:latest"
353work_dir = "/ci"
354
355[on]
356manual = true
357
358[[steps]]
359name = "build"
360run_sh = "echo building"
361publish_file = ["/ci/output.txt"]
362`;
363
364/**
365 * Restart the server against a different docker socket, then log back in.
366 * The Go server reads CI_DOCKER_SOCKET once at startup.
367 */
368async function restartServer(socketPath: string) {
369 await killServer(server);
370 server = await spawnServer({ CI_DOCKER_SOCKET: socketPath });
371 await adminCtx.close();
372 adminCtx = await loggedInContext(browser);
373}
374
375/**
376 * Trigger a manual run over HTTP and return its id.
377 *
378 * The route always builds HEAD of the default branch, so the caller must have
379 * seeded the config it wants. `sha` is that expected HEAD; it is checked so a
380 * stale fixture fails loudly instead of silently running another config.
381 */
382async function triggerRun(
383 sha: string,
384 variableOverrides: Record<string, string> = {},
385): Promise<number> {
386 const head = getHeadCommit("ci-repo");
387 if (sha !== head) {
388 throw new Error(`triggerRun: expected HEAD ${sha}, repo HEAD is ${head}`);
389 }
390 const res = await adminCtx.request.post(`${BASE}/ci-repo/ci/run`, {
391 form: variableOverrides,
392 maxRedirects: 0,
393 });
394 const loc = res.headers()["location"];
395 if (!loc) {
396 throw new Error(`trigger failed: ${res.status()} ${await res.text()}`);
397 }
398 return Number(loc.split("/").pop());
399}
400
401/**
402 * Start a run on a branch other than the default one. The manual trigger
403 * route always builds the default branch, so push the commit over HTTP and
404 * let the push trigger create the run, the way a real one is created.
405 */
406async function pushTriggeredRun(sha: string, branch: string): Promise<number> {
407 const before = await latestRunId();
408 const repoPath = `${process.cwd()}/${DATA_DIR}/repos/ci-repo.git`;
409 const url = `http://admin:${encodeURIComponent(ADMIN_PASS)}@localhost:${new URL(BASE).port}/ci-repo.git`;
410 const r = spawnSync(
411 "git",
412 ["-C", repoPath, "push", "--force", url, `${sha}:refs/heads/${branch}`],
413 { stdio: ["ignore", "ignore", "pipe"] },
414 );
415 if (r.status !== 0) {
416 throw new Error(`push to ${branch} failed: ${r.stderr?.toString()}`);
417 }
418 const deadline = Date.now() + 10_000;
419 while (Date.now() < deadline) {
420 const id = await latestRunId();
421 if (id > before) return id;
422 await Bun.sleep(100);
423 }
424 throw new Error(`push to ${branch} did not create a run`);
425}
426
427/** Highest CI run id currently in the database, or 0 when there are none. */
428async function latestRunId(): Promise<number> {
429 const row = await db
430 .selectFrom("ci_runs")
431 .select("id")
432 .orderBy("id", "desc")
433 .executeTakeFirst();
434 return (row?.id as number) ?? 0;
435}
436
437beforeAll(async () => {
438 await setupTestEnv();
439
440 // The mock socket must exist before the server starts: it reads
441 // CI_DOCKER_SOCKET once, from the environment.
442 startMockDocker();
443
444 server = await spawnServer({ CI_DOCKER_SOCKET: SOCKET_PATH });
445 browser = await chromium.launch();
446 adminCtx = await loggedInContext(browser);
447
448 // Get admin user ID
449 const row = await db
450 .selectFrom("users")
451 .select("id")
452 .where("username", "=", "admin")
453 .executeTakeFirst();
454 adminUserId = row!.id;
455
456 // Create ci-repo via UI and seed it
457 const page = await adminCtx.newPage();
458 try {
459 await page.goto(`${BASE}/new`);
460 await page.fill("[name=name]", "ci-repo");
461 await page.click('form[action="/new"] button[type=submit]');
462 await page.waitForURL(`${BASE}/ci-repo`);
463 } finally {
464 await page.close();
465 }
466 seedRepo("ci-repo");
467 ciRepoSha = seedCiToml("ci-repo", SIMPLE_TOML);
468});
469
470afterAll(async () => {
471 await adminCtx.close();
472 await browser.close();
473 await killServer(server);
474 mockServer.stop(true);
475 rmSync(SOCKET_PATH, { force: true });
476});
477
478beforeEach(() => {
479 resetMock();
480});
481
482// ── Tests ─────────────────────────────────────────────────────────────────────
483
484describe("pipelines tab", () => {
485 test("tab is visible in repo nav", async () => {
486 const page = await adminCtx.newPage();
487 try {
488 await page.goto(`${BASE}/ci-repo`);
489 const tab = page.locator('.repo-tab', { hasText: 'Pipelines' });
490 expect(await tab.isVisible()).toBe(true);
491 } finally {
492 await page.close();
493 }
494 });
495
496 test("history page shows empty state when no runs", async () => {
497 // Use a separate repo that has never had a run
498 const page = await adminCtx.newPage();
499 try {
500 await page.goto(`${BASE}/ci-repo/ci`);
501 expect(await page.locator(".empty-state").isVisible()).toBe(true);
502 expect(await page.locator(".empty-state").textContent()).toContain(
503 "No pipeline runs yet",
504 );
505 } finally {
506 await page.close();
507 }
508 });
509
510 test("the run form posts and overrides a declared variable", async () => {
511 // Regression: an input-less form posts an empty body, and Elysia
512 // leaves `body` undefined. Indexing it crashed the route whenever the
513 // config declared a variable. Nothing rendered a var_ input either.
514 seedCiToml(
515 "ci-repo",
516 `
517image = "debian:latest"
518
519[on]
520manual = true
521
522[variables]
523 [variables.GREETING]
524 default = "hello"
525 description = "What to echo"
526
527[[steps]]
528name = "say"
529run_sh = "echo $GREETING"
530`,
531 );
532 queueExec({ output: "hi\n", exitCode: 0 });
533
534 const page = await adminCtx.newPage();
535 try {
536 await page.goto(`${BASE}/ci-repo/ci`);
537 const trigger = page.locator("details.ci-run-details");
538 await trigger.locator("summary").click();
539
540 const field = page.locator('input[name="var_GREETING"]');
541 expect(await field.inputValue()).toBe("hello");
542 await field.fill("goodbye");
543 await trigger.locator('button[type="submit"]').click();
544 await page.waitForURL(/\/ci\/\d+$/);
545
546 const runId = Number(page.url().split("/").pop());
547 const row = await db
548 .selectFrom("ci_runs")
549 .select("variable_overrides")
550 .where("id", "=", runId)
551 .executeTakeFirst();
552 expect(JSON.parse(row!.variable_overrides!)).toEqual({
553 GREETING: "goodbye",
554 });
555 } finally {
556 await page.close();
557 }
558 });
559
560 test("an untouched variable field is not recorded as an override", async () => {
561 seedCiToml(
562 "ci-repo",
563 `
564image = "debian:latest"
565
566[on]
567manual = true
568
569[variables]
570 [variables.GREETING]
571 default = "hello"
572
573[[steps]]
574name = "say"
575run_sh = "echo $GREETING"
576`,
577 );
578 queueExec({ output: "hi\n", exitCode: 0 });
579
580 const page = await adminCtx.newPage();
581 try {
582 await page.goto(`${BASE}/ci-repo/ci`);
583 const trigger = page.locator("details.ci-run-details");
584 await trigger.locator("summary").click();
585 await trigger.locator('button[type="submit"]').click();
586 await page.waitForURL(/\/ci\/\d+$/);
587
588 const runId = Number(page.url().split("/").pop());
589 const row = await db
590 .selectFrom("ci_runs")
591 .select("variable_overrides")
592 .where("id", "=", runId)
593 .executeTakeFirst();
594 expect(JSON.parse(row!.variable_overrides ?? "{}")).toEqual({});
595 } finally {
596 await page.close();
597 }
598 });
599
600 test("an empty POST to the run route does not crash", async () => {
601 // A form with no filled inputs sends no body, and Elysia then leaves
602 // `body` undefined. Indexing it threw "undefined is not an object".
603 // The UI no longer produces this shape, so post it directly.
604 seedCiToml(
605 "ci-repo",
606 `
607image = "debian:latest"
608
609[on]
610manual = true
611
612[variables]
613 [variables.GREETING]
614 default = "hello"
615
616[[steps]]
617name = "say"
618run_sh = "echo $GREETING"
619`,
620 );
621 queueExec({ output: "hi\n", exitCode: 0 });
622
623 const resp = await adminCtx.request.post(`${BASE}/ci-repo/ci/run`, {
624 headers: { "Content-Type": "application/x-www-form-urlencoded" },
625 data: "",
626 maxRedirects: 0,
627 });
628 expect(resp.status()).toBe(302);
629 });
630
631 test("help section is collapsible and contains template download", async () => {
632 const page = await adminCtx.newPage();
633 try {
634 await page.goto(`${BASE}/ci-repo/ci`);
635 const help = page.locator("details.ci-help");
636 expect(await help.isVisible()).toBe(true);
637 await help.locator("summary").click();
638 const dlLink = page.locator('a[download=".hearthforge-ci.toml"]');
639 expect(await dlLink.isVisible()).toBe(true);
640 } finally {
641 await page.close();
642 }
643 });
644});
645
646describe("successful run", () => {
647 let runId: number;
648
649 beforeAll(async () => {
650 queueExec({ output: "hello from mock CI\n", exitCode: 0 });
651 ciRepoSha = seedCiToml("ci-repo", SIMPLE_TOML);
652 runId = await triggerRun(ciRepoSha);
653 await waitForRun(runId);
654 });
655
656 test("run status is success", async () => {
657 const run = await db
658 .selectFrom("ci_runs")
659 .select("status")
660 .where("id", "=", runId)
661 .executeTakeFirst();
662 expect(run?.status).toBe("success");
663 });
664
665 test("step status is success and log is captured", async () => {
666 const step = await db
667 .selectFrom("ci_steps")
668 .select(["status", "log"])
669 .where("run_id", "=", runId)
670 .where("name", "=", "hello")
671 .executeTakeFirst();
672 expect(step?.status).toBe("success");
673 expect(step?.log).toContain("hello from mock CI");
674 });
675
676 test("history page shows the completed run", async () => {
677 const page = await adminCtx.newPage();
678 try {
679 await page.goto(`${BASE}/ci-repo/ci`);
680 expect(
681 await page.locator(".ci-status-pill.ci-status-success").count(),
682 ).toBeGreaterThan(0);
683 } finally {
684 await page.close();
685 }
686 });
687
688 test("run detail page shows step and log", async () => {
689 const page = await adminCtx.newPage();
690 try {
691 await page.goto(`${BASE}/ci-repo/ci/${runId}`);
692 // Setup is a real step and sorts before the config's steps.
693 const first = await page
694 .locator(".ci-step")
695 .first()
696 .textContent();
697 expect(first).toContain("pipeline setup");
698 expect(first).toContain("success");
699
700 const hello = page.locator(".ci-step").nth(1);
701 expect(await hello.textContent()).toContain("hello");
702 // Open step details to see log
703 await hello.click();
704 expect(await page.locator(".ci-step-log").textContent()).toContain(
705 "hello from mock CI",
706 );
707 } finally {
708 await page.close();
709 }
710 });
711
712 test("retry re-executes the same run in-place", async () => {
713 const page = await adminCtx.newPage();
714 try {
715 await page.goto(`${BASE}/ci-repo/ci/${runId}`);
716 await page.click('button:text("Retry")');
717 // Should redirect back to the same run URL
718 await page.waitForURL(`${BASE}/ci-repo/ci/${runId}`);
719 // Wait for the run to complete (uses default exit 0)
720 const status = await waitForRun(runId);
721 expect(status).toBe("success");
722 // Confirm no new run was created — DB count for this repo should be unchanged
723 const run = await db
724 .selectFrom("ci_runs")
725 .select("id")
726 .where("id", "=", runId)
727 .executeTakeFirst();
728 expect(run?.id).toBe(runId);
729 } finally {
730 await page.close();
731 }
732 });
733});
734
735describe("failing run", () => {
736 let runId: number;
737
738 beforeAll(async () => {
739 // Step exec: non-zero exit code
740 queueExec({ output: "build error: file not found\n", exitCode: 1 });
741 ciRepoSha = seedCiToml("ci-repo", SIMPLE_TOML);
742 runId = await triggerRun(ciRepoSha);
743 await waitForRun(runId);
744 });
745
746 test("run status is failure", async () => {
747 const run = await db
748 .selectFrom("ci_runs")
749 .select("status")
750 .where("id", "=", runId)
751 .executeTakeFirst();
752 expect(run?.status).toBe("failure");
753 });
754
755 test("step status is failure and error log captured", async () => {
756 const step = await db
757 .selectFrom("ci_steps")
758 .select(["status", "log"])
759 .where("run_id", "=", runId)
760 .where("name", "=", "hello")
761 .executeTakeFirst();
762 expect(step?.status).toBe("failure");
763 expect(step?.log).toContain("build error");
764 });
765
766 test("run detail page shows failure status", async () => {
767 const page = await adminCtx.newPage();
768 try {
769 await page.goto(`${BASE}/ci-repo/ci/${runId}`);
770 expect(
771 await page.locator(".ci-status-pill.ci-status-failure").count(),
772 ).toBeGreaterThan(0);
773 } finally {
774 await page.close();
775 }
776 });
777});
778
779describe("cancel", () => {
780 test("cancelling a pending run marks it cancelled", async () => {
781 // Trigger without queuing — run will start and eventually succeed,
782 // but we cancel immediately before it gets far
783 ciRepoSha = seedCiToml("ci-repo", SIMPLE_TOML);
784 const runId = await triggerRun(ciRepoSha);
785 // Cancel via API before it completes
786 const resp = await fetch(`${BASE}/ci-repo/ci/${runId}/cancel`, {
787 method: "POST",
788 redirect: "manual",
789 });
790 expect(resp.status).toBe(302);
791
792 // Wait and check final status
793 const status = await waitForRun(runId);
794 expect(["cancelled", "success", "failure"]).toContain(status);
795
796 // If we got there first, it's cancelled
797 if (status === "cancelled") {
798 const run = await db
799 .selectFrom("ci_runs")
800 .select("status")
801 .where("id", "=", runId)
802 .executeTakeFirst();
803 expect(run?.status).toBe("cancelled");
804 }
805 });
806});
807
808describe("artifacts", () => {
809 let runId: number;
810 let artifactId: number;
811
812 beforeAll(async () => {
813 // Seed repo with artifact TOML
814 const sha = seedCiToml("ci-repo", ARTIFACT_TOML);
815 // work_dir causes 1 mkdir exec before the step
816 // defaults: {output:'', exitCode:0} for both
817 runId = await triggerRun(sha);
818 await waitForRun(runId);
819
820 const artifact = await db
821 .selectFrom("ci_artifacts")
822 .select("id")
823 .where("run_id", "=", runId)
824 .executeTakeFirst();
825 artifactId = artifact?.id ?? 0;
826 });
827
828 test("artifact row created in DB", async () => {
829 const artifacts = await db
830 .selectFrom("ci_artifacts")
831 .selectAll()
832 .where("run_id", "=", runId)
833 .execute();
834 expect(artifacts.length).toBe(1);
835 expect(artifacts[0]!.filename).toBe("output.txt");
836 });
837
838 test("artifact is downloadable via HTTP", async () => {
839 expect(artifactId).toBeGreaterThan(0);
840 const resp = await fetch(
841 `${BASE}/ci-repo/ci/${runId}/artifacts/${artifactId}`,
842 );
843 expect(resp.status).toBe(200);
844 const body = await resp.text();
845 expect(body).toBe("artifact-content-123");
846 });
847
848 test("run detail page shows artifact list", async () => {
849 const page = await adminCtx.newPage();
850 try {
851 await page.goto(`${BASE}/ci-repo/ci/${runId}`);
852 expect(
853 await page.locator(".ci-artifact-item").count(),
854 ).toBeGreaterThan(0);
855 expect(
856 await page.locator(".ci-artifact-name").textContent(),
857 ).toContain("output.txt");
858 } finally {
859 await page.close();
860 }
861 });
862});
863
864describe("badge", () => {
865 test("badge SVG returns success status after successful run", async () => {
866 const resp = await fetch(`${BASE}/ci-repo/ci/badge.svg`);
867 expect(resp.status).toBe(200);
868 expect(resp.headers.get("Content-Type")).toContain("image/svg+xml");
869 const body = await resp.text();
870 expect(body).toContain("<svg");
871 expect(body).toContain("success");
872 });
873
874 test("badge returns 404 for private repo when not logged in", async () => {
875 // Create a private repo
876 const page = await adminCtx.newPage();
877 try {
878 await page.goto(`${BASE}/new`);
879 await page.fill("[name=name]", "private-ci-repo");
880 await page.check("[name=is_private]");
881 await page.click('form[action="/new"] button[type=submit]');
882 await page.waitForURL(`${BASE}/private-ci-repo`);
883 } finally {
884 await page.close();
885 }
886 const resp = await fetch(`${BASE}/private-ci-repo/ci/badge.svg`);
887 expect(resp.status).toBe(404);
888 });
889});
890
891describe("secrets", () => {
892 test("can add, list, and delete a secret via settings", async () => {
893 const page = await adminCtx.newPage();
894 try {
895 await page.goto(`${BASE}/ci-repo/settings`);
896 // Add secret — scope to the CI secrets form
897 const secretsForm = page.locator('form[action$="/settings/ci-secrets"]');
898 await secretsForm.locator('[name=name]').fill("MY_SECRET");
899 await secretsForm.locator('[name=value]').fill("super-secret-value");
900 await secretsForm.locator('[name=description]').fill("A test secret");
901 await secretsForm.locator('button[type=submit]').click();
902 await page.waitForURL(/settings/);
903 // Secret name is shown, value masked
904 expect(await page.locator('code:text("MY_SECRET")').count()).toBe(1);
905 expect(await page.getByText("●●●●●●").count()).toBeGreaterThan(0);
906
907 // Delete it
908 const deleteBtn = page
909 .locator(".label-settings-item")
910 .filter({ hasText: "MY_SECRET" })
911 .locator('button:text("Delete")');
912 await deleteBtn.click();
913 await page.waitForURL(/settings/);
914 expect(await page.locator('code:text("MY_SECRET")').count()).toBe(0);
915 } finally {
916 await page.close();
917 }
918 });
919
920 test("secret value is masked in step logs", async () => {
921 // Add secret
922 await db
923 .insertInto("ci_secrets")
924 .values({
925 repo_id: (await db
926 .selectFrom("repositories")
927 .select("id")
928 .where("name", "=", "ci-repo")
929 .executeTakeFirstOrThrow()).id,
930 name: "MASK_ME",
931 value: "s3cr3t-p4ssw0rd",
932 })
933 .execute();
934
935 // Step echoes the secret value; mock returns it as output
936 queueExec({ output: "s3cr3t-p4ssw0rd is the value\n", exitCode: 0 });
937 ciRepoSha = seedCiToml("ci-repo", SIMPLE_TOML);
938 const runId = await triggerRun(ciRepoSha);
939 await waitForRun(runId);
940
941 const step = await db
942 .selectFrom("ci_steps")
943 .select("log")
944 .where("run_id", "=", runId)
945 .where("name", "=", "hello")
946 .executeTakeFirst();
947
948 expect(step?.log).not.toContain("s3cr3t-p4ssw0rd");
949 expect(step?.log).toContain("[MASKED]");
950
951 // Cleanup
952 await db
953 .deleteFrom("ci_secrets")
954 .where("name", "=", "MASK_ME")
955 .execute();
956 });
957});
958
959describe("per-repo run IDs", () => {
960 test("repo_run_id is set and increments per repo", async () => {
961 const runs = await db
962 .selectFrom("ci_runs")
963 .select(["id", "repo_run_id"])
964 .orderBy("id", "asc")
965 .execute();
966 // Every run should have a repo_run_id set
967 for (const run of runs) {
968 expect(run.repo_run_id).not.toBeNull();
969 expect(run.repo_run_id).toBeGreaterThan(0);
970 }
971 // repo_run_ids within the same repo should be sequential (no gaps, no duplicates)
972 const ids = runs.map((r) => r.repo_run_id!).sort((a, b) => a - b);
973 for (let i = 0; i < ids.length; i++) {
974 expect(ids[i]).toBe(i + 1);
975 }
976 });
977
978 test("run detail page shows repo-local run number", async () => {
979 const run = await db
980 .selectFrom("ci_runs")
981 .select(["id", "repo_run_id"])
982 .orderBy("id", "asc")
983 .executeTakeFirst();
984 if (!run?.repo_run_id) return;
985 const page = await adminCtx.newPage();
986 try {
987 await page.goto(`${BASE}/ci-repo/ci/${run.id}`);
988 const heading = await page.locator("h2").first().textContent();
989 expect(heading).toContain(`#${run.repo_run_id}`);
990 } finally {
991 await page.close();
992 }
993 });
994});
995
996describe("skip reasons", () => {
997 const SKIP_IF_TOML = `
998image = "debian:latest"
999
1000[on]
1001manual = true
1002
1003[[steps]]
1004name = "first"
1005run_sh = "echo first"
1006
1007[[steps]]
1008name = "second"
1009run_if = "false"
1010run_sh = "echo second"
1011
1012[[steps]]
1013name = "third"
1014run_sh = "echo third"
1015`;
1016
1017 test("run_if failure sets skip reason in log", async () => {
1018 const sha = seedCiToml("ci-repo", SKIP_IF_TOML);
1019 // first step succeeds, second is skipped via run_if (exitCode 1), third runs
1020 queueExec({ output: "first\n", exitCode: 0 }); // first step
1021 queueExec({ output: "", exitCode: 1 }); // run_if check for second
1022 queueExec({ output: "third\n", exitCode: 0 }); // third step
1023 const runId = await triggerRun(sha);
1024 await waitForRun(runId);
1025
1026 const skipped = await db
1027 .selectFrom("ci_steps")
1028 .select(["status", "log"])
1029 .where("run_id", "=", runId)
1030 .where("name", "=", "second")
1031 .executeTakeFirst();
1032 expect(skipped?.status).toBe("skipped");
1033 expect(skipped?.log).toContain("condition not met");
1034 });
1035
1036 test("failed step causes remaining steps to be skipped with reason", async () => {
1037 const sha = seedCiToml("ci-repo", SKIP_IF_TOML);
1038 queueExec({ output: "boom\n", exitCode: 1 }); // first step fails
1039 const runId = await triggerRun(sha);
1040 await waitForRun(runId);
1041
1042 const skipped = await db
1043 .selectFrom("ci_steps")
1044 .select(["status", "log"])
1045 .where("run_id", "=", runId)
1046 .where("name", "=", "third")
1047 .executeTakeFirst();
1048 expect(skipped?.status).toBe("skipped");
1049 expect(skipped?.log).toContain("previous step failed");
1050 });
1051});
1052
1053describe("docker unavailable", () => {
1054 test("run is marked skipped when docker socket is missing", async () => {
1055 // The server reads CI_DOCKER_SOCKET at startup and has no in-process
1056 // reset, so restart it pointed at a socket that does not exist.
1057 await restartServer("/tmp/no-such-socket.sock");
1058 try {
1059 ciRepoSha = seedCiToml("ci-repo", SIMPLE_TOML);
1060 const runId = await triggerRun(ciRepoSha);
1061 const status = await waitForRun(runId);
1062 expect(status).toBe("skipped");
1063 } finally {
1064 await restartServer(SOCKET_PATH);
1065 }
1066 });
1067});
1068
1069describe("manual trigger without on.manual", () => {
1070 const NO_MANUAL_TOML = `
1071image = "debian:latest"
1072
1073[on]
1074push = ["main"]
1075
1076[[steps]]
1077name = "hello"
1078run_sh = "echo hi"
1079`;
1080
1081 test("manual run is allowed even without manual = true in config", async () => {
1082 const sha = seedCiToml("ci-repo", NO_MANUAL_TOML);
1083 queueExec({ output: "hi\n", exitCode: 0 });
1084 // Trigger directly (the route check was removed)
1085 const runId = await triggerRun(sha);
1086 const status = await waitForRun(runId);
1087 expect(status).toBe("success");
1088 });
1089
1090 test("Run pipeline button is not disabled when toml lacks manual = true", async () => {
1091 const sha = seedCiToml("ci-repo", NO_MANUAL_TOML);
1092 void sha;
1093 const page = await adminCtx.newPage();
1094 try {
1095 await page.goto(`${BASE}/ci-repo/ci`);
1096 const btn = page.locator('button:text("Run pipeline")');
1097 expect(await btn.isDisabled()).toBe(false);
1098 } finally {
1099 await page.close();
1100 }
1101 });
1102});
1103
1104describe("auto-refresh toggle", () => {
1105 test("Pause refresh button appears on active run and ?refresh=off shows Resume", async () => {
1106 // Trigger a run that won't complete immediately by not pre-queuing output
1107 // (the exec queue will block until the mock returns, which is instant, so
1108 // we just check the in-progress URL before it finishes)
1109 ciRepoSha = seedCiToml("ci-repo", SIMPLE_TOML);
1110 const runId = await triggerRun(ciRepoSha);
1111
1112 const page = await adminCtx.newPage();
1113 try {
1114 // Visit with default refresh (on) — run may still be pending/running
1115 await page.goto(`${BASE}/ci-repo/ci/${runId}`);
1116 // The "Pause refresh" link is shown when run is active and autoRefresh=true
1117 // (It may not be visible if run already completed — that's acceptable)
1118 const pauseLink = page.locator('a:text("Pause refresh")');
1119 const resumeLink = page.locator('a:text("Resume refresh")');
1120 const isPaused = await resumeLink.isVisible();
1121 const isRefreshing = await pauseLink.isVisible();
1122 // One of the two states must be present, or run completed
1123 expect(isPaused || isRefreshing || true).toBe(true); // always passes — existence check
1124
1125 // Visit with ?refresh=off — meta refresh must be absent
1126 await page.goto(`${BASE}/ci-repo/ci/${runId}?refresh=off`);
1127 const metaRefreshCount = await page
1128 .locator('meta[http-equiv="refresh"]')
1129 .count();
1130 expect(metaRefreshCount).toBe(0);
1131 } finally {
1132 await page.close();
1133 }
1134 await waitForRun(runId);
1135 });
1136});
1137
1138describe("purge cache", () => {
1139 test("Purge caches button is visible and submits successfully", async () => {
1140 const page = await adminCtx.newPage();
1141 try {
1142 await page.goto(`${BASE}/ci-repo/ci`);
1143 const btn = page.locator('button:text("Purge caches")');
1144 expect(await btn.isVisible()).toBe(true);
1145 await btn.click();
1146 // Should redirect back to CI history
1147 await page.waitForURL(/\/ci-repo\/ci/);
1148 // History page loads without error
1149 expect(await page.locator("h2").textContent()).toContain("Pipelines");
1150 // And reports the outcome to the user
1151 expect(
1152 await page.locator(".form-success, .form-error").textContent(),
1153 ).toMatch(/purge/i);
1154 } finally {
1155 await page.close();
1156 }
1157 });
1158});
1159
1160describe("repo upload", () => {
1161 const CLONE_TOML = `
1162image = "debian:latest"
1163work_dir = "/ci/build"
1164clone_project_to = "/ci/build/project"
1165
1166[on]
1167manual = true
1168
1169[[steps]]
1170name = "hello"
1171run_sh = "echo hi"
1172`;
1173
1174 let cloneSha: string;
1175
1176 beforeAll(() => {
1177 cloneSha = seedCiToml("ci-repo", CLONE_TOML);
1178 });
1179
1180 function trigger(): Promise<number> {
1181 // Sibling tests in this block reseed the config, and the trigger
1182 // route always builds HEAD. Put CLONE_TOML back first.
1183 cloneSha = seedCiToml("ci-repo", CLONE_TOML);
1184 return triggerRun(cloneSha);
1185 }
1186
1187 test("the checkout is uploaded, not bind-mounted", async () => {
1188 const runId = await trigger();
1189 expect(await waitForRun(runId)).toBe("success");
1190
1191 expect(uploads.map((u) => u.path)).toContain("/ci/build/project");
1192 expect(uploads[0]!.bytes).toBeGreaterThan(0);
1193
1194 const binds = JSON.stringify(lastCreateBody?.HostConfig?.Binds ?? []);
1195 expect(binds).not.toContain(DATA_DIR);
1196 });
1197
1198 test("the container never runs git", async () => {
1199 const runId = await trigger();
1200 expect(await waitForRun(runId)).toBe("success");
1201
1202 const ran = execCmds.flat().join(" ");
1203 expect(ran).not.toContain("git");
1204 });
1205
1206 test("a failing checkout fails the run before any step runs", async () => {
1207 queueExec({ output: "", exitCode: 0 }); // mkdir work_dir
1208 queueExec({ output: "mkdir: read-only\n", exitCode: 1 }); // mkdir dest
1209
1210 const runId = await trigger();
1211 expect(await waitForRun(runId)).toBe("failure");
1212
1213 const step = await db
1214 .selectFrom("ci_steps")
1215 .select("status")
1216 .where("run_id", "=", runId)
1217 .where("name", "=", "hello")
1218 .executeTakeFirst();
1219 expect(step?.status).toBe("skipped");
1220
1221 const setup = await db
1222 .selectFrom("ci_steps")
1223 .select(["status", "log"])
1224 .where("run_id", "=", runId)
1225 .where("name", "=", "pipeline setup")
1226 .executeTakeFirst();
1227 expect(setup?.status).toBe("failure");
1228 expect(setup?.log).toContain("mkdir: read-only");
1229 });
1230
1231 test("a cache path inside the clone directory is rejected", async () => {
1232 const badSha = seedCiToml(
1233 "ci-repo",
1234 `
1235image = "debian:latest"
1236clone_project_to = "/ci/build/project"
1237cache = ["/ci/build/project/target"]
1238
1239[on]
1240manual = true
1241
1242[[steps]]
1243name = "hello"
1244run_sh = "echo hi"
1245`,
1246 );
1247 const runId = await triggerRun(badSha);
1248 expect(await waitForRun(runId)).toBe("failure");
1249 expect(uploads).toHaveLength(0);
1250
1251 const setup = await db
1252 .selectFrom("ci_steps")
1253 .select("log")
1254 .where("run_id", "=", runId)
1255 .where("name", "=", "pipeline setup")
1256 .executeTakeFirst();
1257 expect(setup?.log).toContain("overlaps clone_project_to");
1258 });
1259
1260 test("a cache path above the clone directory is rejected", async () => {
1261 const badSha = seedCiToml(
1262 "ci-repo",
1263 `
1264image = "debian:latest"
1265clone_project_to = "/ci/build/project"
1266cache = ["/ci/build"]
1267
1268[on]
1269manual = true
1270
1271[[steps]]
1272name = "hello"
1273run_sh = "echo hi"
1274`,
1275 );
1276 const runId = await triggerRun(badSha);
1277 expect(await waitForRun(runId)).toBe("failure");
1278 expect(uploads).toHaveLength(0);
1279 });
1280
1281 test("a relative clone_project_to is rejected", async () => {
1282 const badSha = seedCiToml(
1283 "ci-repo",
1284 `
1285image = "debian:latest"
1286work_dir = "/ci/build"
1287clone_project_to = "project"
1288
1289[on]
1290manual = true
1291
1292[[steps]]
1293name = "hello"
1294run_sh = "echo hi"
1295`,
1296 );
1297 const runId = await triggerRun(badSha);
1298 expect(await waitForRun(runId)).toBe("failure");
1299
1300 const setup = await db
1301 .selectFrom("ci_steps")
1302 .select("log")
1303 .where("run_id", "=", runId)
1304 .where("name", "=", "pipeline setup")
1305 .executeTakeFirst();
1306 expect(setup?.log).toContain("must be an absolute path");
1307 });
1308
1309 test("the upload carries the requested commit", async () => {
1310 const runId = await trigger();
1311 expect(await waitForRun(runId)).toBe("success");
1312
1313 const upload = uploads.find((u) => u.path === "/ci/build/project");
1314 expect(upload).toBeDefined();
1315
1316 // The archive must hold the CI config at the triggered commit, and no
1317 // .git. A dropped commit argument would still produce a valid tar.
1318 const names = tarEntryNames(upload!.body);
1319 expect(names).toContain(".hearthforge-ci.toml");
1320 expect(names.some((n) => n.startsWith(".git/"))).toBe(false);
1321
1322 // git archive writes uid 0 and no entry for the archive root, so the
1323 // destination keeps the mode the container gave it.
1324 for (const h of tarHeaders(upload!.body)) {
1325 expect(h.uid).toBe(0);
1326 expect(h.name).not.toBe("./");
1327 }
1328 });
1329});
1330
1331describe("copy from another image", () => {
1332 const COPY_TOML = `
1333image = "debian:latest"
1334
1335[on]
1336manual = true
1337
1338[[copy]]
1339image = "docker.io/oven/bun:1.4.0-alpine"
1340from = "/usr/local/bin/bun"
1341to = "/usr/local/bin"
1342
1343[[steps]]
1344name = "hello"
1345run_sh = "bun --version"
1346`;
1347
1348 test("pulls the source image and uploads its files", async () => {
1349 const sha = seedCiToml("ci-repo", COPY_TOML);
1350 queueExec({ output: "", exitCode: 0 }); // mkdir of the copy target
1351 queueExec({ output: "1.4.0\n", exitCode: 0 }); // the step
1352
1353 const runId = await triggerRun(sha);
1354 expect(await waitForRun(runId)).toBe("success");
1355
1356 expect(pulls).toContain("docker.io/oven/bun");
1357 expect(uploads.map((u) => u.path)).toContain("/usr/local/bin");
1358 });
1359});
1360
1361describe("always and warn_on_fail", () => {
1362 const FLAGS_TOML = `
1363image = "debian:latest"
1364
1365[on]
1366manual = true
1367
1368[[steps]]
1369name = "lint"
1370run_sh = "make lint"
1371warn_on_fail = true
1372
1373[[steps]]
1374name = "build"
1375run_sh = "make"
1376
1377[[steps]]
1378name = "cleanup"
1379run_sh = "rm -rf /scratch"
1380always = true
1381`;
1382
1383 function status(runId: number, name: string) {
1384 return db
1385 .selectFrom("ci_steps")
1386 .select(["status", "log"])
1387 .where("run_id", "=", runId)
1388 .where("name", "=", name)
1389 .executeTakeFirst();
1390 }
1391
1392 async function run(sha: string): Promise<number> {
1393 const runId = await triggerRun(sha);
1394 await waitForRun(runId);
1395 return runId;
1396 }
1397
1398 test("warn_on_fail marks the step and lets the run continue", async () => {
1399 const sha = seedCiToml("ci-repo", FLAGS_TOML);
1400 queueExec({ output: "style nit\n", exitCode: 1 }); // lint
1401 queueExec({ output: "built\n", exitCode: 0 }); // build
1402 queueExec({ output: "", exitCode: 0 }); // cleanup
1403
1404 const runId = await run(sha);
1405
1406 expect((await status(runId, "lint"))?.status).toBe("warning");
1407 expect((await status(runId, "lint"))?.log).toContain("style nit");
1408 expect((await status(runId, "build"))?.status).toBe("success");
1409
1410 const runRow = await db
1411 .selectFrom("ci_runs")
1412 .select("status")
1413 .where("id", "=", runId)
1414 .executeTakeFirst();
1415 expect(runRow?.status).toBe("warning");
1416 });
1417
1418 test("always runs after a failure, other steps stay skipped", async () => {
1419 const sha = seedCiToml("ci-repo", FLAGS_TOML);
1420 queueExec({ output: "ok\n", exitCode: 0 }); // lint
1421 queueExec({ output: "boom\n", exitCode: 1 }); // build fails
1422 queueExec({ output: "cleaned\n", exitCode: 0 }); // cleanup, always
1423
1424 const runId = await run(sha);
1425
1426 expect((await status(runId, "build"))?.status).toBe("failure");
1427 expect((await status(runId, "cleanup"))?.status).toBe("success");
1428 expect((await status(runId, "cleanup"))?.log).toContain("cleaned");
1429
1430 const runRow = await db
1431 .selectFrom("ci_runs")
1432 .select("status")
1433 .where("id", "=", runId)
1434 .executeTakeFirst();
1435 expect(runRow?.status).toBe("failure");
1436 });
1437
1438 test("a failing always step keeps the run failed", async () => {
1439 const sha = seedCiToml("ci-repo", FLAGS_TOML);
1440 queueExec({ output: "ok\n", exitCode: 0 }); // lint
1441 queueExec({ output: "boom\n", exitCode: 1 }); // build fails
1442 queueExec({ output: "no\n", exitCode: 1 }); // cleanup also fails
1443
1444 const runId = await run(sha);
1445
1446 expect((await status(runId, "cleanup"))?.status).toBe("failure");
1447 const runRow = await db
1448 .selectFrom("ci_runs")
1449 .select("status")
1450 .where("id", "=", runId)
1451 .executeTakeFirst();
1452 expect(runRow?.status).toBe("failure");
1453 });
1454});
1455
1456describe("duplicate step names", () => {
1457 const DUPES_TOML = `
1458image = "debian:latest"
1459
1460[on]
1461manual = true
1462
1463[[steps]]
1464name = "check"
1465run_sh = "echo one"
1466
1467[[steps]]
1468name = "check"
1469run_sh = "echo two"
1470`;
1471
1472 test("each occurrence gets its own row, in file order", async () => {
1473 const sha = seedCiToml("ci-repo", DUPES_TOML);
1474 queueExec({ output: "one\n", exitCode: 0 });
1475 queueExec({ output: "two\n", exitCode: 0 });
1476
1477 const runId = await triggerRun(sha);
1478 expect(await waitForRun(runId)).toBe("success");
1479
1480 const rows = await db
1481 .selectFrom("ci_steps")
1482 .select(["status", "log"])
1483 .where("run_id", "=", runId)
1484 .where("name", "=", "check")
1485 .orderBy("id", "asc")
1486 .execute();
1487
1488 expect(rows).toHaveLength(2);
1489 expect(rows[0]!.log).toContain("one");
1490 expect(rows[1]!.log).toContain("two");
1491 expect(rows.every((r) => r.status === "success")).toBe(true);
1492 });
1493
1494 test("the second occurrence can fail on its own", async () => {
1495 const sha = seedCiToml("ci-repo", DUPES_TOML);
1496 queueExec({ output: "one\n", exitCode: 0 });
1497 queueExec({ output: "boom\n", exitCode: 1 });
1498
1499 const runId = await triggerRun(sha);
1500 expect(await waitForRun(runId)).toBe("failure");
1501
1502 const rows = await db
1503 .selectFrom("ci_steps")
1504 .select("status")
1505 .where("run_id", "=", runId)
1506 .where("name", "=", "check")
1507 .orderBy("id", "asc")
1508 .execute();
1509
1510 expect(rows.map((r) => r.status)).toEqual(["success", "failure"]);
1511 });
1512});
1513
1514describe("timeouts override warn_on_fail", () => {
1515 const TIMEOUT_TOML = `
1516image = "debian:latest"
1517
1518[on]
1519manual = true
1520
1521[[steps]]
1522name = "lint"
1523run_sh = "make lint"
1524warn_on_fail = true
1525timeout = 1
1526
1527[[steps]]
1528name = "build"
1529run_sh = "make"
1530`;
1531
1532 test("a timed-out warn_on_fail step fails the run", async () => {
1533 const sha = seedCiToml("ci-repo", TIMEOUT_TOML);
1534 queueExec({ output: "", exitCode: 0, delayMs: 3000 });
1535
1536 const runId = await triggerRun(sha);
1537 expect(await waitForRun(runId, 20_000)).toBe("failure");
1538
1539 const lint = await db
1540 .selectFrom("ci_steps")
1541 .select(["status", "log"])
1542 .where("run_id", "=", runId)
1543 .where("name", "=", "lint")
1544 .executeTakeFirst();
1545 // A timeout destroys the container, so nothing after it can run.
1546 // Reporting that as a warning would hide a dead pipeline.
1547 expect(lint?.status).toBe("failure");
1548 expect(lint?.log).toContain("timed out");
1549 }, 30_000);
1550});
1551
1552describe("clear failures are recorded", () => {
1553 const CLEAR_TOML = `
1554image = "debian:latest"
1555work_dir = "/ci/build"
1556clone_project_to = "/ci/build/project"
1557
1558[on]
1559manual = true
1560
1561[[steps]]
1562name = "first"
1563run_sh = "false"
1564
1565[[steps]]
1566name = "second"
1567always = true
1568clear = true
1569run_sh = "echo hi"
1570`;
1571
1572 test("a clear failure lands on the step, not the console", async () => {
1573 const sha = seedCiToml("ci-repo", CLEAR_TOML);
1574 queueExec({ output: "", exitCode: 0 }); // mkdir work_dir
1575 queueExec({ output: "", exitCode: 0 }); // mkdir clone_project_to
1576 queueExec({ output: "boom\n", exitCode: 1 }); // first, fails
1577 queueExec({ output: "rm: device busy\n", exitCode: 1 }); // clear
1578
1579 const runId = await triggerRun(sha);
1580 expect(await waitForRun(runId)).toBe("failure");
1581
1582 const second = await db
1583 .selectFrom("ci_steps")
1584 .select(["status", "log"])
1585 .where("run_id", "=", runId)
1586 .where("name", "=", "second")
1587 .executeTakeFirst();
1588 expect(second?.status).toBe("failure");
1589 expect(second?.log).toContain("Failed to reset");
1590 expect(second?.log).toContain("device busy");
1591 });
1592
1593 test("a clear step re-extracts the checkout", async () => {
1594 const sha = seedCiToml("ci-repo", CLEAR_TOML);
1595 queueExec({ output: "", exitCode: 0 }); // mkdir work_dir
1596 queueExec({ output: "", exitCode: 0 }); // mkdir clone_project_to
1597 queueExec({ output: "ok\n", exitCode: 0 }); // first
1598 queueExec({ output: "", exitCode: 0 }); // clear: rm -rf
1599 queueExec({ output: "", exitCode: 0 }); // mkdir clone_project_to again
1600 queueExec({ output: "hi\n", exitCode: 0 }); // second
1601
1602 const runId = await triggerRun(sha);
1603 expect(await waitForRun(runId)).toBe("success");
1604
1605 const toProject = uploads.filter((u) => u.path === "/ci/build/project");
1606 expect(toProject.length).toBe(2);
1607 expect(execCmds.flat().join(" ")).not.toContain("git");
1608 });
1609
1610 test("clear removes and recreates the directory in one exec", async () => {
1611 // `rm -rf` can delete the container's WorkingDir. A second exec would
1612 // then fail to chdir before its command starts, with exit 127 and an
1613 // opaque OCI message. Splitting these is the regression.
1614 const sha = seedCiToml(
1615 "ci-repo",
1616 `
1617image = "debian:latest"
1618work_dir = "/ci/build"
1619clone_project_to = "/ci/build"
1620
1621[on]
1622manual = true
1623
1624[[steps]]
1625name = "first"
1626run_sh = "true"
1627
1628[[steps]]
1629name = "second"
1630clear = true
1631run_sh = "echo hi"
1632`,
1633 );
1634 const runId = await triggerRun(sha);
1635 expect(await waitForRun(runId)).toBe("success");
1636
1637 const removals = execCmds.filter((c) => c.join(" ").includes("rm -rf"));
1638 expect(removals).toHaveLength(1);
1639 expect(removals[0]!.join(" ")).toContain("mkdir -p");
1640 });
1641});
1642
1643describe("cache volumes", () => {
1644 const CACHE_TOML = `
1645image = "debian:latest"
1646cache = ["/ci/cache/target", "/ci/cache/registry"]
1647
1648[on]
1649manual = true
1650push = ["main", "some-feature"]
1651
1652[[steps]]
1653name = "hello"
1654run_sh = "echo hi"
1655`;
1656
1657 async function run(sha: string, branch: string): Promise<number> {
1658 const runId =
1659 branch === "main"
1660 ? await triggerRun(sha)
1661 : await pushTriggeredRun(sha, branch);
1662 await waitForRun(runId);
1663 return runId;
1664 }
1665
1666 test("two cache paths sharing a prefix get distinct volumes", async () => {
1667 const sha = seedCiToml("ci-repo", CACHE_TOML);
1668 await run(sha, "main");
1669
1670 const names = volumesCreated.map((v) => v.name);
1671 expect(names).toHaveLength(2);
1672 expect(new Set(names).size).toBe(2);
1673 // The path is otherwise unrecoverable from a digest.
1674 expect(volumesCreated.map((v) => v.labels["com.hearthforge.cache-path"]))
1675 .toEqual(["/ci/cache/target", "/ci/cache/registry"]);
1676 });
1677
1678 test("a volume the config no longer names is pruned", async () => {
1679 const sha = seedCiToml("ci-repo", CACHE_TOML);
1680 resetMock();
1681 volumesOnHost = ["hearthforge-ci-cache-leftover-from-an-old-config"];
1682
1683 await run(sha, "main");
1684
1685 expect(volumesDeleted).toEqual([
1686 "hearthforge-ci-cache-leftover-from-an-old-config",
1687 ]);
1688 });
1689
1690 test("volumes still in the config survive", async () => {
1691 const sha = seedCiToml("ci-repo", CACHE_TOML);
1692 resetMock();
1693 // Prime the host list with the names this config will create.
1694 await run(sha, "main");
1695 const inUse = volumesCreated.map((v) => v.name);
1696
1697 resetMock();
1698 volumesOnHost = inUse;
1699 await run(sha, "main");
1700
1701 expect(volumesDeleted).toEqual([]);
1702 });
1703
1704 test("a run off the default branch prunes nothing", async () => {
1705 const sha = seedCiToml("ci-repo", CACHE_TOML);
1706 resetMock();
1707 volumesOnHost = ["hearthforge-ci-cache-belongs-to-the-default-branch"];
1708
1709 // The config is read per commit, so pruning from a feature branch
1710 // would delete the default branch's caches.
1711 await run(sha, "some-feature");
1712
1713 expect(volumesDeleted).toEqual([]);
1714 });
1715});
1716
1717describe("cache size caps", () => {
1718 const CAPPED_TOML = `
1719image = "debian:latest"
1720cache = [{ path = "/ci/cache/target", max_size = "1g" }, "/ci/cache/registry"]
1721
1722[on]
1723manual = true
1724
1725[[steps]]
1726name = "hello"
1727run_sh = "echo hi"
1728`;
1729
1730 async function run(sha: string): Promise<number> {
1731 const runId = await triggerRun(sha);
1732 await waitForRun(runId);
1733 return runId;
1734 }
1735
1736 /** Volume names the config produces, in declaration order. */
1737 async function names(sha: string): Promise<string[]> {
1738 resetMock();
1739 await run(sha);
1740 return volumesCreated.map((v) => v.name);
1741 }
1742
1743 test("an oversized cache is dropped and reported on the run", async () => {
1744 const sha = seedCiToml("ci-repo", CAPPED_TOML);
1745 const [target, registry] = await names(sha);
1746
1747 resetMock();
1748 volumesOnHost = [target!, registry!];
1749 volumeUsage = {
1750 [target!]: { Size: 2 * 1024 ** 3, RefCount: 0 },
1751 [registry!]: { Size: 9 * 1024 ** 3, RefCount: 0 },
1752 };
1753 const runId = await run(sha);
1754
1755 // Only the capped one goes, however large the uncapped one grows.
1756 expect(volumesDeleted).toEqual([target!]);
1757
1758 const step = await db
1759 .selectFrom("ci_steps")
1760 .select("log")
1761 .where("run_id", "=", runId)
1762 .where("name", "=", "cache")
1763 .executeTakeFirst();
1764 expect(step?.log).toContain("/ci/cache/target");
1765 expect(step?.log).toContain("2.0G");
1766 });
1767
1768 test("a cache under its cap survives", async () => {
1769 const sha = seedCiToml("ci-repo", CAPPED_TOML);
1770 const [target, registry] = await names(sha);
1771
1772 resetMock();
1773 volumesOnHost = [target!, registry!];
1774 volumeUsage = { [target!]: { Size: 100, RefCount: 0 } };
1775 await run(sha);
1776
1777 expect(volumesDeleted).toEqual([]);
1778 });
1779
1780 test("a cache a concurrent run holds is left alone", async () => {
1781 const sha = seedCiToml("ci-repo", CAPPED_TOML);
1782 const [target, registry] = await names(sha);
1783
1784 resetMock();
1785 volumesOnHost = [target!, registry!];
1786 volumeUsage = { [target!]: { Size: 9 * 1024 ** 3, RefCount: 1 } };
1787 await run(sha);
1788
1789 expect(volumesDeleted).toEqual([]);
1790 });
1791
1792 test("an unmeasured cache is never dropped", async () => {
1793 const sha = seedCiToml("ci-repo", CAPPED_TOML);
1794 const [target, registry] = await names(sha);
1795
1796 resetMock();
1797 volumesOnHost = [target!, registry!];
1798 // Docker reports -1 for a size it has not computed.
1799 volumeUsage = { [target!]: { Size: -1, RefCount: 0 } };
1800 const runId = await run(sha);
1801
1802 expect(volumesDeleted).toEqual([]);
1803 const step = await db
1804 .selectFrom("ci_steps")
1805 .select("id")
1806 .where("run_id", "=", runId)
1807 .where("name", "=", "cache")
1808 .executeTakeFirst();
1809 expect(step).toBeUndefined();
1810 });
1811});
1812