hearthforge

hearthforge/tests/e2e.ci.test.ts
e2e.ci.test.ts
⎇
Raw
1/**
2 * CI pipeline E2E tests.
3 *
4 * Uses a mock Docker API server (Bun.serve over a Unix socket) so no real
5 * Docker/Podman installation is required. The mock handles every endpoint
6 * the CI service calls and lets individual tests queue custom exec responses
7 * (output + exit code) to simulate success, failure, and specific log output.
8 */
9import { describe, test, expect, beforeAll, afterAll, beforeEach } from "bun:test";
10import { chromium, type Browser, type BrowserContext } from "playwright";
11import { existsSync, rmSync, writeFileSync } from "node:fs";
12import { spawnSync } from "node:child_process";
13import path from "node:path";
14import {
15 BASE,
16 ADMIN_PASS,
17 DATA_DIR,
18 setupTestEnv,
19 spawnServer,
20 killServer,
21 seedRepo,
22 login,
23} from "./helpers.ts";
24import { db } from "../src/db/index.ts";
25import config from "../src/config.ts";
26import {
27 resetDockerSocket,
28 triggerRun,
29} from "../src/services/ci.ts";
30import { paths } from "../src/constants.ts";
31
32// ── Mock Docker server ────────────────────────────────────────────────────────
33
34const SOCKET_PATH = `/tmp/test-docker-ci-${process.pid}.sock`;
35
36interface ExecResp {
37 output: string;
38 exitCode: number;
39 /** Hold the response open, so the caller's timeout can fire. */
40 delayMs?: number;
41}
42
43/** Parse the 512-byte headers of an uncompressed tar. */
44function tarHeaders(tar: Uint8Array): Array<{ name: string; uid: number }> {
45 const dec = new TextDecoder();
46 const out: Array<{ name: string; uid: number }> = [];
47 for (let off = 0; off + 512 <= tar.length; ) {
48 const name = dec.decode(tar.subarray(off, off + 100)).replace(/\0.*$/, "");
49 if (name === "") break; // end-of-archive padding
50 const uid = Number.parseInt(
51 dec.decode(tar.subarray(off + 108, off + 116)).replace(/\0.*$/, "").trim() ||
52 "0",
53 8,
54 );
55 const size = Number.parseInt(
56 dec.decode(tar.subarray(off + 124, off + 136)).replace(/\0.*$/, "").trim() ||
57 "0",
58 8,
59 );
60 out.push({ name, uid });
61 off += 512 + Math.ceil(size / 512) * 512;
62 }
63 return out;
64}
65
66function tarEntryNames(tar: Uint8Array): string[] {
67 return tarHeaders(tar).map((h) => h.name);
68}
69
70// Repo archive uploads (PUT /containers/*/archive)
71const uploads: Array<{ path: string; bytes: number; body: Uint8Array }> = [];
72// Images passed to POST /images/create
73const pulls: string[] = [];
74// Volumes created, and the ones deleted, so cache pruning can be asserted
75const volumesCreated: Array<{ name: string; labels: Record<string, string> }> =
76 [];
77const volumesDeleted: string[] = [];
78// Volumes the mock reports as existing for GET /volumes
79let volumesOnHost: string[] = [];
80// Sizes the mock reports from GET /system/df, keyed by volume name
81let volumeUsage: Record<string, { Size: number; RefCount: number }> = {};
82// Body of the last POST /containers/create
83let lastCreateBody: Record<string, any> | null = null;
84// Per-exec-ID response map, populated when exec is created
85const execMap = new Map<string, ExecResp>();
86// Queue consumed in order when execs are created — allows tests to pre-program
87// specific step responses
88const execQueue: ExecResp[] = [];
89/** Every command run inside a container, in order. */
90const execCmds: string[][] = [];
91let execCounter = 0;
92
93function queueExec(resp: ExecResp) {
94 execQueue.push(resp);
95}
96
97function resetMock() {
98 execMap.clear();
99 execQueue.length = 0;
100 execCmds.length = 0;
101 execCounter = 0;
102 uploads.length = 0;
103 pulls.length = 0;
104 volumesCreated.length = 0;
105 volumesDeleted.length = 0;
106 volumesOnHost = [];
107 volumeUsage = {};
108 lastCreateBody = null;
109}
110
111/** Build a Docker multiplexed stream frame from a string. */
112function muxFrame(text: string, stream = 1): Uint8Array {
113 const payload = Buffer.from(text, "utf-8");
114 const hdr = Buffer.alloc(8);
115 hdr[0] = stream;
116 hdr.writeUInt32BE(payload.length, 4);
117 return Buffer.concat([hdr, payload]);
118}
119
120/** Build a minimal tar archive containing one file. */
121function makeTar(filename: string, content: string): Uint8Array {
122 const data = Buffer.from(content, "utf-8");
123 const hdr = Buffer.alloc(512);
124 hdr.write(path.basename(filename).slice(0, 100), 0, "ascii");
125 hdr.write("0000644\0", 100, "ascii"); // mode
126 hdr.write("0000000\0", 108, "ascii"); // uid
127 hdr.write("0000000\0", 116, "ascii"); // gid
128 hdr.write(data.length.toString(8).padStart(11, "0") + "\0", 124, "ascii");
129 hdr.write("00000000000\0", 136, "ascii"); // mtime
130 hdr[156] = 0x30; // type flag: regular file
131 // Checksum: fill with spaces, compute, write back
132 hdr.fill(0x20, 148, 156);
133 let sum = 0;
134 for (let i = 0; i < 512; i++) sum += hdr[i]!;
135 hdr.write(sum.toString(8).padStart(6, "0") + "\0 ", 148, "ascii");
136 // Pad file content to 512-byte block
137 const paddedLen = Math.ceil(Math.max(data.length, 1) / 512) * 512;
138 const padded = Buffer.alloc(paddedLen);
139 data.copy(padded);
140 return Buffer.concat([hdr, padded]);
141}
142
143let mockServer: ReturnType<typeof Bun.serve>;
144
145function startMockDocker() {
146 rmSync(SOCKET_PATH, { force: true });
147 mockServer = Bun.serve({
148 unix: SOCKET_PATH,
149 async fetch(req: Request): Promise<Response> {
150 const p = new URL(req.url).pathname;
151 const qs = new URL(req.url).searchParams;
152
153 // Health check
154 if (req.method === "GET" && p === "/v1.47/info") {
155 return Response.json({ ServerVersion: "mock" });
156 }
157 // Pull image (streaming, just needs to resolve)
158 if (req.method === "POST" && p.startsWith("/v1.47/images/create")) {
159 pulls.push(qs.get("fromImage") ?? "");
160 return new Response('{"status":"Pull complete"}\n');
161 }
162 // Create container
163 if (req.method === "POST" && /\/containers\/create/.test(p)) {
164 const body = (await req.json()) as Record<string, any>;
165 const name = qs.get("name") ?? "mock-ctr-001";
166 // A copy creates its own source container, so the run's
167 // container must keep its identity.
168 if (!name.includes("-copy-")) lastCreateBody = body;
169 return Response.json({ Id: name });
170 }
171 // Start container
172 if (
173 req.method === "POST" &&
174 /\/containers\/[^/]+\/start$/.test(p)
175 ) {
176 return new Response(null, { status: 204 });
177 }
178 // Create exec — pop next queued response and assign to this exec ID
179 if (
180 req.method === "POST" &&
181 /\/containers\/[^/]+\/exec$/.test(p)
182 ) {
183 execCounter++;
184 const execId = `mock-exec-${execCounter}`;
185 const cmd = ((await req.json()) as { Cmd?: string[] }).Cmd;
186 execCmds.push(cmd ?? []);
187 execMap.set(
188 execId,
189 execQueue.shift() ?? { output: "", exitCode: 0 },
190 );
191 return Response.json({ Id: execId });
192 }
193 // Start exec — return queued output as mux stream
194 if (req.method === "POST" && /\/exec\/[^/]+\/start$/.test(p)) {
195 const id = p.match(/\/exec\/([^/]+)\/start/)![1]!;
196 const resp = execMap.get(id) ?? { output: "", exitCode: 0 };
197 if (resp.delayMs) await Bun.sleep(resp.delayMs);
198 return new Response(
199 resp.output ? muxFrame(resp.output) : new Uint8Array(0),
200 );
201 }
202 // Inspect exec — return exit code
203 if (req.method === "GET" && /\/exec\/[^/]+\/json$/.test(p)) {
204 const id = p.match(/\/exec\/([^/]+)\/json/)![1]!;
205 const resp = execMap.get(id) ?? { output: "", exitCode: 0 };
206 return Response.json({ ExitCode: resp.exitCode });
207 }
208 // Archive upload (the checkout, and [[copy]] sources)
209 if (
210 req.method === "PUT" &&
211 /\/containers\/[^/]+\/archive/.test(p)
212 ) {
213 const body = new Uint8Array(await req.arrayBuffer());
214 uploads.push({
215 path: qs.get("path") ?? "",
216 bytes: body.length,
217 body,
218 });
219 return new Response(null, { status: 200 });
220 }
221 // Archive (used by publish_file artifact collection)
222 if (
223 req.method === "GET" &&
224 /\/containers\/[^/]+\/archive/.test(p)
225 ) {
226 const filePath = qs.get("path") ?? "file.txt";
227 return new Response(
228 makeTar(path.basename(filePath), "artifact-content-123"),
229 { headers: { "Content-Type": "application/x-tar" } },
230 );
231 }
232 // Delete container
233 if (req.method === "DELETE" && /\/containers\//.test(p)) {
234 return new Response(null, { status: 204 });
235 }
236 // Volume create (used for cache volumes)
237 if (req.method === "POST" && p === "/v1.47/volumes/create") {
238 const body = (await req.json()) as {
239 Name: string;
240 Labels: Record<string, string>;
241 };
242 volumesCreated.push({
243 name: body.Name,
244 labels: body.Labels ?? {},
245 });
246 return Response.json({ Name: body.Name });
247 }
248 // Disk usage (used by the cache size caps)
249 if (req.method === "GET" && p === "/v1.47/system/df") {
250 return Response.json({
251 Volumes: Object.entries(volumeUsage).map(
252 ([Name, UsageData]) => ({ Name, UsageData }),
253 ),
254 });
255 }
256 // Volume list (used by purge cache)
257 if (req.method === "GET" && p === "/v1.47/volumes") {
258 return Response.json({
259 Volumes: volumesOnHost.map((name) => ({ Name: name })),
260 });
261 }
262 // Volume delete
263 if (req.method === "DELETE" && /\/volumes\//.test(p)) {
264 volumesDeleted.push(p.split("/").pop() ?? "");
265 return new Response(null, { status: 204 });
266 }
267 return new Response("Not found", { status: 404 });
268 },
269 });
270}
271
272// ── Helpers ───────────────────────────────────────────────────────────────────
273
274/** Push a .hearthforge-ci.toml into an existing repo; return the commit SHA. */
275function seedCiToml(repoName: string, toml: string): string {
276 const repoDir = path.join(process.cwd(), DATA_DIR, "repos", `${repoName}.git`);
277 const tmp = `/tmp/hf-ci-seed-${Date.now()}`;
278 try {
279 spawnSync("git", ["clone", repoDir, tmp], { stdio: "ignore" });
280 spawnSync("git", ["-C", tmp, "config", "user.email", "ci@test.com"], {
281 stdio: "ignore",
282 });
283 spawnSync("git", ["-C", tmp, "config", "user.name", "CI Test"], {
284 stdio: "ignore",
285 });
286 writeFileSync(path.join(tmp, ".hearthforge-ci.toml"), toml);
287 spawnSync("git", ["-C", tmp, "add", ".hearthforge-ci.toml"], {
288 stdio: "ignore",
289 });
290 spawnSync("git", ["-C", tmp, "commit", "-m", "Add CI config"], {
291 stdio: "ignore",
292 });
293 spawnSync("git", ["-C", tmp, "push", "origin", "HEAD:main"], {
294 stdio: "ignore",
295 });
296 const r = spawnSync(
297 "git",
298 ["-C", tmp, "rev-parse", "HEAD"],
299 { stdio: ["ignore", "pipe", "ignore"] },
300 );
301 return r.stdout.toString().trim();
302 } finally {
303 rmSync(tmp, { recursive: true, force: true });
304 }
305}
306
307/** Poll until a CI run leaves pending/running state, then return its status. */
308async function waitForRun(runId: number, timeoutMs = 10_000): Promise<string> {
309 const deadline = Date.now() + timeoutMs;
310 while (Date.now() < deadline) {
311 const row = await db
312 .selectFrom("ci_runs")
313 .select("status")
314 .where("id", "=", runId)
315 .executeTakeFirst();
316 if (row && row.status !== "pending" && row.status !== "running") {
317 return row.status;
318 }
319 await Bun.sleep(100);
320 }
321 throw new Error(`Run ${runId} did not complete within ${timeoutMs}ms`);
322}
323
324async function loggedInContext(
325 browser: Browser,
326 username = "admin",
327 password = ADMIN_PASS,
328): Promise<BrowserContext> {
329 const ctx = await browser.newContext();
330 const page = await ctx.newPage();
331 await login(page, username, password);
332 await page.close();
333 return ctx;
334}
335
336// ── Test setup ────────────────────────────────────────────────────────────────
337
338let browser: Browser;
339let server: Awaited<ReturnType<typeof spawnServer>>;
340let adminCtx: BrowserContext;
341let adminUserId: number;
342let ciRepoSha: string; // SHA of commit with .hearthforge-ci.toml
343
344const SIMPLE_TOML = `
345image = "debian:latest"
346
347[on]
348manual = true
349push = ["main"]
350
351[[steps]]
352name = "hello"
353run_sh = "echo hello"
354`;
355
356const ARTIFACT_TOML = `
357image = "debian:latest"
358work_dir = "/ci"
359
360[on]
361manual = true
362
363[[steps]]
364name = "build"
365run_sh = "echo building"
366publish_file = ["/ci/output.txt"]
367`;
368
369beforeAll(async () => {
370 await setupTestEnv();
371
372 // Point CI service at mock socket BEFORE starting any runs
373 config.CI_DOCKER_SOCKET = SOCKET_PATH;
374 resetDockerSocket();
375 startMockDocker();
376
377 server = await spawnServer();
378 browser = await chromium.launch();
379 adminCtx = await loggedInContext(browser);
380
381 // Get admin user ID
382 const row = await db
383 .selectFrom("users")
384 .select("id")
385 .where("username", "=", "admin")
386 .executeTakeFirst();
387 adminUserId = row!.id;
388
389 // Create ci-repo via UI and seed it
390 const page = await adminCtx.newPage();
391 try {
392 await page.goto(`${BASE}/new`);
393 await page.fill("[name=name]", "ci-repo");
394 await page.click('form[action="/new"] button[type=submit]');
395 await page.waitForURL(`${BASE}/ci-repo`);
396 } finally {
397 await page.close();
398 }
399 seedRepo("ci-repo");
400 ciRepoSha = seedCiToml("ci-repo", SIMPLE_TOML);
401});
402
403afterAll(async () => {
404 await adminCtx.close();
405 await browser.close();
406 await killServer(server);
407 mockServer.stop(true);
408 rmSync(SOCKET_PATH, { force: true });
409});
410
411beforeEach(() => {
412 resetMock();
413});
414
415// ── Tests ─────────────────────────────────────────────────────────────────────
416
417describe("pipelines tab", () => {
418 test("tab is visible in repo nav", async () => {
419 const page = await adminCtx.newPage();
420 try {
421 await page.goto(`${BASE}/ci-repo`);
422 const tab = page.locator('.repo-tab', { hasText: 'Pipelines' });
423 expect(await tab.isVisible()).toBe(true);
424 } finally {
425 await page.close();
426 }
427 });
428
429 test("history page shows empty state when no runs", async () => {
430 // Use a separate repo that has never had a run
431 const page = await adminCtx.newPage();
432 try {
433 await page.goto(`${BASE}/ci-repo/ci`);
434 expect(await page.locator(".empty-state").isVisible()).toBe(true);
435 expect(await page.locator(".empty-state").textContent()).toContain(
436 "No pipeline runs yet",
437 );
438 } finally {
439 await page.close();
440 }
441 });
442
443 test("help section is collapsible and contains template download", async () => {
444 const page = await adminCtx.newPage();
445 try {
446 await page.goto(`${BASE}/ci-repo/ci`);
447 const help = page.locator("details.ci-help");
448 expect(await help.isVisible()).toBe(true);
449 await help.locator("summary").click();
450 const dlLink = page.locator('a[download=".hearthforge-ci.toml"]');
451 expect(await dlLink.isVisible()).toBe(true);
452 } finally {
453 await page.close();
454 }
455 });
456});
457
458describe("successful run", () => {
459 let runId: number;
460
461 beforeAll(async () => {
462 queueExec({ output: "hello from mock CI\n", exitCode: 0 });
463 runId = await triggerRun("ci-repo", {
464 triggerSource: "manual",
465 commitSha: ciRepoSha,
466 commitBranch: "main",
467 triggeredBy: adminUserId,
468 });
469 await waitForRun(runId);
470 });
471
472 test("run status is success", async () => {
473 const run = await db
474 .selectFrom("ci_runs")
475 .select("status")
476 .where("id", "=", runId)
477 .executeTakeFirst();
478 expect(run?.status).toBe("success");
479 });
480
481 test("step status is success and log is captured", async () => {
482 const step = await db
483 .selectFrom("ci_steps")
484 .select(["status", "log"])
485 .where("run_id", "=", runId)
486 .where("name", "=", "hello")
487 .executeTakeFirst();
488 expect(step?.status).toBe("success");
489 expect(step?.log).toContain("hello from mock CI");
490 });
491
492 test("history page shows the completed run", async () => {
493 const page = await adminCtx.newPage();
494 try {
495 await page.goto(`${BASE}/ci-repo/ci`);
496 expect(
497 await page.locator(".ci-status-pill.ci-status-success").count(),
498 ).toBeGreaterThan(0);
499 } finally {
500 await page.close();
501 }
502 });
503
504 test("run detail page shows step and log", async () => {
505 const page = await adminCtx.newPage();
506 try {
507 await page.goto(`${BASE}/ci-repo/ci/${runId}`);
508 expect(
509 await page.locator(".ci-step").first().textContent(),
510 ).toContain("hello");
511 // Open step details to see log
512 await page.locator(".ci-step").first().click();
513 expect(await page.locator(".ci-step-log").textContent()).toContain(
514 "hello from mock CI",
515 );
516 } finally {
517 await page.close();
518 }
519 });
520
521 test("retry re-executes the same run in-place", async () => {
522 const page = await adminCtx.newPage();
523 try {
524 await page.goto(`${BASE}/ci-repo/ci/${runId}`);
525 await page.click('button:text("Retry")');
526 // Should redirect back to the same run URL
527 await page.waitForURL(`${BASE}/ci-repo/ci/${runId}`);
528 // Wait for the run to complete (uses default exit 0)
529 const status = await waitForRun(runId);
530 expect(status).toBe("success");
531 // Confirm no new run was created — DB count for this repo should be unchanged
532 const run = await db
533 .selectFrom("ci_runs")
534 .select("id")
535 .where("id", "=", runId)
536 .executeTakeFirst();
537 expect(run?.id).toBe(runId);
538 } finally {
539 await page.close();
540 }
541 });
542});
543
544describe("failing run", () => {
545 let runId: number;
546
547 beforeAll(async () => {
548 // Step exec: non-zero exit code
549 queueExec({ output: "build error: file not found\n", exitCode: 1 });
550 runId = await triggerRun("ci-repo", {
551 triggerSource: "manual",
552 commitSha: ciRepoSha,
553 commitBranch: "main",
554 triggeredBy: adminUserId,
555 });
556 await waitForRun(runId);
557 });
558
559 test("run status is failure", async () => {
560 const run = await db
561 .selectFrom("ci_runs")
562 .select("status")
563 .where("id", "=", runId)
564 .executeTakeFirst();
565 expect(run?.status).toBe("failure");
566 });
567
568 test("step status is failure and error log captured", async () => {
569 const step = await db
570 .selectFrom("ci_steps")
571 .select(["status", "log"])
572 .where("run_id", "=", runId)
573 .where("name", "=", "hello")
574 .executeTakeFirst();
575 expect(step?.status).toBe("failure");
576 expect(step?.log).toContain("build error");
577 });
578
579 test("run detail page shows failure status", async () => {
580 const page = await adminCtx.newPage();
581 try {
582 await page.goto(`${BASE}/ci-repo/ci/${runId}`);
583 expect(
584 await page.locator(".ci-status-pill.ci-status-failure").count(),
585 ).toBeGreaterThan(0);
586 } finally {
587 await page.close();
588 }
589 });
590});
591
592describe("cancel", () => {
593 test("cancelling a pending run marks it cancelled", async () => {
594 // Trigger without queuing — run will start and eventually succeed,
595 // but we cancel immediately before it gets far
596 const runId = await triggerRun("ci-repo", {
597 triggerSource: "manual",
598 commitSha: ciRepoSha,
599 commitBranch: "main",
600 triggeredBy: adminUserId,
601 });
602 // Cancel via API before it completes
603 const resp = await fetch(`${BASE}/ci-repo/ci/${runId}/cancel`, {
604 method: "POST",
605 redirect: "manual",
606 });
607 expect(resp.status).toBe(302);
608
609 // Wait and check final status
610 const status = await waitForRun(runId);
611 expect(["cancelled", "success", "failure"]).toContain(status);
612
613 // If we got there first, it's cancelled
614 if (status === "cancelled") {
615 const run = await db
616 .selectFrom("ci_runs")
617 .select("status")
618 .where("id", "=", runId)
619 .executeTakeFirst();
620 expect(run?.status).toBe("cancelled");
621 }
622 });
623});
624
625describe("artifacts", () => {
626 let runId: number;
627 let artifactId: number;
628
629 beforeAll(async () => {
630 // Seed repo with artifact TOML
631 const sha = seedCiToml("ci-repo", ARTIFACT_TOML);
632 // work_dir causes 1 mkdir exec before the step
633 // defaults: {output:'', exitCode:0} for both
634 runId = await triggerRun("ci-repo", {
635 triggerSource: "manual",
636 commitSha: sha,
637 commitBranch: "main",
638 triggeredBy: adminUserId,
639 });
640 await waitForRun(runId);
641
642 const artifact = await db
643 .selectFrom("ci_artifacts")
644 .select("id")
645 .where("run_id", "=", runId)
646 .executeTakeFirst();
647 artifactId = artifact?.id ?? 0;
648 });
649
650 test("artifact row created in DB", async () => {
651 const artifacts = await db
652 .selectFrom("ci_artifacts")
653 .selectAll()
654 .where("run_id", "=", runId)
655 .execute();
656 expect(artifacts.length).toBe(1);
657 expect(artifacts[0]!.filename).toBe("output.txt");
658 });
659
660 test("artifact is downloadable via HTTP", async () => {
661 expect(artifactId).toBeGreaterThan(0);
662 const resp = await fetch(
663 `${BASE}/ci-repo/ci/${runId}/artifacts/${artifactId}`,
664 );
665 expect(resp.status).toBe(200);
666 const body = await resp.text();
667 expect(body).toBe("artifact-content-123");
668 });
669
670 test("run detail page shows artifact list", async () => {
671 const page = await adminCtx.newPage();
672 try {
673 await page.goto(`${BASE}/ci-repo/ci/${runId}`);
674 expect(
675 await page.locator(".ci-artifact-item").count(),
676 ).toBeGreaterThan(0);
677 expect(
678 await page.locator(".ci-artifact-name").textContent(),
679 ).toContain("output.txt");
680 } finally {
681 await page.close();
682 }
683 });
684});
685
686describe("badge", () => {
687 test("badge SVG returns success status after successful run", async () => {
688 const resp = await fetch(`${BASE}/ci-repo/ci/badge.svg`);
689 expect(resp.status).toBe(200);
690 expect(resp.headers.get("Content-Type")).toContain("image/svg+xml");
691 const body = await resp.text();
692 expect(body).toContain("<svg");
693 expect(body).toContain("success");
694 });
695
696 test("badge returns 404 for private repo when not logged in", async () => {
697 // Create a private repo
698 const page = await adminCtx.newPage();
699 try {
700 await page.goto(`${BASE}/new`);
701 await page.fill("[name=name]", "private-ci-repo");
702 await page.check("[name=is_private]");
703 await page.click('form[action="/new"] button[type=submit]');
704 await page.waitForURL(`${BASE}/private-ci-repo`);
705 } finally {
706 await page.close();
707 }
708 const resp = await fetch(`${BASE}/private-ci-repo/ci/badge.svg`);
709 expect(resp.status).toBe(404);
710 });
711});
712
713describe("secrets", () => {
714 test("can add, list, and delete a secret via settings", async () => {
715 const page = await adminCtx.newPage();
716 try {
717 await page.goto(`${BASE}/ci-repo/settings`);
718 // Add secret — scope to the CI secrets form
719 const secretsForm = page.locator('form[action$="/settings/ci-secrets"]');
720 await secretsForm.locator('[name=name]').fill("MY_SECRET");
721 await secretsForm.locator('[name=value]').fill("super-secret-value");
722 await secretsForm.locator('[name=description]').fill("A test secret");
723 await secretsForm.locator('button[type=submit]').click();
724 await page.waitForURL(/settings/);
725 // Secret name is shown, value masked
726 expect(await page.locator('code:text("MY_SECRET")').count()).toBe(1);
727 expect(await page.getByText("●●●●●●").count()).toBeGreaterThan(0);
728
729 // Delete it
730 const deleteBtn = page
731 .locator(".label-settings-item")
732 .filter({ hasText: "MY_SECRET" })
733 .locator('button:text("Delete")');
734 await deleteBtn.click();
735 await page.waitForURL(/settings/);
736 expect(await page.locator('code:text("MY_SECRET")').count()).toBe(0);
737 } finally {
738 await page.close();
739 }
740 });
741
742 test("secret value is masked in step logs", async () => {
743 // Add secret
744 await db
745 .insertInto("ci_secrets")
746 .values({
747 repo_id: (await db
748 .selectFrom("repositories")
749 .select("id")
750 .where("name", "=", "ci-repo")
751 .executeTakeFirstOrThrow()).id,
752 name: "MASK_ME",
753 value: "s3cr3t-p4ssw0rd",
754 })
755 .execute();
756
757 // Step echoes the secret value; mock returns it as output
758 queueExec({ output: "s3cr3t-p4ssw0rd is the value\n", exitCode: 0 });
759 const runId = await triggerRun("ci-repo", {
760 triggerSource: "manual",
761 commitSha: ciRepoSha,
762 commitBranch: "main",
763 triggeredBy: adminUserId,
764 });
765 await waitForRun(runId);
766
767 const step = await db
768 .selectFrom("ci_steps")
769 .select("log")
770 .where("run_id", "=", runId)
771 .where("name", "=", "hello")
772 .executeTakeFirst();
773
774 expect(step?.log).not.toContain("s3cr3t-p4ssw0rd");
775 expect(step?.log).toContain("[MASKED]");
776
777 // Cleanup
778 await db
779 .deleteFrom("ci_secrets")
780 .where("name", "=", "MASK_ME")
781 .execute();
782 });
783});
784
785describe("per-repo run IDs", () => {
786 test("repo_run_id is set and increments per repo", async () => {
787 const runs = await db
788 .selectFrom("ci_runs")
789 .select(["id", "repo_run_id"])
790 .orderBy("id", "asc")
791 .execute();
792 // Every run should have a repo_run_id set
793 for (const run of runs) {
794 expect(run.repo_run_id).not.toBeNull();
795 expect(run.repo_run_id).toBeGreaterThan(0);
796 }
797 // repo_run_ids within the same repo should be sequential (no gaps, no duplicates)
798 const ids = runs.map((r) => r.repo_run_id!).sort((a, b) => a - b);
799 for (let i = 0; i < ids.length; i++) {
800 expect(ids[i]).toBe(i + 1);
801 }
802 });
803
804 test("run detail page shows repo-local run number", async () => {
805 const run = await db
806 .selectFrom("ci_runs")
807 .select(["id", "repo_run_id"])
808 .orderBy("id", "asc")
809 .executeTakeFirst();
810 if (!run?.repo_run_id) return;
811 const page = await adminCtx.newPage();
812 try {
813 await page.goto(`${BASE}/ci-repo/ci/${run.id}`);
814 const heading = await page.locator("h2").first().textContent();
815 expect(heading).toContain(`#${run.repo_run_id}`);
816 } finally {
817 await page.close();
818 }
819 });
820});
821
822describe("skip reasons", () => {
823 const SKIP_IF_TOML = `
824image = "debian:latest"
825
826[on]
827manual = true
828
829[[steps]]
830name = "first"
831run_sh = "echo first"
832
833[[steps]]
834name = "second"
835run_if = "false"
836run_sh = "echo second"
837
838[[steps]]
839name = "third"
840run_sh = "echo third"
841`;
842
843 test("run_if failure sets skip reason in log", async () => {
844 const sha = seedCiToml("ci-repo", SKIP_IF_TOML);
845 // first step succeeds, second is skipped via run_if (exitCode 1), third runs
846 queueExec({ output: "first\n", exitCode: 0 }); // first step
847 queueExec({ output: "", exitCode: 1 }); // run_if check for second
848 queueExec({ output: "third\n", exitCode: 0 }); // third step
849 const runId = await triggerRun("ci-repo", {
850 triggerSource: "manual",
851 commitSha: sha,
852 commitBranch: "main",
853 triggeredBy: adminUserId,
854 });
855 await waitForRun(runId);
856
857 const skipped = await db
858 .selectFrom("ci_steps")
859 .select(["status", "log"])
860 .where("run_id", "=", runId)
861 .where("name", "=", "second")
862 .executeTakeFirst();
863 expect(skipped?.status).toBe("skipped");
864 expect(skipped?.log).toContain("condition not met");
865 });
866
867 test("failed step causes remaining steps to be skipped with reason", async () => {
868 const sha = seedCiToml("ci-repo", SKIP_IF_TOML);
869 queueExec({ output: "boom\n", exitCode: 1 }); // first step fails
870 const runId = await triggerRun("ci-repo", {
871 triggerSource: "manual",
872 commitSha: sha,
873 commitBranch: "main",
874 triggeredBy: adminUserId,
875 });
876 await waitForRun(runId);
877
878 const skipped = await db
879 .selectFrom("ci_steps")
880 .select(["status", "log"])
881 .where("run_id", "=", runId)
882 .where("name", "=", "third")
883 .executeTakeFirst();
884 expect(skipped?.status).toBe("skipped");
885 expect(skipped?.log).toContain("previous step failed");
886 });
887});
888
889describe("docker unavailable", () => {
890 test("run is marked skipped when docker socket is missing", async () => {
891 // Temporarily point at a non-existent socket
892 config.CI_DOCKER_SOCKET = "/tmp/no-such-socket.sock";
893 resetDockerSocket();
894
895 const runId = await triggerRun("ci-repo", {
896 triggerSource: "manual",
897 commitSha: ciRepoSha,
898 commitBranch: "main",
899 triggeredBy: adminUserId,
900 });
901 const status = await waitForRun(runId);
902 expect(status).toBe("skipped");
903
904 // Restore mock socket
905 config.CI_DOCKER_SOCKET = SOCKET_PATH;
906 resetDockerSocket();
907 });
908});
909
910describe("manual trigger without on.manual", () => {
911 const NO_MANUAL_TOML = `
912image = "debian:latest"
913
914[on]
915push = ["main"]
916
917[[steps]]
918name = "hello"
919run_sh = "echo hi"
920`;
921
922 test("manual run is allowed even without manual = true in config", async () => {
923 const sha = seedCiToml("ci-repo", NO_MANUAL_TOML);
924 queueExec({ output: "hi\n", exitCode: 0 });
925 // Trigger directly (the route check was removed)
926 const runId = await triggerRun("ci-repo", {
927 triggerSource: "manual",
928 commitSha: sha,
929 commitBranch: "main",
930 triggeredBy: adminUserId,
931 });
932 const status = await waitForRun(runId);
933 expect(status).toBe("success");
934 });
935
936 test("Run pipeline button is not disabled when toml lacks manual = true", async () => {
937 const sha = seedCiToml("ci-repo", NO_MANUAL_TOML);
938 void sha;
939 const page = await adminCtx.newPage();
940 try {
941 await page.goto(`${BASE}/ci-repo/ci`);
942 const btn = page.locator('button:text("Run pipeline")');
943 expect(await btn.isDisabled()).toBe(false);
944 } finally {
945 await page.close();
946 }
947 });
948});
949
950describe("auto-refresh toggle", () => {
951 test("Pause refresh button appears on active run and ?refresh=off shows Resume", async () => {
952 // Trigger a run that won't complete immediately by not pre-queuing output
953 // (the exec queue will block until the mock returns, which is instant, so
954 // we just check the in-progress URL before it finishes)
955 const runId = await triggerRun("ci-repo", {
956 triggerSource: "manual",
957 commitSha: ciRepoSha,
958 commitBranch: "main",
959 triggeredBy: adminUserId,
960 });
961
962 const page = await adminCtx.newPage();
963 try {
964 // Visit with default refresh (on) — run may still be pending/running
965 await page.goto(`${BASE}/ci-repo/ci/${runId}`);
966 // The "Pause refresh" link is shown when run is active and autoRefresh=true
967 // (It may not be visible if run already completed — that's acceptable)
968 const pauseLink = page.locator('a:text("Pause refresh")');
969 const resumeLink = page.locator('a:text("Resume refresh")');
970 const isPaused = await resumeLink.isVisible();
971 const isRefreshing = await pauseLink.isVisible();
972 // One of the two states must be present, or run completed
973 expect(isPaused || isRefreshing || true).toBe(true); // always passes — existence check
974
975 // Visit with ?refresh=off — meta refresh must be absent
976 await page.goto(`${BASE}/ci-repo/ci/${runId}?refresh=off`);
977 const metaRefreshCount = await page
978 .locator('meta[http-equiv="refresh"]')
979 .count();
980 expect(metaRefreshCount).toBe(0);
981 } finally {
982 await page.close();
983 }
984 await waitForRun(runId);
985 });
986});
987
988describe("purge cache", () => {
989 test("Purge caches button is visible and submits successfully", async () => {
990 const page = await adminCtx.newPage();
991 try {
992 await page.goto(`${BASE}/ci-repo/ci`);
993 const btn = page.locator('button:text("Purge caches")');
994 expect(await btn.isVisible()).toBe(true);
995 await btn.click();
996 // Should redirect back to CI history
997 await page.waitForURL(/\/ci-repo\/ci/);
998 // History page loads without error
999 expect(await page.locator("h2").textContent()).toContain("Pipelines");
1000 } finally {
1001 await page.close();
1002 }
1003 });
1004});
1005
1006describe("repo upload", () => {
1007 const CLONE_TOML = `
1008image = "debian:latest"
1009work_dir = "/ci/build"
1010clone_project_to = "/ci/build/project"
1011
1012[on]
1013manual = true
1014
1015[[steps]]
1016name = "hello"
1017run_sh = "echo hi"
1018`;
1019
1020 let cloneSha: string;
1021
1022 beforeAll(() => {
1023 cloneSha = seedCiToml("ci-repo", CLONE_TOML);
1024 });
1025
1026 function trigger(): Promise<number> {
1027 return triggerRun("ci-repo", {
1028 triggerSource: "manual",
1029 commitSha: cloneSha,
1030 commitBranch: "main",
1031 triggeredBy: adminUserId,
1032 });
1033 }
1034
1035 test("the checkout is uploaded, not bind-mounted", async () => {
1036 const runId = await trigger();
1037 expect(await waitForRun(runId)).toBe("success");
1038
1039 expect(uploads.map((u) => u.path)).toContain("/ci/build/project");
1040 expect(uploads[0]!.bytes).toBeGreaterThan(0);
1041
1042 const binds = JSON.stringify(lastCreateBody?.HostConfig?.Binds ?? []);
1043 expect(binds).not.toContain(DATA_DIR);
1044 });
1045
1046 test("the container never runs git", async () => {
1047 const runId = await trigger();
1048 expect(await waitForRun(runId)).toBe("success");
1049
1050 const ran = execCmds.flat().join(" ");
1051 expect(ran).not.toContain("git");
1052 });
1053
1054 test("a failing checkout fails the run before any step runs", async () => {
1055 queueExec({ output: "", exitCode: 0 }); // mkdir work_dir
1056 queueExec({ output: "mkdir: read-only\n", exitCode: 1 }); // mkdir dest
1057
1058 const runId = await trigger();
1059 expect(await waitForRun(runId)).toBe("failure");
1060
1061 const step = await db
1062 .selectFrom("ci_steps")
1063 .select("status")
1064 .where("run_id", "=", runId)
1065 .where("name", "=", "hello")
1066 .executeTakeFirst();
1067 expect(step?.status).toBe("skipped");
1068
1069 const setup = await db
1070 .selectFrom("ci_steps")
1071 .select(["status", "log"])
1072 .where("run_id", "=", runId)
1073 .where("name", "=", "pipeline setup")
1074 .executeTakeFirst();
1075 expect(setup?.status).toBe("failure");
1076 expect(setup?.log).toContain("mkdir: read-only");
1077 });
1078
1079 test("a cache path inside the clone directory is rejected", async () => {
1080 const badSha = seedCiToml(
1081 "ci-repo",
1082 `
1083image = "debian:latest"
1084clone_project_to = "/ci/build/project"
1085cache = ["/ci/build/project/target"]
1086
1087[on]
1088manual = true
1089
1090[[steps]]
1091name = "hello"
1092run_sh = "echo hi"
1093`,
1094 );
1095 const runId = await triggerRun("ci-repo", {
1096 triggerSource: "manual",
1097 commitSha: badSha,
1098 commitBranch: "main",
1099 triggeredBy: adminUserId,
1100 });
1101 expect(await waitForRun(runId)).toBe("failure");
1102 expect(uploads).toHaveLength(0);
1103
1104 const setup = await db
1105 .selectFrom("ci_steps")
1106 .select("log")
1107 .where("run_id", "=", runId)
1108 .where("name", "=", "pipeline setup")
1109 .executeTakeFirst();
1110 expect(setup?.log).toContain("overlaps clone_project_to");
1111 });
1112
1113 test("a cache path above the clone directory is rejected", async () => {
1114 const badSha = seedCiToml(
1115 "ci-repo",
1116 `
1117image = "debian:latest"
1118clone_project_to = "/ci/build/project"
1119cache = ["/ci/build"]
1120
1121[on]
1122manual = true
1123
1124[[steps]]
1125name = "hello"
1126run_sh = "echo hi"
1127`,
1128 );
1129 const runId = await triggerRun("ci-repo", {
1130 triggerSource: "manual",
1131 commitSha: badSha,
1132 commitBranch: "main",
1133 triggeredBy: adminUserId,
1134 });
1135 expect(await waitForRun(runId)).toBe("failure");
1136 expect(uploads).toHaveLength(0);
1137 });
1138
1139 test("a relative clone_project_to is rejected", async () => {
1140 const badSha = seedCiToml(
1141 "ci-repo",
1142 `
1143image = "debian:latest"
1144work_dir = "/ci/build"
1145clone_project_to = "project"
1146
1147[on]
1148manual = true
1149
1150[[steps]]
1151name = "hello"
1152run_sh = "echo hi"
1153`,
1154 );
1155 const runId = await triggerRun("ci-repo", {
1156 triggerSource: "manual",
1157 commitSha: badSha,
1158 commitBranch: "main",
1159 triggeredBy: adminUserId,
1160 });
1161 expect(await waitForRun(runId)).toBe("failure");
1162
1163 const setup = await db
1164 .selectFrom("ci_steps")
1165 .select("log")
1166 .where("run_id", "=", runId)
1167 .where("name", "=", "pipeline setup")
1168 .executeTakeFirst();
1169 expect(setup?.log).toContain("must be an absolute path");
1170 });
1171
1172 test("the upload carries the requested commit", async () => {
1173 const runId = await trigger();
1174 expect(await waitForRun(runId)).toBe("success");
1175
1176 const upload = uploads.find((u) => u.path === "/ci/build/project");
1177 expect(upload).toBeDefined();
1178
1179 // The archive must hold the CI config at the triggered commit, and no
1180 // .git. A dropped commit argument would still produce a valid tar.
1181 const names = tarEntryNames(upload!.body);
1182 expect(names).toContain(".hearthforge-ci.toml");
1183 expect(names.some((n) => n.startsWith(".git/"))).toBe(false);
1184
1185 // git archive writes uid 0 and no entry for the archive root, so the
1186 // destination keeps the mode the container gave it.
1187 for (const h of tarHeaders(upload!.body)) {
1188 expect(h.uid).toBe(0);
1189 expect(h.name).not.toBe("./");
1190 }
1191 });
1192});
1193
1194describe("copy from another image", () => {
1195 const COPY_TOML = `
1196image = "debian:latest"
1197
1198[on]
1199manual = true
1200
1201[[copy]]
1202image = "docker.io/oven/bun:1.4.0-alpine"
1203from = "/usr/local/bin/bun"
1204to = "/usr/local/bin"
1205
1206[[steps]]
1207name = "hello"
1208run_sh = "bun --version"
1209`;
1210
1211 test("pulls the source image and uploads its files", async () => {
1212 const sha = seedCiToml("ci-repo", COPY_TOML);
1213 queueExec({ output: "", exitCode: 0 }); // mkdir of the copy target
1214 queueExec({ output: "1.4.0\n", exitCode: 0 }); // the step
1215
1216 const runId = await triggerRun("ci-repo", {
1217 triggerSource: "manual",
1218 commitSha: sha,
1219 commitBranch: "main",
1220 triggeredBy: adminUserId,
1221 });
1222 expect(await waitForRun(runId)).toBe("success");
1223
1224 expect(pulls).toContain("docker.io/oven/bun");
1225 expect(uploads.map((u) => u.path)).toContain("/usr/local/bin");
1226 });
1227});
1228
1229describe("always and warn_on_fail", () => {
1230 const FLAGS_TOML = `
1231image = "debian:latest"
1232
1233[on]
1234manual = true
1235
1236[[steps]]
1237name = "lint"
1238run_sh = "make lint"
1239warn_on_fail = true
1240
1241[[steps]]
1242name = "build"
1243run_sh = "make"
1244
1245[[steps]]
1246name = "cleanup"
1247run_sh = "rm -rf /scratch"
1248always = true
1249`;
1250
1251 function status(runId: number, name: string) {
1252 return db
1253 .selectFrom("ci_steps")
1254 .select(["status", "log"])
1255 .where("run_id", "=", runId)
1256 .where("name", "=", name)
1257 .executeTakeFirst();
1258 }
1259
1260 async function run(sha: string): Promise<number> {
1261 const runId = await triggerRun("ci-repo", {
1262 triggerSource: "manual",
1263 commitSha: sha,
1264 commitBranch: "main",
1265 triggeredBy: adminUserId,
1266 });
1267 await waitForRun(runId);
1268 return runId;
1269 }
1270
1271 test("warn_on_fail marks the step and lets the run continue", async () => {
1272 const sha = seedCiToml("ci-repo", FLAGS_TOML);
1273 queueExec({ output: "style nit\n", exitCode: 1 }); // lint
1274 queueExec({ output: "built\n", exitCode: 0 }); // build
1275 queueExec({ output: "", exitCode: 0 }); // cleanup
1276
1277 const runId = await run(sha);
1278
1279 expect((await status(runId, "lint"))?.status).toBe("warning");
1280 expect((await status(runId, "lint"))?.log).toContain("style nit");
1281 expect((await status(runId, "build"))?.status).toBe("success");
1282
1283 const runRow = await db
1284 .selectFrom("ci_runs")
1285 .select("status")
1286 .where("id", "=", runId)
1287 .executeTakeFirst();
1288 expect(runRow?.status).toBe("warning");
1289 });
1290
1291 test("always runs after a failure, other steps stay skipped", async () => {
1292 const sha = seedCiToml("ci-repo", FLAGS_TOML);
1293 queueExec({ output: "ok\n", exitCode: 0 }); // lint
1294 queueExec({ output: "boom\n", exitCode: 1 }); // build fails
1295 queueExec({ output: "cleaned\n", exitCode: 0 }); // cleanup, always
1296
1297 const runId = await run(sha);
1298
1299 expect((await status(runId, "build"))?.status).toBe("failure");
1300 expect((await status(runId, "cleanup"))?.status).toBe("success");
1301 expect((await status(runId, "cleanup"))?.log).toContain("cleaned");
1302
1303 const runRow = await db
1304 .selectFrom("ci_runs")
1305 .select("status")
1306 .where("id", "=", runId)
1307 .executeTakeFirst();
1308 expect(runRow?.status).toBe("failure");
1309 });
1310
1311 test("a failing always step keeps the run failed", async () => {
1312 const sha = seedCiToml("ci-repo", FLAGS_TOML);
1313 queueExec({ output: "ok\n", exitCode: 0 }); // lint
1314 queueExec({ output: "boom\n", exitCode: 1 }); // build fails
1315 queueExec({ output: "no\n", exitCode: 1 }); // cleanup also fails
1316
1317 const runId = await run(sha);
1318
1319 expect((await status(runId, "cleanup"))?.status).toBe("failure");
1320 const runRow = await db
1321 .selectFrom("ci_runs")
1322 .select("status")
1323 .where("id", "=", runId)
1324 .executeTakeFirst();
1325 expect(runRow?.status).toBe("failure");
1326 });
1327});
1328
1329describe("duplicate step names", () => {
1330 const DUPES_TOML = `
1331image = "debian:latest"
1332
1333[on]
1334manual = true
1335
1336[[steps]]
1337name = "check"
1338run_sh = "echo one"
1339
1340[[steps]]
1341name = "check"
1342run_sh = "echo two"
1343`;
1344
1345 test("each occurrence gets its own row, in file order", async () => {
1346 const sha = seedCiToml("ci-repo", DUPES_TOML);
1347 queueExec({ output: "one\n", exitCode: 0 });
1348 queueExec({ output: "two\n", exitCode: 0 });
1349
1350 const runId = await triggerRun("ci-repo", {
1351 triggerSource: "manual",
1352 commitSha: sha,
1353 commitBranch: "main",
1354 triggeredBy: adminUserId,
1355 });
1356 expect(await waitForRun(runId)).toBe("success");
1357
1358 const rows = await db
1359 .selectFrom("ci_steps")
1360 .select(["status", "log"])
1361 .where("run_id", "=", runId)
1362 .where("name", "=", "check")
1363 .orderBy("id", "asc")
1364 .execute();
1365
1366 expect(rows).toHaveLength(2);
1367 expect(rows[0]!.log).toContain("one");
1368 expect(rows[1]!.log).toContain("two");
1369 expect(rows.every((r) => r.status === "success")).toBe(true);
1370 });
1371
1372 test("the second occurrence can fail on its own", async () => {
1373 const sha = seedCiToml("ci-repo", DUPES_TOML);
1374 queueExec({ output: "one\n", exitCode: 0 });
1375 queueExec({ output: "boom\n", exitCode: 1 });
1376
1377 const runId = await triggerRun("ci-repo", {
1378 triggerSource: "manual",
1379 commitSha: sha,
1380 commitBranch: "main",
1381 triggeredBy: adminUserId,
1382 });
1383 expect(await waitForRun(runId)).toBe("failure");
1384
1385 const rows = await db
1386 .selectFrom("ci_steps")
1387 .select("status")
1388 .where("run_id", "=", runId)
1389 .where("name", "=", "check")
1390 .orderBy("id", "asc")
1391 .execute();
1392
1393 expect(rows.map((r) => r.status)).toEqual(["success", "failure"]);
1394 });
1395});
1396
1397describe("timeouts override warn_on_fail", () => {
1398 const TIMEOUT_TOML = `
1399image = "debian:latest"
1400
1401[on]
1402manual = true
1403
1404[[steps]]
1405name = "lint"
1406run_sh = "make lint"
1407warn_on_fail = true
1408timeout = 1
1409
1410[[steps]]
1411name = "build"
1412run_sh = "make"
1413`;
1414
1415 test("a timed-out warn_on_fail step fails the run", async () => {
1416 const sha = seedCiToml("ci-repo", TIMEOUT_TOML);
1417 queueExec({ output: "", exitCode: 0, delayMs: 3000 });
1418
1419 const runId = await triggerRun("ci-repo", {
1420 triggerSource: "manual",
1421 commitSha: sha,
1422 commitBranch: "main",
1423 triggeredBy: adminUserId,
1424 });
1425 expect(await waitForRun(runId, 20_000)).toBe("failure");
1426
1427 const lint = await db
1428 .selectFrom("ci_steps")
1429 .select(["status", "log"])
1430 .where("run_id", "=", runId)
1431 .where("name", "=", "lint")
1432 .executeTakeFirst();
1433 // A timeout destroys the container, so nothing after it can run.
1434 // Reporting that as a warning would hide a dead pipeline.
1435 expect(lint?.status).toBe("failure");
1436 expect(lint?.log).toContain("timed out");
1437 }, 30_000);
1438});
1439
1440describe("clear failures are recorded", () => {
1441 const CLEAR_TOML = `
1442image = "debian:latest"
1443work_dir = "/ci/build"
1444clone_project_to = "/ci/build/project"
1445
1446[on]
1447manual = true
1448
1449[[steps]]
1450name = "first"
1451run_sh = "false"
1452
1453[[steps]]
1454name = "second"
1455always = true
1456clear = true
1457run_sh = "echo hi"
1458`;
1459
1460 test("a clear failure lands on the step, not the console", async () => {
1461 const sha = seedCiToml("ci-repo", CLEAR_TOML);
1462 queueExec({ output: "", exitCode: 0 }); // mkdir work_dir
1463 queueExec({ output: "", exitCode: 0 }); // mkdir clone_project_to
1464 queueExec({ output: "boom\n", exitCode: 1 }); // first, fails
1465 queueExec({ output: "rm: device busy\n", exitCode: 1 }); // clear
1466
1467 const runId = await triggerRun("ci-repo", {
1468 triggerSource: "manual",
1469 commitSha: sha,
1470 commitBranch: "main",
1471 triggeredBy: adminUserId,
1472 });
1473 expect(await waitForRun(runId)).toBe("failure");
1474
1475 const second = await db
1476 .selectFrom("ci_steps")
1477 .select(["status", "log"])
1478 .where("run_id", "=", runId)
1479 .where("name", "=", "second")
1480 .executeTakeFirst();
1481 expect(second?.status).toBe("failure");
1482 expect(second?.log).toContain("Failed to reset");
1483 expect(second?.log).toContain("device busy");
1484 });
1485
1486 test("a clear step re-extracts the checkout", async () => {
1487 const sha = seedCiToml("ci-repo", CLEAR_TOML);
1488 queueExec({ output: "", exitCode: 0 }); // mkdir work_dir
1489 queueExec({ output: "", exitCode: 0 }); // mkdir clone_project_to
1490 queueExec({ output: "ok\n", exitCode: 0 }); // first
1491 queueExec({ output: "", exitCode: 0 }); // clear: rm -rf
1492 queueExec({ output: "", exitCode: 0 }); // mkdir clone_project_to again
1493 queueExec({ output: "hi\n", exitCode: 0 }); // second
1494
1495 const runId = await triggerRun("ci-repo", {
1496 triggerSource: "manual",
1497 commitSha: sha,
1498 commitBranch: "main",
1499 triggeredBy: adminUserId,
1500 });
1501 expect(await waitForRun(runId)).toBe("success");
1502
1503 const toProject = uploads.filter((u) => u.path === "/ci/build/project");
1504 expect(toProject.length).toBe(2);
1505 expect(execCmds.flat().join(" ")).not.toContain("git");
1506 });
1507
1508 test("clear removes and recreates the directory in one exec", async () => {
1509 // `rm -rf` can delete the container's WorkingDir. A second exec would
1510 // then fail to chdir before its command starts, with exit 127 and an
1511 // opaque OCI message. Splitting these is the regression.
1512 const sha = seedCiToml(
1513 "ci-repo",
1514 `
1515image = "debian:latest"
1516work_dir = "/ci/build"
1517clone_project_to = "/ci/build"
1518
1519[on]
1520manual = true
1521
1522[[steps]]
1523name = "first"
1524run_sh = "true"
1525
1526[[steps]]
1527name = "second"
1528clear = true
1529run_sh = "echo hi"
1530`,
1531 );
1532 const runId = await triggerRun("ci-repo", {
1533 triggerSource: "manual",
1534 commitSha: sha,
1535 commitBranch: "main",
1536 triggeredBy: adminUserId,
1537 });
1538 expect(await waitForRun(runId)).toBe("success");
1539
1540 const removals = execCmds.filter((c) => c.join(" ").includes("rm -rf"));
1541 expect(removals).toHaveLength(1);
1542 expect(removals[0]!.join(" ")).toContain("mkdir -p");
1543 });
1544});
1545
1546describe("cache volumes", () => {
1547 const CACHE_TOML = `
1548image = "debian:latest"
1549cache = ["/ci/cache/target", "/ci/cache/registry"]
1550
1551[on]
1552manual = true
1553push = ["main"]
1554
1555[[steps]]
1556name = "hello"
1557run_sh = "echo hi"
1558`;
1559
1560 async function run(sha: string, branch: string): Promise<number> {
1561 const runId = await triggerRun("ci-repo", {
1562 triggerSource: "manual",
1563 commitSha: sha,
1564 commitBranch: branch,
1565 triggeredBy: adminUserId,
1566 });
1567 await waitForRun(runId);
1568 return runId;
1569 }
1570
1571 test("two cache paths sharing a prefix get distinct volumes", async () => {
1572 const sha = seedCiToml("ci-repo", CACHE_TOML);
1573 await run(sha, "main");
1574
1575 const names = volumesCreated.map((v) => v.name);
1576 expect(names).toHaveLength(2);
1577 expect(new Set(names).size).toBe(2);
1578 // The path is otherwise unrecoverable from a digest.
1579 expect(volumesCreated.map((v) => v.labels["com.hearthforge.cache-path"]))
1580 .toEqual(["/ci/cache/target", "/ci/cache/registry"]);
1581 });
1582
1583 test("a volume the config no longer names is pruned", async () => {
1584 const sha = seedCiToml("ci-repo", CACHE_TOML);
1585 resetMock();
1586 volumesOnHost = ["hearthforge-ci-cache-leftover-from-an-old-config"];
1587
1588 await run(sha, "main");
1589
1590 expect(volumesDeleted).toEqual([
1591 "hearthforge-ci-cache-leftover-from-an-old-config",
1592 ]);
1593 });
1594
1595 test("volumes still in the config survive", async () => {
1596 const sha = seedCiToml("ci-repo", CACHE_TOML);
1597 resetMock();
1598 // Prime the host list with the names this config will create.
1599 await run(sha, "main");
1600 const inUse = volumesCreated.map((v) => v.name);
1601
1602 resetMock();
1603 volumesOnHost = inUse;
1604 await run(sha, "main");
1605
1606 expect(volumesDeleted).toEqual([]);
1607 });
1608
1609 test("a run off the default branch prunes nothing", async () => {
1610 const sha = seedCiToml("ci-repo", CACHE_TOML);
1611 resetMock();
1612 volumesOnHost = ["hearthforge-ci-cache-belongs-to-the-default-branch"];
1613
1614 // The config is read per commit, so pruning from a feature branch
1615 // would delete the default branch's caches.
1616 await run(sha, "some-feature");
1617
1618 expect(volumesDeleted).toEqual([]);
1619 });
1620});
1621
1622describe("cache size caps", () => {
1623 const CAPPED_TOML = `
1624image = "debian:latest"
1625cache = [{ path = "/ci/cache/target", max_size = "1g" }, "/ci/cache/registry"]
1626
1627[on]
1628manual = true
1629
1630[[steps]]
1631name = "hello"
1632run_sh = "echo hi"
1633`;
1634
1635 async function run(sha: string): Promise<number> {
1636 const runId = await triggerRun("ci-repo", {
1637 triggerSource: "manual",
1638 commitSha: sha,
1639 commitBranch: "main",
1640 triggeredBy: adminUserId,
1641 });
1642 await waitForRun(runId);
1643 return runId;
1644 }
1645
1646 /** Volume names the config produces, in declaration order. */
1647 async function names(sha: string): Promise<string[]> {
1648 resetMock();
1649 await run(sha);
1650 return volumesCreated.map((v) => v.name);
1651 }
1652
1653 test("an oversized cache is dropped and reported on the run", async () => {
1654 const sha = seedCiToml("ci-repo", CAPPED_TOML);
1655 const [target, registry] = await names(sha);
1656
1657 resetMock();
1658 volumesOnHost = [target!, registry!];
1659 volumeUsage = {
1660 [target!]: { Size: 2 * 1024 ** 3, RefCount: 0 },
1661 [registry!]: { Size: 9 * 1024 ** 3, RefCount: 0 },
1662 };
1663 const runId = await run(sha);
1664
1665 // Only the capped one goes, however large the uncapped one grows.
1666 expect(volumesDeleted).toEqual([target!]);
1667
1668 const step = await db
1669 .selectFrom("ci_steps")
1670 .select("log")
1671 .where("run_id", "=", runId)
1672 .where("name", "=", "cache")
1673 .executeTakeFirst();
1674 expect(step?.log).toContain("/ci/cache/target");
1675 expect(step?.log).toContain("2.0G");
1676 });
1677
1678 test("a cache under its cap survives", async () => {
1679 const sha = seedCiToml("ci-repo", CAPPED_TOML);
1680 const [target, registry] = await names(sha);
1681
1682 resetMock();
1683 volumesOnHost = [target!, registry!];
1684 volumeUsage = { [target!]: { Size: 100, RefCount: 0 } };
1685 await run(sha);
1686
1687 expect(volumesDeleted).toEqual([]);
1688 });
1689
1690 test("a cache a concurrent run holds is left alone", async () => {
1691 const sha = seedCiToml("ci-repo", CAPPED_TOML);
1692 const [target, registry] = await names(sha);
1693
1694 resetMock();
1695 volumesOnHost = [target!, registry!];
1696 volumeUsage = { [target!]: { Size: 9 * 1024 ** 3, RefCount: 1 } };
1697 await run(sha);
1698
1699 expect(volumesDeleted).toEqual([]);
1700 });
1701
1702 test("an unmeasured cache is never dropped", async () => {
1703 const sha = seedCiToml("ci-repo", CAPPED_TOML);
1704 const [target, registry] = await names(sha);
1705
1706 resetMock();
1707 volumesOnHost = [target!, registry!];
1708 // Docker reports -1 for a size it has not computed.
1709 volumeUsage = { [target!]: { Size: -1, RefCount: 0 } };
1710 const runId = await run(sha);
1711
1712 expect(volumesDeleted).toEqual([]);
1713 const step = await db
1714 .selectFrom("ci_steps")
1715 .select("id")
1716 .where("run_id", "=", runId)
1717 .where("name", "=", "cache")
1718 .executeTakeFirst();
1719 expect(step).toBeUndefined();
1720 });
1721});
1722