ci: steps array, image copies, step flags, and bounded caches
Breaking: steps move from top-level tables to a [[steps]] array. Each one needs a name. Order used to come from Object.entries. JavaScript enumerates integer-like keys first, so a step named "2024" ran first wherever it sat in the file. An array carries document order instead. Names no longer share a namespace with config keys, so RESERVED_TABLES is gone. They need not be unique either: step rows are addressed by position now, which also drops one query per step. [[copy]] lifts a file or directory out of another image. It is the pipeline form of COPY --from. A source container is created but never started, and the archive streams straight into the run's container. Some tools ship no installer for the target libc, and a single-container pipeline has no build stage to copy from. Cache volumes were named after the first 18 bytes of "<repo>:<path>". So /ci/cache/target and /ci/build/project/target named one volume and silently shared it. Names are digests now. The path moves to a label, because a digest tells a reader nothing. - ci: `always` runs a step after an earlier one failed, for cleanup - ci: `warn_on_fail` marks a step "warning" and the run carries on. The run reports "warning" as well, so the badge does not read green - ci: cache entries accept a max_size cap. Sizes come from /system/df, so nothing runs inside the CI container. A timed-out run has no container left and can still be measured. A volume is dropped only on a size the daemon actually reported - ci: cache volumes the config no longer names are deleted after a default-branch run. The config is read per commit, so pruning off a feature branch would delete the default branch's caches - ci: a timeout is never a warning. It destroys the container, so every later step is skipped. Reporting that as a warning hid a dead pipeline behind a yellow badge - ci: a failed `clear` reset is recorded on its step instead of thrown. Thrown, it was swallowed whenever an earlier step had already failed - ci: drain tar's stderr in the same turn as the wait in uploadRepo. A tar that fills the pipe buffer blocks forever. That path has no timeout, so the run held its concurrency slot for good - ci: leave the copy source container unnamed. The startup sweep deletes by exact name, and a retry reuses the run id, so one leaked container would fail every later retry with a name conflict - template, help panel: document [[steps]], [[copy]], the step flags and the cache caps
Mpublic/assets/hearthforge-ci-template.toml
@@ -9,7 +9,15 @@ shell_setup = "set -euo pipefail"
# timeout = 3600 # overall run timeout in seconds
# cpu_limit = 2.0 # CPU cores limit
# memory_limit = "2g" # memory limit (k/m/g suffix)
# cache = ["/root/.cargo", "/root/.npm"] # persist between runs
# Caches persist between runs. An entry is a bare path, or a table with a
# max_size cap (k/m/g suffix). A capped cache is deleted after the run that
# takes it over, so the next run starts cold. Uncapped caches grow forever.
# cache = [
# { path = "/root/.cargo", max_size = "8g" },
# "/root/.npm",
# ]
# A cache path must sit outside clone_project_to. The volume is mounted
# before the clone runs, and git refuses to clone into a non-empty directory.
[on]
push = ["main"] # trigger on push to these branches; use ["*"] for all
@@ -21,22 +29,43 @@ tag = false # trigger on tag push
# default = "hello"
# description = "A custom variable, overridable from the UI"
# Steps are executed in file order. Each [section] is one step.
# Reserved section names: [on], [variables]
# Files taken from another image before any step runs, like COPY --from.
# `to` is a directory and the source basename is kept, so the example below
# lands at /usr/local/bin/bun. HearthForge creates `to` when it is missing.
# Match the libc: an -alpine tag is musl and will not run on this glibc image.
# A "/." suffix on `from` copies the contents instead of the directory itself.
#
# [[copy]]
# image = "docker.io/oven/bun:1.4.0"
# from = "/usr/local/bin/bun"
# to = "/usr/local/bin"
[setup]
# Steps run in file order. Names are labels only, repeats are allowed.
[[steps]]
name = "setup"
run_sh = "apt-get update -qq && apt-get install -y --no-install-recommends build-essential"
timeout = 180
[build]
[[steps]]
name = "lint"
run_sh = "make -C project lint"
# warn_on_fail marks the step "warning" instead of failing the run, and the
# steps after it still run.
# warn_on_fail = true
[[steps]]
name = "build"
run_sh = "make -C project all"
[test]
[[steps]]
name = "test"
# run_if is a shell expression; the step is skipped if it returns non-zero
# run_if = 'test -n "${CI_COMMIT_TAG}"'
run_sh = "make -C project test"
[package]
[[steps]]
name = "package"
run_sh = "make -C project dist"
# Publish artifacts — these can appear in any step
# publish_file: copy a single file directly
@@ -49,3 +78,9 @@ run_sh = "make -C project dist"
# publish_zip = ["/ci/build/project/dist/"]
# publish_zstd: create a .tar.zst archive (requires tar + zstd in image)
# publish_zstd = ["/ci/build/project/dist/"]
[[steps]]
name = "cleanup"
# always runs the step even when an earlier one failed.
always = true
run_sh = "rm -rf /ci/build/scratch"
Msrc/routes/ci.tsx
@@ -23,6 +23,7 @@ import { html } from "../views/render.tsx";
function makeBadge(status: string): string {
const colors: Record<string, string> = {
success: "#4c1",
warning: "#dfb317",
failure: "#e05d44",
running: "#007ec6",
pending: "#9f9f9f",
Msrc/services/ci.ts
@@ -1,3 +1,4 @@
import { createHash } from "node:crypto";
import { existsSync, mkdirSync, writeFileSync } from "node:fs";
import path from "node:path";
import config from "../config.ts";
@@ -15,6 +16,10 @@ interface CiVariableDef {
interface CiStepConfig {
run_sh?: string;
run_if?: string;
/** Run even after an earlier step failed. */
always?: boolean;
/** A non-zero exit marks the step "warning" and the run carries on. */
warn_on_fail?: boolean;
clear?: boolean;
timeout?: number;
publish_file?: string | string[];
@@ -28,6 +33,23 @@ export interface CiStep extends CiStepConfig {
name: string;
}
/**
* One cache path, normalised from either form `cache` accepts: a bare string,
* or a table carrying a size cap.
*/
export interface CiCache {
path: string;
/** Bytes. Absent means unbounded. */
maxSize?: number;
}
/** One file or directory lifted out of another image, like COPY --from. */
export interface CiCopy {
image: string;
from: string;
to: string;
}
export interface CiConfig {
image: string;
work_dir?: string;
@@ -37,7 +59,8 @@ export interface CiConfig {
timeout?: number;
cpu_limit?: number;
memory_limit?: string;
cache?: string[];
cache?: CiCache[];
copy?: CiCopy[];
on?: {
push?: string[] | boolean;
tag?: boolean;
@@ -56,9 +79,6 @@ export interface TriggerOpts {
variableOverrides?: Record<string, string>;
}
// Reserved TOML table names that are not steps
const RESERVED_TABLES = new Set(["on", "variables"]);
const CONTAINER_REPO_PATH = "/hearthforge-repo.git";
// In-memory map of running tasks for cancellation
@@ -118,16 +138,54 @@ export function parseCiConfig(tomlStr: string): CiConfig | null {
const image = raw.image;
if (typeof image !== "string" || !image) return null;
const steps: CiStep[] = [];
for (const [key, val] of Object.entries(raw)) {
if (RESERVED_TABLES.has(key)) continue;
if (typeof val !== "object" || val === null || Array.isArray(val))
// Not a table per step: JavaScript enumerates integer-like keys first, so
// a step named "2024" would jump to the front. An array keeps file order.
if (!Array.isArray(raw.steps)) return null;
const steps = raw.steps as CiStep[];
// A string or array element fails this the same way a missing name does.
if (steps.some((s) => typeof s?.name !== "string" || !s.name)) return null;
// `cache` takes a bare path or a table with a cap, so the common entry
// stays one line. Normalised here, so nothing downstream sees the union.
if (raw.cache !== undefined && !Array.isArray(raw.cache)) return null;
const cache: CiCache[] = [];
for (const entry of raw.cache ?? []) {
if (typeof entry === "string") {
if (!entry) return null;
cache.push({ path: entry });
continue;
// It's a table section — treat as a step
const stepCfg = val as Record<string, unknown>;
steps.push({ name: key, ...(stepCfg as CiStepConfig) });
}
if (typeof entry !== "object" || entry === null) return null;
const c = entry as Record<string, unknown>;
if (typeof c.path !== "string" || !c.path) return null;
if (c.max_size === undefined) {
cache.push({ path: c.path });
continue;
}
if (typeof c.max_size !== "string") return null;
const maxSize = parseMemoryBytes(c.max_size);
// parseMemoryBytes answers 0 for anything it cannot read. Left alone
// that is a limit every cache exceeds, so the cache would be wiped
// after every run and look broken rather than misconfigured.
if (maxSize <= 0) return null;
cache.push({ path: c.path, maxSize });
}
if (raw.copy !== undefined && !Array.isArray(raw.copy)) return null;
const copy = (raw.copy ?? []) as CiCopy[];
if (
copy.some(
(c) =>
typeof c?.image !== "string" ||
!c.image ||
typeof c.from !== "string" ||
!c.from ||
typeof c.to !== "string" ||
!c.to,
)
)
return null;
const rawOn = raw.on as Record<string, unknown> | undefined;
const rawVars = raw.variables as
| Record<string, Record<string, unknown>>
@@ -165,7 +223,8 @@ export function parseCiConfig(tomlStr: string): CiConfig | null {
typeof raw.cpu_limit === "number" ? raw.cpu_limit : undefined,
memory_limit:
typeof raw.memory_limit === "string" ? raw.memory_limit : undefined,
cache: Array.isArray(raw.cache) ? (raw.cache as string[]) : undefined,
cache,
copy: copy.length ? copy : undefined,
on: rawOn
? {
push: Array.isArray(rawOn.push)
@@ -190,17 +249,27 @@ export function parseCiConfig(tomlStr: string): CiConfig | null {
* generic failure. Returns an error string, or null when the config is usable.
*/
export function validateCiConfig(cfg: CiConfig): string | null {
for (const c of cfg.copy ?? []) {
// `to` is a directory and the basename is kept, so a `to` that repeats
// the basename means someone expected a rename. Left alone it silently
// produces to/<name>/<name>, which only shows up as a missing tool
// several steps later.
if (path.basename(c.to) === path.basename(c.from)) {
return `[[copy]] "to" is a directory, so ${c.from} lands at ${path.join(c.to, path.basename(c.from))}. Drop the last path segment from "to".`;
}
}
if (!cfg.clone_project_to || !cfg.cache) return null;
const clone = path.resolve(cfg.clone_project_to);
for (const entry of cfg.cache) {
const cachePath = path.resolve(entry);
const cachePath = path.resolve(entry.path);
if (
cachePath === clone ||
cachePath.startsWith(`${clone}${path.sep}`)
) {
return (
`cache path "${entry}" is inside clone_project_to ("${cfg.clone_project_to}"). ` +
`cache path "${entry.path}" is inside clone_project_to ("${cfg.clone_project_to}"). ` +
"The cache volume is mounted before the clone runs, which leaves the " +
"directory non-empty, and git refuses to clone into it. Move the cache " +
"outside the clone directory."
@@ -317,6 +386,17 @@ async function pullImage(image: string): Promise<void> {
}
}
function formatBytes(n: number): string {
const units = ["B", "K", "M", "G", "T"];
let i = 0;
let v = n;
while (v >= 1024 && i < units.length - 1) {
v /= 1024;
i++;
}
return `${i === 0 ? v : v.toFixed(1)}${units[i]}`;
}
function parseMemoryBytes(s: string): number {
const m = s.match(/^(\d+(?:\.\d+)?)\s*([kmgKMG]?)b?$/);
if (!m) return 0;
@@ -333,18 +413,130 @@ function parseMemoryBytes(s: string): number {
}
}
async function ensureVolume(volName: string, repoName: string): Promise<void> {
/**
* Volume name for one cache path.
*
* Hashed, not encoded: base64 of the plain string keeps only its first bytes
* once truncated, so `/ci/cache/target` and `/ci/build/project/target` used to
* name the same volume and silently share it.
*/
function cacheVolumeName(repoName: string, cachePath: string): string {
const digest = createHash("sha256")
.update(`${repoName}:${cachePath}`)
.digest("base64url");
return `hearthforge-ci-cache-${digest.slice(0, 24)}`;
}
async function ensureVolume(
volName: string,
repoName: string,
cachePath: string,
): Promise<void> {
const resp = await dockerFetch("/volumes/create", {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({
Name: volName,
Labels: { "com.hearthforge.repo": repoName },
Labels: {
"com.hearthforge.repo": repoName,
// The name is a digest, so without this nothing can say which
// path a volume belongs to.
"com.hearthforge.cache-path": cachePath,
},
}),
});
await resp.body?.cancel();
}
/**
* Delete this repo's cache volumes that the given config no longer mentions.
*
* Only safe to call for a default-branch run. The config is read per commit,
* so pruning against a feature branch would delete the default branch's
* volumes and the two would rebuild each other's caches forever.
*/
async function pruneStaleCaches(
repoName: string,
cache: CiCache[] | undefined,
): Promise<void> {
const keep = new Set(
(cache ?? []).map((c) => cacheVolumeName(repoName, c.path)),
);
const filters = encodeURIComponent(
JSON.stringify({ label: [`com.hearthforge.repo=${repoName}`] }),
);
const resp = await dockerFetch(`/volumes?filters=${filters}`);
if (!resp.ok) {
await resp.body?.cancel();
return;
}
const data = (await resp.json()) as { Volumes?: Array<{ Name: string }> };
for (const vol of data.Volumes ?? []) {
if (keep.has(vol.Name)) continue;
// A volume a concurrent run still holds refuses to go. That is fine,
// the next default-branch run picks it up.
const del = await dockerFetch(`/volumes/${vol.Name}`, {
method: "DELETE",
});
await del.body?.cancel();
}
}
/**
* Delete cache volumes that outgrew their `max_size`.
*
* Sizes come from the daemon, not from a `du` in the CI container: the image
* need not ship one, and a timed-out run has no container left to exec in.
* Returns what was dropped, so the run can say so.
*
* A cache is only dropped on a size the daemon actually reported. Treating a
* missing measurement as "over the limit" would clear the cache on every run
* and read as caching being broken rather than misconfigured.
*/
async function enforceCacheLimits(
repoName: string,
cache: CiCache[] | undefined,
): Promise<Array<{ path: string; size: number; maxSize: number }>> {
const capped = new Map(
(cache ?? [])
.filter((c) => c.maxSize !== undefined)
.map((c) => [
cacheVolumeName(repoName, c.path),
{ path: c.path, maxSize: c.maxSize! },
]),
);
if (capped.size === 0) return [];
const resp = await dockerFetch("/system/df");
if (!resp.ok) {
await resp.body?.cancel();
return [];
}
const data = (await resp.json()) as {
Volumes?: Array<{
Name: string;
UsageData?: { Size: number; RefCount: number };
}>;
};
const dropped: Array<{ path: string; size: number; maxSize: number }> = [];
for (const vol of data.Volumes ?? []) {
const cap = capped.get(vol.Name);
if (!cap) continue;
const size = vol.UsageData?.Size ?? -1;
if (size < 0 || size <= cap.maxSize) continue;
// A concurrent run still has it mounted, so the delete would fail.
if ((vol.UsageData?.RefCount ?? 0) > 0) continue;
const del = await dockerFetch(`/volumes/${vol.Name}`, {
method: "DELETE",
});
await del.body?.cancel();
if (del.ok) dropped.push({ ...cap, size });
}
return dropped;
}
async function createContainer(
runId: number,
repoName: string,
@@ -354,12 +546,10 @@ async function createContainer(
// No bind for the repo: the Docker daemon resolves bind sources on the
// host, where HearthForge's own paths do not exist. uploadRepo copies it in.
const binds: string[] = [];
if (cfg.cache) {
for (const cachePath of cfg.cache) {
const volName = `hearthforge-ci-cache-${Buffer.from(`${repoName}:${cachePath}`).toString("base64url").slice(0, 24)}`;
await ensureVolume(volName, repoName);
binds.push(`${volName}:${cachePath}`);
}
for (const { path: cachePath } of cfg.cache ?? []) {
const volName = cacheVolumeName(repoName, cachePath);
await ensureVolume(volName, repoName, cachePath);
binds.push(`${volName}:${cachePath}`);
}
const hostConfig: Record<string, unknown> = { Binds: binds };
@@ -540,22 +730,10 @@ async function uploadRepo(
);
}
// Flatten ownership: host uids mean nothing here and trip git's
// ownership check on the clone below.
const tar = Bun.spawn(
[
"tar",
"-cf",
"-",
"--owner=0",
"--group=0",
"--numeric-owner",
"-C",
repoAbsPath,
".",
],
{ stdout: "pipe", stderr: "pipe" },
);
const tar = Bun.spawn(["tar", "-cf", "-", "-C", repoAbsPath, "."], {
stdout: "pipe",
stderr: "pipe",
});
const resp = await dockerFetch(
`/containers/${containerId}/archive?path=${encodeURIComponent(CONTAINER_REPO_PATH)}`,
@@ -565,7 +743,17 @@ async function uploadRepo(
body: tar.stdout,
},
);
const tarExit = await tar.exited;
// The PUT stopped reading, so tar would block writing into a full pipe.
if (!resp.ok) tar.kill();
// stderr must be drained in the same turn as the wait. A tar that fills
// the pipe buffer blocks on the write, and awaiting `exited` first would
// then hang the run for good: this path has no timeout and no signal.
const [tarExit, tarErr] = await Promise.all([
tar.exited,
new Response(tar.stderr).text(),
]);
if (!resp.ok) {
const detail = (await resp.text().catch(() => "")).trim();
@@ -575,13 +763,80 @@ async function uploadRepo(
}
await resp.body?.cancel();
if (tarExit !== 0) {
const err = (await new Response(tar.stderr).text()).trim();
const err = tarErr.trim();
throw new Error(
`Failed to read the repository: tar exited ${tarExit}${err ? ` ${err}` : ""}`,
);
}
}
/**
* Lift a file or directory out of another image, like `COPY --from`.
*
* The source container is created but never started: the archive endpoint
* reads its filesystem either way.
*
* `to` is a directory and the source basename is preserved. Renaming would
* mean rewriting tar headers in flight, which turns a stream into a parser.
* A `/.` suffix on `from` copies the contents instead, as `docker cp` does.
*/
async function copyFromImage(containerId: string, spec: CiCopy): Promise<void> {
await pullImage(spec.image);
// Unnamed on purpose. The startup sweep deletes by exact name, so a name
// here would not get cleaned up, and a retry reuses the run id: one leaked
// container would then fail every retry with a name conflict.
const createResp = await dockerFetch("/containers/create", {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ Image: spec.image, Cmd: ["true"] }),
});
if (!createResp.ok) {
const detail = (await createResp.text().catch(() => "")).trim();
throw new Error(
`copy: cannot create a container from ${spec.image}: HTTP ${createResp.status}${detail ? ` ${detail}` : ""}`,
);
}
const sourceId = ((await createResp.json()) as { Id: string }).Id;
try {
// Docker rejects the upload unless the destination already exists.
const mk = await execInContainer(containerId, ["mkdir", "-p", spec.to]);
if (mk.exitCode !== 0) {
throw new Error(`copy: cannot create ${spec.to}: ${mk.log.trim()}`);
}
const get = await dockerFetch(
`/containers/${sourceId}/archive?path=${encodeURIComponent(spec.from)}`,
);
if (!get.ok) {
await get.body?.cancel();
throw new Error(
`copy: cannot read ${spec.from} from ${spec.image}: HTTP ${get.status}`,
);
}
const put = await dockerFetch(
`/containers/${containerId}/archive?path=${encodeURIComponent(spec.to)}`,
{
method: "PUT",
headers: { "Content-Type": "application/x-tar" },
body: get.body,
},
);
if (!put.ok) {
await get.body?.cancel();
const detail = (await put.text().catch(() => "")).trim();
throw new Error(
`copy: cannot write ${spec.from} to ${spec.to}: HTTP ${put.status}${detail ? ` ${detail}` : ""}`,
);
}
await put.body?.cancel();
} finally {
await removeContainer(sourceId);
}
}
async function removeContainer(containerId: string): Promise<void> {
try {
const resp = await dockerFetch(
@@ -794,6 +1049,10 @@ async function collectArtifacts(
async function executeRun(runId: number, signal: AbortSignal): Promise<void> {
const now = () => new Date().toISOString();
let containerId: string | undefined;
// Set once the config is known, and only for a default-branch run. The
// finally block cannot see cfg, and pruning off another branch is wrong.
let pruneCaches: { repoName: string; cache?: CiCache[] } | null = null;
let cacheLimits: { repoName: string; cache?: CiCache[] } | null = null;
try {
// Mark as running
@@ -813,7 +1072,7 @@ async function executeRun(runId: number, signal: AbortSignal): Promise<void> {
const repo = await db
.selectFrom("repositories")
.select(["id", "name"])
.select(["id", "name", "default_branch"])
.where("id", "=", run.repo_id)
.executeTakeFirst();
if (!repo) throw new Error("Repo not found");
@@ -832,6 +1091,14 @@ async function executeRun(runId: number, signal: AbortSignal): Promise<void> {
if (invalid)
throw new Error(`Invalid .hearthforge-ci.toml: ${invalid}`);
// Size caps apply on any branch: an oversized volume is oversized
// whoever noticed. Dropping stale volumes is default-branch only,
// because the config that names them is read per commit.
cacheLimits = { repoName: repo.name, cache: cfg.cache };
if (run.commit_branch && run.commit_branch === repo.default_branch) {
pruneCaches = { repoName: repo.name, cache: cfg.cache };
}
// Load secrets for log masking
const secrets = await db
.selectFrom("ci_secrets")
@@ -858,16 +1125,20 @@ async function executeRun(runId: number, signal: AbortSignal): Promise<void> {
secrets,
);
// Create step rows in DB
// Step names need not be unique, so a name cannot identify a row.
// Keep the ids in config order instead.
const stepIds: number[] = [];
for (const step of cfg.steps) {
await db
const row = await db
.insertInto("ci_steps")
.values({
run_id: runId,
name: step.name,
status: "pending",
})
.execute();
.returning("id")
.executeTakeFirstOrThrow();
stepIds.push(row.id);
}
// Pull image
@@ -886,6 +1157,13 @@ async function executeRun(runId: number, signal: AbortSignal): Promise<void> {
await execInContainer(containerId, ["mkdir", "-p", cfg.work_dir]);
}
// Copies run before the clone, so a step can rely on the tools they
// bring in, and so they can supply a shell the image lacks.
for (const spec of cfg.copy ?? []) {
await copyFromImage(containerId, spec);
if (signal.aborted) throw new Error("Cancelled");
}
// Clone project if requested
if (cfg.clone_project_to && run.commit_sha) {
await uploadRepo(containerId, repoPath(repo.name));
@@ -894,9 +1172,8 @@ async function executeRun(runId: number, signal: AbortSignal): Promise<void> {
[
"sh",
"-c",
// safe.directory: the upload lands as root, but the step
// user is whatever the image defaults to. Without this git
// refuses the repo as "dubious ownership".
// safe.directory: the upload carries the host's uids, and
// git refuses a repo it does not appear to own.
`git -c safe.directory=${CONTAINER_REPO_PATH} clone ${CONTAINER_REPO_PATH} ${cfg.clone_project_to} && git -C ${cfg.clone_project_to} checkout --detach ${run.commit_sha}`,
],
cfg.work_dir,
@@ -912,21 +1189,23 @@ async function executeRun(runId: number, signal: AbortSignal): Promise<void> {
// Execute steps
const shell = cfg.shell ?? ["/bin/sh", "-c"];
let runFailed = false;
let sawWarning = false;
for (const step of cfg.steps) {
for (const [stepIndex, step] of cfg.steps.entries()) {
if (signal.aborted) {
runFailed = true;
break;
}
const stepRow = await db
.selectFrom("ci_steps")
.select("id")
.where("run_id", "=", runId)
.where("name", "=", step.name)
.executeTakeFirst();
if (!stepRow) continue;
const stepId = stepRow.id;
// After a failure, the skipped steps stay pending and are marked
// skipped below.
if (runFailed && !step.always) continue;
// A step timeout removes the container to kill the command, so
// there is nothing left to run an `always` step in.
if (!containerId) continue;
const stepId = stepIds[stepIndex]!;
// Check run_if condition
if (step.run_if) {
@@ -964,9 +1243,21 @@ async function executeRun(runId: number, signal: AbortSignal): Promise<void> {
envArray,
);
if (cleared.exitCode !== 0) {
throw new Error(
`Failed to reset ${cfg.clone_project_to} before step "${step.name}": ${cleared.log.trim()}`,
);
// Not thrown: the outer handler only records a message
// when no step has failed yet, so after an earlier failure
// it would vanish. The step row always survives.
await db
.updateTable("ci_steps")
.set({
status: "failure",
started_at: now(),
finished_at: now(),
log: `Failed to reset ${cfg.clone_project_to}: ${cleared.log.trim()}\n`,
})
.where("id", "=", stepId)
.execute();
runFailed = true;
continue;
}
}
@@ -977,7 +1268,9 @@ async function executeRun(runId: number, signal: AbortSignal): Promise<void> {
.execute();
let stepLog = "";
let stepStatus: "success" | "failure" = "success";
let stepStatus: "success" | "failure" | "warning" = "success";
// Applies to every way a step can fail, a timeout included.
const onFail = step.warn_on_fail ? "warning" : "failure";
if (step.run_sh) {
const command = cfg.shell_setup
@@ -1011,30 +1304,33 @@ async function executeRun(runId: number, signal: AbortSignal): Promise<void> {
},
);
stepLog = maskSecrets(log, secretValues);
if (exitCode !== 0) {
stepStatus = "failure";
runFailed = true;
}
if (exitCode !== 0) stepStatus = onFail;
} catch (err) {
// A run cancellation is reported as "cancelled" by the outer
// catch — don't relabel it as a step failure here.
if (signal.aborted) throw err;
stepStatus = "failure";
runFailed = true;
if (timeoutSignal.aborted) {
// Not subject to warn_on_fail. The container is about
// to be destroyed, so every later step is skipped no
// matter what. A run that cannot continue is a failure.
stepStatus = "failure";
stepLog = `Step timed out after ${stepTimeout}s\n`;
// Kill the container now so the timed-out command stops
// immediately rather than lingering until cleanup.
await removeContainer(containerId);
containerId = "";
} else {
stepStatus = onFail;
stepLog = `Step failed: ${err instanceof Error ? err.message : String(err)}\n`;
}
}
}
if (stepStatus === "failure") runFailed = true;
if (stepStatus === "warning") sawWarning = true;
// Collect artifacts for this step
if (!runFailed || stepStatus === "success") {
if (stepStatus !== "failure") {
await collectArtifacts(
runId,
containerId,
@@ -1053,8 +1349,6 @@ async function executeRun(runId: number, signal: AbortSignal): Promise<void> {
})
.where("id", "=", stepId)
.execute();
if (runFailed) break;
}
// Mark remaining steps as skipped
@@ -1070,7 +1364,11 @@ async function executeRun(runId: number, signal: AbortSignal): Promise<void> {
.where("status", "=", "pending")
.execute();
const finalStatus = runFailed ? "failure" : "success";
const finalStatus = runFailed
? "failure"
: sawWarning
? "warning"
: "success";
await db
.updateTable("ci_runs")
.set({ status: finalStatus, finished_at: now() })
@@ -1141,6 +1439,41 @@ async function executeRun(runId: number, signal: AbortSignal): Promise<void> {
.where("id", "=", runId)
.executeTakeFirst();
if (run) await pruneHistory(run.repo_id).catch(() => {});
if (pruneCaches) {
await pruneStaleCaches(
pruneCaches.repoName,
pruneCaches.cache,
).catch(() => {});
}
// After removeContainer above: the daemon will not delete a volume
// that is still mounted.
if (cacheLimits) {
const dropped = await enforceCacheLimits(
cacheLimits.repoName,
cacheLimits.cache,
).catch(() => []);
if (dropped.length) {
// Recorded on the run, because the only other symptom is the
// next build being mysteriously slow.
await db
.insertInto("ci_steps")
.values({
run_id: runId,
name: "cache",
status: "success",
started_at: new Date().toISOString(),
finished_at: new Date().toISOString(),
log: `${dropped
.map(
(d) =>
`Dropped cache ${d.path}: ${formatBytes(d.size)} over the ${formatBytes(d.maxSize)} limit`,
)
.join("\n")}\n`,
})
.execute()
.catch(() => {});
}
}
// A slot just freed up — start the next queued run if any.
pumpQueue();
}
Msrc/styles/components.css
@@ -1991,6 +1991,10 @@
background: #cf222e;
color: #fff;
}
.ci-status-warning {
background: #bf8700;
color: #fff;
}
.ci-status-cancelled {
background: var(--color-border);
color: var(--color-text-muted);
Msrc/views/ci/CiHistory.tsx
@@ -77,9 +77,11 @@ function CiHelp({ repo }: { repo: RepositoryRow }) {
<div class="ci-help-body">
<p class="ci-help-desc">
Add <code>.hearthforge-ci.toml</code> to your repository
root. Each <code>[section]</code> is a step executed in file
order. Reserved tables: <code>[on]</code> (triggers) and{" "}
<code>[variables]</code> (user-overridable inputs).
root. Each <code>[[steps]]</code> entry is a step, executed
in file order, and needs a <code>name</code>. Other tables:{" "}
<code>[on]</code> (triggers), <code>[variables]</code>{" "}
(user-overridable inputs) and <code>[[copy]]</code> (files
taken from another image, like <code>COPY --from</code>).
</p>
<div class="ci-help-sections">
<div class="ci-help-section">
@@ -104,6 +106,35 @@ function CiHelp({ repo }: { repo: RepositoryRow }) {
class="ci-help-badge-code"
safe
>{``}</code>
<h4
class="ci-help-section-title"
style="margin-top: var(--space-4)"
>
Step options
</h4>
<dl class="ci-help-vars">
{[
["run_sh", "shell command to run"],
["run_if", "skip the step unless this exits 0"],
[
"always",
"run even after an earlier step failed",
],
[
"warn_on_fail",
"step warns instead of failing; the run reports warning",
],
["clear", "reset the clone before the step"],
["timeout", "per-step timeout in seconds"],
].map(([k, v]) => (
<>
<dt>
<code safe>{k}</code>
</dt>
<dd safe>{v}</dd>
</>
))}
</dl>
<h4
class="ci-help-section-title"
style="margin-top: var(--space-4)"
Msrc/views/ci/CiStatusPill.tsx
@@ -4,6 +4,7 @@ const statusStyles: Record<string, string> = {
running: "ci-status-running",
success: "ci-status-success",
failure: "ci-status-failure",
warning: "ci-status-warning",
cancelled: "ci-status-cancelled",
skipped: "ci-status-skipped",
};
Mtests/ci.unit.test.ts
@@ -10,7 +10,8 @@ describe("parseCiConfig", () => {
test("parses a minimal valid config", () => {
const cfg = parseCiConfig(`
image = "debian:latest"
[build]
[[steps]]
name = "build"
run_sh = "make all"
`);
expect(cfg).not.toBeNull();
@@ -21,45 +22,81 @@ run_sh = "make all"
});
test("returns null when image is missing", () => {
expect(parseCiConfig(`[build]\nrun_sh = "make"`)).toBeNull();
expect(
parseCiConfig(`[[steps]]\nname = "build"\nrun_sh = "make"`),
).toBeNull();
});
test("returns null on invalid TOML", () => {
expect(parseCiConfig("image = [unclosed")).toBeNull();
});
test("excludes reserved table names from steps", () => {
const cfg = parseCiConfig(`
image = "alpine"
[on]
push = ["main"]
[variables]
[variables.FOO]
default = "bar"
[step1]
run_sh = "echo hi"
`);
expect(cfg).not.toBeNull();
expect(cfg!.steps.map((s) => s.name)).toEqual(["step1"]);
test("returns null when steps are missing", () => {
expect(parseCiConfig(`image = "alpine"`)).toBeNull();
});
test("returns null when a step has no name", () => {
expect(
parseCiConfig(`image = "alpine"\n[[steps]]\nrun_sh = "make"`),
).toBeNull();
});
test("preserves step order", () => {
test("preserves step order, including numeric names", () => {
// An object keyed by step name would reorder this: JavaScript
// enumerates integer-like keys first.
const cfg = parseCiConfig(`
image = "alpine"
[setup]
[[steps]]
name = "setup"
run_sh = "apt install"
[compile]
run_sh = "make"
[test]
[[steps]]
name = "2024"
run_sh = "echo year"
[[steps]]
name = "test"
run_sh = "./test.sh"
`);
expect(cfg!.steps.map((s) => s.name)).toEqual([
"setup",
"compile",
"2024",
"test",
]);
});
test("parses copy entries", () => {
const cfg = parseCiConfig(`
image = "alpine"
[[copy]]
image = "docker.io/oven/bun:1.4.0-alpine"
from = "/usr/local/bin/bun"
to = "/usr/local/bin"
[[steps]]
name = "build"
run_sh = "make"
`);
expect(cfg!.copy).toEqual([
{
image: "docker.io/oven/bun:1.4.0-alpine",
from: "/usr/local/bin/bun",
to: "/usr/local/bin",
},
]);
});
test("returns null when a copy entry is incomplete", () => {
expect(
parseCiConfig(`
image = "alpine"
[[copy]]
image = "busybox"
from = "/bin/busybox"
[[steps]]
name = "build"
run_sh = "make"
`),
).toBeNull();
});
test("parses optional top-level fields", () => {
const cfg = parseCiConfig(`
image = "alpine"
@@ -70,6 +107,9 @@ timeout = 1800
cpu_limit = 2.0
memory_limit = "1g"
cache = ["/root/.npm"]
[[steps]]
name = "build"
run_sh = "make"
`);
expect(cfg!.work_dir).toBe("/ci");
expect(cfg!.clone_project_to).toBe("/ci/repo");
@@ -77,7 +117,7 @@ cache = ["/root/.npm"]
expect(cfg!.timeout).toBe(1800);
expect(cfg!.cpu_limit).toBe(2.0);
expect(cfg!.memory_limit).toBe("1g");
expect(cfg!.cache).toEqual(["/root/.npm"]);
expect(cfg!.cache).toEqual([{ path: "/root/.npm" }]);
});
test("parses on.push as array", () => {
@@ -85,6 +125,9 @@ cache = ["/root/.npm"]
image = "alpine"
[on]
push = ["main", "develop"]
[[steps]]
name = "build"
run_sh = "make"
`);
expect(cfg!.on?.push).toEqual(["main", "develop"]);
});
@@ -95,6 +138,9 @@ image = "alpine"
[on]
tag = true
manual = true
[[steps]]
name = "build"
run_sh = "make"
`);
expect(cfg!.on?.tag).toBe(true);
expect(cfg!.on?.manual).toBe(true);
@@ -109,6 +155,9 @@ image = "alpine"
description = "Target environment"
[variables.VERSION]
default = "1.0.0"
[[steps]]
name = "build"
run_sh = "make"
`);
expect(cfg!.variables?.DEPLOY_ENV).toEqual({
default: "staging",
@@ -120,7 +169,8 @@ image = "alpine"
test("parses step-level fields", () => {
const cfg = parseCiConfig(`
image = "alpine"
[test]
[[steps]]
name = "test"
run_sh = "./run_tests"
run_if = 'test -n "\${CI_COMMIT_TAG}"'
clear = true
@@ -136,12 +186,29 @@ publish_gzip = ["/dist/"]
expect(step.publish_file).toEqual(["/dist/binary"]);
expect(step.publish_gzip).toEqual(["/dist/"]);
});
test("parses always and warn_on_fail", () => {
const cfg = parseCiConfig(`
image = "alpine"
[[steps]]
name = "lint"
run_sh = "make lint"
warn_on_fail = true
[[steps]]
name = "cleanup"
run_sh = "rm -rf /scratch"
always = true
`);
expect(cfg!.steps[0]!.warn_on_fail).toBe(true);
expect(cfg!.steps[0]!.always).toBeUndefined();
expect(cfg!.steps[1]!.always).toBe(true);
});
});
describe("shouldTriggerPush", () => {
function cfg(push: string[] | boolean) {
return parseCiConfig(
`image="alpine"\n[on]\npush=${JSON.stringify(push)}`,
`image="alpine"\n[[steps]]\nname="b"\n[on]\npush=${JSON.stringify(push)}`,
)!;
}
@@ -163,7 +230,7 @@ describe("shouldTriggerPush", () => {
});
test("returns false when on.push is absent", () => {
const c = parseCiConfig('image="alpine"')!;
const c = parseCiConfig('image="alpine"\n[[steps]]\nname="b"')!;
expect(shouldTriggerPush(c, "main")).toBe(false);
});
@@ -176,17 +243,17 @@ describe("shouldTriggerPush", () => {
describe("shouldTriggerTag", () => {
test("returns true when on.tag = true", () => {
const c = parseCiConfig('image="alpine"\n[on]\ntag=true')!;
const c = parseCiConfig('image="alpine"\n[[steps]]\nname="b"\n[on]\ntag=true')!;
expect(shouldTriggerTag(c)).toBe(true);
});
test("returns false when on.tag = false", () => {
const c = parseCiConfig('image="alpine"\n[on]\ntag=false')!;
const c = parseCiConfig('image="alpine"\n[[steps]]\nname="b"\n[on]\ntag=false')!;
expect(shouldTriggerTag(c)).toBe(false);
});
test("returns false when on.tag is absent", () => {
const c = parseCiConfig('image="alpine"')!;
const c = parseCiConfig('image="alpine"\n[[steps]]\nname="b"')!;
expect(shouldTriggerTag(c)).toBe(false);
});
});
@@ -199,6 +266,9 @@ describe("validateCiConfig", () => {
image = "alpine"
clone_project_to = "/ci/build/project"
cache = ["/ci/cache/target", "/usr/local/cargo/registry"]
[[steps]]
name = "build"
run_sh = "make"
`);
expect(validateCiConfig(cfg)).toBeNull();
});
@@ -208,6 +278,9 @@ cache = ["/ci/cache/target", "/usr/local/cargo/registry"]
image = "alpine"
clone_project_to = "/ci/build/project"
cache = ["/ci/build/project/target"]
[[steps]]
name = "build"
run_sh = "make"
`);
expect(validateCiConfig(cfg)).toContain("/ci/build/project/target");
});
@@ -217,6 +290,9 @@ cache = ["/ci/build/project/target"]
image = "alpine"
clone_project_to = "/ci/build/project"
cache = ["/ci/build/project"]
[[steps]]
name = "build"
run_sh = "make"
`);
expect(validateCiConfig(cfg)).not.toBeNull();
});
@@ -226,6 +302,9 @@ cache = ["/ci/build/project"]
image = "alpine"
clone_project_to = "/ci/build/project"
cache = ["/ci/build/project-cache"]
[[steps]]
name = "build"
run_sh = "make"
`);
expect(validateCiConfig(cfg)).toBeNull();
});
@@ -234,7 +313,93 @@ cache = ["/ci/build/project-cache"]
const cfg = parse(`
image = "alpine"
cache = ["/anything"]
[[steps]]
name = "build"
run_sh = "make"
`);
expect(validateCiConfig(cfg)).toBeNull();
});
});
describe("validateCiConfig, copy destinations", () => {
const parse = (copyTo: string) =>
parseCiConfig(`
image = "alpine"
[[copy]]
image = "busybox"
from = "/bin/busybox"
to = "${copyTo}"
[[steps]]
name = "build"
run_sh = "make"
`)!;
test("accepts a directory destination", () => {
expect(validateCiConfig(parse("/usr/local/bin"))).toBeNull();
});
test("rejects a destination that repeats the source basename", () => {
// Looks like a rename, silently produces /usr/local/bin/busybox/busybox.
const err = validateCiConfig(parse("/usr/local/bin/busybox"));
expect(err).toContain("/usr/local/bin/busybox/busybox");
});
});
describe("cache entries", () => {
test("accepts bare paths and tables in one array", () => {
const cfg = parseCiConfig(`
image = "alpine"
cache = [
{ path = "/ci/cache/target", max_size = "12g" },
"/usr/local/cargo/registry",
{ path = "/root/.bun/install/cache" },
]
[[steps]]
name = "build"
run_sh = "make"
`);
expect(cfg!.cache).toEqual([
{ path: "/ci/cache/target", maxSize: 12 * 1024 ** 3 },
{ path: "/usr/local/cargo/registry" },
{ path: "/root/.bun/install/cache" },
]);
});
test("rejects a size it cannot parse", () => {
// parseMemoryBytes answers 0 here, which would be a limit every cache
// exceeds, wiping it after every run.
expect(
parseCiConfig(`
image = "alpine"
cache = [{ path = "/c", max_size = "12 GiB" }]
[[steps]]
name = "build"
run_sh = "make"
`),
).toBeNull();
});
test("rejects a table with no path", () => {
expect(
parseCiConfig(`
image = "alpine"
cache = [{ max_size = "1g" }]
[[steps]]
name = "build"
run_sh = "make"
`),
).toBeNull();
});
test("still rejects a capped cache inside the clone directory", () => {
const cfg = parseCiConfig(`
image = "alpine"
clone_project_to = "/ci/build/project"
cache = [{ path = "/ci/build/project/target", max_size = "1g" }]
[[steps]]
name = "build"
run_sh = "make"
`)!;
expect(validateCiConfig(cfg)).toContain("/ci/build/project/target");
});
});
Mtests/e2e.ci.test.ts
@@ -36,10 +36,22 @@ const SOCKET_PATH = `/tmp/test-docker-ci-${process.pid}.sock`;
interface ExecResp {
output: string;
exitCode: number;
/** Hold the response open, so the caller's timeout can fire. */
delayMs?: number;
}
// Repo archive uploads (PUT /containers/*/archive)
const uploads: Array<{ path: string; bytes: number }> = [];
// Images passed to POST /images/create
const pulls: string[] = [];
// Volumes created, and the ones deleted, so cache pruning can be asserted
const volumesCreated: Array<{ name: string; labels: Record<string, string> }> =
[];
const volumesDeleted: string[] = [];
// Volumes the mock reports as existing for GET /volumes
let volumesOnHost: string[] = [];
// Sizes the mock reports from GET /system/df, keyed by volume name
let volumeUsage: Record<string, { Size: number; RefCount: number }> = {};
// Body of the last POST /containers/create
let lastCreateBody: Record<string, any> | null = null;
// Per-exec-ID response map, populated when exec is created
@@ -58,6 +70,11 @@ function resetMock() {
execQueue.length = 0;
execCounter = 0;
uploads.length = 0;
pulls.length = 0;
volumesCreated.length = 0;
volumesDeleted.length = 0;
volumesOnHost = [];
volumeUsage = {};
lastCreateBody = null;
}
@@ -109,12 +126,17 @@ function startMockDocker() {
}
// Pull image (streaming, just needs to resolve)
if (req.method === "POST" && p.startsWith("/v1.47/images/create")) {
pulls.push(qs.get("fromImage") ?? "");
return new Response('{"status":"Pull complete"}\n');
}
// Create container
if (req.method === "POST" && /\/containers\/create/.test(p)) {
lastCreateBody = (await req.json()) as Record<string, any>;
return Response.json({ Id: "mock-ctr-001" });
const body = (await req.json()) as Record<string, any>;
const name = qs.get("name") ?? "mock-ctr-001";
// A copy creates its own source container, so the run's
// container must keep its identity.
if (!name.includes("-copy-")) lastCreateBody = body;
return Response.json({ Id: name });
}
// Start container
if (
@@ -140,6 +162,7 @@ function startMockDocker() {
if (req.method === "POST" && /\/exec\/[^/]+\/start$/.test(p)) {
const id = p.match(/\/exec\/([^/]+)\/start/)![1]!;
const resp = execMap.get(id) ?? { output: "", exitCode: 0 };
if (resp.delayMs) await Bun.sleep(resp.delayMs);
return new Response(
resp.output ? muxFrame(resp.output) : new Uint8Array(0),
);
@@ -179,14 +202,33 @@ function startMockDocker() {
}
// Volume create (used for cache volumes)
if (req.method === "POST" && p === "/v1.47/volumes/create") {
return Response.json({ Name: "mock-volume" });
const body = (await req.json()) as {
Name: string;
Labels: Record<string, string>;
};
volumesCreated.push({
name: body.Name,
labels: body.Labels ?? {},
});
return Response.json({ Name: body.Name });
}
// Disk usage (used by the cache size caps)
if (req.method === "GET" && p === "/v1.47/system/df") {
return Response.json({
Volumes: Object.entries(volumeUsage).map(
([Name, UsageData]) => ({ Name, UsageData }),
),
});
}
// Volume list (used by purge cache)
if (req.method === "GET" && p === "/v1.47/volumes") {
return Response.json({ Volumes: [] });
return Response.json({
Volumes: volumesOnHost.map((name) => ({ Name: name })),
});
}
// Volume delete
if (req.method === "DELETE" && /\/volumes\//.test(p)) {
volumesDeleted.push(p.split("/").pop() ?? "");
return new Response(null, { status: 204 });
}
return new Response("Not found", { status: 404 });
@@ -273,7 +315,8 @@ image = "debian:latest"
manual = true
push = ["main"]
[hello]
[[steps]]
name = "hello"
run_sh = "echo hello"
`;
@@ -284,7 +327,8 @@ work_dir = "/ci"
[on]
manual = true
[build]
[[steps]]
name = "build"
run_sh = "echo building"
publish_file = ["/ci/output.txt"]
`;
@@ -749,14 +793,17 @@ image = "debian:latest"
[on]
manual = true
[first]
[[steps]]
name = "first"
run_sh = "echo first"
[second]
[[steps]]
name = "second"
run_if = "false"
run_sh = "echo second"
[third]
[[steps]]
name = "third"
run_sh = "echo third"
`;
@@ -834,7 +881,8 @@ image = "debian:latest"
[on]
push = ["main"]
[hello]
[[steps]]
name = "hello"
run_sh = "echo hi"
`;
@@ -931,7 +979,8 @@ clone_project_to = "/ci/build/project"
[on]
manual = true
[hello]
[[steps]]
name = "hello"
run_sh = "echo hi"
`;
@@ -998,7 +1047,8 @@ cache = ["/ci/build/project/target"]
[on]
manual = true
[hello]
[[steps]]
name = "hello"
run_sh = "echo hi"
`,
);
@@ -1020,3 +1070,474 @@ run_sh = "echo hi"
expect(setup?.log).toContain("is inside clone_project_to");
});
});
describe("copy from another image", () => {
const COPY_TOML = `
image = "debian:latest"
[on]
manual = true
[[copy]]
image = "docker.io/oven/bun:1.4.0-alpine"
from = "/usr/local/bin/bun"
to = "/usr/local/bin"
[[steps]]
name = "hello"
run_sh = "bun --version"
`;
test("pulls the source image and uploads its files", async () => {
const sha = seedCiToml("ci-repo", COPY_TOML);
queueExec({ output: "", exitCode: 0 }); // mkdir of the copy target
queueExec({ output: "1.4.0\n", exitCode: 0 }); // the step
const runId = await triggerRun("ci-repo", {
triggerSource: "manual",
commitSha: sha,
commitBranch: "main",
triggeredBy: adminUserId,
});
expect(await waitForRun(runId)).toBe("success");
expect(pulls).toContain("docker.io/oven/bun");
expect(uploads.map((u) => u.path)).toContain("/usr/local/bin");
});
});
describe("always and warn_on_fail", () => {
const FLAGS_TOML = `
image = "debian:latest"
[on]
manual = true
[[steps]]
name = "lint"
run_sh = "make lint"
warn_on_fail = true
[[steps]]
name = "build"
run_sh = "make"
[[steps]]
name = "cleanup"
run_sh = "rm -rf /scratch"
always = true
`;
function status(runId: number, name: string) {
return db
.selectFrom("ci_steps")
.select(["status", "log"])
.where("run_id", "=", runId)
.where("name", "=", name)
.executeTakeFirst();
}
async function run(sha: string): Promise<number> {
const runId = await triggerRun("ci-repo", {
triggerSource: "manual",
commitSha: sha,
commitBranch: "main",
triggeredBy: adminUserId,
});
await waitForRun(runId);
return runId;
}
test("warn_on_fail marks the step and lets the run continue", async () => {
const sha = seedCiToml("ci-repo", FLAGS_TOML);
queueExec({ output: "style nit\n", exitCode: 1 }); // lint
queueExec({ output: "built\n", exitCode: 0 }); // build
queueExec({ output: "", exitCode: 0 }); // cleanup
const runId = await run(sha);
expect((await status(runId, "lint"))?.status).toBe("warning");
expect((await status(runId, "lint"))?.log).toContain("style nit");
expect((await status(runId, "build"))?.status).toBe("success");
const runRow = await db
.selectFrom("ci_runs")
.select("status")
.where("id", "=", runId)
.executeTakeFirst();
expect(runRow?.status).toBe("warning");
});
test("always runs after a failure, other steps stay skipped", async () => {
const sha = seedCiToml("ci-repo", FLAGS_TOML);
queueExec({ output: "ok\n", exitCode: 0 }); // lint
queueExec({ output: "boom\n", exitCode: 1 }); // build fails
queueExec({ output: "cleaned\n", exitCode: 0 }); // cleanup, always
const runId = await run(sha);
expect((await status(runId, "build"))?.status).toBe("failure");
expect((await status(runId, "cleanup"))?.status).toBe("success");
expect((await status(runId, "cleanup"))?.log).toContain("cleaned");
const runRow = await db
.selectFrom("ci_runs")
.select("status")
.where("id", "=", runId)
.executeTakeFirst();
expect(runRow?.status).toBe("failure");
});
test("a failing always step keeps the run failed", async () => {
const sha = seedCiToml("ci-repo", FLAGS_TOML);
queueExec({ output: "ok\n", exitCode: 0 }); // lint
queueExec({ output: "boom\n", exitCode: 1 }); // build fails
queueExec({ output: "no\n", exitCode: 1 }); // cleanup also fails
const runId = await run(sha);
expect((await status(runId, "cleanup"))?.status).toBe("failure");
const runRow = await db
.selectFrom("ci_runs")
.select("status")
.where("id", "=", runId)
.executeTakeFirst();
expect(runRow?.status).toBe("failure");
});
});
describe("duplicate step names", () => {
const DUPES_TOML = `
image = "debian:latest"
[on]
manual = true
[[steps]]
name = "check"
run_sh = "echo one"
[[steps]]
name = "check"
run_sh = "echo two"
`;
test("each occurrence gets its own row, in file order", async () => {
const sha = seedCiToml("ci-repo", DUPES_TOML);
queueExec({ output: "one\n", exitCode: 0 });
queueExec({ output: "two\n", exitCode: 0 });
const runId = await triggerRun("ci-repo", {
triggerSource: "manual",
commitSha: sha,
commitBranch: "main",
triggeredBy: adminUserId,
});
expect(await waitForRun(runId)).toBe("success");
const rows = await db
.selectFrom("ci_steps")
.select(["status", "log"])
.where("run_id", "=", runId)
.where("name", "=", "check")
.orderBy("id", "asc")
.execute();
expect(rows).toHaveLength(2);
expect(rows[0]!.log).toContain("one");
expect(rows[1]!.log).toContain("two");
expect(rows.every((r) => r.status === "success")).toBe(true);
});
test("the second occurrence can fail on its own", async () => {
const sha = seedCiToml("ci-repo", DUPES_TOML);
queueExec({ output: "one\n", exitCode: 0 });
queueExec({ output: "boom\n", exitCode: 1 });
const runId = await triggerRun("ci-repo", {
triggerSource: "manual",
commitSha: sha,
commitBranch: "main",
triggeredBy: adminUserId,
});
expect(await waitForRun(runId)).toBe("failure");
const rows = await db
.selectFrom("ci_steps")
.select("status")
.where("run_id", "=", runId)
.where("name", "=", "check")
.orderBy("id", "asc")
.execute();
expect(rows.map((r) => r.status)).toEqual(["success", "failure"]);
});
});
describe("timeouts override warn_on_fail", () => {
const TIMEOUT_TOML = `
image = "debian:latest"
[on]
manual = true
[[steps]]
name = "lint"
run_sh = "make lint"
warn_on_fail = true
timeout = 1
[[steps]]
name = "build"
run_sh = "make"
`;
test("a timed-out warn_on_fail step fails the run", async () => {
const sha = seedCiToml("ci-repo", TIMEOUT_TOML);
queueExec({ output: "", exitCode: 0, delayMs: 3000 });
const runId = await triggerRun("ci-repo", {
triggerSource: "manual",
commitSha: sha,
commitBranch: "main",
triggeredBy: adminUserId,
});
expect(await waitForRun(runId, 20_000)).toBe("failure");
const lint = await db
.selectFrom("ci_steps")
.select(["status", "log"])
.where("run_id", "=", runId)
.where("name", "=", "lint")
.executeTakeFirst();
// A timeout destroys the container, so nothing after it can run.
// Reporting that as a warning would hide a dead pipeline.
expect(lint?.status).toBe("failure");
expect(lint?.log).toContain("timed out");
}, 30_000);
});
describe("clear failures are recorded", () => {
const CLEAR_TOML = `
image = "debian:latest"
work_dir = "/ci/build"
clone_project_to = "/ci/build/project"
[on]
manual = true
[[steps]]
name = "first"
run_sh = "false"
[[steps]]
name = "second"
always = true
clear = true
run_sh = "echo hi"
`;
test("a clear failure lands on the step, not the console", async () => {
const sha = seedCiToml("ci-repo", CLEAR_TOML);
queueExec({ output: "", exitCode: 0 }); // mkdir work_dir
queueExec({ output: "", exitCode: 0 }); // mkdir repo path
queueExec({ output: "", exitCode: 0 }); // clone
queueExec({ output: "boom\n", exitCode: 1 }); // first, fails
queueExec({ output: "index.lock exists\n", exitCode: 1 }); // clear
const runId = await triggerRun("ci-repo", {
triggerSource: "manual",
commitSha: sha,
commitBranch: "main",
triggeredBy: adminUserId,
});
expect(await waitForRun(runId)).toBe("failure");
const second = await db
.selectFrom("ci_steps")
.select(["status", "log"])
.where("run_id", "=", runId)
.where("name", "=", "second")
.executeTakeFirst();
expect(second?.status).toBe("failure");
expect(second?.log).toContain("Failed to reset");
expect(second?.log).toContain("index.lock");
});
});
describe("cache volumes", () => {
const CACHE_TOML = `
image = "debian:latest"
cache = ["/ci/cache/target", "/ci/cache/registry"]
[on]
manual = true
push = ["main"]
[[steps]]
name = "hello"
run_sh = "echo hi"
`;
async function run(sha: string, branch: string): Promise<number> {
const runId = await triggerRun("ci-repo", {
triggerSource: "manual",
commitSha: sha,
commitBranch: branch,
triggeredBy: adminUserId,
});
await waitForRun(runId);
return runId;
}
test("two cache paths sharing a prefix get distinct volumes", async () => {
const sha = seedCiToml("ci-repo", CACHE_TOML);
await run(sha, "main");
const names = volumesCreated.map((v) => v.name);
expect(names).toHaveLength(2);
expect(new Set(names).size).toBe(2);
// The path is otherwise unrecoverable from a digest.
expect(volumesCreated.map((v) => v.labels["com.hearthforge.cache-path"]))
.toEqual(["/ci/cache/target", "/ci/cache/registry"]);
});
test("a volume the config no longer names is pruned", async () => {
const sha = seedCiToml("ci-repo", CACHE_TOML);
resetMock();
volumesOnHost = ["hearthforge-ci-cache-leftover-from-an-old-config"];
await run(sha, "main");
expect(volumesDeleted).toEqual([
"hearthforge-ci-cache-leftover-from-an-old-config",
]);
});
test("volumes still in the config survive", async () => {
const sha = seedCiToml("ci-repo", CACHE_TOML);
resetMock();
// Prime the host list with the names this config will create.
await run(sha, "main");
const inUse = volumesCreated.map((v) => v.name);
resetMock();
volumesOnHost = inUse;
await run(sha, "main");
expect(volumesDeleted).toEqual([]);
});
test("a run off the default branch prunes nothing", async () => {
const sha = seedCiToml("ci-repo", CACHE_TOML);
resetMock();
volumesOnHost = ["hearthforge-ci-cache-belongs-to-the-default-branch"];
// The config is read per commit, so pruning from a feature branch
// would delete the default branch's caches.
await run(sha, "some-feature");
expect(volumesDeleted).toEqual([]);
});
});
describe("cache size caps", () => {
const CAPPED_TOML = `
image = "debian:latest"
cache = [{ path = "/ci/cache/target", max_size = "1g" }, "/ci/cache/registry"]
[on]
manual = true
[[steps]]
name = "hello"
run_sh = "echo hi"
`;
async function run(sha: string): Promise<number> {
const runId = await triggerRun("ci-repo", {
triggerSource: "manual",
commitSha: sha,
commitBranch: "main",
triggeredBy: adminUserId,
});
await waitForRun(runId);
return runId;
}
/** Volume names the config produces, in declaration order. */
async function names(sha: string): Promise<string[]> {
resetMock();
await run(sha);
return volumesCreated.map((v) => v.name);
}
test("an oversized cache is dropped and reported on the run", async () => {
const sha = seedCiToml("ci-repo", CAPPED_TOML);
const [target, registry] = await names(sha);
resetMock();
volumesOnHost = [target!, registry!];
volumeUsage = {
[target!]: { Size: 2 * 1024 ** 3, RefCount: 0 },
[registry!]: { Size: 9 * 1024 ** 3, RefCount: 0 },
};
const runId = await run(sha);
// Only the capped one goes, however large the uncapped one grows.
expect(volumesDeleted).toEqual([target!]);
const step = await db
.selectFrom("ci_steps")
.select("log")
.where("run_id", "=", runId)
.where("name", "=", "cache")
.executeTakeFirst();
expect(step?.log).toContain("/ci/cache/target");
expect(step?.log).toContain("2.0G");
});
test("a cache under its cap survives", async () => {
const sha = seedCiToml("ci-repo", CAPPED_TOML);
const [target, registry] = await names(sha);
resetMock();
volumesOnHost = [target!, registry!];
volumeUsage = { [target!]: { Size: 100, RefCount: 0 } };
await run(sha);
expect(volumesDeleted).toEqual([]);
});
test("a cache a concurrent run holds is left alone", async () => {
const sha = seedCiToml("ci-repo", CAPPED_TOML);
const [target, registry] = await names(sha);
resetMock();
volumesOnHost = [target!, registry!];
volumeUsage = { [target!]: { Size: 9 * 1024 ** 3, RefCount: 1 } };
await run(sha);
expect(volumesDeleted).toEqual([]);
});
test("an unmeasured cache is never dropped", async () => {
const sha = seedCiToml("ci-repo", CAPPED_TOML);
const [target, registry] = await names(sha);
resetMock();
volumesOnHost = [target!, registry!];
// Docker reports -1 for a size it has not computed.
volumeUsage = { [target!]: { Size: -1, RefCount: 0 } };
const runId = await run(sha);
expect(volumesDeleted).toEqual([]);
const step = await db
.selectFrom("ci_steps")
.select("id")
.where("run_id", "=", runId)
.where("name", "=", "cache")
.executeTakeFirst();
expect(step).toBeUndefined();
});
});