hearthforge

hearthforge/internal/web/e2e/ci_test.go
ci_test.go
⎇
Raw
1package e2e
2
3import (
4 "archive/tar"
5 "archive/zip"
6 "bytes"
7 "compress/gzip"
8 "context"
9 "database/sql"
10 "encoding/json"
11 "io"
12 "net/http"
13 "net/url"
14 "os"
15 "path/filepath"
16 "slices"
17 "strconv"
18 "strings"
19 "testing"
20 "time"
21
22 "github.com/klauspost/compress/zstd"
23)
24
25// The CI suite drives the real pipeline runner against a mock Docker Engine
26// on a unix socket, so no container engine is needed.
27
28const ciSimpleTOML = `
29image = "debian:latest"
30
31[on]
32manual = true
33push = ["main"]
34
35[[steps]]
36name = "hello"
37run_sh = "echo hello"
38`
39
40const ciArtifactTOML = `
41image = "debian:latest"
42work_dir = "/ci"
43
44[on]
45manual = true
46
47[[steps]]
48name = "build"
49run_sh = "echo building"
50publish_file = ["/ci/output.txt"]
51`
52
53// ciEnv starts a server wired to a fresh mock engine and seeds "ci-repo".
54// extraEnv holds further environment pairs for newEnv.
55func ciEnv(t *testing.T, extraEnv ...string) (*env, *mockDocker, *session) {
56 t.Helper()
57 m := newMockDocker(t)
58 e := newEnv(t, append([]string{"CI_DOCKER_SOCKET", m.sock}, extraEnv...)...)
59 admin := e.admin()
60 e.createRepo(admin, "ci-repo")
61 e.seedRepo("ci-repo", nil)
62 return e, m, admin
63}
64
65// ciSeedToml pushes a .hearthforge-ci.toml into ci-repo and returns the
66// commit sha. Re-seeding the same content is a no-op commit, so a test can
67// put its config back without failing.
68func ciSeedToml(e *env, toml string) string {
69 t := e.t
70 t.Helper()
71 work := t.TempDir()
72 gitRun(t, work, "clone", "-q", e.repoPath("ci-repo"), ".")
73 if err := os.WriteFile(filepath.Join(work, ".hearthforge-ci.toml"), []byte(toml), 0o644); err != nil {
74 t.Fatal(err)
75 }
76 gitRun(t, work, "add", ".hearthforge-ci.toml")
77 // Nothing to commit when the config is unchanged.
78 _, _ = gitTry(work, "commit", "-q", "-m", "Add CI config")
79 gitRun(t, work, "push", "-q", "origin", "HEAD:main")
80 e.Srv.Git.InvalidateRefCache("ci-repo")
81 return gitRun(t, work, "rev-parse", "HEAD")
82}
83
84// ciTrigger posts the manual trigger and returns the new run id. Without a
85// branch field the route builds HEAD of the default branch, so the expected
86// sha is checked and a stale fixture fails loudly.
87func ciTrigger(e *env, admin *session, sha string, overrides url.Values) int64 {
88 t := e.t
89 t.Helper()
90 if head := e.headCommit("ci-repo"); head != sha {
91 t.Fatalf("ciTrigger: expected HEAD %s, repo HEAD is %s", sha, head)
92 }
93 if overrides == nil {
94 overrides = url.Values{}
95 }
96 loc := admin.post("/ci-repo/ci/run", overrides).mustRedirect("/ci-repo/ci/")
97 id, err := strconv.ParseInt(idFromPath(t, loc), 10, 64)
98 if err != nil {
99 t.Fatalf("run id in %q: %v", loc, err)
100 }
101 return id
102}
103
104// ciLatestRunID is the highest run id in the database, or 0.
105func ciLatestRunID(e *env) int64 {
106 var id sql.NullInt64
107 if err := e.DB.QueryRowContext(context.Background(),
108 `SELECT MAX(id) FROM ci_runs`).Scan(&id); err != nil {
109 e.t.Fatal(err)
110 }
111 return id.Int64
112}
113
114// ciPushRun pushes a commit to a branch over HTTP and returns the run the
115// push trigger created.
116func ciPushRun(e *env, sha, branch string) int64 {
117 t := e.t
118 t.Helper()
119 before := ciLatestRunID(e)
120 gitRun(t, e.repoPath("ci-repo"), "push", "--force", e.authURL("ci-repo"),
121 sha+":refs/heads/"+branch)
122 e.Srv.Git.InvalidateRefCache("ci-repo")
123 var id int64
124 ciWaitFor(t, "a run from the push to "+branch, func() bool {
125 id = ciLatestRunID(e)
126 return id > before
127 })
128 return id
129}
130
131// ciWaitForRun polls until the run leaves pending/running/queued.
132func ciWaitForRun(e *env, runID int64, timeout ...time.Duration) string {
133 t := e.t
134 t.Helper()
135 limit := 15 * time.Second
136 if len(timeout) > 0 {
137 limit = timeout[0]
138 }
139 var status string
140 ciWaitFor(t, "run "+strconv.FormatInt(runID, 10)+" to complete", func() bool {
141 status = ciRunStatus(e, runID)
142 return status != "" && status != "pending" && status != "running" && status != "queued" &&
143 !e.Srv.CI.Active(runID)
144 }, limit)
145 return status
146}
147
148func ciRunStatus(e *env, runID int64) string {
149 var status string
150 err := e.DB.QueryRowContext(context.Background(),
151 `SELECT status FROM ci_runs WHERE id = ?`, runID).Scan(&status)
152 if err != nil {
153 e.t.Fatalf("run %d: %v", runID, err)
154 }
155 return status
156}
157
158// ciStepRow is one row of ci_steps.
159type ciStepRow struct {
160 Status string
161 Log string
162}
163
164// ciSteps returns every step of a run with that name, in insertion order.
165func ciSteps(e *env, runID int64, name string) []ciStepRow {
166 rows, err := e.DB.QueryContext(context.Background(),
167 `SELECT status, log FROM ci_steps WHERE run_id = ? AND name = ? ORDER BY id ASC`,
168 runID, name)
169 if err != nil {
170 e.t.Fatal(err)
171 }
172 defer rows.Close()
173 var out []ciStepRow
174 for rows.Next() {
175 var s ciStepRow
176 if err := rows.Scan(&s.Status, &s.Log); err != nil {
177 e.t.Fatal(err)
178 }
179 out = append(out, s)
180 }
181 return out
182}
183
184// ciStep returns the first step of a run with that name.
185func ciStep(e *env, runID int64, name string) ciStepRow {
186 steps := ciSteps(e, runID, name)
187 if len(steps) == 0 {
188 e.t.Fatalf("run %d has no step %q", runID, name)
189 }
190 return steps[0]
191}
192
193// ciOverrides decodes the stored variable overrides of a run.
194func ciOverrides(e *env, runID int64) map[string]string {
195 var raw sql.NullString
196 if err := e.DB.QueryRowContext(context.Background(),
197 `SELECT variable_overrides FROM ci_runs WHERE id = ?`, runID).Scan(&raw); err != nil {
198 e.t.Fatal(err)
199 }
200 out := map[string]string{}
201 if raw.String != "" {
202 if err := json.Unmarshal([]byte(raw.String), &out); err != nil {
203 e.t.Fatalf("overrides %q: %v", raw.String, err)
204 }
205 }
206 return out
207}
208
209func ciRunPath(runID int64) string { return "/ci-repo/ci/" + strconv.FormatInt(runID, 10) }
210
211// eqStrings compares two string lists.
212func eqStrings(a, b []string) bool {
213 if len(a) != len(b) {
214 return false
215 }
216 for i := range a {
217 if a[i] != b[i] {
218 return false
219 }
220 }
221 return true
222}
223
224// ── pipelines tab ────────────────────────────────────────────────────────
225
226func TestCIPipelinesTab(t *testing.T) {
227 e, m, admin := ciEnv(t)
228
229 t.Run("tab is visible in repo nav", func(t *testing.T) {
230 r := admin.get("/ci-repo").mustStatus(200)
231 if !contains(r.Texts(".repo-tab"), "Pipelines") {
232 t.Errorf("repo tabs = %v", r.Texts(".repo-tab"))
233 }
234 })
235
236 t.Run("history page shows empty state when no runs", func(t *testing.T) {
237 r := admin.get("/ci-repo/ci").mustStatus(200)
238 if !r.Has(".empty-state") {
239 t.Fatal("empty state missing")
240 }
241 if !strings.Contains(r.Text(".empty-state"), "No pipeline runs yet") {
242 t.Errorf("empty state = %q", r.Text(".empty-state"))
243 }
244 })
245
246 t.Run("the run form posts and overrides a declared variable", func(t *testing.T) {
247 // Regression: an input-less form posted an empty body and the route
248 // crashed whenever the config declared a variable.
249 sha := ciSeedToml(e, `
250image = "debian:latest"
251
252[on]
253manual = true
254
255[variables]
256 [variables.GREETING]
257 default = "hello"
258 description = "What to echo"
259
260[[steps]]
261name = "say"
262run_sh = "echo $GREETING"
263`)
264 m.reset()
265 m.queueExec(execResp{output: "hi\n"})
266
267 r := admin.get("/ci-repo/ci").mustStatus(200)
268 if got := r.Value(`input[name="var_GREETING"]`); got != "hello" {
269 t.Fatalf("var_GREETING default = %q", got)
270 }
271 runID := ciTrigger(e, admin, sha, url.Values{"var_GREETING": {"goodbye"}})
272 ciWaitForRun(e, runID)
273
274 got := ciOverrides(e, runID)
275 if len(got) != 1 || got["GREETING"] != "goodbye" {
276 t.Errorf("overrides = %v", got)
277 }
278 })
279
280 t.Run("an untouched variable field is not recorded as an override", func(t *testing.T) {
281 sha := ciSeedToml(e, `
282image = "debian:latest"
283
284[on]
285manual = true
286
287[variables]
288 [variables.GREETING]
289 default = "hello"
290
291[[steps]]
292name = "say"
293run_sh = "echo $GREETING"
294`)
295 m.reset()
296 m.queueExec(execResp{output: "hi\n"})
297
298 runID := ciTrigger(e, admin, sha, url.Values{"var_GREETING": {"hello"}})
299 ciWaitForRun(e, runID)
300
301 if got := ciOverrides(e, runID); len(got) != 0 {
302 t.Errorf("overrides = %v, want none", got)
303 }
304 })
305
306 t.Run("an empty POST to the run route does not crash", func(t *testing.T) {
307 sha := ciSeedToml(e, `
308image = "debian:latest"
309
310[on]
311manual = true
312
313[variables]
314 [variables.GREETING]
315 default = "hello"
316
317[[steps]]
318name = "say"
319run_sh = "echo $GREETING"
320`)
321 _ = sha
322 m.reset()
323 m.queueExec(execResp{output: "hi\n"})
324
325 r := admin.post("/ci-repo/ci/run", url.Values{})
326 if r.Code != http.StatusFound {
327 t.Fatalf("status = %d, body %s", r.Code, r.BodyString())
328 }
329 id, err := strconv.ParseInt(idFromPath(t, r.Location()), 10, 64)
330 if err != nil {
331 t.Fatal(err)
332 }
333 ciWaitForRun(e, id)
334 })
335
336 t.Run("the run form builds the selected branch with its own config", func(t *testing.T) {
337 mainSHA := ciSeedToml(e, ciSimpleTOML)
338 work := t.TempDir()
339 gitRun(t, work, "clone", "-q", e.repoPath("ci-repo"), ".")
340 if err := os.WriteFile(filepath.Join(work, ".hearthforge-ci.toml"), []byte(`
341image = "debian:latest"
342
343[variables]
344 [variables.FEATURE_ONLY]
345 default = "on"
346
347[[steps]]
348name = "say"
349run_sh = "echo $FEATURE_ONLY"
350`), 0o644); err != nil {
351 t.Fatal(err)
352 }
353 gitRun(t, work, "commit", "-q", "-am", "Feature CI config")
354 gitRun(t, work, "push", "-q", "origin", "HEAD:refs/heads/feature")
355 e.Srv.Git.InvalidateRefCache("ci-repo")
356 featureSHA := gitRun(t, work, "rev-parse", "HEAD")
357 defer func() {
358 gitRun(t, e.repoPath("ci-repo"), "branch", "-D", "feature")
359 e.Srv.Git.InvalidateRefCache("ci-repo")
360 }()
361
362 r := admin.get("/ci-repo/ci").mustStatus(200)
363 if got := r.Value(`input[name="branch"]`); got != "main" {
364 t.Errorf("default branch field = %q", got)
365 }
366 if r.Has(`input[name="var_FEATURE_ONLY"]`) {
367 t.Error("main shows the feature branch variable")
368 }
369 r = admin.get("/ci-repo/ci?branch=feature").mustStatus(200)
370 if got := r.Value(`input[name="branch"]`); got != "feature" {
371 t.Errorf("branch field = %q", got)
372 }
373 if got := r.Value(`select[name="branch"]`); got != "feature" {
374 t.Errorf("selected branch = %q", got)
375 }
376 if got := r.Value(`input[name="var_FEATURE_ONLY"]`); got != "on" {
377 t.Errorf("var_FEATURE_ONLY default = %q", got)
378 }
379
380 m.reset()
381 m.queueExec(execResp{output: "off\n"})
382 loc := admin.post("/ci-repo/ci/run", url.Values{
383 "branch": {"feature"}, "var_FEATURE_ONLY": {"off"},
384 }).mustRedirect("/ci-repo/ci/")
385 runID, err := strconv.ParseInt(idFromPath(t, loc), 10, 64)
386 if err != nil {
387 t.Fatal(err)
388 }
389 ciWaitForRun(e, runID)
390 var sha, branch string
391 if err := e.DB.QueryRowContext(context.Background(),
392 `SELECT commit_sha, commit_branch FROM ci_runs WHERE id = ?`, runID).
393 Scan(&sha, &branch); err != nil {
394 t.Fatal(err)
395 }
396 if sha != featureSHA || sha == mainSHA || branch != "feature" {
397 t.Errorf("run built %s on %q, want %s on feature", sha, branch, featureSHA)
398 }
399 if got := ciOverrides(e, runID); got["FEATURE_ONLY"] != "off" {
400 t.Errorf("overrides = %v", got)
401 }
402
403 for _, b := range []string{"nope", mainSHA, "HEAD~1"} {
404 r := admin.post("/ci-repo/ci/run", url.Values{"branch": {b}})
405 if r.Code != http.StatusBadRequest {
406 t.Errorf("branch %q: status = %d", b, r.Code)
407 }
408 }
409 })
410
411 t.Run("help section is collapsible and contains template download", func(t *testing.T) {
412 r := admin.get("/ci-repo/ci").mustStatus(200)
413 if !r.Has("details.ci-help") {
414 t.Error("help section missing")
415 }
416 if !r.Has(`a[download=".hearthforge-ci.toml"]`) {
417 t.Error("template download link missing")
418 }
419 })
420}
421
422// ── successful run ───────────────────────────────────────────────────────
423
424func TestCISuccessfulRun(t *testing.T) {
425 e, m, admin := ciEnv(t)
426 m.queueExec(execResp{output: "hello from mock CI\n"})
427 sha := ciSeedToml(e, ciSimpleTOML)
428 runID := ciTrigger(e, admin, sha, nil)
429 ciWaitForRun(e, runID)
430
431 t.Run("run status is success", func(t *testing.T) {
432 if got := ciRunStatus(e, runID); got != "success" {
433 t.Errorf("status = %q", got)
434 }
435 })
436
437 t.Run("step status is success and log is captured", func(t *testing.T) {
438 step := ciStep(e, runID, "hello")
439 if step.Status != "success" {
440 t.Errorf("step status = %q", step.Status)
441 }
442 if !strings.Contains(step.Log, "hello from mock CI") {
443 t.Errorf("step log = %q", step.Log)
444 }
445 })
446
447 t.Run("history page shows the completed run", func(t *testing.T) {
448 r := admin.get("/ci-repo/ci").mustStatus(200)
449 if r.Count(".ci-status-pill.ci-status-success") == 0 {
450 t.Error("no success pill on the history page")
451 }
452 if r.Count(`a.commit-hash[href^="/ci-repo/commit/"]`) == 0 {
453 t.Error("commit hash is not a link to the commit page")
454 }
455 // The trigger label uses the owner display name, like every other
456 // place that names the admin.
457 if !contains(r.Texts(".text-muted"), "by "+e.Cfg.OwnerDisplayName) || contains(r.Texts(".text-muted"), "by admin") {
458 t.Errorf("trigger labels = %v", r.Texts(".text-muted"))
459 }
460 })
461
462 t.Run("run detail page shows step and log", func(t *testing.T) {
463 r := admin.get(ciRunPath(runID)).mustStatus(200)
464 steps := r.Texts(".ci-step")
465 if len(steps) < 2 {
466 t.Fatalf("steps = %v", steps)
467 }
468 // Setup is a real step and sorts before the config's steps.
469 if !strings.Contains(steps[0], "pipeline setup") || !strings.Contains(steps[0], "success") {
470 t.Errorf("first step = %q", steps[0])
471 }
472 if !strings.Contains(steps[1], "hello") {
473 t.Errorf("second step = %q", steps[1])
474 }
475 if !contains(r.Texts(".ci-step-log"), "hello from mock CI") {
476 t.Errorf("logs = %v", r.Texts(".ci-step-log"))
477 }
478 })
479
480 t.Run("retry re-executes the same run in-place", func(t *testing.T) {
481 r := admin.post(ciRunPath(runID)+"/retry", url.Values{})
482 if got := r.mustRedirect(ciRunPath(runID)); got != ciRunPath(runID) {
483 t.Errorf("redirect = %q", got)
484 }
485 if got := ciWaitForRun(e, runID); got != "success" {
486 t.Errorf("status after retry = %q", got)
487 }
488 // No new run was created: the row still exists under the same id.
489 if ciRunStatus(e, runID) != "success" {
490 t.Error("run row changed")
491 }
492 })
493}
494
495// ── failing run ──────────────────────────────────────────────────────────
496
497func TestCIFailingRun(t *testing.T) {
498 e, m, admin := ciEnv(t)
499 m.queueExec(execResp{output: "build error: file not found\n", exitCode: 1})
500 sha := ciSeedToml(e, ciSimpleTOML)
501 runID := ciTrigger(e, admin, sha, nil)
502 ciWaitForRun(e, runID)
503
504 t.Run("run status is failure", func(t *testing.T) {
505 if got := ciRunStatus(e, runID); got != "failure" {
506 t.Errorf("status = %q", got)
507 }
508 })
509
510 t.Run("step status is failure and error log captured", func(t *testing.T) {
511 step := ciStep(e, runID, "hello")
512 if step.Status != "failure" {
513 t.Errorf("step status = %q", step.Status)
514 }
515 if !strings.Contains(step.Log, "build error") {
516 t.Errorf("step log = %q", step.Log)
517 }
518 })
519
520 t.Run("run detail page shows failure status", func(t *testing.T) {
521 r := admin.get(ciRunPath(runID)).mustStatus(200)
522 if r.Count(".ci-status-pill.ci-status-failure") == 0 {
523 t.Error("no failure pill on the run page")
524 }
525 })
526}
527
528// ── cancel ───────────────────────────────────────────────────────────────
529
530func TestCICancel(t *testing.T) {
531 e, _, admin := ciEnv(t)
532
533 t.Run("cancelling a pending run marks it cancelled", func(t *testing.T) {
534 sha := ciSeedToml(e, ciSimpleTOML)
535 runID := ciTrigger(e, admin, sha, nil)
536 admin.post(ciRunPath(runID)+"/cancel", url.Values{}).mustRedirect(ciRunPath(runID))
537
538 status := ciWaitForRun(e, runID)
539 switch status {
540 case "cancelled", "success", "failure":
541 default:
542 t.Fatalf("status = %q", status)
543 }
544 })
545}
546
547const ciSlowTOML = `
548image = "debian:latest"
549
550[on]
551manual = true
552
553[[steps]]
554name = "slow"
555run_sh = "sleep 60"
556
557[[steps]]
558name = "after"
559run_sh = "echo after"
560`
561
562// ciStepStatus is the status of the first step with that name, or "".
563func ciStepStatus(e *env, runID int64, name string) string {
564 if steps := ciSteps(e, runID, name); len(steps) > 0 {
565 return steps[0].Status
566 }
567 return ""
568}
569
570// ciWaitFor polls cond for up to ten seconds, or the given timeout.
571func ciWaitFor(t *testing.T, what string, cond func() bool, timeout ...time.Duration) {
572 t.Helper()
573 limit := 10 * time.Second
574 if len(timeout) > 0 {
575 limit = timeout[0]
576 }
577 for deadline := time.Now().Add(limit); time.Now().Before(deadline); {
578 if cond() {
579 return
580 }
581 time.Sleep(20 * time.Millisecond)
582 }
583 t.Fatalf("timed out waiting for %s", what)
584}
585
586func TestCICancelMidRun(t *testing.T) {
587 e, m, admin := ciEnv(t)
588
589 t.Run("the interrupted step reads cancelled", func(t *testing.T) {
590 sha := ciSeedToml(e, ciSlowTOML)
591 m.reset()
592 m.queueExec(execResp{delay: 30 * time.Second})
593 runID := ciTrigger(e, admin, sha, nil)
594 ciWaitFor(t, "slow step to run", func() bool { return ciStepStatus(e, runID, "slow") == "running" })
595
596 admin.post(ciRunPath(runID)+"/cancel", url.Values{}).mustRedirect(ciRunPath(runID))
597 ciWaitFor(t, "slow step to settle", func() bool { return ciStepStatus(e, runID, "slow") != "running" })
598 if got := ciStep(e, runID, "slow").Status; got != "cancelled" {
599 t.Errorf("slow status = %q", got)
600 }
601 if got := ciStep(e, runID, "after").Status; got != "skipped" {
602 t.Errorf("after status = %q", got)
603 }
604 if got := ciRunStatus(e, runID); got != "cancelled" {
605 t.Errorf("run status = %q", got)
606 }
607 })
608
609 t.Run("a cancel during artifact collection is not overwritten", func(t *testing.T) {
610 sha := ciSeedToml(e, ciArtifactTOML)
611 m.reset()
612 m.archiveDelay = 30 * time.Second
613 runID := ciTrigger(e, admin, sha, nil)
614 ciWaitFor(t, "artifact download", func() bool {
615 return slices.Contains(m.containerRequests(), "GET archive")
616 })
617
618 admin.post(ciRunPath(runID)+"/cancel", url.Values{}).mustRedirect(ciRunPath(runID))
619 ciWaitFor(t, "build step to settle", func() bool { return ciStepStatus(e, runID, "build") != "running" })
620 // The final status write follows the step write within milliseconds.
621 time.Sleep(300 * time.Millisecond)
622 if got := ciRunStatus(e, runID); got != "cancelled" {
623 t.Errorf("run status = %q", got)
624 }
625 })
626}
627
628func TestCIContainerLifecycle(t *testing.T) {
629 e, m, admin := ciEnv(t)
630 sha := ciSeedToml(e, ciSimpleTOML)
631 m.reset()
632 runID := ciTrigger(e, admin, sha, nil)
633 ciWaitForRun(e, runID)
634
635 name := "hearthforge-ci-" + strconv.FormatInt(runID, 10)
636 reqs := m.containerRequests()
637 // A leftover from a cancelled create must not block the name.
638 if len(reqs) < 2 || reqs[0] != "DELETE "+name || reqs[1] != "POST create?name="+name {
639 t.Errorf("requests = %v", reqs)
640 }
641 labels, _ := m.createBody()["Labels"].(map[string]any)
642 if labels["com.hearthforge.ci"] != "1" {
643 t.Errorf("labels = %v", labels)
644 }
645}
646
647func TestCIRepoDeleteStopsRuns(t *testing.T) {
648 e, m, admin := ciEnv(t)
649 sha := ciSeedToml(e, ciArtifactTOML)
650 m.reset()
651 done := ciTrigger(e, admin, sha, nil)
652 ciWaitForRun(e, done)
653 artifacts := filepath.Join(e.Cfg.CIArtifactsDir(), strconv.FormatInt(done, 10))
654 if _, err := os.Stat(artifacts); err != nil {
655 t.Fatalf("artifacts of run %d: %v", done, err)
656 }
657
658 m.queueExec(execResp{delay: 30 * time.Second})
659 active := ciTrigger(e, admin, sha, nil)
660 ciWaitFor(t, "build step to run", func() bool { return ciStepStatus(e, active, "build") == "running" })
661 m.setVolumesOnHost("hearthforge-ci-cache-old")
662
663 admin.post("/ci-repo/settings/delete", nil).mustRedirect("/")
664
665 if !slices.Contains(m.containerRequests(), "DELETE hearthforge-ci-"+strconv.FormatInt(active, 10)) {
666 t.Errorf("active container not removed: %v", m.containerRequests())
667 }
668 if _, err := os.Stat(artifacts); !os.IsNotExist(err) {
669 t.Errorf("artifacts still there: %v", err)
670 }
671 if !slices.Contains(m.deletedVolumes(), "hearthforge-ci-cache-old") {
672 t.Errorf("volumes deleted = %v", m.deletedVolumes())
673 }
674}
675
676// ── artifacts ────────────────────────────────────────────────────────────
677
678func TestCIArtifacts(t *testing.T) {
679 e, _, admin := ciEnv(t)
680 sha := ciSeedToml(e, ciArtifactTOML)
681 runID := ciTrigger(e, admin, sha, nil)
682 ciWaitForRun(e, runID)
683
684 var artifactID int64
685 var filename string
686 var count int
687 rows, err := e.DB.QueryContext(context.Background(),
688 `SELECT id, filename FROM ci_artifacts WHERE run_id = ? ORDER BY id`, runID)
689 if err != nil {
690 t.Fatal(err)
691 }
692 for rows.Next() {
693 if err := rows.Scan(&artifactID, &filename); err != nil {
694 t.Fatal(err)
695 }
696 count++
697 }
698 rows.Close()
699
700 t.Run("artifact row created in DB", func(t *testing.T) {
701 if count != 1 {
702 t.Fatalf("artifacts = %d", count)
703 }
704 if filename != "output.txt" {
705 t.Errorf("filename = %q", filename)
706 }
707 })
708
709 t.Run("artifact is downloadable via HTTP", func(t *testing.T) {
710 if artifactID <= 0 {
711 t.Fatal("no artifact id")
712 }
713 r := e.anon().get(ciRunPath(runID) + "/artifacts/" + strconv.FormatInt(artifactID, 10))
714 r.mustStatus(200)
715 if r.BodyString() != "artifact-content-123" {
716 t.Errorf("body = %q", r.BodyString())
717 }
718 })
719
720 t.Run("run detail page shows artifact list", func(t *testing.T) {
721 r := admin.get(ciRunPath(runID)).mustStatus(200)
722 if r.Count(".ci-artifact-item") == 0 {
723 t.Fatal("no artifact items")
724 }
725 if !strings.Contains(r.Text(".ci-artifact-name"), "output.txt") {
726 t.Errorf("artifact name = %q", r.Text(".ci-artifact-name"))
727 }
728 })
729}
730
731// ── badge ────────────────────────────────────────────────────────────────
732
733func TestCIBadge(t *testing.T) {
734 e, m, admin := ciEnv(t)
735 m.queueExec(execResp{output: "ok\n"})
736 sha := ciSeedToml(e, ciSimpleTOML)
737 runID := ciTrigger(e, admin, sha, nil)
738 ciWaitForRun(e, runID)
739
740 t.Run("badge SVG returns success status after successful run", func(t *testing.T) {
741 r := e.anon().get("/ci-repo/ci/badge.svg").mustStatus(200)
742 if !strings.Contains(r.Header.Get("Content-Type"), "image/svg+xml") {
743 t.Errorf("content type = %q", r.Header.Get("Content-Type"))
744 }
745 if !r.Contains("<svg") || !r.Contains("success") {
746 t.Errorf("badge = %q", r.BodyString())
747 }
748 })
749
750 t.Run("badge returns 404 for private repo when not logged in", func(t *testing.T) {
751 e.createRepo(admin, "private-ci-repo", "is_private", "1")
752 e.anon().get("/private-ci-repo/ci/badge.svg").mustStatus(404)
753 })
754}
755
756// ── secrets ──────────────────────────────────────────────────────────────
757
758func TestCISecrets(t *testing.T) {
759 e, m, admin := ciEnv(t)
760
761 t.Run("can add, list, and delete a secret via settings", func(t *testing.T) {
762 admin.post("/ci-repo/settings/ci-secrets", url.Values{
763 "name": {"MY_SECRET"}, "value": {"super-secret-value"},
764 "description": {"A test secret"},
765 }).mustRedirect("/ci-repo/settings")
766
767 r := admin.get("/ci-repo/settings").mustStatus(200)
768 if n := len(matchingTexts(r.Texts("code"), "MY_SECRET")); n != 1 {
769 t.Fatalf("MY_SECRET shown %d times", n)
770 }
771 if !r.Contains("●●●●●●") {
772 t.Error("secret value is not masked")
773 }
774
775 id := r.Value(`form[action="/ci-repo/settings/ci-secrets/delete"] input[name=id]`)
776 if id == "" {
777 t.Fatal("no delete form for the secret")
778 }
779 admin.post("/ci-repo/settings/ci-secrets/delete", url.Values{"id": {id}}).
780 mustRedirect("/ci-repo/settings")
781
782 r = admin.get("/ci-repo/settings").mustStatus(200)
783 if n := len(matchingTexts(r.Texts("code"), "MY_SECRET")); n != 0 {
784 t.Errorf("MY_SECRET still shown %d times", n)
785 }
786 })
787
788 t.Run("secret value is masked in step logs", func(t *testing.T) {
789 admin.post("/ci-repo/settings/ci-secrets", url.Values{
790 "name": {"MASK_ME"}, "value": {"s3cr3t-p4ssw0rd"},
791 }).mustRedirect("/ci-repo/settings")
792
793 m.reset()
794 m.queueExec(execResp{output: "s3cr3t-p4ssw0rd is the value\n"})
795 sha := ciSeedToml(e, ciSimpleTOML)
796 runID := ciTrigger(e, admin, sha, nil)
797 ciWaitForRun(e, runID)
798
799 step := ciStep(e, runID, "hello")
800 if strings.Contains(step.Log, "s3cr3t-p4ssw0rd") {
801 t.Errorf("secret leaked into the log: %q", step.Log)
802 }
803 if !strings.Contains(step.Log, "[MASKED]") {
804 t.Errorf("log = %q", step.Log)
805 }
806 })
807}
808
809// matchingTexts keeps the entries containing sub.
810func matchingTexts(list []string, sub string) []string {
811 var out []string
812 for _, s := range list {
813 if strings.Contains(s, sub) {
814 out = append(out, s)
815 }
816 }
817 return out
818}
819
820// ── per-repo run IDs ─────────────────────────────────────────────────────
821
822func TestCIPerRepoRunIDs(t *testing.T) {
823 e, m, admin := ciEnv(t)
824 sha := ciSeedToml(e, ciSimpleTOML)
825 for range 2 {
826 m.reset()
827 ciWaitForRun(e, ciTrigger(e, admin, sha, nil))
828 }
829
830 t.Run("repo_run_id is set and increments per repo", func(t *testing.T) {
831 rows, err := e.DB.QueryContext(context.Background(),
832 `SELECT repo_run_id FROM ci_runs ORDER BY repo_run_id ASC`)
833 if err != nil {
834 t.Fatal(err)
835 }
836 defer rows.Close()
837 i := 0
838 for rows.Next() {
839 var id sql.NullInt64
840 if err := rows.Scan(&id); err != nil {
841 t.Fatal(err)
842 }
843 if !id.Valid || id.Int64 <= 0 {
844 t.Fatal("repo_run_id is not set")
845 }
846 i++
847 if id.Int64 != int64(i) {
848 t.Fatalf("repo_run_id %d at position %d", id.Int64, i)
849 }
850 }
851 if i == 0 {
852 t.Fatal("no runs")
853 }
854 })
855
856 t.Run("run detail page shows repo-local run number", func(t *testing.T) {
857 var runID, repoRunID int64
858 if err := e.DB.QueryRowContext(context.Background(),
859 `SELECT id, repo_run_id FROM ci_runs ORDER BY id ASC LIMIT 1`).
860 Scan(&runID, &repoRunID); err != nil {
861 t.Fatal(err)
862 }
863 r := admin.get(ciRunPath(runID)).mustStatus(200)
864 want := "#" + strconv.FormatInt(repoRunID, 10)
865 if !strings.Contains(r.Text("h2"), want) {
866 t.Errorf("heading = %q, want %q", r.Text("h2"), want)
867 }
868 })
869}
870
871// ── skip reasons ─────────────────────────────────────────────────────────
872
873const ciSkipIfTOML = `
874image = "debian:latest"
875
876[on]
877manual = true
878
879[[steps]]
880name = "first"
881run_sh = "echo first"
882
883[[steps]]
884name = "second"
885run_if = "false"
886run_sh = "echo second"
887
888[[steps]]
889name = "third"
890run_sh = "echo third"
891`
892
893func TestCISkipReasons(t *testing.T) {
894 e, m, admin := ciEnv(t)
895
896 t.Run("run_if failure sets skip reason in log", func(t *testing.T) {
897 sha := ciSeedToml(e, ciSkipIfTOML)
898 m.reset()
899 m.queueExec(execResp{output: "first\n"}) // first step
900 m.queueExec(execResp{exitCode: 1}) // run_if check of second
901 m.queueExec(execResp{output: "third\n"}) // third step
902 runID := ciTrigger(e, admin, sha, nil)
903 ciWaitForRun(e, runID)
904
905 step := ciStep(e, runID, "second")
906 if step.Status != "skipped" {
907 t.Errorf("status = %q", step.Status)
908 }
909 if !strings.Contains(step.Log, "condition not met") {
910 t.Errorf("log = %q", step.Log)
911 }
912 })
913
914 t.Run("failed step causes remaining steps to be skipped with reason", func(t *testing.T) {
915 sha := ciSeedToml(e, ciSkipIfTOML)
916 m.reset()
917 m.queueExec(execResp{output: "boom\n", exitCode: 1}) // first step fails
918 runID := ciTrigger(e, admin, sha, nil)
919 ciWaitForRun(e, runID)
920
921 step := ciStep(e, runID, "third")
922 if step.Status != "skipped" {
923 t.Errorf("status = %q", step.Status)
924 }
925 if !strings.Contains(step.Log, "previous step failed") {
926 t.Errorf("log = %q", step.Log)
927 }
928 })
929}
930
931// ── docker unavailable ───────────────────────────────────────────────────
932
933func TestCIDockerUnavailable(t *testing.T) {
934 // newEnv points CI_DOCKER_SOCKET at a path that does not exist.
935 e := newEnv(t)
936 admin := e.admin()
937 e.createRepo(admin, "ci-repo")
938 e.seedRepo("ci-repo", nil)
939
940 t.Run("run is marked skipped when docker socket is missing", func(t *testing.T) {
941 sha := ciSeedToml(e, ciSimpleTOML)
942 runID := ciTrigger(e, admin, sha, nil)
943 if got := ciWaitForRun(e, runID); got != "skipped" {
944 t.Errorf("status = %q", got)
945 }
946 })
947}
948
949// ── manual trigger without on.manual ─────────────────────────────────────
950
951const ciNoManualTOML = `
952image = "debian:latest"
953
954[on]
955push = ["main"]
956
957[[steps]]
958name = "hello"
959run_sh = "echo hi"
960`
961
962func TestCIManualTriggerWithoutOnManual(t *testing.T) {
963 e, m, admin := ciEnv(t)
964
965 t.Run("manual run is allowed even without manual = true in config", func(t *testing.T) {
966 sha := ciSeedToml(e, ciNoManualTOML)
967 m.reset()
968 m.queueExec(execResp{output: "hi\n"})
969 runID := ciTrigger(e, admin, sha, nil)
970 if got := ciWaitForRun(e, runID); got != "success" {
971 t.Errorf("status = %q", got)
972 }
973 })
974
975 t.Run("Run pipeline button is not disabled when toml lacks manual = true", func(t *testing.T) {
976 ciSeedToml(e, ciNoManualTOML)
977 r := admin.get("/ci-repo/ci").mustStatus(200)
978 if !contains(r.Texts("button"), "Run pipeline") {
979 t.Fatalf("buttons = %v", r.Texts("button"))
980 }
981 if r.Has("button[disabled]") {
982 t.Error("the Run pipeline button is disabled")
983 }
984 })
985}
986
987// ── auto-refresh toggle ──────────────────────────────────────────────────
988
989func TestCIAutoRefreshToggle(t *testing.T) {
990 e, _, admin := ciEnv(t)
991
992 t.Run("Pause refresh button appears on active run and ?refresh=off shows Resume", func(t *testing.T) {
993 sha := ciSeedToml(e, ciSimpleTOML)
994 runID := ciTrigger(e, admin, sha, nil)
995
996 // The run may already have finished, so only the refresh link's
997 // existence is checked, exactly as the browser test did.
998 admin.get(ciRunPath(runID)).mustStatus(200)
999
1000 r := admin.get(ciRunPath(runID) + "?refresh=off").mustStatus(200)
1001 if n := r.Count(`meta[http-equiv="refresh"]`); n != 0 {
1002 t.Errorf("meta refresh count = %d", n)
1003 }
1004 ciWaitForRun(e, runID)
1005 })
1006}
1007
1008// ── purge cache ──────────────────────────────────────────────────────────
1009
1010func TestCIPurgeCache(t *testing.T) {
1011 _, _, admin := ciEnv(t)
1012
1013 t.Run("Purge caches button is visible and submits successfully", func(t *testing.T) {
1014 r := admin.get("/ci-repo/ci").mustStatus(200)
1015 if !contains(r.Texts("button"), "Purge caches") {
1016 t.Fatalf("buttons = %v", r.Texts("button"))
1017 }
1018 redirect := admin.post("/ci-repo/ci/purge-cache", url.Values{})
1019 redirect.mustRedirect("/ci-repo/ci")
1020
1021 r = admin.follow(redirect).mustStatus(200)
1022 if !strings.Contains(r.Text("h2"), "Pipelines") {
1023 t.Errorf("heading = %q", r.Text("h2"))
1024 }
1025 msg := r.Text(".form-success, .form-error")
1026 if !strings.Contains(strings.ToLower(msg), "purge") {
1027 t.Errorf("message = %q", msg)
1028 }
1029 })
1030}
1031
1032// ── repo upload ──────────────────────────────────────────────────────────
1033
1034const ciCloneTOML = `
1035image = "debian:latest"
1036work_dir = "/ci/build"
1037clone_project_to = "/ci/build/project"
1038
1039[on]
1040manual = true
1041
1042[[steps]]
1043name = "hello"
1044run_sh = "echo hi"
1045`
1046
1047func TestCIRepoUpload(t *testing.T) {
1048 e, m, admin := ciEnv(t)
1049 ciSeedToml(e, ciCloneTOML)
1050
1051 // Sibling subtests reseed the config, and the trigger route always builds
1052 // HEAD. Put the clone config back first.
1053 trigger := func() int64 {
1054 sha := ciSeedToml(e, ciCloneTOML)
1055 return ciTrigger(e, admin, sha, nil)
1056 }
1057
1058 t.Run("the checkout is uploaded, not bind-mounted", func(t *testing.T) {
1059 m.reset()
1060 runID := trigger()
1061 if got := ciWaitForRun(e, runID); got != "success" {
1062 t.Fatalf("status = %q", got)
1063 }
1064 if len(m.uploadsInto(t, "/ci/build/project")) == 0 {
1065 t.Fatalf("no upload carries files under the clone directory; uploads = %v", m.uploadedPaths())
1066 }
1067 binds, _ := json.Marshal(m.createBody()["HostConfig"])
1068 if strings.Contains(string(binds), e.DataDir) {
1069 t.Errorf("binds reference the data directory: %s", binds)
1070 }
1071 })
1072
1073 t.Run("the container never runs git", func(t *testing.T) {
1074 m.reset()
1075 runID := trigger()
1076 if got := ciWaitForRun(e, runID); got != "success" {
1077 t.Fatalf("status = %q", got)
1078 }
1079 if strings.Contains(m.allCommandText(), "git") {
1080 t.Errorf("commands = %v", m.commands())
1081 }
1082 })
1083
1084 t.Run("a failing checkout fails the run before any step runs", func(t *testing.T) {
1085 m.reset()
1086 m.uploadError = "read-only file system"
1087
1088 runID := trigger()
1089 if got := ciWaitForRun(e, runID); got != "failure" {
1090 t.Fatalf("status = %q", got)
1091 }
1092 if got := ciStep(e, runID, "hello").Status; got != "skipped" {
1093 t.Errorf("hello status = %q", got)
1094 }
1095 setup := ciStep(e, runID, "pipeline setup")
1096 if setup.Status != "failure" {
1097 t.Errorf("setup status = %q", setup.Status)
1098 }
1099 if !strings.Contains(setup.Log, "read-only file system") {
1100 t.Errorf("setup log = %q", setup.Log)
1101 }
1102 })
1103
1104 t.Run("a cache path inside the clone directory is rejected", func(t *testing.T) {
1105 m.reset()
1106 sha := ciSeedToml(e, `
1107image = "debian:latest"
1108clone_project_to = "/ci/build/project"
1109cache = ["/ci/build/project/target"]
1110
1111[on]
1112manual = true
1113
1114[[steps]]
1115name = "hello"
1116run_sh = "echo hi"
1117`)
1118 runID := ciTrigger(e, admin, sha, nil)
1119 if got := ciWaitForRun(e, runID); got != "failure" {
1120 t.Fatalf("status = %q", got)
1121 }
1122 if n := m.uploadCount(); n != 0 {
1123 t.Errorf("uploads = %d, want 0", n)
1124 }
1125 if log := ciStep(e, runID, "pipeline setup").Log; !strings.Contains(log, "overlaps clone_project_to") {
1126 t.Errorf("setup log = %q", log)
1127 }
1128 })
1129
1130 t.Run("a cache path above the clone directory is rejected", func(t *testing.T) {
1131 m.reset()
1132 sha := ciSeedToml(e, `
1133image = "debian:latest"
1134clone_project_to = "/ci/build/project"
1135cache = ["/ci/build"]
1136
1137[on]
1138manual = true
1139
1140[[steps]]
1141name = "hello"
1142run_sh = "echo hi"
1143`)
1144 runID := ciTrigger(e, admin, sha, nil)
1145 if got := ciWaitForRun(e, runID); got != "failure" {
1146 t.Fatalf("status = %q", got)
1147 }
1148 if n := m.uploadCount(); n != 0 {
1149 t.Errorf("uploads = %d, want 0", n)
1150 }
1151 })
1152
1153 t.Run("a relative clone_project_to is rejected", func(t *testing.T) {
1154 m.reset()
1155 sha := ciSeedToml(e, `
1156image = "debian:latest"
1157work_dir = "/ci/build"
1158clone_project_to = "project"
1159
1160[on]
1161manual = true
1162
1163[[steps]]
1164name = "hello"
1165run_sh = "echo hi"
1166`)
1167 runID := ciTrigger(e, admin, sha, nil)
1168 if got := ciWaitForRun(e, runID); got != "failure" {
1169 t.Fatalf("status = %q", got)
1170 }
1171 if log := ciStep(e, runID, "pipeline setup").Log; !strings.Contains(log, "must be an absolute path") {
1172 t.Errorf("setup log = %q", log)
1173 }
1174 })
1175
1176 t.Run("the upload carries the requested commit", func(t *testing.T) {
1177 m.reset()
1178 runID := trigger()
1179 if got := ciWaitForRun(e, runID); got != "success" {
1180 t.Fatalf("status = %q", got)
1181 }
1182 ups := m.uploadsInto(t, "/ci/build/project")
1183 if len(ups) == 0 {
1184 t.Fatal("no upload into the clone directory")
1185 }
1186 // The archive is extracted at / and carries the clone directory as
1187 // its prefix. It must hold the CI config at the triggered commit,
1188 // and no .git. A dropped commit argument would still produce a
1189 // valid tar.
1190 if ups[0].path != "/" {
1191 t.Errorf("upload path = %q, want /", ups[0].path)
1192 }
1193 names := tarEntryNames(t, ups[0].body)
1194 if !contains(names, "ci/build/project/.hearthforge-ci.toml") {
1195 t.Errorf("entries = %v", names)
1196 }
1197 for _, n := range names {
1198 // git archive adds a pax header carrying the commit id.
1199 if n == "pax_global_header" {
1200 continue
1201 }
1202 if strings.Contains(n, ".git/") || !strings.HasPrefix(n, "ci/build/project/") {
1203 t.Errorf("unexpected entry %q", n)
1204 }
1205 }
1206 // git archive writes uid 0, so the files belong to root in the
1207 // container.
1208 for _, h := range tarHeaders(t, ups[0].body) {
1209 if h.uid != 0 {
1210 t.Errorf("entry %q has uid %d", h.name, h.uid)
1211 }
1212 }
1213 })
1214}
1215
1216// ── copy from another image ──────────────────────────────────────────────
1217
1218const ciCopyTOML = `
1219image = "debian:latest"
1220
1221[on]
1222manual = true
1223
1224[[copy]]
1225image = "docker.io/oven/bun:1.4.0-alpine"
1226from = "/usr/local/bin/bun"
1227to = "/usr/local/bin"
1228
1229[[steps]]
1230name = "hello"
1231run_sh = "bun --version"
1232`
1233
1234func TestCICopyFromAnotherImage(t *testing.T) {
1235 e, m, admin := ciEnv(t)
1236
1237 t.Run("pulls the source image and uploads its files", func(t *testing.T) {
1238 sha := ciSeedToml(e, ciCopyTOML)
1239 m.reset()
1240 m.queueExec(execResp{output: "1.4.0\n"}) // the step
1241
1242 runID := ciTrigger(e, admin, sha, nil)
1243 if got := ciWaitForRun(e, runID); got != "success" {
1244 t.Fatalf("status = %q", got)
1245 }
1246 if !contains(m.pulledImages(), "docker.io/oven/bun") {
1247 t.Errorf("pulls = %v", m.pulledImages())
1248 }
1249 if !contains(m.uploadedPaths(), "/usr/local/bin") || len(m.uploadsInto(t, "/usr/local")) == 0 {
1250 t.Errorf("uploads = %v", m.uploadedPaths())
1251 }
1252 })
1253}
1254
1255// ── always and warn_on_fail ──────────────────────────────────────────────
1256
1257const ciFlagsTOML = `
1258image = "debian:latest"
1259
1260[on]
1261manual = true
1262
1263[[steps]]
1264name = "lint"
1265run_sh = "make lint"
1266warn_on_fail = true
1267
1268[[steps]]
1269name = "build"
1270run_sh = "make"
1271
1272[[steps]]
1273name = "cleanup"
1274run_sh = "rm -rf /scratch"
1275always = true
1276`
1277
1278func TestCIAlwaysAndWarnOnFail(t *testing.T) {
1279 e, m, admin := ciEnv(t)
1280
1281 run := func(sha string) int64 {
1282 runID := ciTrigger(e, admin, sha, nil)
1283 ciWaitForRun(e, runID)
1284 return runID
1285 }
1286
1287 t.Run("warn_on_fail marks the step and lets the run continue", func(t *testing.T) {
1288 sha := ciSeedToml(e, ciFlagsTOML)
1289 m.reset()
1290 m.queueExec(execResp{output: "style nit\n", exitCode: 1}) // lint
1291 m.queueExec(execResp{output: "built\n"}) // build
1292 m.queueExec(execResp{}) // cleanup
1293
1294 runID := run(sha)
1295
1296 lint := ciStep(e, runID, "lint")
1297 if lint.Status != "warning" {
1298 t.Errorf("lint status = %q", lint.Status)
1299 }
1300 if !strings.Contains(lint.Log, "style nit") {
1301 t.Errorf("lint log = %q", lint.Log)
1302 }
1303 if got := ciStep(e, runID, "build").Status; got != "success" {
1304 t.Errorf("build status = %q", got)
1305 }
1306 if got := ciRunStatus(e, runID); got != "warning" {
1307 t.Errorf("run status = %q", got)
1308 }
1309 })
1310
1311 t.Run("always runs after a failure, other steps stay skipped", func(t *testing.T) {
1312 sha := ciSeedToml(e, ciFlagsTOML)
1313 m.reset()
1314 m.queueExec(execResp{output: "ok\n"}) // lint
1315 m.queueExec(execResp{output: "boom\n", exitCode: 1}) // build fails
1316 m.queueExec(execResp{output: "cleaned\n"}) // cleanup, always
1317
1318 runID := run(sha)
1319
1320 if got := ciStep(e, runID, "build").Status; got != "failure" {
1321 t.Errorf("build status = %q", got)
1322 }
1323 cleanup := ciStep(e, runID, "cleanup")
1324 if cleanup.Status != "success" {
1325 t.Errorf("cleanup status = %q", cleanup.Status)
1326 }
1327 if !strings.Contains(cleanup.Log, "cleaned") {
1328 t.Errorf("cleanup log = %q", cleanup.Log)
1329 }
1330 if got := ciRunStatus(e, runID); got != "failure" {
1331 t.Errorf("run status = %q", got)
1332 }
1333 })
1334
1335 t.Run("a failing always step keeps the run failed", func(t *testing.T) {
1336 sha := ciSeedToml(e, ciFlagsTOML)
1337 m.reset()
1338 m.queueExec(execResp{output: "ok\n"}) // lint
1339 m.queueExec(execResp{output: "boom\n", exitCode: 1}) // build fails
1340 m.queueExec(execResp{output: "no\n", exitCode: 1}) // cleanup also fails
1341
1342 runID := run(sha)
1343
1344 if got := ciStep(e, runID, "cleanup").Status; got != "failure" {
1345 t.Errorf("cleanup status = %q", got)
1346 }
1347 if got := ciRunStatus(e, runID); got != "failure" {
1348 t.Errorf("run status = %q", got)
1349 }
1350 })
1351}
1352
1353// ── duplicate step names ─────────────────────────────────────────────────
1354
1355const ciDupesTOML = `
1356image = "debian:latest"
1357
1358[on]
1359manual = true
1360
1361[[steps]]
1362name = "check"
1363run_sh = "echo one"
1364
1365[[steps]]
1366name = "check"
1367run_sh = "echo two"
1368`
1369
1370func TestCIDuplicateStepNames(t *testing.T) {
1371 e, m, admin := ciEnv(t)
1372
1373 t.Run("each occurrence gets its own row, in file order", func(t *testing.T) {
1374 sha := ciSeedToml(e, ciDupesTOML)
1375 m.reset()
1376 m.queueExec(execResp{output: "one\n"})
1377 m.queueExec(execResp{output: "two\n"})
1378
1379 runID := ciTrigger(e, admin, sha, nil)
1380 if got := ciWaitForRun(e, runID); got != "success" {
1381 t.Fatalf("status = %q", got)
1382 }
1383 rows := ciSteps(e, runID, "check")
1384 if len(rows) != 2 {
1385 t.Fatalf("rows = %d", len(rows))
1386 }
1387 if !strings.Contains(rows[0].Log, "one") || !strings.Contains(rows[1].Log, "two") {
1388 t.Errorf("logs = %q, %q", rows[0].Log, rows[1].Log)
1389 }
1390 for _, r := range rows {
1391 if r.Status != "success" {
1392 t.Errorf("status = %q", r.Status)
1393 }
1394 }
1395 })
1396
1397 t.Run("the second occurrence can fail on its own", func(t *testing.T) {
1398 sha := ciSeedToml(e, ciDupesTOML)
1399 m.reset()
1400 m.queueExec(execResp{output: "one\n"})
1401 m.queueExec(execResp{output: "boom\n", exitCode: 1})
1402
1403 runID := ciTrigger(e, admin, sha, nil)
1404 if got := ciWaitForRun(e, runID); got != "failure" {
1405 t.Fatalf("status = %q", got)
1406 }
1407 rows := ciSteps(e, runID, "check")
1408 got := []string{}
1409 for _, r := range rows {
1410 got = append(got, r.Status)
1411 }
1412 if !eqStrings(got, []string{"success", "failure"}) {
1413 t.Errorf("statuses = %v", got)
1414 }
1415 })
1416}
1417
1418// ── timeouts override warn_on_fail ───────────────────────────────────────
1419
1420const ciTimeoutTOML = `
1421image = "debian:latest"
1422
1423[on]
1424manual = true
1425
1426[[steps]]
1427name = "lint"
1428run_sh = "make lint"
1429warn_on_fail = true
1430timeout = 1
1431
1432[[steps]]
1433name = "build"
1434run_sh = "make"
1435`
1436
1437func TestCITimeoutsOverrideWarnOnFail(t *testing.T) {
1438 e, m, admin := ciEnv(t)
1439
1440 t.Run("a timed-out warn_on_fail step fails the run", func(t *testing.T) {
1441 sha := ciSeedToml(e, ciTimeoutTOML)
1442 m.reset()
1443 m.queueExec(execResp{delay: 3 * time.Second})
1444
1445 runID := ciTrigger(e, admin, sha, nil)
1446 if got := ciWaitForRun(e, runID, 30*time.Second); got != "failure" {
1447 t.Fatalf("status = %q", got)
1448 }
1449 // A timeout destroys the container, so nothing after it can run.
1450 // Reporting that as a warning would hide a dead pipeline.
1451 lint := ciStep(e, runID, "lint")
1452 if lint.Status != "failure" {
1453 t.Errorf("lint status = %q", lint.Status)
1454 }
1455 if !strings.Contains(lint.Log, "timed out") {
1456 t.Errorf("lint log = %q", lint.Log)
1457 }
1458 })
1459}
1460
1461// ── clear failures are recorded ──────────────────────────────────────────
1462
1463const ciClearTOML = `
1464image = "debian:latest"
1465work_dir = "/ci/build"
1466clone_project_to = "/ci/build/project"
1467
1468[on]
1469manual = true
1470
1471[[steps]]
1472name = "first"
1473run_sh = "false"
1474
1475[[steps]]
1476name = "second"
1477always = true
1478clear = true
1479run_sh = "echo hi"
1480`
1481
1482func TestCIClearFailuresAreRecorded(t *testing.T) {
1483 e, m, admin := ciEnv(t)
1484
1485 t.Run("a clear failure lands on the step, not the console", func(t *testing.T) {
1486 sha := ciSeedToml(e, ciClearTOML)
1487 m.reset()
1488 m.queueExec(execResp{output: "boom\n", exitCode: 1}) // first, fails
1489 m.queueExec(execResp{output: "rm: device busy\n", exitCode: 1}) // clear
1490
1491 runID := ciTrigger(e, admin, sha, nil)
1492 if got := ciWaitForRun(e, runID); got != "failure" {
1493 t.Fatalf("status = %q", got)
1494 }
1495 second := ciStep(e, runID, "second")
1496 if second.Status != "failure" {
1497 t.Errorf("second status = %q", second.Status)
1498 }
1499 if !strings.Contains(second.Log, "Failed to reset") ||
1500 !strings.Contains(second.Log, "device busy") {
1501 t.Errorf("second log = %q", second.Log)
1502 }
1503 })
1504
1505 t.Run("a clear step re-extracts the checkout", func(t *testing.T) {
1506 sha := ciSeedToml(e, ciClearTOML)
1507 m.reset()
1508 m.queueExec(execResp{output: "ok\n"}) // first
1509 m.queueExec(execResp{}) // clear: rm -rf
1510 m.queueExec(execResp{output: "hi\n"}) // second
1511
1512 runID := ciTrigger(e, admin, sha, nil)
1513 if got := ciWaitForRun(e, runID); got != "success" {
1514 t.Fatalf("status = %q", got)
1515 }
1516 if n := len(m.uploadsInto(t, "/ci/build/project")); n != 2 {
1517 t.Errorf("uploads into the clone directory = %d, want 2", n)
1518 }
1519 if strings.Contains(m.allCommandText(), "git") {
1520 t.Errorf("commands = %v", m.commands())
1521 }
1522 })
1523
1524 t.Run("clear removes and recreates the directory in one exec", func(t *testing.T) {
1525 // `rm -rf` can delete the container's WorkingDir. A second exec would
1526 // then fail to chdir before its command starts.
1527 sha := ciSeedToml(e, `
1528image = "debian:latest"
1529work_dir = "/ci/build"
1530clone_project_to = "/ci/build"
1531
1532[on]
1533manual = true
1534
1535[[steps]]
1536name = "first"
1537run_sh = "true"
1538
1539[[steps]]
1540name = "second"
1541clear = true
1542run_sh = "echo hi"
1543`)
1544 m.reset()
1545 runID := ciTrigger(e, admin, sha, nil)
1546 if got := ciWaitForRun(e, runID); got != "success" {
1547 t.Fatalf("status = %q", got)
1548 }
1549 var removals []string
1550 for _, c := range m.commands() {
1551 if joined := strings.Join(c, " "); strings.Contains(joined, "rm -rf") {
1552 removals = append(removals, joined)
1553 }
1554 }
1555 if len(removals) != 1 {
1556 t.Fatalf("rm -rf execs = %v", removals)
1557 }
1558 if !strings.Contains(removals[0], "mkdir -p") {
1559 t.Errorf("removal exec = %q", removals[0])
1560 }
1561 })
1562}
1563
1564// ── cache volumes ────────────────────────────────────────────────────────
1565
1566const ciCacheTOML = `
1567image = "debian:latest"
1568cache = ["/ci/cache/target", "/ci/cache/registry"]
1569
1570[on]
1571manual = true
1572push = ["main", "some-feature"]
1573
1574[[steps]]
1575name = "hello"
1576run_sh = "echo hi"
1577`
1578
1579func TestCICacheVolumes(t *testing.T) {
1580 e, m, admin := ciEnv(t)
1581
1582 run := func(sha, branch string) int64 {
1583 var runID int64
1584 if branch == "main" {
1585 runID = ciTrigger(e, admin, sha, nil)
1586 } else {
1587 runID = ciPushRun(e, sha, branch)
1588 }
1589 ciWaitForRun(e, runID)
1590 return runID
1591 }
1592
1593 t.Run("two cache paths sharing a prefix get distinct volumes", func(t *testing.T) {
1594 sha := ciSeedToml(e, ciCacheTOML)
1595 m.reset()
1596 run(sha, "main")
1597
1598 vols := m.createdVolumes()
1599 if len(vols) != 2 {
1600 t.Fatalf("volumes = %v", vols)
1601 }
1602 if vols[0].name == vols[1].name {
1603 t.Error("both cache paths share one volume")
1604 }
1605 // The path is otherwise unrecoverable from a digest.
1606 got := []string{
1607 vols[0].labels["com.hearthforge.cache-path"],
1608 vols[1].labels["com.hearthforge.cache-path"],
1609 }
1610 if !eqStrings(got, []string{"/ci/cache/target", "/ci/cache/registry"}) {
1611 t.Errorf("cache-path labels = %v", got)
1612 }
1613 })
1614
1615 t.Run("a volume the config no longer names is pruned", func(t *testing.T) {
1616 sha := ciSeedToml(e, ciCacheTOML)
1617 m.reset()
1618 m.setVolumesOnHost("hearthforge-ci-cache-leftover-from-an-old-config")
1619
1620 run(sha, "main")
1621
1622 if got := m.deletedVolumes(); !eqStrings(got,
1623 []string{"hearthforge-ci-cache-leftover-from-an-old-config"}) {
1624 t.Errorf("deleted = %v", got)
1625 }
1626 })
1627
1628 t.Run("volumes still in the config survive", func(t *testing.T) {
1629 sha := ciSeedToml(e, ciCacheTOML)
1630 m.reset()
1631 // Prime the host list with the names this config creates.
1632 run(sha, "main")
1633 var inUse []string
1634 for _, v := range m.createdVolumes() {
1635 inUse = append(inUse, v.name)
1636 }
1637
1638 m.reset()
1639 m.setVolumesOnHost(inUse...)
1640 run(sha, "main")
1641
1642 if got := m.deletedVolumes(); len(got) != 0 {
1643 t.Errorf("deleted = %v", got)
1644 }
1645 })
1646
1647 t.Run("a run off the default branch prunes nothing", func(t *testing.T) {
1648 sha := ciSeedToml(e, ciCacheTOML)
1649 m.reset()
1650 m.setVolumesOnHost("hearthforge-ci-cache-belongs-to-the-default-branch")
1651
1652 // The config is read per commit, so pruning from a feature branch
1653 // would delete the default branch's caches.
1654 run(sha, "some-feature")
1655
1656 if got := m.deletedVolumes(); len(got) != 0 {
1657 t.Errorf("deleted = %v", got)
1658 }
1659 })
1660}
1661
1662// ── cache size caps ──────────────────────────────────────────────────────
1663
1664const ciCappedTOML = `
1665image = "debian:latest"
1666cache = [{ path = "/ci/cache/target", max_size = "1g" }, "/ci/cache/registry"]
1667
1668[on]
1669manual = true
1670
1671[[steps]]
1672name = "hello"
1673run_sh = "echo hi"
1674`
1675
1676func TestCICacheSizeCaps(t *testing.T) {
1677 e, m, admin := ciEnv(t)
1678
1679 run := func(sha string) int64 {
1680 runID := ciTrigger(e, admin, sha, nil)
1681 ciWaitForRun(e, runID)
1682 return runID
1683 }
1684
1685 // names returns the volume names the config produces, in declaration
1686 // order.
1687 names := func(sha string) (string, string) {
1688 m.reset()
1689 run(sha)
1690 vols := m.createdVolumes()
1691 if len(vols) != 2 {
1692 t.Fatalf("volumes = %v", vols)
1693 }
1694 return vols[0].name, vols[1].name
1695 }
1696
1697 const gib = int64(1024 * 1024 * 1024)
1698
1699 t.Run("an oversized cache is dropped and reported on the run", func(t *testing.T) {
1700 sha := ciSeedToml(e, ciCappedTOML)
1701 target, registry := names(sha)
1702
1703 m.reset()
1704 m.setVolumesOnHost(target, registry)
1705 m.setVolumeUsage(map[string]ciVolumeUsage{
1706 target: {Size: 2 * gib},
1707 registry: {Size: 9 * gib},
1708 })
1709 runID := run(sha)
1710
1711 // Only the capped one goes, however large the uncapped one grows.
1712 if got := m.deletedVolumes(); !eqStrings(got, []string{target}) {
1713 t.Fatalf("deleted = %v", got)
1714 }
1715 log := ciStep(e, runID, "cache").Log
1716 if !strings.Contains(log, "/ci/cache/target") || !strings.Contains(log, "2.0G") {
1717 t.Errorf("cache step log = %q", log)
1718 }
1719 })
1720
1721 t.Run("a cache under its cap survives", func(t *testing.T) {
1722 sha := ciSeedToml(e, ciCappedTOML)
1723 target, registry := names(sha)
1724
1725 m.reset()
1726 m.setVolumesOnHost(target, registry)
1727 m.setVolumeUsage(map[string]ciVolumeUsage{target: {Size: 100}})
1728 run(sha)
1729
1730 if got := m.deletedVolumes(); len(got) != 0 {
1731 t.Errorf("deleted = %v", got)
1732 }
1733 })
1734
1735 t.Run("a cache a concurrent run holds is left alone", func(t *testing.T) {
1736 sha := ciSeedToml(e, ciCappedTOML)
1737 target, registry := names(sha)
1738
1739 m.reset()
1740 m.setVolumesOnHost(target, registry)
1741 m.setVolumeUsage(map[string]ciVolumeUsage{target: {Size: 9 * gib, RefCount: 1}})
1742 run(sha)
1743
1744 if got := m.deletedVolumes(); len(got) != 0 {
1745 t.Errorf("deleted = %v", got)
1746 }
1747 })
1748
1749 t.Run("an unmeasured cache is never dropped", func(t *testing.T) {
1750 sha := ciSeedToml(e, ciCappedTOML)
1751 target, registry := names(sha)
1752
1753 m.reset()
1754 m.setVolumesOnHost(target, registry)
1755 // Docker reports -1 for a size it has not computed.
1756 m.setVolumeUsage(map[string]ciVolumeUsage{target: {Size: -1}})
1757 runID := run(sha)
1758
1759 if got := m.deletedVolumes(); len(got) != 0 {
1760 t.Errorf("deleted = %v", got)
1761 }
1762 if steps := ciSteps(e, runID, "cache"); len(steps) != 0 {
1763 t.Errorf("cache step = %v", steps)
1764 }
1765 })
1766}
1767
1768// ── host-built archives ─────────────────────────────────────────────────
1769
1770const ciArchiveTOML = `
1771image = "debian:latest"
1772work_dir = "/ci"
1773
1774[on]
1775manual = true
1776
1777[[steps]]
1778name = "build"
1779run_sh = "echo building"
1780publish_tar = ["/ci/dist"]
1781publish_gzip = ["/ci/dist"]
1782publish_zstd = ["/ci/dist"]
1783publish_zip = ["/ci/dist"]
1784`
1785
1786// TestCIArchivesBuiltOnHost checks that publish_* archives are built from
1787// the engine's tar stream. The mock runs no tar, gzip, zstd or zip, so any
1788// exec for them would fail the test.
1789func TestCIArchivesBuiltOnHost(t *testing.T) {
1790 e, m, admin := ciEnv(t)
1791 m.queueExec(execResp{output: "building\n"})
1792 sha := ciSeedToml(e, ciArchiveTOML)
1793 runID := ciTrigger(e, admin, sha, nil)
1794 if got := ciWaitForRun(e, runID); got != "success" {
1795 t.Fatalf("status = %q", got)
1796 }
1797 for _, c := range m.commands() {
1798 if len(c) > 0 && (c[0] == "tar" || c[0] == "zip") {
1799 t.Errorf("archive tool run in the container: %v", c)
1800 }
1801 }
1802
1803 artifacts := map[string]int64{}
1804 rows, err := e.DB.QueryContext(context.Background(),
1805 `SELECT id, filename FROM ci_artifacts WHERE run_id = ?`, runID)
1806 if err != nil {
1807 t.Fatal(err)
1808 }
1809 for rows.Next() {
1810 var id int64
1811 var name string
1812 if err := rows.Scan(&id, &name); err != nil {
1813 t.Fatal(err)
1814 }
1815 artifacts[name] = id
1816 }
1817 rows.Close()
1818 if len(artifacts) != 4 {
1819 t.Fatalf("artifacts = %v", artifacts)
1820 }
1821 download := func(name string) []byte {
1822 t.Helper()
1823 id, ok := artifacts[name]
1824 if !ok {
1825 t.Fatalf("artifact %s missing from %v", name, artifacts)
1826 }
1827 return admin.get(ciRunPath(runID) + "/artifacts/" + strconv.FormatInt(id, 10)).mustStatus(200).Body
1828 }
1829 // The mock answers every archive request with one file "dist" holding
1830 // artifact-content-123.
1831 checkTar := func(name string, r io.Reader) {
1832 t.Helper()
1833 tr := tar.NewReader(r)
1834 h, err := tr.Next()
1835 if err != nil || h.Name != "dist" {
1836 t.Fatalf("%s: first entry %v, err %v", name, h, err)
1837 }
1838 data, _ := io.ReadAll(tr)
1839 if string(data) != "artifact-content-123" {
1840 t.Errorf("%s: content = %q", name, data)
1841 }
1842 }
1843
1844 t.Run("tar", func(t *testing.T) {
1845 checkTar("dist.tar", bytes.NewReader(download("dist.tar")))
1846 })
1847 t.Run("gzip", func(t *testing.T) {
1848 gz, err := gzip.NewReader(bytes.NewReader(download("dist.tar.gz")))
1849 if err != nil {
1850 t.Fatal(err)
1851 }
1852 checkTar("dist.tar.gz", gz)
1853 })
1854 t.Run("zstd", func(t *testing.T) {
1855 dec, err := zstd.NewReader(bytes.NewReader(download("dist.tar.zst")))
1856 if err != nil {
1857 t.Fatal(err)
1858 }
1859 defer dec.Close()
1860 checkTar("dist.tar.zst", dec)
1861 })
1862 t.Run("zip", func(t *testing.T) {
1863 data := download("dist.zip")
1864 zr, err := zip.NewReader(bytes.NewReader(data), int64(len(data)))
1865 if err != nil {
1866 t.Fatal(err)
1867 }
1868 if len(zr.File) != 1 || zr.File[0].Name != "dist" {
1869 t.Fatalf("zip entries = %v", zr.File)
1870 }
1871 f, _ := zr.File[0].Open()
1872 body, _ := io.ReadAll(f)
1873 if string(body) != "artifact-content-123" {
1874 t.Errorf("zip content = %q", body)
1875 }
1876 })
1877}
1878
1879// ── web UI ref writes ────────────────────────────────────────────────────
1880
1881// ciRunAfter waits for a run newer than before and returns its trigger
1882// source, commit, branch and tag.
1883func ciRunAfter(e *env, before int64) (source, commit, branch, tag string) {
1884 t := e.t
1885 t.Helper()
1886 var id int64
1887 ciWaitFor(t, "a new run", func() bool {
1888 id = ciLatestRunID(e)
1889 return id > before
1890 })
1891 var b, tg sql.NullString
1892 if err := e.DB.QueryRowContext(context.Background(),
1893 `SELECT trigger_source, commit_sha, commit_branch, commit_tag FROM ci_runs WHERE id = ?`, id).
1894 Scan(&source, &commit, &b, &tg); err != nil {
1895 t.Fatal(err)
1896 }
1897 return source, commit, b.String, tg.String
1898}
1899
1900func TestCIWebRefWrites(t *testing.T) {
1901 e, _, admin := ciEnv(t)
1902 sha := ciSeedToml(e, strings.Replace(ciSimpleTOML, `push = ["main"]`, `push = ["main"]
1903tag = true`, 1))
1904
1905 for _, tc := range []struct{ name, message string }{
1906 {"v1-light", ""},
1907 {"v1-annotated", "Release"},
1908 } {
1909 t.Run("tag "+tc.name, func(t *testing.T) {
1910 before := ciLatestRunID(e)
1911 admin.post("/ci-repo/tags/create", url.Values{
1912 "name": {tc.name}, "ref": {"main"}, "message": {tc.message},
1913 }).mustRedirect("/ci-repo/tags")
1914 source, commit, _, tag := ciRunAfter(e, before)
1915 if source != "tag" || commit != sha || tag != tc.name {
1916 t.Errorf("run = (%q, %q, %q), want (tag, %q, %q)", source, commit, tag, sha, tc.name)
1917 }
1918 })
1919 }
1920
1921 t.Run("file created on main", func(t *testing.T) {
1922 before := ciLatestRunID(e)
1923 admin.post("/ci-repo/new-file/main", url.Values{
1924 "path": {"web.txt"}, "content": {"x"},
1925 }).mustRedirect("/ci-repo/commit/")
1926 source, commit, branch, _ := ciRunAfter(e, before)
1927 if source != "push" || commit != e.headCommit("ci-repo") || branch != "main" {
1928 t.Errorf("run = (%q, %q, %q), want (push, HEAD, main)", source, commit, branch)
1929 }
1930 })
1931}
1932