hearthforge

hearthforge/internal/web/e2e/ci_test.go
ci_test.go
⎇
Raw
1package e2e
2
3import (
4 "archive/tar"
5 "archive/zip"
6 "bytes"
7 "compress/gzip"
8 "context"
9 "database/sql"
10 "encoding/json"
11 "io"
12 "net/http"
13 "net/url"
14 "os"
15 "path/filepath"
16 "slices"
17 "strconv"
18 "strings"
19 "testing"
20 "time"
21
22 "github.com/klauspost/compress/zstd"
23)
24
25// The CI suite drives the real pipeline runner against a mock Docker Engine
26// on a unix socket, so no container engine is needed.
27
28const ciSimpleTOML = `
29image = "debian:latest"
30
31[on]
32manual = true
33push = ["main"]
34
35[[steps]]
36name = "hello"
37run_sh = "echo hello"
38`
39
40const ciArtifactTOML = `
41image = "debian:latest"
42work_dir = "/ci"
43
44[on]
45manual = true
46
47[[steps]]
48name = "build"
49run_sh = "echo building"
50publish_file = ["/ci/output.txt"]
51`
52
53// ciEnv starts a server wired to a fresh mock engine and seeds "ci-repo".
54// extraEnv holds further environment pairs for newEnv.
55func ciEnv(t *testing.T, extraEnv ...string) (*env, *mockDocker, *session) {
56 t.Helper()
57 m := newMockDocker(t)
58 e := newEnv(t, append([]string{"CI_DOCKER_SOCKET", m.sock}, extraEnv...)...)
59 admin := e.admin()
60 e.createRepo(admin, "ci-repo")
61 e.seedRepo("ci-repo", nil)
62 return e, m, admin
63}
64
65// ciSeedToml pushes a .hearthforge-ci.toml into ci-repo and returns the
66// commit sha. Re-seeding the same content is a no-op commit, so a test can
67// put its config back without failing.
68func ciSeedToml(e *env, toml string) string {
69 t := e.t
70 t.Helper()
71 work := t.TempDir()
72 gitRun(t, work, "clone", "-q", e.repoPath("ci-repo"), ".")
73 if err := os.WriteFile(filepath.Join(work, ".hearthforge-ci.toml"), []byte(toml), 0o644); err != nil {
74 t.Fatal(err)
75 }
76 gitRun(t, work, "add", ".hearthforge-ci.toml")
77 // Nothing to commit when the config is unchanged.
78 _, _ = gitTry(work, "commit", "-q", "-m", "Add CI config")
79 gitRun(t, work, "push", "-q", "origin", "HEAD:main")
80 e.Srv.Git.InvalidateRefCache("ci-repo")
81 return gitRun(t, work, "rev-parse", "HEAD")
82}
83
84// ciTrigger posts the manual trigger and returns the new run id. The route
85// always builds HEAD of the default branch, so the expected sha is checked
86// and a stale fixture fails loudly.
87func ciTrigger(e *env, admin *session, sha string, overrides url.Values) int64 {
88 t := e.t
89 t.Helper()
90 if head := e.headCommit("ci-repo"); head != sha {
91 t.Fatalf("ciTrigger: expected HEAD %s, repo HEAD is %s", sha, head)
92 }
93 if overrides == nil {
94 overrides = url.Values{}
95 }
96 loc := admin.post("/ci-repo/ci/run", overrides).mustRedirect("/ci-repo/ci/")
97 id, err := strconv.ParseInt(idFromPath(t, loc), 10, 64)
98 if err != nil {
99 t.Fatalf("run id in %q: %v", loc, err)
100 }
101 return id
102}
103
104// ciLatestRunID is the highest run id in the database, or 0.
105func ciLatestRunID(e *env) int64 {
106 var id sql.NullInt64
107 if err := e.DB.QueryRowContext(context.Background(),
108 `SELECT MAX(id) FROM ci_runs`).Scan(&id); err != nil {
109 e.t.Fatal(err)
110 }
111 return id.Int64
112}
113
114// ciPushRun pushes a commit to a branch over HTTP and returns the run the
115// push trigger created. The manual route always builds the default branch.
116func ciPushRun(e *env, sha, branch string) int64 {
117 t := e.t
118 t.Helper()
119 before := ciLatestRunID(e)
120 gitRun(t, e.repoPath("ci-repo"), "push", "--force", e.authURL("ci-repo"),
121 sha+":refs/heads/"+branch)
122 e.Srv.Git.InvalidateRefCache("ci-repo")
123 var id int64
124 ciWaitFor(t, "a run from the push to "+branch, func() bool {
125 id = ciLatestRunID(e)
126 return id > before
127 })
128 return id
129}
130
131// ciWaitForRun polls until the run leaves pending/running/queued.
132func ciWaitForRun(e *env, runID int64, timeout ...time.Duration) string {
133 t := e.t
134 t.Helper()
135 limit := 15 * time.Second
136 if len(timeout) > 0 {
137 limit = timeout[0]
138 }
139 var status string
140 ciWaitFor(t, "run "+strconv.FormatInt(runID, 10)+" to complete", func() bool {
141 status = ciRunStatus(e, runID)
142 return status != "" && status != "pending" && status != "running" && status != "queued" &&
143 !e.Srv.CI.Active(runID)
144 }, limit)
145 return status
146}
147
148func ciRunStatus(e *env, runID int64) string {
149 var status string
150 err := e.DB.QueryRowContext(context.Background(),
151 `SELECT status FROM ci_runs WHERE id = ?`, runID).Scan(&status)
152 if err != nil {
153 e.t.Fatalf("run %d: %v", runID, err)
154 }
155 return status
156}
157
158// ciStepRow is one row of ci_steps.
159type ciStepRow struct {
160 Status string
161 Log string
162}
163
164// ciSteps returns every step of a run with that name, in insertion order.
165func ciSteps(e *env, runID int64, name string) []ciStepRow {
166 rows, err := e.DB.QueryContext(context.Background(),
167 `SELECT status, log FROM ci_steps WHERE run_id = ? AND name = ? ORDER BY id ASC`,
168 runID, name)
169 if err != nil {
170 e.t.Fatal(err)
171 }
172 defer rows.Close()
173 var out []ciStepRow
174 for rows.Next() {
175 var s ciStepRow
176 if err := rows.Scan(&s.Status, &s.Log); err != nil {
177 e.t.Fatal(err)
178 }
179 out = append(out, s)
180 }
181 return out
182}
183
184// ciStep returns the first step of a run with that name.
185func ciStep(e *env, runID int64, name string) ciStepRow {
186 steps := ciSteps(e, runID, name)
187 if len(steps) == 0 {
188 e.t.Fatalf("run %d has no step %q", runID, name)
189 }
190 return steps[0]
191}
192
193// ciOverrides decodes the stored variable overrides of a run.
194func ciOverrides(e *env, runID int64) map[string]string {
195 var raw sql.NullString
196 if err := e.DB.QueryRowContext(context.Background(),
197 `SELECT variable_overrides FROM ci_runs WHERE id = ?`, runID).Scan(&raw); err != nil {
198 e.t.Fatal(err)
199 }
200 out := map[string]string{}
201 if raw.String != "" {
202 if err := json.Unmarshal([]byte(raw.String), &out); err != nil {
203 e.t.Fatalf("overrides %q: %v", raw.String, err)
204 }
205 }
206 return out
207}
208
209func ciRunPath(runID int64) string { return "/ci-repo/ci/" + strconv.FormatInt(runID, 10) }
210
211// eqStrings compares two string lists.
212func eqStrings(a, b []string) bool {
213 if len(a) != len(b) {
214 return false
215 }
216 for i := range a {
217 if a[i] != b[i] {
218 return false
219 }
220 }
221 return true
222}
223
224// ── pipelines tab ────────────────────────────────────────────────────────
225
226func TestCIPipelinesTab(t *testing.T) {
227 e, m, admin := ciEnv(t)
228
229 t.Run("tab is visible in repo nav", func(t *testing.T) {
230 r := admin.get("/ci-repo").mustStatus(200)
231 if !contains(r.Texts(".repo-tab"), "Pipelines") {
232 t.Errorf("repo tabs = %v", r.Texts(".repo-tab"))
233 }
234 })
235
236 t.Run("history page shows empty state when no runs", func(t *testing.T) {
237 r := admin.get("/ci-repo/ci").mustStatus(200)
238 if !r.Has(".empty-state") {
239 t.Fatal("empty state missing")
240 }
241 if !strings.Contains(r.Text(".empty-state"), "No pipeline runs yet") {
242 t.Errorf("empty state = %q", r.Text(".empty-state"))
243 }
244 })
245
246 t.Run("the run form posts and overrides a declared variable", func(t *testing.T) {
247 // Regression: an input-less form posted an empty body and the route
248 // crashed whenever the config declared a variable.
249 sha := ciSeedToml(e, `
250image = "debian:latest"
251
252[on]
253manual = true
254
255[variables]
256 [variables.GREETING]
257 default = "hello"
258 description = "What to echo"
259
260[[steps]]
261name = "say"
262run_sh = "echo $GREETING"
263`)
264 m.reset()
265 m.queueExec(execResp{output: "hi\n"})
266
267 r := admin.get("/ci-repo/ci").mustStatus(200)
268 if got := r.Value(`input[name="var_GREETING"]`); got != "hello" {
269 t.Fatalf("var_GREETING default = %q", got)
270 }
271 runID := ciTrigger(e, admin, sha, url.Values{"var_GREETING": {"goodbye"}})
272 ciWaitForRun(e, runID)
273
274 got := ciOverrides(e, runID)
275 if len(got) != 1 || got["GREETING"] != "goodbye" {
276 t.Errorf("overrides = %v", got)
277 }
278 })
279
280 t.Run("an untouched variable field is not recorded as an override", func(t *testing.T) {
281 sha := ciSeedToml(e, `
282image = "debian:latest"
283
284[on]
285manual = true
286
287[variables]
288 [variables.GREETING]
289 default = "hello"
290
291[[steps]]
292name = "say"
293run_sh = "echo $GREETING"
294`)
295 m.reset()
296 m.queueExec(execResp{output: "hi\n"})
297
298 runID := ciTrigger(e, admin, sha, url.Values{"var_GREETING": {"hello"}})
299 ciWaitForRun(e, runID)
300
301 if got := ciOverrides(e, runID); len(got) != 0 {
302 t.Errorf("overrides = %v, want none", got)
303 }
304 })
305
306 t.Run("an empty POST to the run route does not crash", func(t *testing.T) {
307 sha := ciSeedToml(e, `
308image = "debian:latest"
309
310[on]
311manual = true
312
313[variables]
314 [variables.GREETING]
315 default = "hello"
316
317[[steps]]
318name = "say"
319run_sh = "echo $GREETING"
320`)
321 _ = sha
322 m.reset()
323 m.queueExec(execResp{output: "hi\n"})
324
325 r := admin.post("/ci-repo/ci/run", url.Values{})
326 if r.Code != http.StatusFound {
327 t.Fatalf("status = %d, body %s", r.Code, r.BodyString())
328 }
329 id, err := strconv.ParseInt(idFromPath(t, r.Location()), 10, 64)
330 if err != nil {
331 t.Fatal(err)
332 }
333 ciWaitForRun(e, id)
334 })
335
336 t.Run("help section is collapsible and contains template download", func(t *testing.T) {
337 r := admin.get("/ci-repo/ci").mustStatus(200)
338 if !r.Has("details.ci-help") {
339 t.Error("help section missing")
340 }
341 if !r.Has(`a[download=".hearthforge-ci.toml"]`) {
342 t.Error("template download link missing")
343 }
344 })
345}
346
347// ── successful run ───────────────────────────────────────────────────────
348
349func TestCISuccessfulRun(t *testing.T) {
350 e, m, admin := ciEnv(t)
351 m.queueExec(execResp{output: "hello from mock CI\n"})
352 sha := ciSeedToml(e, ciSimpleTOML)
353 runID := ciTrigger(e, admin, sha, nil)
354 ciWaitForRun(e, runID)
355
356 t.Run("run status is success", func(t *testing.T) {
357 if got := ciRunStatus(e, runID); got != "success" {
358 t.Errorf("status = %q", got)
359 }
360 })
361
362 t.Run("step status is success and log is captured", func(t *testing.T) {
363 step := ciStep(e, runID, "hello")
364 if step.Status != "success" {
365 t.Errorf("step status = %q", step.Status)
366 }
367 if !strings.Contains(step.Log, "hello from mock CI") {
368 t.Errorf("step log = %q", step.Log)
369 }
370 })
371
372 t.Run("history page shows the completed run", func(t *testing.T) {
373 r := admin.get("/ci-repo/ci").mustStatus(200)
374 if r.Count(".ci-status-pill.ci-status-success") == 0 {
375 t.Error("no success pill on the history page")
376 }
377 if r.Count(`a.commit-hash[href^="/ci-repo/commit/"]`) == 0 {
378 t.Error("commit hash is not a link to the commit page")
379 }
380 // The trigger label uses the owner display name, like every other
381 // place that names the admin.
382 if !contains(r.Texts(".text-muted"), "by "+e.Cfg.OwnerDisplayName) || contains(r.Texts(".text-muted"), "by admin") {
383 t.Errorf("trigger labels = %v", r.Texts(".text-muted"))
384 }
385 })
386
387 t.Run("run detail page shows step and log", func(t *testing.T) {
388 r := admin.get(ciRunPath(runID)).mustStatus(200)
389 steps := r.Texts(".ci-step")
390 if len(steps) < 2 {
391 t.Fatalf("steps = %v", steps)
392 }
393 // Setup is a real step and sorts before the config's steps.
394 if !strings.Contains(steps[0], "pipeline setup") || !strings.Contains(steps[0], "success") {
395 t.Errorf("first step = %q", steps[0])
396 }
397 if !strings.Contains(steps[1], "hello") {
398 t.Errorf("second step = %q", steps[1])
399 }
400 if !contains(r.Texts(".ci-step-log"), "hello from mock CI") {
401 t.Errorf("logs = %v", r.Texts(".ci-step-log"))
402 }
403 })
404
405 t.Run("retry re-executes the same run in-place", func(t *testing.T) {
406 r := admin.post(ciRunPath(runID)+"/retry", url.Values{})
407 if got := r.mustRedirect(ciRunPath(runID)); got != ciRunPath(runID) {
408 t.Errorf("redirect = %q", got)
409 }
410 if got := ciWaitForRun(e, runID); got != "success" {
411 t.Errorf("status after retry = %q", got)
412 }
413 // No new run was created: the row still exists under the same id.
414 if ciRunStatus(e, runID) != "success" {
415 t.Error("run row changed")
416 }
417 })
418}
419
420// ── failing run ──────────────────────────────────────────────────────────
421
422func TestCIFailingRun(t *testing.T) {
423 e, m, admin := ciEnv(t)
424 m.queueExec(execResp{output: "build error: file not found\n", exitCode: 1})
425 sha := ciSeedToml(e, ciSimpleTOML)
426 runID := ciTrigger(e, admin, sha, nil)
427 ciWaitForRun(e, runID)
428
429 t.Run("run status is failure", func(t *testing.T) {
430 if got := ciRunStatus(e, runID); got != "failure" {
431 t.Errorf("status = %q", got)
432 }
433 })
434
435 t.Run("step status is failure and error log captured", func(t *testing.T) {
436 step := ciStep(e, runID, "hello")
437 if step.Status != "failure" {
438 t.Errorf("step status = %q", step.Status)
439 }
440 if !strings.Contains(step.Log, "build error") {
441 t.Errorf("step log = %q", step.Log)
442 }
443 })
444
445 t.Run("run detail page shows failure status", func(t *testing.T) {
446 r := admin.get(ciRunPath(runID)).mustStatus(200)
447 if r.Count(".ci-status-pill.ci-status-failure") == 0 {
448 t.Error("no failure pill on the run page")
449 }
450 })
451}
452
453// ── cancel ───────────────────────────────────────────────────────────────
454
455func TestCICancel(t *testing.T) {
456 e, _, admin := ciEnv(t)
457
458 t.Run("cancelling a pending run marks it cancelled", func(t *testing.T) {
459 sha := ciSeedToml(e, ciSimpleTOML)
460 runID := ciTrigger(e, admin, sha, nil)
461 admin.post(ciRunPath(runID)+"/cancel", url.Values{}).mustRedirect(ciRunPath(runID))
462
463 status := ciWaitForRun(e, runID)
464 switch status {
465 case "cancelled", "success", "failure":
466 default:
467 t.Fatalf("status = %q", status)
468 }
469 })
470}
471
472const ciSlowTOML = `
473image = "debian:latest"
474
475[on]
476manual = true
477
478[[steps]]
479name = "slow"
480run_sh = "sleep 60"
481
482[[steps]]
483name = "after"
484run_sh = "echo after"
485`
486
487// ciStepStatus is the status of the first step with that name, or "".
488func ciStepStatus(e *env, runID int64, name string) string {
489 if steps := ciSteps(e, runID, name); len(steps) > 0 {
490 return steps[0].Status
491 }
492 return ""
493}
494
495// ciWaitFor polls cond for up to ten seconds, or the given timeout.
496func ciWaitFor(t *testing.T, what string, cond func() bool, timeout ...time.Duration) {
497 t.Helper()
498 limit := 10 * time.Second
499 if len(timeout) > 0 {
500 limit = timeout[0]
501 }
502 for deadline := time.Now().Add(limit); time.Now().Before(deadline); {
503 if cond() {
504 return
505 }
506 time.Sleep(20 * time.Millisecond)
507 }
508 t.Fatalf("timed out waiting for %s", what)
509}
510
511func TestCICancelMidRun(t *testing.T) {
512 e, m, admin := ciEnv(t)
513
514 t.Run("the interrupted step reads cancelled", func(t *testing.T) {
515 sha := ciSeedToml(e, ciSlowTOML)
516 m.reset()
517 m.queueExec(execResp{delay: 30 * time.Second})
518 runID := ciTrigger(e, admin, sha, nil)
519 ciWaitFor(t, "slow step to run", func() bool { return ciStepStatus(e, runID, "slow") == "running" })
520
521 admin.post(ciRunPath(runID)+"/cancel", url.Values{}).mustRedirect(ciRunPath(runID))
522 ciWaitFor(t, "slow step to settle", func() bool { return ciStepStatus(e, runID, "slow") != "running" })
523 if got := ciStep(e, runID, "slow").Status; got != "cancelled" {
524 t.Errorf("slow status = %q", got)
525 }
526 if got := ciStep(e, runID, "after").Status; got != "skipped" {
527 t.Errorf("after status = %q", got)
528 }
529 if got := ciRunStatus(e, runID); got != "cancelled" {
530 t.Errorf("run status = %q", got)
531 }
532 })
533
534 t.Run("a cancel during artifact collection is not overwritten", func(t *testing.T) {
535 sha := ciSeedToml(e, ciArtifactTOML)
536 m.reset()
537 m.archiveDelay = 30 * time.Second
538 runID := ciTrigger(e, admin, sha, nil)
539 ciWaitFor(t, "artifact download", func() bool {
540 return slices.Contains(m.containerRequests(), "GET archive")
541 })
542
543 admin.post(ciRunPath(runID)+"/cancel", url.Values{}).mustRedirect(ciRunPath(runID))
544 ciWaitFor(t, "build step to settle", func() bool { return ciStepStatus(e, runID, "build") != "running" })
545 // The final status write follows the step write within milliseconds.
546 time.Sleep(300 * time.Millisecond)
547 if got := ciRunStatus(e, runID); got != "cancelled" {
548 t.Errorf("run status = %q", got)
549 }
550 })
551}
552
553func TestCIContainerLifecycle(t *testing.T) {
554 e, m, admin := ciEnv(t)
555 sha := ciSeedToml(e, ciSimpleTOML)
556 m.reset()
557 runID := ciTrigger(e, admin, sha, nil)
558 ciWaitForRun(e, runID)
559
560 name := "hearthforge-ci-" + strconv.FormatInt(runID, 10)
561 reqs := m.containerRequests()
562 // A leftover from a cancelled create must not block the name.
563 if len(reqs) < 2 || reqs[0] != "DELETE "+name || reqs[1] != "POST create?name="+name {
564 t.Errorf("requests = %v", reqs)
565 }
566 labels, _ := m.createBody()["Labels"].(map[string]any)
567 if labels["com.hearthforge.ci"] != "1" {
568 t.Errorf("labels = %v", labels)
569 }
570}
571
572func TestCIRepoDeleteStopsRuns(t *testing.T) {
573 e, m, admin := ciEnv(t)
574 sha := ciSeedToml(e, ciArtifactTOML)
575 m.reset()
576 done := ciTrigger(e, admin, sha, nil)
577 ciWaitForRun(e, done)
578 artifacts := filepath.Join(e.Cfg.CIArtifactsDir(), strconv.FormatInt(done, 10))
579 if _, err := os.Stat(artifacts); err != nil {
580 t.Fatalf("artifacts of run %d: %v", done, err)
581 }
582
583 m.queueExec(execResp{delay: 30 * time.Second})
584 active := ciTrigger(e, admin, sha, nil)
585 ciWaitFor(t, "build step to run", func() bool { return ciStepStatus(e, active, "build") == "running" })
586 m.setVolumesOnHost("hearthforge-ci-cache-old")
587
588 admin.post("/ci-repo/settings/delete", nil).mustRedirect("/")
589
590 if !slices.Contains(m.containerRequests(), "DELETE hearthforge-ci-"+strconv.FormatInt(active, 10)) {
591 t.Errorf("active container not removed: %v", m.containerRequests())
592 }
593 if _, err := os.Stat(artifacts); !os.IsNotExist(err) {
594 t.Errorf("artifacts still there: %v", err)
595 }
596 if !slices.Contains(m.deletedVolumes(), "hearthforge-ci-cache-old") {
597 t.Errorf("volumes deleted = %v", m.deletedVolumes())
598 }
599}
600
601// ── artifacts ────────────────────────────────────────────────────────────
602
603func TestCIArtifacts(t *testing.T) {
604 e, _, admin := ciEnv(t)
605 sha := ciSeedToml(e, ciArtifactTOML)
606 runID := ciTrigger(e, admin, sha, nil)
607 ciWaitForRun(e, runID)
608
609 var artifactID int64
610 var filename string
611 var count int
612 rows, err := e.DB.QueryContext(context.Background(),
613 `SELECT id, filename FROM ci_artifacts WHERE run_id = ? ORDER BY id`, runID)
614 if err != nil {
615 t.Fatal(err)
616 }
617 for rows.Next() {
618 if err := rows.Scan(&artifactID, &filename); err != nil {
619 t.Fatal(err)
620 }
621 count++
622 }
623 rows.Close()
624
625 t.Run("artifact row created in DB", func(t *testing.T) {
626 if count != 1 {
627 t.Fatalf("artifacts = %d", count)
628 }
629 if filename != "output.txt" {
630 t.Errorf("filename = %q", filename)
631 }
632 })
633
634 t.Run("artifact is downloadable via HTTP", func(t *testing.T) {
635 if artifactID <= 0 {
636 t.Fatal("no artifact id")
637 }
638 r := e.anon().get(ciRunPath(runID) + "/artifacts/" + strconv.FormatInt(artifactID, 10))
639 r.mustStatus(200)
640 if r.BodyString() != "artifact-content-123" {
641 t.Errorf("body = %q", r.BodyString())
642 }
643 })
644
645 t.Run("run detail page shows artifact list", func(t *testing.T) {
646 r := admin.get(ciRunPath(runID)).mustStatus(200)
647 if r.Count(".ci-artifact-item") == 0 {
648 t.Fatal("no artifact items")
649 }
650 if !strings.Contains(r.Text(".ci-artifact-name"), "output.txt") {
651 t.Errorf("artifact name = %q", r.Text(".ci-artifact-name"))
652 }
653 })
654}
655
656// ── badge ────────────────────────────────────────────────────────────────
657
658func TestCIBadge(t *testing.T) {
659 e, m, admin := ciEnv(t)
660 m.queueExec(execResp{output: "ok\n"})
661 sha := ciSeedToml(e, ciSimpleTOML)
662 runID := ciTrigger(e, admin, sha, nil)
663 ciWaitForRun(e, runID)
664
665 t.Run("badge SVG returns success status after successful run", func(t *testing.T) {
666 r := e.anon().get("/ci-repo/ci/badge.svg").mustStatus(200)
667 if !strings.Contains(r.Header.Get("Content-Type"), "image/svg+xml") {
668 t.Errorf("content type = %q", r.Header.Get("Content-Type"))
669 }
670 if !r.Contains("<svg") || !r.Contains("success") {
671 t.Errorf("badge = %q", r.BodyString())
672 }
673 })
674
675 t.Run("badge returns 404 for private repo when not logged in", func(t *testing.T) {
676 e.createRepo(admin, "private-ci-repo", "is_private", "1")
677 e.anon().get("/private-ci-repo/ci/badge.svg").mustStatus(404)
678 })
679}
680
681// ── secrets ──────────────────────────────────────────────────────────────
682
683func TestCISecrets(t *testing.T) {
684 e, m, admin := ciEnv(t)
685
686 t.Run("can add, list, and delete a secret via settings", func(t *testing.T) {
687 admin.post("/ci-repo/settings/ci-secrets", url.Values{
688 "name": {"MY_SECRET"}, "value": {"super-secret-value"},
689 "description": {"A test secret"},
690 }).mustRedirect("/ci-repo/settings")
691
692 r := admin.get("/ci-repo/settings").mustStatus(200)
693 if n := len(matchingTexts(r.Texts("code"), "MY_SECRET")); n != 1 {
694 t.Fatalf("MY_SECRET shown %d times", n)
695 }
696 if !r.Contains("●●●●●●") {
697 t.Error("secret value is not masked")
698 }
699
700 id := r.Value(`form[action="/ci-repo/settings/ci-secrets/delete"] input[name=id]`)
701 if id == "" {
702 t.Fatal("no delete form for the secret")
703 }
704 admin.post("/ci-repo/settings/ci-secrets/delete", url.Values{"id": {id}}).
705 mustRedirect("/ci-repo/settings")
706
707 r = admin.get("/ci-repo/settings").mustStatus(200)
708 if n := len(matchingTexts(r.Texts("code"), "MY_SECRET")); n != 0 {
709 t.Errorf("MY_SECRET still shown %d times", n)
710 }
711 })
712
713 t.Run("secret value is masked in step logs", func(t *testing.T) {
714 admin.post("/ci-repo/settings/ci-secrets", url.Values{
715 "name": {"MASK_ME"}, "value": {"s3cr3t-p4ssw0rd"},
716 }).mustRedirect("/ci-repo/settings")
717
718 m.reset()
719 m.queueExec(execResp{output: "s3cr3t-p4ssw0rd is the value\n"})
720 sha := ciSeedToml(e, ciSimpleTOML)
721 runID := ciTrigger(e, admin, sha, nil)
722 ciWaitForRun(e, runID)
723
724 step := ciStep(e, runID, "hello")
725 if strings.Contains(step.Log, "s3cr3t-p4ssw0rd") {
726 t.Errorf("secret leaked into the log: %q", step.Log)
727 }
728 if !strings.Contains(step.Log, "[MASKED]") {
729 t.Errorf("log = %q", step.Log)
730 }
731 })
732}
733
734// matchingTexts keeps the entries containing sub.
735func matchingTexts(list []string, sub string) []string {
736 var out []string
737 for _, s := range list {
738 if strings.Contains(s, sub) {
739 out = append(out, s)
740 }
741 }
742 return out
743}
744
745// ── per-repo run IDs ─────────────────────────────────────────────────────
746
747func TestCIPerRepoRunIDs(t *testing.T) {
748 e, m, admin := ciEnv(t)
749 sha := ciSeedToml(e, ciSimpleTOML)
750 for range 2 {
751 m.reset()
752 ciWaitForRun(e, ciTrigger(e, admin, sha, nil))
753 }
754
755 t.Run("repo_run_id is set and increments per repo", func(t *testing.T) {
756 rows, err := e.DB.QueryContext(context.Background(),
757 `SELECT repo_run_id FROM ci_runs ORDER BY repo_run_id ASC`)
758 if err != nil {
759 t.Fatal(err)
760 }
761 defer rows.Close()
762 i := 0
763 for rows.Next() {
764 var id sql.NullInt64
765 if err := rows.Scan(&id); err != nil {
766 t.Fatal(err)
767 }
768 if !id.Valid || id.Int64 <= 0 {
769 t.Fatal("repo_run_id is not set")
770 }
771 i++
772 if id.Int64 != int64(i) {
773 t.Fatalf("repo_run_id %d at position %d", id.Int64, i)
774 }
775 }
776 if i == 0 {
777 t.Fatal("no runs")
778 }
779 })
780
781 t.Run("run detail page shows repo-local run number", func(t *testing.T) {
782 var runID, repoRunID int64
783 if err := e.DB.QueryRowContext(context.Background(),
784 `SELECT id, repo_run_id FROM ci_runs ORDER BY id ASC LIMIT 1`).
785 Scan(&runID, &repoRunID); err != nil {
786 t.Fatal(err)
787 }
788 r := admin.get(ciRunPath(runID)).mustStatus(200)
789 want := "#" + strconv.FormatInt(repoRunID, 10)
790 if !strings.Contains(r.Text("h2"), want) {
791 t.Errorf("heading = %q, want %q", r.Text("h2"), want)
792 }
793 })
794}
795
796// ── skip reasons ─────────────────────────────────────────────────────────
797
798const ciSkipIfTOML = `
799image = "debian:latest"
800
801[on]
802manual = true
803
804[[steps]]
805name = "first"
806run_sh = "echo first"
807
808[[steps]]
809name = "second"
810run_if = "false"
811run_sh = "echo second"
812
813[[steps]]
814name = "third"
815run_sh = "echo third"
816`
817
818func TestCISkipReasons(t *testing.T) {
819 e, m, admin := ciEnv(t)
820
821 t.Run("run_if failure sets skip reason in log", func(t *testing.T) {
822 sha := ciSeedToml(e, ciSkipIfTOML)
823 m.reset()
824 m.queueExec(execResp{output: "first\n"}) // first step
825 m.queueExec(execResp{exitCode: 1}) // run_if check of second
826 m.queueExec(execResp{output: "third\n"}) // third step
827 runID := ciTrigger(e, admin, sha, nil)
828 ciWaitForRun(e, runID)
829
830 step := ciStep(e, runID, "second")
831 if step.Status != "skipped" {
832 t.Errorf("status = %q", step.Status)
833 }
834 if !strings.Contains(step.Log, "condition not met") {
835 t.Errorf("log = %q", step.Log)
836 }
837 })
838
839 t.Run("failed step causes remaining steps to be skipped with reason", func(t *testing.T) {
840 sha := ciSeedToml(e, ciSkipIfTOML)
841 m.reset()
842 m.queueExec(execResp{output: "boom\n", exitCode: 1}) // first step fails
843 runID := ciTrigger(e, admin, sha, nil)
844 ciWaitForRun(e, runID)
845
846 step := ciStep(e, runID, "third")
847 if step.Status != "skipped" {
848 t.Errorf("status = %q", step.Status)
849 }
850 if !strings.Contains(step.Log, "previous step failed") {
851 t.Errorf("log = %q", step.Log)
852 }
853 })
854}
855
856// ── docker unavailable ───────────────────────────────────────────────────
857
858func TestCIDockerUnavailable(t *testing.T) {
859 // newEnv points CI_DOCKER_SOCKET at a path that does not exist.
860 e := newEnv(t)
861 admin := e.admin()
862 e.createRepo(admin, "ci-repo")
863 e.seedRepo("ci-repo", nil)
864
865 t.Run("run is marked skipped when docker socket is missing", func(t *testing.T) {
866 sha := ciSeedToml(e, ciSimpleTOML)
867 runID := ciTrigger(e, admin, sha, nil)
868 if got := ciWaitForRun(e, runID); got != "skipped" {
869 t.Errorf("status = %q", got)
870 }
871 })
872}
873
874// ── manual trigger without on.manual ─────────────────────────────────────
875
876const ciNoManualTOML = `
877image = "debian:latest"
878
879[on]
880push = ["main"]
881
882[[steps]]
883name = "hello"
884run_sh = "echo hi"
885`
886
887func TestCIManualTriggerWithoutOnManual(t *testing.T) {
888 e, m, admin := ciEnv(t)
889
890 t.Run("manual run is allowed even without manual = true in config", func(t *testing.T) {
891 sha := ciSeedToml(e, ciNoManualTOML)
892 m.reset()
893 m.queueExec(execResp{output: "hi\n"})
894 runID := ciTrigger(e, admin, sha, nil)
895 if got := ciWaitForRun(e, runID); got != "success" {
896 t.Errorf("status = %q", got)
897 }
898 })
899
900 t.Run("Run pipeline button is not disabled when toml lacks manual = true", func(t *testing.T) {
901 ciSeedToml(e, ciNoManualTOML)
902 r := admin.get("/ci-repo/ci").mustStatus(200)
903 if !contains(r.Texts("button"), "Run pipeline") {
904 t.Fatalf("buttons = %v", r.Texts("button"))
905 }
906 if r.Has("button[disabled]") {
907 t.Error("the Run pipeline button is disabled")
908 }
909 })
910}
911
912// ── auto-refresh toggle ──────────────────────────────────────────────────
913
914func TestCIAutoRefreshToggle(t *testing.T) {
915 e, _, admin := ciEnv(t)
916
917 t.Run("Pause refresh button appears on active run and ?refresh=off shows Resume", func(t *testing.T) {
918 sha := ciSeedToml(e, ciSimpleTOML)
919 runID := ciTrigger(e, admin, sha, nil)
920
921 // The run may already have finished, so only the refresh link's
922 // existence is checked, exactly as the browser test did.
923 admin.get(ciRunPath(runID)).mustStatus(200)
924
925 r := admin.get(ciRunPath(runID) + "?refresh=off").mustStatus(200)
926 if n := r.Count(`meta[http-equiv="refresh"]`); n != 0 {
927 t.Errorf("meta refresh count = %d", n)
928 }
929 ciWaitForRun(e, runID)
930 })
931}
932
933// ── purge cache ──────────────────────────────────────────────────────────
934
935func TestCIPurgeCache(t *testing.T) {
936 _, _, admin := ciEnv(t)
937
938 t.Run("Purge caches button is visible and submits successfully", func(t *testing.T) {
939 r := admin.get("/ci-repo/ci").mustStatus(200)
940 if !contains(r.Texts("button"), "Purge caches") {
941 t.Fatalf("buttons = %v", r.Texts("button"))
942 }
943 redirect := admin.post("/ci-repo/ci/purge-cache", url.Values{})
944 redirect.mustRedirect("/ci-repo/ci")
945
946 r = admin.follow(redirect).mustStatus(200)
947 if !strings.Contains(r.Text("h2"), "Pipelines") {
948 t.Errorf("heading = %q", r.Text("h2"))
949 }
950 msg := r.Text(".form-success, .form-error")
951 if !strings.Contains(strings.ToLower(msg), "purge") {
952 t.Errorf("message = %q", msg)
953 }
954 })
955}
956
957// ── repo upload ──────────────────────────────────────────────────────────
958
959const ciCloneTOML = `
960image = "debian:latest"
961work_dir = "/ci/build"
962clone_project_to = "/ci/build/project"
963
964[on]
965manual = true
966
967[[steps]]
968name = "hello"
969run_sh = "echo hi"
970`
971
972func TestCIRepoUpload(t *testing.T) {
973 e, m, admin := ciEnv(t)
974 ciSeedToml(e, ciCloneTOML)
975
976 // Sibling subtests reseed the config, and the trigger route always builds
977 // HEAD. Put the clone config back first.
978 trigger := func() int64 {
979 sha := ciSeedToml(e, ciCloneTOML)
980 return ciTrigger(e, admin, sha, nil)
981 }
982
983 t.Run("the checkout is uploaded, not bind-mounted", func(t *testing.T) {
984 m.reset()
985 runID := trigger()
986 if got := ciWaitForRun(e, runID); got != "success" {
987 t.Fatalf("status = %q", got)
988 }
989 if len(m.uploadsInto(t, "/ci/build/project")) == 0 {
990 t.Fatalf("no upload carries files under the clone directory; uploads = %v", m.uploadedPaths())
991 }
992 binds, _ := json.Marshal(m.createBody()["HostConfig"])
993 if strings.Contains(string(binds), e.DataDir) {
994 t.Errorf("binds reference the data directory: %s", binds)
995 }
996 })
997
998 t.Run("the container never runs git", func(t *testing.T) {
999 m.reset()
1000 runID := trigger()
1001 if got := ciWaitForRun(e, runID); got != "success" {
1002 t.Fatalf("status = %q", got)
1003 }
1004 if strings.Contains(m.allCommandText(), "git") {
1005 t.Errorf("commands = %v", m.commands())
1006 }
1007 })
1008
1009 t.Run("a failing checkout fails the run before any step runs", func(t *testing.T) {
1010 m.reset()
1011 m.uploadError = "read-only file system"
1012
1013 runID := trigger()
1014 if got := ciWaitForRun(e, runID); got != "failure" {
1015 t.Fatalf("status = %q", got)
1016 }
1017 if got := ciStep(e, runID, "hello").Status; got != "skipped" {
1018 t.Errorf("hello status = %q", got)
1019 }
1020 setup := ciStep(e, runID, "pipeline setup")
1021 if setup.Status != "failure" {
1022 t.Errorf("setup status = %q", setup.Status)
1023 }
1024 if !strings.Contains(setup.Log, "read-only file system") {
1025 t.Errorf("setup log = %q", setup.Log)
1026 }
1027 })
1028
1029 t.Run("a cache path inside the clone directory is rejected", func(t *testing.T) {
1030 m.reset()
1031 sha := ciSeedToml(e, `
1032image = "debian:latest"
1033clone_project_to = "/ci/build/project"
1034cache = ["/ci/build/project/target"]
1035
1036[on]
1037manual = true
1038
1039[[steps]]
1040name = "hello"
1041run_sh = "echo hi"
1042`)
1043 runID := ciTrigger(e, admin, sha, nil)
1044 if got := ciWaitForRun(e, runID); got != "failure" {
1045 t.Fatalf("status = %q", got)
1046 }
1047 if n := m.uploadCount(); n != 0 {
1048 t.Errorf("uploads = %d, want 0", n)
1049 }
1050 if log := ciStep(e, runID, "pipeline setup").Log; !strings.Contains(log, "overlaps clone_project_to") {
1051 t.Errorf("setup log = %q", log)
1052 }
1053 })
1054
1055 t.Run("a cache path above the clone directory is rejected", func(t *testing.T) {
1056 m.reset()
1057 sha := ciSeedToml(e, `
1058image = "debian:latest"
1059clone_project_to = "/ci/build/project"
1060cache = ["/ci/build"]
1061
1062[on]
1063manual = true
1064
1065[[steps]]
1066name = "hello"
1067run_sh = "echo hi"
1068`)
1069 runID := ciTrigger(e, admin, sha, nil)
1070 if got := ciWaitForRun(e, runID); got != "failure" {
1071 t.Fatalf("status = %q", got)
1072 }
1073 if n := m.uploadCount(); n != 0 {
1074 t.Errorf("uploads = %d, want 0", n)
1075 }
1076 })
1077
1078 t.Run("a relative clone_project_to is rejected", func(t *testing.T) {
1079 m.reset()
1080 sha := ciSeedToml(e, `
1081image = "debian:latest"
1082work_dir = "/ci/build"
1083clone_project_to = "project"
1084
1085[on]
1086manual = true
1087
1088[[steps]]
1089name = "hello"
1090run_sh = "echo hi"
1091`)
1092 runID := ciTrigger(e, admin, sha, nil)
1093 if got := ciWaitForRun(e, runID); got != "failure" {
1094 t.Fatalf("status = %q", got)
1095 }
1096 if log := ciStep(e, runID, "pipeline setup").Log; !strings.Contains(log, "must be an absolute path") {
1097 t.Errorf("setup log = %q", log)
1098 }
1099 })
1100
1101 t.Run("the upload carries the requested commit", func(t *testing.T) {
1102 m.reset()
1103 runID := trigger()
1104 if got := ciWaitForRun(e, runID); got != "success" {
1105 t.Fatalf("status = %q", got)
1106 }
1107 ups := m.uploadsInto(t, "/ci/build/project")
1108 if len(ups) == 0 {
1109 t.Fatal("no upload into the clone directory")
1110 }
1111 // The archive is extracted at / and carries the clone directory as
1112 // its prefix. It must hold the CI config at the triggered commit,
1113 // and no .git. A dropped commit argument would still produce a
1114 // valid tar.
1115 if ups[0].path != "/" {
1116 t.Errorf("upload path = %q, want /", ups[0].path)
1117 }
1118 names := tarEntryNames(t, ups[0].body)
1119 if !contains(names, "ci/build/project/.hearthforge-ci.toml") {
1120 t.Errorf("entries = %v", names)
1121 }
1122 for _, n := range names {
1123 // git archive adds a pax header carrying the commit id.
1124 if n == "pax_global_header" {
1125 continue
1126 }
1127 if strings.Contains(n, ".git/") || !strings.HasPrefix(n, "ci/build/project/") {
1128 t.Errorf("unexpected entry %q", n)
1129 }
1130 }
1131 // git archive writes uid 0, so the files belong to root in the
1132 // container.
1133 for _, h := range tarHeaders(t, ups[0].body) {
1134 if h.uid != 0 {
1135 t.Errorf("entry %q has uid %d", h.name, h.uid)
1136 }
1137 }
1138 })
1139}
1140
1141// ── copy from another image ──────────────────────────────────────────────
1142
1143const ciCopyTOML = `
1144image = "debian:latest"
1145
1146[on]
1147manual = true
1148
1149[[copy]]
1150image = "docker.io/oven/bun:1.4.0-alpine"
1151from = "/usr/local/bin/bun"
1152to = "/usr/local/bin"
1153
1154[[steps]]
1155name = "hello"
1156run_sh = "bun --version"
1157`
1158
1159func TestCICopyFromAnotherImage(t *testing.T) {
1160 e, m, admin := ciEnv(t)
1161
1162 t.Run("pulls the source image and uploads its files", func(t *testing.T) {
1163 sha := ciSeedToml(e, ciCopyTOML)
1164 m.reset()
1165 m.queueExec(execResp{output: "1.4.0\n"}) // the step
1166
1167 runID := ciTrigger(e, admin, sha, nil)
1168 if got := ciWaitForRun(e, runID); got != "success" {
1169 t.Fatalf("status = %q", got)
1170 }
1171 if !contains(m.pulledImages(), "docker.io/oven/bun") {
1172 t.Errorf("pulls = %v", m.pulledImages())
1173 }
1174 if !contains(m.uploadedPaths(), "/usr/local/bin") || len(m.uploadsInto(t, "/usr/local")) == 0 {
1175 t.Errorf("uploads = %v", m.uploadedPaths())
1176 }
1177 })
1178}
1179
1180// ── always and warn_on_fail ──────────────────────────────────────────────
1181
1182const ciFlagsTOML = `
1183image = "debian:latest"
1184
1185[on]
1186manual = true
1187
1188[[steps]]
1189name = "lint"
1190run_sh = "make lint"
1191warn_on_fail = true
1192
1193[[steps]]
1194name = "build"
1195run_sh = "make"
1196
1197[[steps]]
1198name = "cleanup"
1199run_sh = "rm -rf /scratch"
1200always = true
1201`
1202
1203func TestCIAlwaysAndWarnOnFail(t *testing.T) {
1204 e, m, admin := ciEnv(t)
1205
1206 run := func(sha string) int64 {
1207 runID := ciTrigger(e, admin, sha, nil)
1208 ciWaitForRun(e, runID)
1209 return runID
1210 }
1211
1212 t.Run("warn_on_fail marks the step and lets the run continue", func(t *testing.T) {
1213 sha := ciSeedToml(e, ciFlagsTOML)
1214 m.reset()
1215 m.queueExec(execResp{output: "style nit\n", exitCode: 1}) // lint
1216 m.queueExec(execResp{output: "built\n"}) // build
1217 m.queueExec(execResp{}) // cleanup
1218
1219 runID := run(sha)
1220
1221 lint := ciStep(e, runID, "lint")
1222 if lint.Status != "warning" {
1223 t.Errorf("lint status = %q", lint.Status)
1224 }
1225 if !strings.Contains(lint.Log, "style nit") {
1226 t.Errorf("lint log = %q", lint.Log)
1227 }
1228 if got := ciStep(e, runID, "build").Status; got != "success" {
1229 t.Errorf("build status = %q", got)
1230 }
1231 if got := ciRunStatus(e, runID); got != "warning" {
1232 t.Errorf("run status = %q", got)
1233 }
1234 })
1235
1236 t.Run("always runs after a failure, other steps stay skipped", func(t *testing.T) {
1237 sha := ciSeedToml(e, ciFlagsTOML)
1238 m.reset()
1239 m.queueExec(execResp{output: "ok\n"}) // lint
1240 m.queueExec(execResp{output: "boom\n", exitCode: 1}) // build fails
1241 m.queueExec(execResp{output: "cleaned\n"}) // cleanup, always
1242
1243 runID := run(sha)
1244
1245 if got := ciStep(e, runID, "build").Status; got != "failure" {
1246 t.Errorf("build status = %q", got)
1247 }
1248 cleanup := ciStep(e, runID, "cleanup")
1249 if cleanup.Status != "success" {
1250 t.Errorf("cleanup status = %q", cleanup.Status)
1251 }
1252 if !strings.Contains(cleanup.Log, "cleaned") {
1253 t.Errorf("cleanup log = %q", cleanup.Log)
1254 }
1255 if got := ciRunStatus(e, runID); got != "failure" {
1256 t.Errorf("run status = %q", got)
1257 }
1258 })
1259
1260 t.Run("a failing always step keeps the run failed", func(t *testing.T) {
1261 sha := ciSeedToml(e, ciFlagsTOML)
1262 m.reset()
1263 m.queueExec(execResp{output: "ok\n"}) // lint
1264 m.queueExec(execResp{output: "boom\n", exitCode: 1}) // build fails
1265 m.queueExec(execResp{output: "no\n", exitCode: 1}) // cleanup also fails
1266
1267 runID := run(sha)
1268
1269 if got := ciStep(e, runID, "cleanup").Status; got != "failure" {
1270 t.Errorf("cleanup status = %q", got)
1271 }
1272 if got := ciRunStatus(e, runID); got != "failure" {
1273 t.Errorf("run status = %q", got)
1274 }
1275 })
1276}
1277
1278// ── duplicate step names ─────────────────────────────────────────────────
1279
1280const ciDupesTOML = `
1281image = "debian:latest"
1282
1283[on]
1284manual = true
1285
1286[[steps]]
1287name = "check"
1288run_sh = "echo one"
1289
1290[[steps]]
1291name = "check"
1292run_sh = "echo two"
1293`
1294
1295func TestCIDuplicateStepNames(t *testing.T) {
1296 e, m, admin := ciEnv(t)
1297
1298 t.Run("each occurrence gets its own row, in file order", func(t *testing.T) {
1299 sha := ciSeedToml(e, ciDupesTOML)
1300 m.reset()
1301 m.queueExec(execResp{output: "one\n"})
1302 m.queueExec(execResp{output: "two\n"})
1303
1304 runID := ciTrigger(e, admin, sha, nil)
1305 if got := ciWaitForRun(e, runID); got != "success" {
1306 t.Fatalf("status = %q", got)
1307 }
1308 rows := ciSteps(e, runID, "check")
1309 if len(rows) != 2 {
1310 t.Fatalf("rows = %d", len(rows))
1311 }
1312 if !strings.Contains(rows[0].Log, "one") || !strings.Contains(rows[1].Log, "two") {
1313 t.Errorf("logs = %q, %q", rows[0].Log, rows[1].Log)
1314 }
1315 for _, r := range rows {
1316 if r.Status != "success" {
1317 t.Errorf("status = %q", r.Status)
1318 }
1319 }
1320 })
1321
1322 t.Run("the second occurrence can fail on its own", func(t *testing.T) {
1323 sha := ciSeedToml(e, ciDupesTOML)
1324 m.reset()
1325 m.queueExec(execResp{output: "one\n"})
1326 m.queueExec(execResp{output: "boom\n", exitCode: 1})
1327
1328 runID := ciTrigger(e, admin, sha, nil)
1329 if got := ciWaitForRun(e, runID); got != "failure" {
1330 t.Fatalf("status = %q", got)
1331 }
1332 rows := ciSteps(e, runID, "check")
1333 got := []string{}
1334 for _, r := range rows {
1335 got = append(got, r.Status)
1336 }
1337 if !eqStrings(got, []string{"success", "failure"}) {
1338 t.Errorf("statuses = %v", got)
1339 }
1340 })
1341}
1342
1343// ── timeouts override warn_on_fail ───────────────────────────────────────
1344
1345const ciTimeoutTOML = `
1346image = "debian:latest"
1347
1348[on]
1349manual = true
1350
1351[[steps]]
1352name = "lint"
1353run_sh = "make lint"
1354warn_on_fail = true
1355timeout = 1
1356
1357[[steps]]
1358name = "build"
1359run_sh = "make"
1360`
1361
1362func TestCITimeoutsOverrideWarnOnFail(t *testing.T) {
1363 e, m, admin := ciEnv(t)
1364
1365 t.Run("a timed-out warn_on_fail step fails the run", func(t *testing.T) {
1366 sha := ciSeedToml(e, ciTimeoutTOML)
1367 m.reset()
1368 m.queueExec(execResp{delay: 3 * time.Second})
1369
1370 runID := ciTrigger(e, admin, sha, nil)
1371 if got := ciWaitForRun(e, runID, 30*time.Second); got != "failure" {
1372 t.Fatalf("status = %q", got)
1373 }
1374 // A timeout destroys the container, so nothing after it can run.
1375 // Reporting that as a warning would hide a dead pipeline.
1376 lint := ciStep(e, runID, "lint")
1377 if lint.Status != "failure" {
1378 t.Errorf("lint status = %q", lint.Status)
1379 }
1380 if !strings.Contains(lint.Log, "timed out") {
1381 t.Errorf("lint log = %q", lint.Log)
1382 }
1383 })
1384}
1385
1386// ── clear failures are recorded ──────────────────────────────────────────
1387
1388const ciClearTOML = `
1389image = "debian:latest"
1390work_dir = "/ci/build"
1391clone_project_to = "/ci/build/project"
1392
1393[on]
1394manual = true
1395
1396[[steps]]
1397name = "first"
1398run_sh = "false"
1399
1400[[steps]]
1401name = "second"
1402always = true
1403clear = true
1404run_sh = "echo hi"
1405`
1406
1407func TestCIClearFailuresAreRecorded(t *testing.T) {
1408 e, m, admin := ciEnv(t)
1409
1410 t.Run("a clear failure lands on the step, not the console", func(t *testing.T) {
1411 sha := ciSeedToml(e, ciClearTOML)
1412 m.reset()
1413 m.queueExec(execResp{output: "boom\n", exitCode: 1}) // first, fails
1414 m.queueExec(execResp{output: "rm: device busy\n", exitCode: 1}) // clear
1415
1416 runID := ciTrigger(e, admin, sha, nil)
1417 if got := ciWaitForRun(e, runID); got != "failure" {
1418 t.Fatalf("status = %q", got)
1419 }
1420 second := ciStep(e, runID, "second")
1421 if second.Status != "failure" {
1422 t.Errorf("second status = %q", second.Status)
1423 }
1424 if !strings.Contains(second.Log, "Failed to reset") ||
1425 !strings.Contains(second.Log, "device busy") {
1426 t.Errorf("second log = %q", second.Log)
1427 }
1428 })
1429
1430 t.Run("a clear step re-extracts the checkout", func(t *testing.T) {
1431 sha := ciSeedToml(e, ciClearTOML)
1432 m.reset()
1433 m.queueExec(execResp{output: "ok\n"}) // first
1434 m.queueExec(execResp{}) // clear: rm -rf
1435 m.queueExec(execResp{output: "hi\n"}) // second
1436
1437 runID := ciTrigger(e, admin, sha, nil)
1438 if got := ciWaitForRun(e, runID); got != "success" {
1439 t.Fatalf("status = %q", got)
1440 }
1441 if n := len(m.uploadsInto(t, "/ci/build/project")); n != 2 {
1442 t.Errorf("uploads into the clone directory = %d, want 2", n)
1443 }
1444 if strings.Contains(m.allCommandText(), "git") {
1445 t.Errorf("commands = %v", m.commands())
1446 }
1447 })
1448
1449 t.Run("clear removes and recreates the directory in one exec", func(t *testing.T) {
1450 // `rm -rf` can delete the container's WorkingDir. A second exec would
1451 // then fail to chdir before its command starts.
1452 sha := ciSeedToml(e, `
1453image = "debian:latest"
1454work_dir = "/ci/build"
1455clone_project_to = "/ci/build"
1456
1457[on]
1458manual = true
1459
1460[[steps]]
1461name = "first"
1462run_sh = "true"
1463
1464[[steps]]
1465name = "second"
1466clear = true
1467run_sh = "echo hi"
1468`)
1469 m.reset()
1470 runID := ciTrigger(e, admin, sha, nil)
1471 if got := ciWaitForRun(e, runID); got != "success" {
1472 t.Fatalf("status = %q", got)
1473 }
1474 var removals []string
1475 for _, c := range m.commands() {
1476 if joined := strings.Join(c, " "); strings.Contains(joined, "rm -rf") {
1477 removals = append(removals, joined)
1478 }
1479 }
1480 if len(removals) != 1 {
1481 t.Fatalf("rm -rf execs = %v", removals)
1482 }
1483 if !strings.Contains(removals[0], "mkdir -p") {
1484 t.Errorf("removal exec = %q", removals[0])
1485 }
1486 })
1487}
1488
1489// ── cache volumes ────────────────────────────────────────────────────────
1490
1491const ciCacheTOML = `
1492image = "debian:latest"
1493cache = ["/ci/cache/target", "/ci/cache/registry"]
1494
1495[on]
1496manual = true
1497push = ["main", "some-feature"]
1498
1499[[steps]]
1500name = "hello"
1501run_sh = "echo hi"
1502`
1503
1504func TestCICacheVolumes(t *testing.T) {
1505 e, m, admin := ciEnv(t)
1506
1507 run := func(sha, branch string) int64 {
1508 var runID int64
1509 if branch == "main" {
1510 runID = ciTrigger(e, admin, sha, nil)
1511 } else {
1512 runID = ciPushRun(e, sha, branch)
1513 }
1514 ciWaitForRun(e, runID)
1515 return runID
1516 }
1517
1518 t.Run("two cache paths sharing a prefix get distinct volumes", func(t *testing.T) {
1519 sha := ciSeedToml(e, ciCacheTOML)
1520 m.reset()
1521 run(sha, "main")
1522
1523 vols := m.createdVolumes()
1524 if len(vols) != 2 {
1525 t.Fatalf("volumes = %v", vols)
1526 }
1527 if vols[0].name == vols[1].name {
1528 t.Error("both cache paths share one volume")
1529 }
1530 // The path is otherwise unrecoverable from a digest.
1531 got := []string{
1532 vols[0].labels["com.hearthforge.cache-path"],
1533 vols[1].labels["com.hearthforge.cache-path"],
1534 }
1535 if !eqStrings(got, []string{"/ci/cache/target", "/ci/cache/registry"}) {
1536 t.Errorf("cache-path labels = %v", got)
1537 }
1538 })
1539
1540 t.Run("a volume the config no longer names is pruned", func(t *testing.T) {
1541 sha := ciSeedToml(e, ciCacheTOML)
1542 m.reset()
1543 m.setVolumesOnHost("hearthforge-ci-cache-leftover-from-an-old-config")
1544
1545 run(sha, "main")
1546
1547 if got := m.deletedVolumes(); !eqStrings(got,
1548 []string{"hearthforge-ci-cache-leftover-from-an-old-config"}) {
1549 t.Errorf("deleted = %v", got)
1550 }
1551 })
1552
1553 t.Run("volumes still in the config survive", func(t *testing.T) {
1554 sha := ciSeedToml(e, ciCacheTOML)
1555 m.reset()
1556 // Prime the host list with the names this config creates.
1557 run(sha, "main")
1558 var inUse []string
1559 for _, v := range m.createdVolumes() {
1560 inUse = append(inUse, v.name)
1561 }
1562
1563 m.reset()
1564 m.setVolumesOnHost(inUse...)
1565 run(sha, "main")
1566
1567 if got := m.deletedVolumes(); len(got) != 0 {
1568 t.Errorf("deleted = %v", got)
1569 }
1570 })
1571
1572 t.Run("a run off the default branch prunes nothing", func(t *testing.T) {
1573 sha := ciSeedToml(e, ciCacheTOML)
1574 m.reset()
1575 m.setVolumesOnHost("hearthforge-ci-cache-belongs-to-the-default-branch")
1576
1577 // The config is read per commit, so pruning from a feature branch
1578 // would delete the default branch's caches.
1579 run(sha, "some-feature")
1580
1581 if got := m.deletedVolumes(); len(got) != 0 {
1582 t.Errorf("deleted = %v", got)
1583 }
1584 })
1585}
1586
1587// ── cache size caps ──────────────────────────────────────────────────────
1588
1589const ciCappedTOML = `
1590image = "debian:latest"
1591cache = [{ path = "/ci/cache/target", max_size = "1g" }, "/ci/cache/registry"]
1592
1593[on]
1594manual = true
1595
1596[[steps]]
1597name = "hello"
1598run_sh = "echo hi"
1599`
1600
1601func TestCICacheSizeCaps(t *testing.T) {
1602 e, m, admin := ciEnv(t)
1603
1604 run := func(sha string) int64 {
1605 runID := ciTrigger(e, admin, sha, nil)
1606 ciWaitForRun(e, runID)
1607 return runID
1608 }
1609
1610 // names returns the volume names the config produces, in declaration
1611 // order.
1612 names := func(sha string) (string, string) {
1613 m.reset()
1614 run(sha)
1615 vols := m.createdVolumes()
1616 if len(vols) != 2 {
1617 t.Fatalf("volumes = %v", vols)
1618 }
1619 return vols[0].name, vols[1].name
1620 }
1621
1622 const gib = int64(1024 * 1024 * 1024)
1623
1624 t.Run("an oversized cache is dropped and reported on the run", func(t *testing.T) {
1625 sha := ciSeedToml(e, ciCappedTOML)
1626 target, registry := names(sha)
1627
1628 m.reset()
1629 m.setVolumesOnHost(target, registry)
1630 m.setVolumeUsage(map[string]ciVolumeUsage{
1631 target: {Size: 2 * gib},
1632 registry: {Size: 9 * gib},
1633 })
1634 runID := run(sha)
1635
1636 // Only the capped one goes, however large the uncapped one grows.
1637 if got := m.deletedVolumes(); !eqStrings(got, []string{target}) {
1638 t.Fatalf("deleted = %v", got)
1639 }
1640 log := ciStep(e, runID, "cache").Log
1641 if !strings.Contains(log, "/ci/cache/target") || !strings.Contains(log, "2.0G") {
1642 t.Errorf("cache step log = %q", log)
1643 }
1644 })
1645
1646 t.Run("a cache under its cap survives", func(t *testing.T) {
1647 sha := ciSeedToml(e, ciCappedTOML)
1648 target, registry := names(sha)
1649
1650 m.reset()
1651 m.setVolumesOnHost(target, registry)
1652 m.setVolumeUsage(map[string]ciVolumeUsage{target: {Size: 100}})
1653 run(sha)
1654
1655 if got := m.deletedVolumes(); len(got) != 0 {
1656 t.Errorf("deleted = %v", got)
1657 }
1658 })
1659
1660 t.Run("a cache a concurrent run holds is left alone", func(t *testing.T) {
1661 sha := ciSeedToml(e, ciCappedTOML)
1662 target, registry := names(sha)
1663
1664 m.reset()
1665 m.setVolumesOnHost(target, registry)
1666 m.setVolumeUsage(map[string]ciVolumeUsage{target: {Size: 9 * gib, RefCount: 1}})
1667 run(sha)
1668
1669 if got := m.deletedVolumes(); len(got) != 0 {
1670 t.Errorf("deleted = %v", got)
1671 }
1672 })
1673
1674 t.Run("an unmeasured cache is never dropped", func(t *testing.T) {
1675 sha := ciSeedToml(e, ciCappedTOML)
1676 target, registry := names(sha)
1677
1678 m.reset()
1679 m.setVolumesOnHost(target, registry)
1680 // Docker reports -1 for a size it has not computed.
1681 m.setVolumeUsage(map[string]ciVolumeUsage{target: {Size: -1}})
1682 runID := run(sha)
1683
1684 if got := m.deletedVolumes(); len(got) != 0 {
1685 t.Errorf("deleted = %v", got)
1686 }
1687 if steps := ciSteps(e, runID, "cache"); len(steps) != 0 {
1688 t.Errorf("cache step = %v", steps)
1689 }
1690 })
1691}
1692
1693// ── host-built archives ─────────────────────────────────────────────────
1694
1695const ciArchiveTOML = `
1696image = "debian:latest"
1697work_dir = "/ci"
1698
1699[on]
1700manual = true
1701
1702[[steps]]
1703name = "build"
1704run_sh = "echo building"
1705publish_tar = ["/ci/dist"]
1706publish_gzip = ["/ci/dist"]
1707publish_zstd = ["/ci/dist"]
1708publish_zip = ["/ci/dist"]
1709`
1710
1711// TestCIArchivesBuiltOnHost checks that publish_* archives are built from
1712// the engine's tar stream. The mock runs no tar, gzip, zstd or zip, so any
1713// exec for them would fail the test.
1714func TestCIArchivesBuiltOnHost(t *testing.T) {
1715 e, m, admin := ciEnv(t)
1716 m.queueExec(execResp{output: "building\n"})
1717 sha := ciSeedToml(e, ciArchiveTOML)
1718 runID := ciTrigger(e, admin, sha, nil)
1719 if got := ciWaitForRun(e, runID); got != "success" {
1720 t.Fatalf("status = %q", got)
1721 }
1722 for _, c := range m.commands() {
1723 if len(c) > 0 && (c[0] == "tar" || c[0] == "zip") {
1724 t.Errorf("archive tool run in the container: %v", c)
1725 }
1726 }
1727
1728 artifacts := map[string]int64{}
1729 rows, err := e.DB.QueryContext(context.Background(),
1730 `SELECT id, filename FROM ci_artifacts WHERE run_id = ?`, runID)
1731 if err != nil {
1732 t.Fatal(err)
1733 }
1734 for rows.Next() {
1735 var id int64
1736 var name string
1737 if err := rows.Scan(&id, &name); err != nil {
1738 t.Fatal(err)
1739 }
1740 artifacts[name] = id
1741 }
1742 rows.Close()
1743 if len(artifacts) != 4 {
1744 t.Fatalf("artifacts = %v", artifacts)
1745 }
1746 download := func(name string) []byte {
1747 t.Helper()
1748 id, ok := artifacts[name]
1749 if !ok {
1750 t.Fatalf("artifact %s missing from %v", name, artifacts)
1751 }
1752 return admin.get(ciRunPath(runID) + "/artifacts/" + strconv.FormatInt(id, 10)).mustStatus(200).Body
1753 }
1754 // The mock answers every archive request with one file "dist" holding
1755 // artifact-content-123.
1756 checkTar := func(name string, r io.Reader) {
1757 t.Helper()
1758 tr := tar.NewReader(r)
1759 h, err := tr.Next()
1760 if err != nil || h.Name != "dist" {
1761 t.Fatalf("%s: first entry %v, err %v", name, h, err)
1762 }
1763 data, _ := io.ReadAll(tr)
1764 if string(data) != "artifact-content-123" {
1765 t.Errorf("%s: content = %q", name, data)
1766 }
1767 }
1768
1769 t.Run("tar", func(t *testing.T) {
1770 checkTar("dist.tar", bytes.NewReader(download("dist.tar")))
1771 })
1772 t.Run("gzip", func(t *testing.T) {
1773 gz, err := gzip.NewReader(bytes.NewReader(download("dist.tar.gz")))
1774 if err != nil {
1775 t.Fatal(err)
1776 }
1777 checkTar("dist.tar.gz", gz)
1778 })
1779 t.Run("zstd", func(t *testing.T) {
1780 dec, err := zstd.NewReader(bytes.NewReader(download("dist.tar.zst")))
1781 if err != nil {
1782 t.Fatal(err)
1783 }
1784 defer dec.Close()
1785 checkTar("dist.tar.zst", dec)
1786 })
1787 t.Run("zip", func(t *testing.T) {
1788 data := download("dist.zip")
1789 zr, err := zip.NewReader(bytes.NewReader(data), int64(len(data)))
1790 if err != nil {
1791 t.Fatal(err)
1792 }
1793 if len(zr.File) != 1 || zr.File[0].Name != "dist" {
1794 t.Fatalf("zip entries = %v", zr.File)
1795 }
1796 f, _ := zr.File[0].Open()
1797 body, _ := io.ReadAll(f)
1798 if string(body) != "artifact-content-123" {
1799 t.Errorf("zip content = %q", body)
1800 }
1801 })
1802}
1803