hearthforge

hearthforge/internal/web/e2e/ci_build_test.go
ci_build_test.go
⎇
Raw
1package e2e
2
3import (
4 "archive/tar"
5 "bytes"
6 "crypto/sha256"
7 "encoding/hex"
8 "encoding/json"
9 "net/http"
10 "net/url"
11 "strconv"
12 "strings"
13 "testing"
14 "time"
15
16 "hearthforge/internal/ci"
17)
18
19// build_image runs in a build VM container. The mock engine plays that
20// container: it prints a log, exits with a code, and serves the image tar.
21
22const ciBuildTOML = `
23image = "debian:latest"
24clone_project_to = "/ci/project"
25
26[on]
27manual = true
28
29[[steps]]
30name = "image"
31build_image = { image = "web", tags = ["$CI_COMMIT_SHORT_SHA", "latest", "$CI_COMMIT_TAG"], args = { REPO = "$CI_REPO_NAME" }, secrets = ["NPM_TOKEN"] }
32`
33
34// dirTar builds a directory tar as the archive endpoint returns it.
35func dirTar(t *testing.T, top string, files map[string]string) []byte {
36 t.Helper()
37 var buf bytes.Buffer
38 tw := tar.NewWriter(&buf)
39 if err := tw.WriteHeader(&tar.Header{Name: top + "/", Typeflag: tar.TypeDir, Mode: 0o755}); err != nil {
40 t.Fatal(err)
41 }
42 for name, body := range files {
43 hdr := &tar.Header{Name: top + "/" + name, Typeflag: tar.TypeReg, Mode: 0o644, Size: int64(len(body))}
44 if err := tw.WriteHeader(hdr); err != nil {
45 t.Fatal(err)
46 }
47 tw.Write([]byte(body))
48 }
49 tw.Close()
50 return buf.Bytes()
51}
52
53func hexOf(b []byte) string { s := sha256.Sum256(b); return hex.EncodeToString(s[:]) }
54
55// extraBlob is a blob the archive carries but the manifest does not name.
56var extraBlob = []byte("not-in-the-manifest")
57
58// builtImage returns a tar of a containers-image dir: layout, as the build
59// VM writes it, and the manifest digest. variant is "tamper" to corrupt the
60// layer, "no-layer" to leave it out, or "" for a good image. Every variant
61// carries extraBlob.
62func builtImage(t *testing.T, variant string) ([]byte, string) {
63 t.Helper()
64 config := []byte(`{"architecture":"amd64","os":"linux","rootfs":{"type":"layers","diff_ids":[]}}`)
65 layer := []byte("layer-bytes")
66 manifest := fmtManifest(hexOf(config), len(config), hexOf(layer), len(layer))
67 files := map[string]string{
68 "manifest.json": string(manifest),
69 "version": "Directory Transport Version: 1.1\n",
70 hexOf(config): string(config),
71 hexOf(layer): string(layer),
72 hexOf(extraBlob): string(extraBlob),
73 }
74 switch variant {
75 case "tamper":
76 files[hexOf(layer)] = "other-bytes"
77 case "no-layer":
78 delete(files, hexOf(layer))
79 }
80 return dirTar(t, ".", files), regDigest(manifest)
81}
82
83// buildContainerRemoved reports whether the run's build VM container was
84// deleted after it was created.
85func buildContainerRemoved(m *mockDocker, runID int64) bool {
86 name := "hearthforge-ci-" + strconv.FormatInt(runID, 10) + "-build"
87 created := false
88 for _, req := range m.containerRequests() {
89 switch req {
90 case "POST create?name=" + name:
91 created = true
92 case "DELETE " + name:
93 if created {
94 return true
95 }
96 }
97 }
98 return false
99}
100
101func fmtManifest(config string, configSize int, layer string, layerSize int) []byte {
102 b, _ := json.Marshal(map[string]any{
103 "schemaVersion": 2,
104 "mediaType": "application/vnd.docker.distribution.manifest.v2+json",
105 "config": map[string]any{
106 "mediaType": "application/vnd.docker.container.image.v1+json",
107 "digest": "sha256:" + config, "size": configSize,
108 },
109 "layers": []map[string]any{{
110 "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip",
111 "digest": "sha256:" + layer, "size": layerSize,
112 }},
113 })
114 return b
115}
116
117// ciBuildEnv seeds the build config, the secret it names, and the context
118// directory in the CI container. extraEnv goes to ciEnv.
119func ciBuildEnv(t *testing.T, extraEnv ...string) (*env, *mockDocker, *session, string) {
120 t.Helper()
121 e, m, admin := ciEnv(t, extraEnv...)
122 admin.post("/ci-repo/settings/ci-secrets", url.Values{
123 "name": {"NPM_TOKEN"}, "value": {"npm-s3cret"},
124 }).mustRedirect("/ci-repo/settings")
125 sha := ciSeedToml(e, ciBuildTOML)
126 m.reset()
127 m.setArchive("/ci/project", dirTar(t, "project", map[string]string{"Containerfile": "FROM scratch\n"}))
128 return e, m, admin, sha
129}
130
131func TestCIBuildImage(t *testing.T) {
132 e, m, admin, sha := ciBuildEnv(t)
133 image, digest := builtImage(t, "")
134 m.programBuild("STEP 1/1: FROM scratch\nnpm-s3cret\n", 0, image)
135
136 runID := ciTrigger(e, admin, sha, nil)
137 if status := ciWaitForRun(e, runID); status != "success" {
138 t.Fatalf("run status = %q, log %q", status, ciStep(e, runID, "image").Log)
139 }
140
141 step := ciStep(e, runID, "image")
142 host := strings.TrimPrefix(e.Base, "http://")
143 for _, want := range []string{
144 "STEP 1/1: FROM scratch",
145 "Pushed " + host + "/ci-repo/web:" + sha[:8],
146 "Pushed " + host + "/ci-repo/web:latest",
147 "Digest " + digest,
148 } {
149 if !strings.Contains(step.Log, want) {
150 t.Errorf("log lacks %q:\n%s", want, step.Log)
151 }
152 }
153 if strings.Contains(step.Log, "npm-s3cret") {
154 t.Error("the secret is not masked in the build log")
155 }
156
157 for _, tag := range []string{"latest", sha[:8]} {
158 r := regAdmin(t, e, http.MethodGet, "/v2/ci-repo/web/manifests/"+tag, nil).mustStatus(200)
159 if got := r.Header.Get("Docker-Content-Digest"); got != digest {
160 t.Errorf("%s digest = %s, want %s", tag, got, digest)
161 }
162 }
163 if tags := regTags(t, e, "ci-repo/web", ""); len(tags) != 2 {
164 t.Errorf("tags = %v, the empty $CI_COMMIT_TAG must be dropped", tags)
165 }
166
167 vmImage, err := ci.DefaultVMImage()
168 if err != nil {
169 t.Fatal(err)
170 }
171
172 t.Run("the VM image is built from the embedded vm/ and logged", func(t *testing.T) {
173 builds := m.builtImages()
174 if len(builds) != 1 || builds[0].tag != vmImage {
175 t.Fatalf("image builds = %v, want one of %s", builds, vmImage)
176 }
177 for _, f := range []string{"Containerfile", "run-vm", "guest/init"} {
178 if !contains(builds[0].files, f) {
179 t.Errorf("build context %v lacks %s", builds[0].files, f)
180 }
181 }
182 for _, want := range []string{"Building the build VM image " + vmImage, "Step 1/20 : ARG ALPINE"} {
183 if !strings.Contains(step.Log, want) {
184 t.Errorf("log lacks %q", want)
185 }
186 }
187 if len(m.pulledImages()) != 1 {
188 t.Errorf("pulls = %v, want only the CI image", m.pulledImages())
189 }
190 })
191
192 t.Run("the VM container gets KVM and nothing of the host", func(t *testing.T) {
193 body := m.buildBody()
194 if body["Image"] != vmImage {
195 t.Errorf("Image = %v", body["Image"])
196 }
197 host, _ := body["HostConfig"].(map[string]any)
198 devices, _ := json.Marshal(host["Devices"])
199 if !strings.Contains(string(devices), `"PathOnHost":"/dev/kvm"`) {
200 t.Errorf("Devices = %s", devices)
201 }
202 if host["Binds"] != nil || host["Privileged"] != nil {
203 t.Errorf("HostConfig = %v", host)
204 }
205 })
206
207 t.Run("the job carries the context, args and secrets", func(t *testing.T) {
208 if got := tarFiles(t, m.uploadTo("/in/context"))["project/Containerfile"]; got != "FROM scratch\n" {
209 t.Errorf("context Containerfile = %q", got)
210 }
211 files := tarFiles(t, m.uploadTo("/in"))
212 want := map[string]string{
213 "job/args/REPO": "ci-repo",
214 "job/secrets/NPM_TOKEN": "npm-s3cret",
215 }
216 for name, body := range want {
217 if files[name] != body {
218 t.Errorf("%s = %q, want %q", name, files[name], body)
219 }
220 }
221 })
222
223 t.Run("a blob the manifest does not name is not stored", func(t *testing.T) {
224 regAdmin(t, e, http.MethodGet, "/v2/ci-repo/web/blobs/sha256:"+hexOf(extraBlob), nil).mustStatus(404)
225 })
226
227 t.Run("the build VM container is removed", func(t *testing.T) {
228 if !buildContainerRemoved(m, runID) {
229 t.Errorf("requests = %v", m.containerRequests())
230 }
231 })
232}
233
234// tarFiles maps the regular files of a tar to their content.
235func tarFiles(t *testing.T, data []byte) map[string]string {
236 t.Helper()
237 out := map[string]string{}
238 tr := tar.NewReader(bytes.NewReader(data))
239 for {
240 h, err := tr.Next()
241 if err != nil {
242 break
243 }
244 var b bytes.Buffer
245 b.ReadFrom(tr)
246 if h.Typeflag == tar.TypeReg {
247 out[h.Name] = b.String()
248 }
249 }
250 return out
251}
252
253func TestCIBuildImageFailures(t *testing.T) {
254 cases := []struct {
255 name string
256 exit int
257 variant string
258 env []string
259 wantLog string
260 }{
261 {"build fails", 1, "", nil, "Image build failed: the build failed"},
262 {"image too large", 3, "", nil, "Image build failed: the image is larger than CI_MAX_IMAGE_BYTES"},
263 {"VM does not start", 2, "", nil, "Image build failed: the build VM did not run (exit code 2)"},
264 {"layer does not match its digest", 0, "tamper", nil, "digest does not match"},
265 {"layer missing from the archive", 0, "no-layer", nil, "not uploaded"},
266 // run-vm enforces the cap too. This is the check that does not trust it.
267 {"image over the cap despite exit 0", 0, "", []string{"CI_MAX_IMAGE_BYTES", "64"}, "larger than CI_MAX_IMAGE_BYTES"},
268 }
269 for _, c := range cases {
270 t.Run(c.name, func(t *testing.T) {
271 e, m, admin, sha := ciBuildEnv(t, append([]string{"CI_VM_IMAGE", "localhost/test-vm:1"}, c.env...)...)
272 m.setLocalImages("localhost/test-vm:1")
273 image, _ := builtImage(t, c.variant)
274 m.programBuild("hearthforge: failure\n", c.exit, image)
275
276 runID := ciTrigger(e, admin, sha, nil)
277 if status := ciWaitForRun(e, runID); status != "failure" {
278 t.Fatalf("run status = %q", status)
279 }
280 if log := ciStep(e, runID, "image").Log; !strings.Contains(log, c.wantLog) {
281 t.Errorf("log = %q, want %q", log, c.wantLog)
282 }
283 regAdmin(t, e, http.MethodGet, "/v2/ci-repo/web/manifests/latest", nil).mustStatus(404)
284 if pulls := m.pulledImages(); contains(pulls, "localhost/test-vm") {
285 t.Errorf("pulls = %v, the local VM image must not be pulled", pulls)
286 }
287 if builds := m.builtImages(); len(builds) != 0 {
288 t.Errorf("image builds = %v, CI_VM_IMAGE must not be built", builds)
289 }
290 if !buildContainerRemoved(m, runID) {
291 t.Errorf("build VM container not removed: %v", m.containerRequests())
292 }
293 })
294 }
295}
296
297func TestCIBuildImageTimeout(t *testing.T) {
298 e, m, admin := ciEnv(t, "CI_VM_IMAGE", "localhost/test-vm:1")
299 sha := ciSeedToml(e, `
300image = "debian:latest"
301clone_project_to = "/ci/project"
302
303[on]
304manual = true
305
306[[steps]]
307name = "image"
308timeout = 1
309build_image = {}
310`)
311 m.reset()
312 m.setLocalImages("localhost/test-vm:1")
313 m.setArchive("/ci/project", dirTar(t, "project", map[string]string{"Containerfile": "FROM scratch\n"}))
314 m.programBuild("STEP 1/1\n", 0, nil)
315 m.delayBuild(time.Minute)
316
317 runID := ciTrigger(e, admin, sha, nil)
318 if status := ciWaitForRun(e, runID); status != "failure" {
319 t.Fatalf("run status = %q", status)
320 }
321 if log := ciStep(e, runID, "image").Log; !strings.Contains(log, "Step timed out after 1s") {
322 t.Errorf("log = %q", log)
323 }
324 if !buildContainerRemoved(m, runID) {
325 t.Errorf("build VM container not removed: %v", m.containerRequests())
326 }
327}
328
329func TestCIBuildImageMissingSecret(t *testing.T) {
330 e, m, admin := ciEnv(t)
331 sha := ciSeedToml(e, ciBuildTOML)
332 m.reset()
333 runID := ciTrigger(e, admin, sha, nil)
334 if status := ciWaitForRun(e, runID); status != "failure" {
335 t.Fatalf("run status = %q", status)
336 }
337 if log := ciStep(e, runID, "image").Log; !strings.Contains(log, "secret NPM_TOKEN is not set") {
338 t.Errorf("log = %q", log)
339 }
340 if m.buildBody() != nil {
341 t.Error("a build VM container was created")
342 }
343}
344
345func TestCIBuildImageVMImage(t *testing.T) {
346 t.Run("a second run reuses the built VM image", func(t *testing.T) {
347 e, m, admin, sha := ciBuildEnv(t)
348 image, _ := builtImage(t, "")
349 for range 2 {
350 m.programBuild("", 0, image)
351 if status := ciWaitForRun(e, ciTrigger(e, admin, sha, nil)); status != "success" {
352 t.Fatalf("run status = %q", status)
353 }
354 }
355 if builds := m.builtImages(); len(builds) != 1 {
356 t.Errorf("image builds = %d, want 1", len(builds))
357 }
358 })
359
360 t.Run("a failed VM image build fails the step", func(t *testing.T) {
361 e, m, admin, sha := ciBuildEnv(t)
362 m.failImageBuild("apk: network unreachable")
363 runID := ciTrigger(e, admin, sha, nil)
364 if status := ciWaitForRun(e, runID); status != "failure" {
365 t.Fatalf("run status = %q", status)
366 }
367 log := ciStep(e, runID, "image").Log
368 if !strings.Contains(log, "cannot build the build VM image: apk: network unreachable") {
369 t.Errorf("log = %q", log)
370 }
371 if m.buildBody() != nil {
372 t.Error("a build VM container was created")
373 }
374 })
375
376 t.Run("a missing CI_VM_IMAGE is pulled, not built", func(t *testing.T) {
377 e, m, admin, sha := ciBuildEnv(t, "CI_VM_IMAGE", "registry.example.com/buildvm:1")
378 image, _ := builtImage(t, "")
379 m.programBuild("", 0, image)
380 runID := ciTrigger(e, admin, sha, nil)
381 if status := ciWaitForRun(e, runID); status != "success" {
382 t.Fatalf("run status = %q", status)
383 }
384 if !contains(m.pulledImages(), "registry.example.com/buildvm") || len(m.builtImages()) != 0 {
385 t.Errorf("pulls = %v, builds = %v", m.pulledImages(), m.builtImages())
386 }
387 if log := ciStep(e, runID, "image").Log; !strings.Contains(log, "Pulling the build VM image registry.example.com/buildvm:1") {
388 t.Errorf("log = %q", log)
389 }
390 })
391}
392