hearthforge

hearthforge/internal/ci/build.go
build.go
⎇
Raw
1package ci
2
3import (
4 "archive/tar"
5 "bytes"
6 "context"
7 "encoding/json"
8 "errors"
9 "fmt"
10 "io"
11 "maps"
12 "net/http"
13 "net/url"
14 "os"
15 "path"
16 "slices"
17 "strconv"
18 "strings"
19 "time"
20
21 "hearthforge/internal/util"
22)
23
24// Exit codes of vm/run-vm.
25const (
26 vmExitBuildFailed = 1
27 vmExitTooLarge = 3
28)
29
30// vmOverheadMB is the container memory on top of the VM's RAM, for QEMU
31// itself and the job tar.
32const vmOverheadMB = 512
33
34func buildContainerName(runID int64) string {
35 return fmt.Sprintf("hearthforge-ci-%d-build", runID)
36}
37
38// buildImage runs one build_image step in a build VM container, a sibling
39// of the CI container that boots QEMU from vm/. It returns the step log.
40// vars expands tags and args. It holds no secrets, because tags are public
41// and args end up in the image history.
42func (r *Runner) buildImage(ctx context.Context, runID int64, ciContainerID, repoName string, cfg *Config,
43 spec *BuildImage, vars map[string]string, secrets []secret, onPartial func(string),
44) (string, error) {
45 if r.ImportImage == nil {
46 return "", errors.New("no image store is configured")
47 }
48 tags, err := expandTags(spec.Tags, vars)
49 if err != nil {
50 return "", err
51 }
52 args := map[string]string{}
53 for name, v := range spec.Args {
54 args[name] = os.Expand(v, func(k string) string { return vars[k] })
55 }
56 secretFiles := map[string]string{}
57 for _, name := range spec.Secrets {
58 i := slices.IndexFunc(secrets, func(s secret) bool { return s.name == name })
59 if i < 0 {
60 return "", fmt.Errorf("secret %s is not set for this repository", name)
61 }
62 secretFiles[name] = secrets[i].value
63 }
64 contextPath := spec.Context
65 if contextPath == "" {
66 contextPath = cfg.CloneProjectTo
67 }
68
69 // The VM image build can take minutes, so its output goes to the step
70 // log as it arrives.
71 var setup logBuffer
72 var lastFlush time.Time
73 say := func(text string) {
74 setup.append(text)
75 if time.Since(lastFlush) >= 2*time.Second {
76 onPartial(setup.String())
77 lastFlush = time.Now()
78 }
79 }
80 vmImage, err := r.prepareVMImage(ctx, say)
81 if err != nil {
82 return setup.String(), err
83 }
84 id, err := r.createBuildContainer(ctx, runID, vmImage)
85 if err != nil {
86 return setup.String(), err
87 }
88 defer r.removeContainer(ctx, id)
89
90 if err := r.uploadBuildJob(ctx, id, ciContainerID, contextPath, spec, args, secretFiles); err != nil {
91 return setup.String(), err
92 }
93 if err := r.startContainer(ctx, id); err != nil {
94 return setup.String(), err
95 }
96 logResp, err := r.do(ctx, http.MethodGet, "/containers/"+id+"/logs?follow=1&stdout=1&stderr=1", nil, "")
97 if err != nil {
98 return setup.String(), err
99 }
100 if logResp.StatusCode >= 300 {
101 discard(logResp)
102 return setup.String(), fmt.Errorf("cannot read the build VM log: HTTP %d", logResp.StatusCode)
103 }
104 buildLog := setup.String() + readMuxFrames(logResp.Body, func(partial string) {
105 onPartial(setup.String() + partial)
106 })
107 discard(logResp)
108 if ctx.Err() != nil {
109 return buildLog, ctx.Err()
110 }
111
112 code, err := r.waitContainer(ctx, id)
113 if err != nil {
114 return buildLog, err
115 }
116 switch code {
117 case 0:
118 case vmExitBuildFailed:
119 return buildLog, errors.New("the build failed")
120 case vmExitTooLarge:
121 return buildLog, fmt.Errorf("the image is larger than CI_MAX_IMAGE_BYTES (%s)", formatBytes(r.cfg.CIMaxImageBytes))
122 default:
123 return buildLog, fmt.Errorf("the build VM did not run (exit code %d)", code)
124 }
125
126 digest, err := r.importBuiltImage(ctx, id, repoName, spec.Image, tags)
127 if err != nil {
128 return buildLog, err
129 }
130 ref := vars["CI_REGISTRY"]
131 if spec.Image != "" {
132 ref += "/" + spec.Image
133 }
134 for _, tag := range tags {
135 buildLog += fmt.Sprintf("Pushed %s:%s\n", ref, tag)
136 }
137 return buildLog + "Digest " + digest + "\n", nil
138}
139
140// expandTags expands each tag and drops the empty results, so a tag like
141// $CI_COMMIT_TAG can sit in a config that also runs on branches.
142func expandTags(tags []string, vars map[string]string) ([]string, error) {
143 if len(tags) == 0 {
144 tags = []string{"$CI_COMMIT_SHORT_SHA"}
145 }
146 var out []string
147 for _, t := range tags {
148 v := os.Expand(t, func(k string) string { return vars[k] })
149 if v == "" || slices.Contains(out, v) {
150 continue
151 }
152 if !util.ValidImageTag(v) {
153 return nil, fmt.Errorf("tag %q expands to %q, which is not a valid image tag", t, v)
154 }
155 out = append(out, v)
156 }
157 if len(out) == 0 {
158 return nil, errors.New("every tag expanded to an empty string")
159 }
160 return out, nil
161}
162
163func (r *Runner) createBuildContainer(ctx context.Context, runID int64, image string) (string, error) {
164 name := buildContainerName(runID)
165 // A retry reuses the name, see createContainer.
166 r.removeContainer(ctx, name)
167 hostConfig := map[string]any{
168 "Devices": []map[string]string{
169 {"PathOnHost": "/dev/kvm", "PathInContainer": "/dev/kvm", "CgroupPermissions": "rwm"},
170 },
171 // Rootless Podman drops supplementary groups, and /dev/kvm is often
172 // open to the kvm group only. crun reads this annotation, other
173 // runtimes ignore it.
174 "Annotations": map[string]string{"run.oci.keep_original_groups": "1"},
175 "Memory": (r.cfg.CIVMMemoryMB + vmOverheadMB) << 20,
176 "NanoCpus": int64(r.cfg.CIVMCPUs) * 1e9,
177 }
178 if r.cfg.CINetwork != "" {
179 hostConfig["NetworkMode"] = r.cfg.CINetwork
180 }
181 resp, body, err := r.doJSON(ctx, http.MethodPost, "/containers/create?name="+name, map[string]any{
182 "Image": image,
183 "Env": []string{
184 "HF_VM_CPUS=" + strconv.Itoa(r.cfg.CIVMCPUs),
185 "HF_VM_MEMORY_MB=" + strconv.FormatInt(r.cfg.CIVMMemoryMB, 10),
186 "HF_VM_DISK_BYTES=" + strconv.FormatInt(r.cfg.CIVMDiskMB<<20, 10),
187 "HF_VM_MAX_IMAGE_BYTES=" + strconv.FormatInt(r.cfg.CIMaxImageBytes, 10),
188 },
189 "HostConfig": hostConfig,
190 "Labels": ciContainerLabels,
191 })
192 if err != nil {
193 return "", err
194 }
195 if resp.StatusCode >= 300 {
196 return "", fmt.Errorf("failed to create the build VM container: %d %s",
197 resp.StatusCode, strings.TrimSpace(string(body)))
198 }
199 var data struct{ Id string }
200 if err := json.Unmarshal(body, &data); err != nil {
201 return "", err
202 }
203 return data.Id, nil
204}
205
206// uploadBuildJob fills /in of the build container: the context under
207// /in/context/<its directory name>, and the build settings as one file each
208// under /in/job. Files keep arbitrary values safe from shell quoting in the
209// guest.
210func (r *Runner) uploadBuildJob(ctx context.Context, buildID, ciContainerID, contextPath string,
211 spec *BuildImage, args, secrets map[string]string,
212) error {
213 resp, err := r.do(ctx, http.MethodGet,
214 "/containers/"+ciContainerID+"/archive?path="+url.QueryEscape(path.Clean(contextPath)), nil, "")
215 if err != nil {
216 return err
217 }
218 defer discard(resp)
219 if resp.StatusCode >= 300 {
220 return fmt.Errorf("cannot read the context %s: HTTP %d", contextPath, resp.StatusCode)
221 }
222 if err := r.putBuildArchive(ctx, buildID, "/in/context", resp.Body); err != nil {
223 return err
224 }
225
226 files := map[string]string{}
227 if spec.File != "" {
228 files["job/file"] = spec.File
229 }
230 if spec.Target != "" {
231 files["job/target"] = spec.Target
232 }
233 for name, v := range args {
234 files["job/args/"+name] = v
235 }
236 for name, v := range secrets {
237 files["job/secrets/"+name] = v
238 }
239 var job bytes.Buffer
240 tw := tar.NewWriter(&job)
241 for _, dir := range []string{"job/", "job/args/", "job/secrets/"} {
242 if err := tw.WriteHeader(&tar.Header{Name: dir, Mode: 0o700, Typeflag: tar.TypeDir}); err != nil {
243 return err
244 }
245 }
246 for _, name := range slices.Sorted(maps.Keys(files)) {
247 hdr := &tar.Header{Name: name, Mode: 0o600, Size: int64(len(files[name])), Typeflag: tar.TypeReg}
248 if err := tw.WriteHeader(hdr); err != nil {
249 return err
250 }
251 if _, err := io.WriteString(tw, files[name]); err != nil {
252 return err
253 }
254 }
255 if err := tw.Close(); err != nil {
256 return err
257 }
258 return r.putBuildArchive(ctx, buildID, "/in", &job)
259}
260
261func (r *Runner) putBuildArchive(ctx context.Context, buildID, dest string, body io.Reader) error {
262 resp, data, err := r.putArchive(ctx, buildID, dest, body)
263 if err != nil {
264 return err
265 }
266 if resp.StatusCode >= 300 {
267 return fmt.Errorf("failed to upload the build job to %s: HTTP %d %s",
268 dest, resp.StatusCode, strings.TrimSpace(string(data)))
269 }
270 return nil
271}
272
273// waitContainer waits for the container to stop and returns its exit code.
274func (r *Runner) waitContainer(ctx context.Context, id string) (int, error) {
275 resp, body, err := r.doJSON(ctx, http.MethodPost, "/containers/"+id+"/wait", nil)
276 if err != nil {
277 return 0, err
278 }
279 if resp.StatusCode >= 300 {
280 return 0, fmt.Errorf("wait for the build VM container: HTTP %d %s",
281 resp.StatusCode, strings.TrimSpace(string(body)))
282 }
283 var data struct{ StatusCode int }
284 if err := json.Unmarshal(body, &data); err != nil {
285 return 0, err
286 }
287 return data.StatusCode, nil
288}
289
290// importBuiltImage streams /out/image.tar out of the stopped build
291// container into the registry. The archive endpoint wraps it in a tar of
292// its own.
293func (r *Runner) importBuiltImage(ctx context.Context, buildID, repoName, image string, tags []string) (string, error) {
294 resp, err := r.do(ctx, http.MethodGet, "/containers/"+buildID+"/archive?path=/out/image.tar", nil, "")
295 if err != nil {
296 return "", err
297 }
298 defer discard(resp)
299 if resp.StatusCode >= 300 {
300 return "", fmt.Errorf("cannot read the built image: HTTP %d", resp.StatusCode)
301 }
302 tr := tar.NewReader(resp.Body)
303 hdr, err := tr.Next()
304 if err != nil {
305 return "", fmt.Errorf("cannot read the built image: %w", err)
306 }
307 if hdr.Typeflag != tar.TypeReg {
308 return "", errors.New("the built image is not a file")
309 }
310 // run-vm enforces the limit too. This check does not trust it.
311 if hdr.Size > r.cfg.CIMaxImageBytes {
312 return "", fmt.Errorf("the image is larger than CI_MAX_IMAGE_BYTES (%s)", formatBytes(r.cfg.CIMaxImageBytes))
313 }
314 return r.ImportImage(ctx, repoName, image, tags, tr)
315}
316