hearthforge

hearthforge/internal/ci/config.go
config.go
⎇
Raw
1// Package ci runs pipelines described by .hearthforge-ci.toml in Docker or
2// Podman containers. See CI.md for the run model.
3package ci
4
5import (
6 "errors"
7 "fmt"
8 "math"
9 "net/url"
10 "path"
11 "path/filepath"
12 "regexp"
13 "strconv"
14 "strings"
15
16 "github.com/BurntSushi/toml"
17
18 "hearthforge/internal/util"
19)
20
21// VariableDef is one entry of the [variables] table. The manual run form
22// renders one input per variable.
23type VariableDef struct {
24 Default string `toml:"default"`
25 Description string `toml:"description"`
26}
27
28// Step is one [[steps]] entry.
29type Step struct {
30 Name string `toml:"name"`
31 RunSh string `toml:"run_sh"`
32 RunIf string `toml:"run_if"`
33 Always bool `toml:"always"`
34 WarnOnFail bool `toml:"warn_on_fail"`
35 Clear bool `toml:"clear"`
36 Timeout int `toml:"timeout"`
37 BuildImage *BuildImage `toml:"build_image"`
38 PublishFile StringList `toml:"publish_file"`
39 PublishTar StringList `toml:"publish_tar"`
40 PublishGzip StringList `toml:"publish_gzip"`
41 PublishZip StringList `toml:"publish_zip"`
42 PublishZstd StringList `toml:"publish_zstd"`
43}
44
45// BuildImage builds a Containerfile in a build VM and pushes the result to
46// the repository's registry namespace.
47type BuildImage struct {
48 // Context is an absolute path in the CI container. Empty means
49 // clone_project_to.
50 Context string `toml:"context"`
51 // File is relative to Context. Empty lets buildah pick Containerfile or
52 // Dockerfile.
53 File string `toml:"file"`
54 Target string `toml:"target"`
55 Args map[string]string `toml:"args"`
56 Secrets []string `toml:"secrets"`
57 Image string `toml:"image"`
58 Tags []string `toml:"tags"`
59}
60
61// StringList accepts either a bare string or an array of strings.
62type StringList []string
63
64func (l *StringList) UnmarshalTOML(v any) error {
65 switch t := v.(type) {
66 case string:
67 *l = StringList{t}
68 return nil
69 case []any:
70 out := make(StringList, 0, len(t))
71 for _, e := range t {
72 s, ok := e.(string)
73 if !ok {
74 return errors.New("expected a string")
75 }
76 out = append(out, s)
77 }
78 *l = out
79 return nil
80 }
81 return errors.New("expected a string or an array of strings")
82}
83
84// Cache is one cache path, normalised from either form `cache` accepts: a
85// bare string, or a table carrying a size cap.
86type Cache struct {
87 Path string
88 // MaxSize is in bytes. Zero means unbounded.
89 MaxSize int64
90}
91
92func (c *Cache) UnmarshalTOML(v any) error {
93 if s, ok := v.(string); ok {
94 if s == "" {
95 return errors.New("cache path is empty")
96 }
97 c.Path = s
98 return nil
99 }
100 m, ok := v.(map[string]any)
101 if !ok {
102 return errors.New("cache entry must be a string or a table")
103 }
104 p, _ := m["path"].(string)
105 if p == "" {
106 return errors.New("cache entry has no path")
107 }
108 c.Path = p
109 raw, present := m["max_size"]
110 if !present {
111 return nil
112 }
113 s, ok := raw.(string)
114 if !ok {
115 return errors.New("cache max_size must be a string")
116 }
117 size := parseMemoryBytes(s)
118 // parseMemoryBytes answers 0 for anything it cannot read. Left alone
119 // that is a limit every cache exceeds, so the cache would be wiped after
120 // every run and look broken rather than misconfigured.
121 if size <= 0 {
122 return fmt.Errorf("cannot read cache max_size %q", s)
123 }
124 c.MaxSize = size
125 return nil
126}
127
128// Copy lifts one file or directory out of another image, like COPY --from.
129type Copy struct {
130 Image string `toml:"image"`
131 From string `toml:"from"`
132 To string `toml:"to"`
133}
134
135// On holds the trigger settings.
136type On struct {
137 Push PushSpec `toml:"push"`
138 Tag bool `toml:"tag"`
139 // Manual is parsed for compatibility. The manual button ignores it.
140 Manual bool `toml:"manual"`
141}
142
143// PushSpec is `push = true` or a list of branch globs.
144type PushSpec struct {
145 All bool
146 Patterns []string
147}
148
149func (p *PushSpec) UnmarshalTOML(v any) error {
150 switch t := v.(type) {
151 case bool:
152 p.All = t
153 return nil
154 case []any:
155 for _, e := range t {
156 s, ok := e.(string)
157 if !ok {
158 return errors.New("push patterns must be strings")
159 }
160 p.Patterns = append(p.Patterns, s)
161 }
162 return nil
163 }
164 return errors.New("on.push must be a boolean or an array of strings")
165}
166
167// Config is a parsed .hearthforge-ci.toml.
168type Config struct {
169 Image string `toml:"image"`
170 WorkDir string `toml:"work_dir"`
171 CloneProjectTo string `toml:"clone_project_to"`
172 Shell []string `toml:"shell"`
173 ShellSetup string `toml:"shell_setup"`
174 Timeout int `toml:"timeout"`
175 CPULimit float64 `toml:"cpu_limit"`
176 MemoryLimit string `toml:"memory_limit"`
177 Cache []Cache `toml:"cache"`
178 Copy []Copy `toml:"copy"`
179 On On `toml:"on"`
180 Variables map[string]VariableDef `toml:"variables"`
181 Steps []Step `toml:"steps"`
182
183 // VariableOrder lists the variable names in file order. A Go map has no
184 // order, and the run form must match the file.
185 VariableOrder []string `toml:"-"`
186}
187
188// ParseCiConfig reads a .hearthforge-ci.toml. It rejects anything a run
189// cannot use, so a broken config fails at parse time and not mid-run.
190func ParseCiConfig(src string) (*Config, error) {
191 var cfg Config
192 md, err := toml.Decode(src, &cfg)
193 if err != nil {
194 return nil, err
195 }
196 if cfg.Image == "" {
197 return nil, errors.New("image is missing")
198 }
199 // Steps stay an array, not a table per step, so file order survives.
200 if len(cfg.Steps) == 0 {
201 return nil, errors.New("no [[steps]] defined")
202 }
203 for _, s := range cfg.Steps {
204 if s.Name == "" {
205 return nil, errors.New("a step has no name")
206 }
207 }
208 for _, c := range cfg.Copy {
209 if c.Image == "" || c.From == "" || c.To == "" {
210 return nil, errors.New("a [[copy]] entry needs image, from and to")
211 }
212 }
213 for _, key := range md.Keys() {
214 if len(key) == 2 && key[0] == "variables" {
215 cfg.VariableOrder = append(cfg.VariableOrder, key[1])
216 }
217 }
218 return &cfg, nil
219}
220
221// ValidateCiConfig reports the run-start checks ParseCiConfig cannot make.
222// It returns an empty string when the config is usable.
223func ValidateCiConfig(cfg *Config) string {
224 for _, c := range cfg.Copy {
225 // `to` is a directory and the basename is kept, so a `to` that
226 // repeats the basename means someone expected a rename. Left alone
227 // it silently produces to/<name>/<name>.
228 if filepath.Base(c.To) == filepath.Base(c.From) {
229 return fmt.Sprintf(
230 "[[copy]] \"to\" is a directory, so %s lands at %s. Drop the last path segment from \"to\".",
231 c.From, filepath.Join(c.To, filepath.Base(c.From)),
232 )
233 }
234 }
235
236 // An unparsable limit would otherwise become 0, which Docker reads as
237 // unlimited.
238 if cfg.MemoryLimit != "" && parseMemoryBytes(cfg.MemoryLimit) <= 0 {
239 return fmt.Sprintf(
240 "memory_limit %q is not a size. Use a number with an optional K, M or G suffix.",
241 cfg.MemoryLimit,
242 )
243 }
244
245 for _, step := range cfg.Steps {
246 if step.BuildImage != nil {
247 if msg := validateBuildImage(cfg, step.BuildImage); msg != "" {
248 return fmt.Sprintf("step %q: build_image %s", step.Name, msg)
249 }
250 }
251 }
252
253 if cfg.CloneProjectTo == "" {
254 return ""
255 }
256 // The archive endpoint resolves `path` against /, while the execs that
257 // create and clear the directory resolve against work_dir. A relative
258 // value would name two different directories.
259 if !filepath.IsAbs(cfg.CloneProjectTo) {
260 return fmt.Sprintf("clone_project_to (%q) must be an absolute path.", cfg.CloneProjectTo)
261 }
262
263 clone := filepath.Clean(cfg.CloneProjectTo)
264 for _, entry := range cfg.Cache {
265 cachePath := filepath.Clean(entry.Path)
266 // Either nesting direction breaks. A cache below the checkout is
267 // overwritten by the extract and deleted by `clear`. A cache above it
268 // carries the previous run's tree back in.
269 if cachePath == clone ||
270 strings.HasPrefix(cachePath, clone+string(filepath.Separator)) ||
271 strings.HasPrefix(clone, cachePath+string(filepath.Separator)) {
272 return fmt.Sprintf(
273 "cache path %q overlaps clone_project_to (%q). "+
274 "The checkout is extracted over that directory and a `clear` step "+
275 "deletes it. Move the cache outside the clone directory.",
276 entry.Path, cfg.CloneProjectTo,
277 )
278 }
279 }
280 return ""
281}
282
283var (
284 // envNameRe also keeps names safe as file names in the job tar.
285 envNameRe = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*$`)
286 buildTargetRe = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]*$`)
287)
288
289// validateBuildImage checks the parts of build_image that do not depend on
290// the run. Tags are checked after expansion, at run time.
291func validateBuildImage(cfg *Config, b *BuildImage) string {
292 if b.Context == "" && cfg.CloneProjectTo == "" {
293 return "needs context when clone_project_to is not set"
294 }
295 if b.Context != "" && !path.IsAbs(b.Context) {
296 return fmt.Sprintf("context %q must be an absolute path", b.Context)
297 }
298 if b.File != "" && !filepath.IsLocal(b.File) {
299 return fmt.Sprintf("file %q must be a relative path inside the context", b.File)
300 }
301 if b.Target != "" && !buildTargetRe.MatchString(b.Target) {
302 return fmt.Sprintf("target %q is not a stage name", b.Target)
303 }
304 for name := range b.Args {
305 if !envNameRe.MatchString(name) {
306 return fmt.Sprintf("arg %q is not a valid name", name)
307 }
308 }
309 for _, name := range b.Secrets {
310 if !envNameRe.MatchString(name) {
311 return fmt.Sprintf("secret %q is not a valid name", name)
312 }
313 }
314 if !util.ValidImagePath(b.Image) {
315 return fmt.Sprintf("image %q must be lowercase path segments", b.Image)
316 }
317 return ""
318}
319
320// shouldTriggerPush reports whether a push to branch starts a run.
321func shouldTriggerPush(cfg *Config, branch string) bool {
322 if cfg.On.Push.All {
323 return true
324 }
325 for _, pattern := range cfg.On.Push.Patterns {
326 if matchGlob(pattern, branch) {
327 return true
328 }
329 }
330 return false
331}
332
333// matchGlob matches a branch against a config pattern. `*` does not cross a
334// `/`, so "release/*" matches "release/1" but not "release/1/x".
335func matchGlob(pattern, value string) bool {
336 if pattern == "*" {
337 return true
338 }
339 ok, err := path.Match(pattern, value)
340 return err == nil && ok
341}
342
343// VariableOverrides picks the manual run form fields that differ from the
344// config default. An untouched field is not an override: sending the default
345// back would pin the run to the value the config had at render time.
346func VariableOverrides(cfg *Config, form url.Values) map[string]string {
347 out := map[string]string{}
348 for name, def := range cfg.Variables {
349 key := "var_" + name
350 if !form.Has(key) {
351 continue
352 }
353 if v := form.Get(key); v != def.Default {
354 out[name] = v
355 }
356 }
357 return out
358}
359
360var memorySizeRe = regexp.MustCompile(`^(\d+(?:\.\d+)?)\s*([kmgKMG]?)b?$`)
361
362// parseMemoryBytes reads sizes like "512m" or "2g". It answers 0 for
363// anything it cannot read.
364func parseMemoryBytes(s string) int64 {
365 m := memorySizeRe.FindStringSubmatch(s)
366 if m == nil {
367 return 0
368 }
369 n, err := strconv.ParseFloat(m[1], 64)
370 if err != nil {
371 return 0
372 }
373 switch strings.ToLower(m[2]) {
374 case "k":
375 n *= 1024
376 case "m":
377 n *= 1024 * 1024
378 case "g":
379 n *= 1024 * 1024 * 1024
380 }
381 return int64(math.Floor(n))
382}
383
384// formatBytes renders a size the way the cache step log shows it.
385func formatBytes(n int64) string {
386 units := []string{"B", "K", "M", "G", "T"}
387 i := 0
388 v := float64(n)
389 for v >= 1024 && i < len(units)-1 {
390 v /= 1024
391 i++
392 }
393 if i == 0 {
394 return strconv.FormatFloat(v, 'f', -1, 64) + units[0]
395 }
396 return strconv.FormatFloat(v, 'f', 1, 64) + units[i]
397}
398