hearthforge

hearthforge/internal/ci/config.go
config.go
⎇
Raw
1// Package ci runs pipelines described by .hearthforge-ci.toml in Docker or
2// Podman containers. See CI.md for the run model.
3package ci
4
5import (
6 "errors"
7 "fmt"
8 "math"
9 "net/url"
10 "path"
11 "path/filepath"
12 "regexp"
13 "slices"
14 "strconv"
15 "strings"
16
17 "github.com/BurntSushi/toml"
18)
19
20// VariableDef is one entry of the [variables] table. The manual run form
21// renders one input per variable.
22type VariableDef struct {
23 Default string `toml:"default"`
24 Description string `toml:"description"`
25}
26
27// Step is one [[steps]] entry.
28type Step struct {
29 Name string `toml:"name"`
30 RunSh string `toml:"run_sh"`
31 RunIf string `toml:"run_if"`
32 Always bool `toml:"always"`
33 WarnOnFail bool `toml:"warn_on_fail"`
34 Clear bool `toml:"clear"`
35 // EngineSocket mounts the Docker/Podman socket and points DOCKER_HOST
36 // at it. Needs CI_ENGINE_SOCKET on the server.
37 EngineSocket bool `toml:"engine_socket"`
38 Timeout int `toml:"timeout"`
39 PublishFile StringList `toml:"publish_file"`
40 PublishTar StringList `toml:"publish_tar"`
41 PublishGzip StringList `toml:"publish_gzip"`
42 PublishZip StringList `toml:"publish_zip"`
43 PublishZstd StringList `toml:"publish_zstd"`
44}
45
46// StringList accepts either a bare string or an array of strings.
47type StringList []string
48
49func (l *StringList) UnmarshalTOML(v any) error {
50 switch t := v.(type) {
51 case string:
52 *l = StringList{t}
53 return nil
54 case []any:
55 out := make(StringList, 0, len(t))
56 for _, e := range t {
57 s, ok := e.(string)
58 if !ok {
59 return errors.New("expected a string")
60 }
61 out = append(out, s)
62 }
63 *l = out
64 return nil
65 }
66 return errors.New("expected a string or an array of strings")
67}
68
69// Cache is one cache path, normalised from either form `cache` accepts: a
70// bare string, or a table carrying a size cap.
71type Cache struct {
72 Path string
73 // MaxSize is in bytes. Zero means unbounded.
74 MaxSize int64
75}
76
77func (c *Cache) UnmarshalTOML(v any) error {
78 if s, ok := v.(string); ok {
79 if s == "" {
80 return errors.New("cache path is empty")
81 }
82 c.Path = s
83 return nil
84 }
85 m, ok := v.(map[string]any)
86 if !ok {
87 return errors.New("cache entry must be a string or a table")
88 }
89 p, _ := m["path"].(string)
90 if p == "" {
91 return errors.New("cache entry has no path")
92 }
93 c.Path = p
94 raw, present := m["max_size"]
95 if !present {
96 return nil
97 }
98 s, ok := raw.(string)
99 if !ok {
100 return errors.New("cache max_size must be a string")
101 }
102 size := parseMemoryBytes(s)
103 // parseMemoryBytes answers 0 for anything it cannot read. Left alone
104 // that is a limit every cache exceeds, so the cache would be wiped after
105 // every run and look broken rather than misconfigured.
106 if size <= 0 {
107 return fmt.Errorf("cannot read cache max_size %q", s)
108 }
109 c.MaxSize = size
110 return nil
111}
112
113// Copy lifts one file or directory out of another image, like COPY --from.
114type Copy struct {
115 Image string `toml:"image"`
116 From string `toml:"from"`
117 To string `toml:"to"`
118}
119
120// On holds the trigger settings.
121type On struct {
122 Push PushSpec `toml:"push"`
123 Tag bool `toml:"tag"`
124 // Manual is parsed for compatibility. The manual button ignores it.
125 Manual bool `toml:"manual"`
126}
127
128// PushSpec is `push = true` or a list of branch globs.
129type PushSpec struct {
130 All bool
131 Patterns []string
132}
133
134func (p *PushSpec) UnmarshalTOML(v any) error {
135 switch t := v.(type) {
136 case bool:
137 p.All = t
138 return nil
139 case []any:
140 for _, e := range t {
141 s, ok := e.(string)
142 if !ok {
143 return errors.New("push patterns must be strings")
144 }
145 p.Patterns = append(p.Patterns, s)
146 }
147 return nil
148 }
149 return errors.New("on.push must be a boolean or an array of strings")
150}
151
152// WantsEngineSocket reports whether any step asked for the engine socket.
153func (c *Config) WantsEngineSocket() bool {
154 return slices.ContainsFunc(c.Steps, func(s Step) bool { return s.EngineSocket })
155}
156
157// Config is a parsed .hearthforge-ci.toml.
158type Config struct {
159 Image string `toml:"image"`
160 WorkDir string `toml:"work_dir"`
161 CloneProjectTo string `toml:"clone_project_to"`
162 Shell []string `toml:"shell"`
163 ShellSetup string `toml:"shell_setup"`
164 Timeout int `toml:"timeout"`
165 CPULimit float64 `toml:"cpu_limit"`
166 MemoryLimit string `toml:"memory_limit"`
167 Cache []Cache `toml:"cache"`
168 Copy []Copy `toml:"copy"`
169 On On `toml:"on"`
170 Variables map[string]VariableDef `toml:"variables"`
171 Steps []Step `toml:"steps"`
172
173 // VariableOrder lists the variable names in file order. A Go map has no
174 // order, and the run form must match the file.
175 VariableOrder []string `toml:"-"`
176}
177
178// ParseCiConfig reads a .hearthforge-ci.toml. It rejects anything a run
179// cannot use, so a broken config fails at parse time and not mid-run.
180func ParseCiConfig(src string) (*Config, error) {
181 var cfg Config
182 md, err := toml.Decode(src, &cfg)
183 if err != nil {
184 return nil, err
185 }
186 if cfg.Image == "" {
187 return nil, errors.New("image is missing")
188 }
189 // Steps stay an array, not a table per step, so file order survives.
190 if len(cfg.Steps) == 0 {
191 return nil, errors.New("no [[steps]] defined")
192 }
193 for _, s := range cfg.Steps {
194 if s.Name == "" {
195 return nil, errors.New("a step has no name")
196 }
197 }
198 for _, c := range cfg.Copy {
199 if c.Image == "" || c.From == "" || c.To == "" {
200 return nil, errors.New("a [[copy]] entry needs image, from and to")
201 }
202 }
203 for _, key := range md.Keys() {
204 if len(key) == 2 && key[0] == "variables" {
205 cfg.VariableOrder = append(cfg.VariableOrder, key[1])
206 }
207 }
208 return &cfg, nil
209}
210
211// ValidateCiConfig reports the run-start checks ParseCiConfig cannot make.
212// It returns an empty string when the config is usable.
213func ValidateCiConfig(cfg *Config) string {
214 for _, c := range cfg.Copy {
215 // `to` is a directory and the basename is kept, so a `to` that
216 // repeats the basename means someone expected a rename. Left alone
217 // it silently produces to/<name>/<name>.
218 if filepath.Base(c.To) == filepath.Base(c.From) {
219 return fmt.Sprintf(
220 "[[copy]] \"to\" is a directory, so %s lands at %s. Drop the last path segment from \"to\".",
221 c.From, filepath.Join(c.To, filepath.Base(c.From)),
222 )
223 }
224 }
225
226 // An unparsable limit would otherwise become 0, which Docker reads as
227 // unlimited.
228 if cfg.MemoryLimit != "" && parseMemoryBytes(cfg.MemoryLimit) <= 0 {
229 return fmt.Sprintf(
230 "memory_limit %q is not a size. Use a number with an optional K, M or G suffix.",
231 cfg.MemoryLimit,
232 )
233 }
234
235 if cfg.CloneProjectTo == "" {
236 return ""
237 }
238 // The archive endpoint resolves `path` against /, while the execs that
239 // create and clear the directory resolve against work_dir. A relative
240 // value would name two different directories.
241 if !filepath.IsAbs(cfg.CloneProjectTo) {
242 return fmt.Sprintf("clone_project_to (%q) must be an absolute path.", cfg.CloneProjectTo)
243 }
244
245 clone := filepath.Clean(cfg.CloneProjectTo)
246 for _, entry := range cfg.Cache {
247 cachePath := filepath.Clean(entry.Path)
248 // Either nesting direction breaks. A cache below the checkout is
249 // overwritten by the extract and deleted by `clear`. A cache above it
250 // carries the previous run's tree back in.
251 if cachePath == clone ||
252 strings.HasPrefix(cachePath, clone+string(filepath.Separator)) ||
253 strings.HasPrefix(clone, cachePath+string(filepath.Separator)) {
254 return fmt.Sprintf(
255 "cache path %q overlaps clone_project_to (%q). "+
256 "The checkout is extracted over that directory and a `clear` step "+
257 "deletes it. Move the cache outside the clone directory.",
258 entry.Path, cfg.CloneProjectTo,
259 )
260 }
261 }
262 return ""
263}
264
265// shouldTriggerPush reports whether a push to branch starts a run.
266func shouldTriggerPush(cfg *Config, branch string) bool {
267 if cfg.On.Push.All {
268 return true
269 }
270 for _, pattern := range cfg.On.Push.Patterns {
271 if matchGlob(pattern, branch) {
272 return true
273 }
274 }
275 return false
276}
277
278// matchGlob matches a branch against a config pattern. `*` does not cross a
279// `/`, so "release/*" matches "release/1" but not "release/1/x".
280func matchGlob(pattern, value string) bool {
281 if pattern == "*" {
282 return true
283 }
284 ok, err := path.Match(pattern, value)
285 return err == nil && ok
286}
287
288// VariableOverrides picks the manual run form fields that differ from the
289// config default. An untouched field is not an override: sending the default
290// back would pin the run to the value the config had at render time.
291func VariableOverrides(cfg *Config, form url.Values) map[string]string {
292 out := map[string]string{}
293 for name, def := range cfg.Variables {
294 key := "var_" + name
295 if !form.Has(key) {
296 continue
297 }
298 if v := form.Get(key); v != def.Default {
299 out[name] = v
300 }
301 }
302 return out
303}
304
305var memorySizeRe = regexp.MustCompile(`^(\d+(?:\.\d+)?)\s*([kmgKMG]?)b?$`)
306
307// parseMemoryBytes reads sizes like "512m" or "2g". It answers 0 for
308// anything it cannot read.
309func parseMemoryBytes(s string) int64 {
310 m := memorySizeRe.FindStringSubmatch(s)
311 if m == nil {
312 return 0
313 }
314 n, err := strconv.ParseFloat(m[1], 64)
315 if err != nil {
316 return 0
317 }
318 switch strings.ToLower(m[2]) {
319 case "k":
320 n *= 1024
321 case "m":
322 n *= 1024 * 1024
323 case "g":
324 n *= 1024 * 1024 * 1024
325 }
326 return int64(math.Floor(n))
327}
328
329// formatBytes renders a size the way the cache step log shows it.
330func formatBytes(n int64) string {
331 units := []string{"B", "K", "M", "G", "T"}
332 i := 0
333 v := float64(n)
334 for v >= 1024 && i < len(units)-1 {
335 v /= 1024
336 i++
337 }
338 if i == 0 {
339 return strconv.FormatFloat(v, 'f', -1, 64) + units[0]
340 }
341 return strconv.FormatFloat(v, 'f', 1, 64) + units[i]
342}
343