hearthforge

hearthforge/internal/ci/config.go
config.go
⎇
Raw
1// Package ci runs pipelines described by .hearthforge-ci.toml in Docker or
2// Podman containers. See CI.md for the run model.
3package ci
4
5import (
6 "errors"
7 "fmt"
8 "math"
9 "net/url"
10 "path"
11 "path/filepath"
12 "regexp"
13 "strconv"
14 "strings"
15
16 "github.com/BurntSushi/toml"
17)
18
19// VariableDef is one entry of the [variables] table. The manual run form
20// renders one input per variable.
21type VariableDef struct {
22 Default string `toml:"default"`
23 Description string `toml:"description"`
24}
25
26// Step is one [[steps]] entry.
27type Step struct {
28 Name string `toml:"name"`
29 RunSh string `toml:"run_sh"`
30 RunIf string `toml:"run_if"`
31 Always bool `toml:"always"`
32 WarnOnFail bool `toml:"warn_on_fail"`
33 Clear bool `toml:"clear"`
34 Timeout int `toml:"timeout"`
35 PublishFile StringList `toml:"publish_file"`
36 PublishTar StringList `toml:"publish_tar"`
37 PublishGzip StringList `toml:"publish_gzip"`
38 PublishZip StringList `toml:"publish_zip"`
39 PublishZstd StringList `toml:"publish_zstd"`
40}
41
42// StringList accepts either a bare string or an array of strings.
43type StringList []string
44
45func (l *StringList) UnmarshalTOML(v any) error {
46 switch t := v.(type) {
47 case string:
48 *l = StringList{t}
49 return nil
50 case []any:
51 out := make(StringList, 0, len(t))
52 for _, e := range t {
53 s, ok := e.(string)
54 if !ok {
55 return errors.New("expected a string")
56 }
57 out = append(out, s)
58 }
59 *l = out
60 return nil
61 }
62 return errors.New("expected a string or an array of strings")
63}
64
65// Cache is one cache path, normalised from either form `cache` accepts: a
66// bare string, or a table carrying a size cap.
67type Cache struct {
68 Path string
69 // MaxSize is in bytes. Zero means unbounded.
70 MaxSize int64
71}
72
73func (c *Cache) UnmarshalTOML(v any) error {
74 if s, ok := v.(string); ok {
75 if s == "" {
76 return errors.New("cache path is empty")
77 }
78 c.Path = s
79 return nil
80 }
81 m, ok := v.(map[string]any)
82 if !ok {
83 return errors.New("cache entry must be a string or a table")
84 }
85 p, _ := m["path"].(string)
86 if p == "" {
87 return errors.New("cache entry has no path")
88 }
89 c.Path = p
90 raw, present := m["max_size"]
91 if !present {
92 return nil
93 }
94 s, ok := raw.(string)
95 if !ok {
96 return errors.New("cache max_size must be a string")
97 }
98 size := parseMemoryBytes(s)
99 // parseMemoryBytes answers 0 for anything it cannot read. Left alone
100 // that is a limit every cache exceeds, so the cache would be wiped after
101 // every run and look broken rather than misconfigured.
102 if size <= 0 {
103 return fmt.Errorf("cannot read cache max_size %q", s)
104 }
105 c.MaxSize = size
106 return nil
107}
108
109// Copy lifts one file or directory out of another image, like COPY --from.
110type Copy struct {
111 Image string `toml:"image"`
112 From string `toml:"from"`
113 To string `toml:"to"`
114}
115
116// On holds the trigger settings.
117type On struct {
118 Push PushSpec `toml:"push"`
119 Tag bool `toml:"tag"`
120 // Manual is parsed for compatibility. The manual button ignores it.
121 Manual bool `toml:"manual"`
122}
123
124// PushSpec is `push = true` or a list of branch globs.
125type PushSpec struct {
126 All bool
127 Patterns []string
128}
129
130func (p *PushSpec) UnmarshalTOML(v any) error {
131 switch t := v.(type) {
132 case bool:
133 p.All = t
134 return nil
135 case []any:
136 for _, e := range t {
137 s, ok := e.(string)
138 if !ok {
139 return errors.New("push patterns must be strings")
140 }
141 p.Patterns = append(p.Patterns, s)
142 }
143 return nil
144 }
145 return errors.New("on.push must be a boolean or an array of strings")
146}
147
148// Config is a parsed .hearthforge-ci.toml.
149type Config struct {
150 Image string `toml:"image"`
151 WorkDir string `toml:"work_dir"`
152 CloneProjectTo string `toml:"clone_project_to"`
153 Shell []string `toml:"shell"`
154 ShellSetup string `toml:"shell_setup"`
155 Timeout int `toml:"timeout"`
156 CPULimit float64 `toml:"cpu_limit"`
157 MemoryLimit string `toml:"memory_limit"`
158 Cache []Cache `toml:"cache"`
159 Copy []Copy `toml:"copy"`
160 On On `toml:"on"`
161 Variables map[string]VariableDef `toml:"variables"`
162 Steps []Step `toml:"steps"`
163
164 // VariableOrder lists the variable names in file order. A Go map has no
165 // order, and the run form must match the file.
166 VariableOrder []string `toml:"-"`
167}
168
169// ParseCiConfig reads a .hearthforge-ci.toml. It rejects anything a run
170// cannot use, so a broken config fails at parse time and not mid-run.
171func ParseCiConfig(src string) (*Config, error) {
172 var cfg Config
173 md, err := toml.Decode(src, &cfg)
174 if err != nil {
175 return nil, err
176 }
177 if cfg.Image == "" {
178 return nil, errors.New("image is missing")
179 }
180 // Steps stay an array, not a table per step, so file order survives.
181 if len(cfg.Steps) == 0 {
182 return nil, errors.New("no [[steps]] defined")
183 }
184 for _, s := range cfg.Steps {
185 if s.Name == "" {
186 return nil, errors.New("a step has no name")
187 }
188 }
189 for _, c := range cfg.Copy {
190 if c.Image == "" || c.From == "" || c.To == "" {
191 return nil, errors.New("a [[copy]] entry needs image, from and to")
192 }
193 }
194 for _, key := range md.Keys() {
195 if len(key) == 2 && key[0] == "variables" {
196 cfg.VariableOrder = append(cfg.VariableOrder, key[1])
197 }
198 }
199 return &cfg, nil
200}
201
202// ValidateCiConfig reports the run-start checks ParseCiConfig cannot make.
203// It returns an empty string when the config is usable.
204func ValidateCiConfig(cfg *Config) string {
205 for _, c := range cfg.Copy {
206 // `to` is a directory and the basename is kept, so a `to` that
207 // repeats the basename means someone expected a rename. Left alone
208 // it silently produces to/<name>/<name>.
209 if filepath.Base(c.To) == filepath.Base(c.From) {
210 return fmt.Sprintf(
211 "[[copy]] \"to\" is a directory, so %s lands at %s. Drop the last path segment from \"to\".",
212 c.From, filepath.Join(c.To, filepath.Base(c.From)),
213 )
214 }
215 }
216
217 // An unparsable limit would otherwise become 0, which Docker reads as
218 // unlimited.
219 if cfg.MemoryLimit != "" && parseMemoryBytes(cfg.MemoryLimit) <= 0 {
220 return fmt.Sprintf(
221 "memory_limit %q is not a size. Use a number with an optional K, M or G suffix.",
222 cfg.MemoryLimit,
223 )
224 }
225
226 if cfg.CloneProjectTo == "" {
227 return ""
228 }
229 // The archive endpoint resolves `path` against /, while the execs that
230 // create and clear the directory resolve against work_dir. A relative
231 // value would name two different directories.
232 if !filepath.IsAbs(cfg.CloneProjectTo) {
233 return fmt.Sprintf("clone_project_to (%q) must be an absolute path.", cfg.CloneProjectTo)
234 }
235
236 clone := filepath.Clean(cfg.CloneProjectTo)
237 for _, entry := range cfg.Cache {
238 cachePath := filepath.Clean(entry.Path)
239 // Either nesting direction breaks. A cache below the checkout is
240 // overwritten by the extract and deleted by `clear`. A cache above it
241 // carries the previous run's tree back in.
242 if cachePath == clone ||
243 strings.HasPrefix(cachePath, clone+string(filepath.Separator)) ||
244 strings.HasPrefix(clone, cachePath+string(filepath.Separator)) {
245 return fmt.Sprintf(
246 "cache path %q overlaps clone_project_to (%q). "+
247 "The checkout is extracted over that directory and a `clear` step "+
248 "deletes it. Move the cache outside the clone directory.",
249 entry.Path, cfg.CloneProjectTo,
250 )
251 }
252 }
253 return ""
254}
255
256// shouldTriggerPush reports whether a push to branch starts a run.
257func shouldTriggerPush(cfg *Config, branch string) bool {
258 if cfg.On.Push.All {
259 return true
260 }
261 for _, pattern := range cfg.On.Push.Patterns {
262 if matchGlob(pattern, branch) {
263 return true
264 }
265 }
266 return false
267}
268
269// matchGlob matches a branch against a config pattern. `*` does not cross a
270// `/`, so "release/*" matches "release/1" but not "release/1/x".
271func matchGlob(pattern, value string) bool {
272 if pattern == "*" {
273 return true
274 }
275 ok, err := path.Match(pattern, value)
276 return err == nil && ok
277}
278
279// VariableOverrides picks the manual run form fields that differ from the
280// config default. An untouched field is not an override: sending the default
281// back would pin the run to the value the config had at render time.
282func VariableOverrides(cfg *Config, form url.Values) map[string]string {
283 out := map[string]string{}
284 for name, def := range cfg.Variables {
285 key := "var_" + name
286 if !form.Has(key) {
287 continue
288 }
289 if v := form.Get(key); v != def.Default {
290 out[name] = v
291 }
292 }
293 return out
294}
295
296var memorySizeRe = regexp.MustCompile(`^(\d+(?:\.\d+)?)\s*([kmgKMG]?)b?$`)
297
298// parseMemoryBytes reads sizes like "512m" or "2g". It answers 0 for
299// anything it cannot read.
300func parseMemoryBytes(s string) int64 {
301 m := memorySizeRe.FindStringSubmatch(s)
302 if m == nil {
303 return 0
304 }
305 n, err := strconv.ParseFloat(m[1], 64)
306 if err != nil {
307 return 0
308 }
309 switch strings.ToLower(m[2]) {
310 case "k":
311 n *= 1024
312 case "m":
313 n *= 1024 * 1024
314 case "g":
315 n *= 1024 * 1024 * 1024
316 }
317 return int64(math.Floor(n))
318}
319
320// formatBytes renders a size the way the cache step log shows it.
321func formatBytes(n int64) string {
322 units := []string{"B", "K", "M", "G", "T"}
323 i := 0
324 v := float64(n)
325 for v >= 1024 && i < len(units)-1 {
326 v /= 1024
327 i++
328 }
329 if i == 0 {
330 return strconv.FormatFloat(v, 'f', -1, 64) + units[0]
331 }
332 return strconv.FormatFloat(v, 'f', 1, 64) + units[i]
333}
334