// Package ci runs pipelines described by .hearthforge-ci.toml in Docker or // Podman containers. See CI.md for the run model. package ci import ( "errors" "fmt" "math" "net/url" "path" "path/filepath" "regexp" "slices" "strconv" "strings" "github.com/BurntSushi/toml" ) // VariableDef is one entry of the [variables] table. The manual run form // renders one input per variable. type VariableDef struct { Default string `toml:"default"` Description string `toml:"description"` } // Step is one [[steps]] entry. type Step struct { Name string `toml:"name"` RunSh string `toml:"run_sh"` RunIf string `toml:"run_if"` Always bool `toml:"always"` WarnOnFail bool `toml:"warn_on_fail"` Clear bool `toml:"clear"` // EngineSocket mounts the Docker/Podman socket and points DOCKER_HOST // at it. Needs CI_ENGINE_SOCKET on the server. EngineSocket bool `toml:"engine_socket"` Timeout int `toml:"timeout"` PublishFile StringList `toml:"publish_file"` PublishTar StringList `toml:"publish_tar"` PublishGzip StringList `toml:"publish_gzip"` PublishZip StringList `toml:"publish_zip"` PublishZstd StringList `toml:"publish_zstd"` } // StringList accepts either a bare string or an array of strings. type StringList []string func (l *StringList) UnmarshalTOML(v any) error { switch t := v.(type) { case string: *l = StringList{t} return nil case []any: out := make(StringList, 0, len(t)) for _, e := range t { s, ok := e.(string) if !ok { return errors.New("expected a string") } out = append(out, s) } *l = out return nil } return errors.New("expected a string or an array of strings") } // Cache is one cache path, normalised from either form `cache` accepts: a // bare string, or a table carrying a size cap. type Cache struct { Path string // MaxSize is in bytes. Zero means unbounded. MaxSize int64 } func (c *Cache) UnmarshalTOML(v any) error { if s, ok := v.(string); ok { if s == "" { return errors.New("cache path is empty") } c.Path = s return nil } m, ok := v.(map[string]any) if !ok { return errors.New("cache entry must be a string or a table") } p, _ := m["path"].(string) if p == "" { return errors.New("cache entry has no path") } c.Path = p raw, present := m["max_size"] if !present { return nil } s, ok := raw.(string) if !ok { return errors.New("cache max_size must be a string") } size := parseMemoryBytes(s) // parseMemoryBytes answers 0 for anything it cannot read. Left alone // that is a limit every cache exceeds, so the cache would be wiped after // every run and look broken rather than misconfigured. if size <= 0 { return fmt.Errorf("cannot read cache max_size %q", s) } c.MaxSize = size return nil } // Copy lifts one file or directory out of another image, like COPY --from. type Copy struct { Image string `toml:"image"` From string `toml:"from"` To string `toml:"to"` } // On holds the trigger settings. type On struct { Push PushSpec `toml:"push"` Tag bool `toml:"tag"` // Manual is parsed for compatibility. The manual button ignores it. Manual bool `toml:"manual"` } // PushSpec is `push = true` or a list of branch globs. type PushSpec struct { All bool Patterns []string } func (p *PushSpec) UnmarshalTOML(v any) error { switch t := v.(type) { case bool: p.All = t return nil case []any: for _, e := range t { s, ok := e.(string) if !ok { return errors.New("push patterns must be strings") } p.Patterns = append(p.Patterns, s) } return nil } return errors.New("on.push must be a boolean or an array of strings") } // WantsEngineSocket reports whether any step asked for the engine socket. func (c *Config) WantsEngineSocket() bool { return slices.ContainsFunc(c.Steps, func(s Step) bool { return s.EngineSocket }) } // Config is a parsed .hearthforge-ci.toml. type Config struct { Image string `toml:"image"` WorkDir string `toml:"work_dir"` CloneProjectTo string `toml:"clone_project_to"` Shell []string `toml:"shell"` ShellSetup string `toml:"shell_setup"` Timeout int `toml:"timeout"` CPULimit float64 `toml:"cpu_limit"` MemoryLimit string `toml:"memory_limit"` Cache []Cache `toml:"cache"` Copy []Copy `toml:"copy"` On On `toml:"on"` Variables map[string]VariableDef `toml:"variables"` Steps []Step `toml:"steps"` // VariableOrder lists the variable names in file order. A Go map has no // order, and the run form must match the file. VariableOrder []string `toml:"-"` } // ParseCiConfig reads a .hearthforge-ci.toml. It rejects anything a run // cannot use, so a broken config fails at parse time and not mid-run. func ParseCiConfig(src string) (*Config, error) { var cfg Config md, err := toml.Decode(src, &cfg) if err != nil { return nil, err } if cfg.Image == "" { return nil, errors.New("image is missing") } // Steps stay an array, not a table per step, so file order survives. if len(cfg.Steps) == 0 { return nil, errors.New("no [[steps]] defined") } for _, s := range cfg.Steps { if s.Name == "" { return nil, errors.New("a step has no name") } } for _, c := range cfg.Copy { if c.Image == "" || c.From == "" || c.To == "" { return nil, errors.New("a [[copy]] entry needs image, from and to") } } for _, key := range md.Keys() { if len(key) == 2 && key[0] == "variables" { cfg.VariableOrder = append(cfg.VariableOrder, key[1]) } } return &cfg, nil } // ValidateCiConfig reports the run-start checks ParseCiConfig cannot make. // It returns an empty string when the config is usable. func ValidateCiConfig(cfg *Config) string { for _, c := range cfg.Copy { // `to` is a directory and the basename is kept, so a `to` that // repeats the basename means someone expected a rename. Left alone // it silently produces to//. if filepath.Base(c.To) == filepath.Base(c.From) { return fmt.Sprintf( "[[copy]] \"to\" is a directory, so %s lands at %s. Drop the last path segment from \"to\".", c.From, filepath.Join(c.To, filepath.Base(c.From)), ) } } // An unparsable limit would otherwise become 0, which Docker reads as // unlimited. if cfg.MemoryLimit != "" && parseMemoryBytes(cfg.MemoryLimit) <= 0 { return fmt.Sprintf( "memory_limit %q is not a size. Use a number with an optional K, M or G suffix.", cfg.MemoryLimit, ) } if cfg.CloneProjectTo == "" { return "" } // The archive endpoint resolves `path` against /, while the execs that // create and clear the directory resolve against work_dir. A relative // value would name two different directories. if !filepath.IsAbs(cfg.CloneProjectTo) { return fmt.Sprintf("clone_project_to (%q) must be an absolute path.", cfg.CloneProjectTo) } clone := filepath.Clean(cfg.CloneProjectTo) for _, entry := range cfg.Cache { cachePath := filepath.Clean(entry.Path) // Either nesting direction breaks. A cache below the checkout is // overwritten by the extract and deleted by `clear`. A cache above it // carries the previous run's tree back in. if cachePath == clone || strings.HasPrefix(cachePath, clone+string(filepath.Separator)) || strings.HasPrefix(clone, cachePath+string(filepath.Separator)) { return fmt.Sprintf( "cache path %q overlaps clone_project_to (%q). "+ "The checkout is extracted over that directory and a `clear` step "+ "deletes it. Move the cache outside the clone directory.", entry.Path, cfg.CloneProjectTo, ) } } return "" } // shouldTriggerPush reports whether a push to branch starts a run. func shouldTriggerPush(cfg *Config, branch string) bool { if cfg.On.Push.All { return true } for _, pattern := range cfg.On.Push.Patterns { if matchGlob(pattern, branch) { return true } } return false } // matchGlob matches a branch against a config pattern. `*` does not cross a // `/`, so "release/*" matches "release/1" but not "release/1/x". func matchGlob(pattern, value string) bool { if pattern == "*" { return true } ok, err := path.Match(pattern, value) return err == nil && ok } // VariableOverrides picks the manual run form fields that differ from the // config default. An untouched field is not an override: sending the default // back would pin the run to the value the config had at render time. func VariableOverrides(cfg *Config, form url.Values) map[string]string { out := map[string]string{} for name, def := range cfg.Variables { key := "var_" + name if !form.Has(key) { continue } if v := form.Get(key); v != def.Default { out[name] = v } } return out } var memorySizeRe = regexp.MustCompile(`^(\d+(?:\.\d+)?)\s*([kmgKMG]?)b?$`) // parseMemoryBytes reads sizes like "512m" or "2g". It answers 0 for // anything it cannot read. func parseMemoryBytes(s string) int64 { m := memorySizeRe.FindStringSubmatch(s) if m == nil { return 0 } n, err := strconv.ParseFloat(m[1], 64) if err != nil { return 0 } switch strings.ToLower(m[2]) { case "k": n *= 1024 case "m": n *= 1024 * 1024 case "g": n *= 1024 * 1024 * 1024 } return int64(math.Floor(n)) } // formatBytes renders a size the way the cache step log shows it. func formatBytes(n int64) string { units := []string{"B", "K", "M", "G", "T"} i := 0 v := float64(n) for v >= 1024 && i < len(units)-1 { v /= 1024 i++ } if i == 0 { return strconv.FormatFloat(v, 'f', -1, 64) + units[0] } return strconv.FormatFloat(v, 'f', 1, 64) + units[i] }