hearthforge

hearthforge/internal/ci/archive.go
archive.go
⎇
Raw
1package ci
2
3import (
4 "archive/tar"
5 "archive/zip"
6 "bytes"
7 "compress/gzip"
8 "context"
9 "errors"
10 "fmt"
11 "io"
12 "net/http"
13 "net/url"
14 "os"
15 "path"
16 "strings"
17
18 "github.com/klauspost/compress/zstd"
19)
20
21// The container image needs no tar, gzip, zstd or zip. The engine streams
22// any path out of a container as a tar, and Hearthforge converts that
23// stream on the host. Directories are created the same way in reverse: a
24// tar with directory entries, extracted at /, needs no mkdir in the image.
25
26// mkdirTar builds a tar that creates dir and its parents when extracted at /.
27func mkdirTar(dir string) []byte {
28 var buf bytes.Buffer
29 tw := tar.NewWriter(&buf)
30 clean := strings.TrimPrefix(path.Clean(dir), "/")
31 if clean != "" && clean != "." {
32 parts := strings.Split(clean, "/")
33 for i := range parts {
34 _ = tw.WriteHeader(&tar.Header{
35 Name: strings.Join(parts[:i+1], "/") + "/", Mode: 0o755, Typeflag: tar.TypeDir,
36 })
37 }
38 }
39 _ = tw.Close()
40 return buf.Bytes()
41}
42
43// mkdirInContainer creates dir and its parents without running a command.
44func (r *Runner) mkdirInContainer(ctx context.Context, containerID, dir string) error {
45 resp, body, err := r.putArchive(ctx, containerID, "/", bytes.NewReader(mkdirTar(dir)))
46 if err != nil {
47 return err
48 }
49 if resp.StatusCode >= 300 {
50 return fmt.Errorf("cannot create %s in the container: HTTP %d %s",
51 dir, resp.StatusCode, strings.TrimSpace(string(body)))
52 }
53 return nil
54}
55
56// archiveFormats maps a publish_* option to its file extension. The order
57// is the order artifacts are collected in.
58var archiveFormats = []struct {
59 pick func(Step) StringList
60 ext string
61}{
62 {func(s Step) StringList { return s.PublishTar }, ".tar"},
63 {func(s Step) StringList { return s.PublishGzip }, ".tar.gz"},
64 {func(s Step) StringList { return s.PublishZstd }, ".tar.zst"},
65 {func(s Step) StringList { return s.PublishZip }, ".zip"},
66}
67
68// storeArchive streams srcPath out of the container and writes it to
69// destPath in the format named by ext. maxBytes caps the written file.
70func (r *Runner) storeArchive(ctx context.Context, containerID, srcPath, destPath, ext string, maxBytes int64) (int64, error) {
71 resp, err := r.do(ctx, http.MethodGet,
72 "/containers/"+containerID+"/archive?path="+url.QueryEscape(srcPath), nil, "")
73 if err != nil {
74 return 0, err
75 }
76 defer discard(resp)
77 if resp.StatusCode >= 300 {
78 return 0, fmt.Errorf("cannot read %s: HTTP %d", srcPath, resp.StatusCode)
79 }
80 f, err := os.Create(destPath)
81 if err != nil {
82 return 0, err
83 }
84 lw := &limitedWriter{w: f, left: maxBytes}
85 err = writeArchive(ext, resp.Body, lw)
86 if cerr := f.Close(); err == nil {
87 err = cerr
88 }
89 if err != nil {
90 os.Remove(destPath)
91 return 0, err
92 }
93 return maxBytes - lw.left, nil
94}
95
96// writeArchive converts a tar stream into the wanted format.
97func writeArchive(ext string, src io.Reader, dst io.Writer) error {
98 switch ext {
99 case ".tar":
100 _, err := io.Copy(dst, src)
101 return err
102 case ".tar.gz":
103 gz := gzip.NewWriter(dst)
104 if _, err := io.Copy(gz, src); err != nil {
105 return err
106 }
107 return gz.Close()
108 case ".tar.zst":
109 enc, err := zstd.NewWriter(dst)
110 if err != nil {
111 return err
112 }
113 if _, err := io.Copy(enc, src); err != nil {
114 return err
115 }
116 return enc.Close()
117 case ".zip":
118 return tarToZip(src, dst)
119 }
120 return fmt.Errorf("unknown archive format %q", ext)
121}
122
123// tarToZip re-packs regular files and directories. Symlinks and devices
124// have no zip equivalent and are skipped.
125func tarToZip(src io.Reader, dst io.Writer) error {
126 tr := tar.NewReader(src)
127 zw := zip.NewWriter(dst)
128 for {
129 hdr, err := tr.Next()
130 if errors.Is(err, io.EOF) {
131 break
132 }
133 if err != nil {
134 return err
135 }
136 switch hdr.Typeflag {
137 case tar.TypeDir:
138 if _, err := zw.CreateHeader(&zip.FileHeader{
139 Name: strings.TrimSuffix(hdr.Name, "/") + "/", Modified: hdr.ModTime,
140 }); err != nil {
141 return err
142 }
143 case tar.TypeReg:
144 w, err := zw.CreateHeader(&zip.FileHeader{
145 Name: hdr.Name, Method: zip.Deflate, Modified: hdr.ModTime,
146 })
147 if err != nil {
148 return err
149 }
150 // The input tar is not compressed and the output is size-capped
151 // by limitedWriter, so no decompression bomb is possible here.
152 if _, err := io.Copy(w, tr); err != nil { //nolint:gosec
153 return err
154 }
155 }
156 }
157 return zw.Close()
158}
159
160// errArtifactTooLarge is what a capped write reports.
161var errArtifactTooLarge = errors.New("artifact exceeds CI_MAX_ARTIFACT_BYTES")
162
163// limitedWriter fails once more than left bytes were written.
164type limitedWriter struct {
165 w io.Writer
166 left int64
167}
168
169func (l *limitedWriter) Write(p []byte) (int, error) {
170 if int64(len(p)) > l.left {
171 return 0, errArtifactTooLarge
172 }
173 n, err := l.w.Write(p)
174 l.left -= int64(n)
175 return n, err
176}
177