dovenest

dovenest/server/src/archive.rs
archive.rs
⎇
Raw
1//! Streaming archive builders (zip / tar / tar.gz / tar.zst).
2//!
3//! Everything writes directly into an `impl std::io::Write` sink — no temp
4//! files, no full in-memory buffering. The sink is typically a channel that
5//! feeds the HTTP response body, so bytes reach the client while the tree is
6//! still being walked.
7
8use std::io::{self, Write};
9use std::path::{Path, PathBuf};
10use std::time::UNIX_EPOCH;
11
12/// The archive formats offered for folder downloads.
13#[derive(Clone, Copy, PartialEq, Eq, Debug)]
14pub enum ArchiveFormat {
15 Zip,
16 Tar,
17 TarGz,
18 TarZst,
19}
20
21impl ArchiveFormat {
22 /// Parse the `format` query parameter. Unknown values are rejected.
23 pub fn parse(s: &str) -> Option<Self> {
24 match s {
25 "zip" => Some(Self::Zip),
26 "tar" => Some(Self::Tar),
27 "tar.gz" | "tgz" => Some(Self::TarGz),
28 "tar.zst" | "tzst" => Some(Self::TarZst),
29 _ => None,
30 }
31 }
32
33 /// The file-name suffix for the produced archive.
34 pub fn extension(self) -> &'static str {
35 match self {
36 Self::Zip => "zip",
37 Self::Tar => "tar",
38 Self::TarGz => "tar.gz",
39 Self::TarZst => "tar.zst",
40 }
41 }
42
43 /// MIME type for the produced archive.
44 pub fn mime(self) -> &'static str {
45 match self {
46 Self::Zip => "application/zip",
47 Self::Tar => "application/x-tar",
48 Self::TarGz => "application/gzip",
49 Self::TarZst => "application/zstd",
50 }
51 }
52}
53
54/// Build `dir` (top-level entry named `top_name`) as `format`, streaming into
55/// `sink`. Blocking — call from `spawn_blocking`.
56pub fn build(
57 format: ArchiveFormat,
58 dir: &Path,
59 top_name: &str,
60 sink: impl Write,
61) -> io::Result<()> {
62 match format {
63 ArchiveFormat::Tar => build_tar(dir, top_name, sink),
64 ArchiveFormat::TarGz => build_tar_gz(dir, top_name, sink),
65 ArchiveFormat::TarZst => build_tar_zst(dir, top_name, sink),
66 ArchiveFormat::Zip => build_zip(dir, top_name, sink),
67 }
68}
69
70fn mtime_secs(p: &Path) -> u64 {
71 std::fs::metadata(p)
72 .and_then(|m| m.modified())
73 .ok()
74 .and_then(|t| t.duration_since(UNIX_EPOCH).ok())
75 .map(|d| d.as_secs())
76 .unwrap_or(0)
77}
78
79/// Depth-first walk. Invokes `f(entry_name, abs_path, is_dir)` for the
80/// directory itself and every descendant. Directory entry names carry no
81/// trailing slash; each format appends it as needed. Deterministic order
82/// (case-insensitive name) so archives are reproducible.
83fn walk<F: FnMut(&str, &Path, bool) -> io::Result<()>>(
84 abs_dir: &Path,
85 entry_prefix: &str,
86 f: &mut F,
87) -> io::Result<()> {
88 f(entry_prefix, abs_dir, true)?;
89 let rd = std::fs::read_dir(abs_dir)?;
90 let mut children: Vec<(String, PathBuf, bool)> = Vec::new();
91 for e in rd.flatten() {
92 let name = e.file_name().to_string_lossy().into_owned();
93 let p = e.path();
94 // Follows symlinks (a broken link shows up as an empty file).
95 let is_dir = p.is_dir();
96 children.push((name, p, is_dir));
97 }
98 children.sort_by(|a, b| a.0.to_lowercase().cmp(&b.0.to_lowercase()));
99 for (name, p, is_dir) in children {
100 let child = format!("{entry_prefix}/{name}");
101 if is_dir {
102 walk(&p, &child, f)?;
103 } else {
104 f(&child, &p, false)?;
105 }
106 }
107 Ok(())
108}
109
110// ---------------------------------------------------------------------------
111// tar
112// ---------------------------------------------------------------------------
113
114fn tar_add<W: Write>(
115 tar: &mut tar::Builder<W>,
116 entry: &str,
117 abs: &Path,
118 is_dir: bool,
119) -> io::Result<()> {
120 let mut header = tar::Header::new_gnu();
121 let meta = std::fs::metadata(abs)?;
122 header.set_mode(if is_dir { 0o755 } else { 0o644 });
123 header.set_mtime(mtime_secs(abs));
124 let name = if is_dir {
125 format!("{entry}/")
126 } else {
127 entry.to_string()
128 };
129 if is_dir {
130 header.set_entry_type(tar::EntryType::Directory);
131 header.set_size(0);
132 tar.append_data(&mut header, name, io::empty())?;
133 } else {
134 header.set_entry_type(tar::EntryType::Regular);
135 header.set_size(meta.len());
136 let f = std::fs::File::open(abs)?;
137 tar.append_data(&mut header, name, f)?;
138 }
139 Ok(())
140}
141
142fn build_tar<W: Write>(dir: &Path, top: &str, sink: W) -> io::Result<()> {
143 let mut tar = tar::Builder::new(sink);
144 let mut add = |entry: &str, abs: &Path, is_dir: bool| -> io::Result<()> {
145 tar_add(&mut tar, entry, abs, is_dir)
146 };
147 walk(dir, top, &mut add)?;
148 tar.finish()?;
149 Ok(())
150}
151
152fn build_tar_gz<W: Write>(dir: &Path, top: &str, sink: W) -> io::Result<()> {
153 let enc = flate2::write::GzEncoder::new(sink, flate2::Compression::default());
154 let mut tar = tar::Builder::new(enc);
155 let mut add = |entry: &str, abs: &Path, is_dir: bool| -> io::Result<()> {
156 tar_add(&mut tar, entry, abs, is_dir)
157 };
158 walk(dir, top, &mut add)?;
159 let enc = tar.into_inner()?;
160 enc.finish()?;
161 Ok(())
162}
163
164fn build_tar_zst<W: Write>(dir: &Path, top: &str, sink: W) -> io::Result<()> {
165 let enc = zstd::stream::write::Encoder::new(sink, 3)?;
166 let mut tar = tar::Builder::new(enc);
167 let mut add = |entry: &str, abs: &Path, is_dir: bool| -> io::Result<()> {
168 tar_add(&mut tar, entry, abs, is_dir)
169 };
170 walk(dir, top, &mut add)?;
171 let enc = tar.into_inner()?;
172 enc.finish()?;
173 Ok(())
174}
175
176// ---------------------------------------------------------------------------
177// zip
178// ---------------------------------------------------------------------------
179
180fn build_zip<W: Write>(dir: &Path, top: &str, sink: W) -> io::Result<()> {
181 // Streaming mode: no `Seek` needed, entries use data descriptors.
182 let mut zip = zip::write::ZipWriter::new_stream(sink);
183 let mut add = |entry: &str, abs: &Path, is_dir: bool| -> io::Result<()> {
184 let opts = zip::write::SimpleFileOptions::default();
185 let name = if is_dir {
186 format!("{entry}/")
187 } else {
188 entry.to_string()
189 };
190 if is_dir {
191 zip.add_directory(&name, opts)?;
192 } else {
193 zip.start_file(&name, opts)?;
194 let mut f = std::fs::File::open(abs)?;
195 io::copy(&mut f, &mut zip)?;
196 }
197 Ok(())
198 };
199 walk(dir, top, &mut add)?;
200 zip.finish()?;
201 Ok(())
202}
203
204// ---------------------------------------------------------------------------
205// Tests
206// ---------------------------------------------------------------------------
207
208#[cfg(test)]
209mod tests {
210 use super::*;
211 use std::collections::BTreeMap;
212 use std::io::Read as _;
213
214 fn sample_dir() -> (tempfile::TempDir, PathBuf) {
215 let tmp = tempfile::tempdir().unwrap();
216 let dir = tmp.path().to_path_buf();
217 std::fs::write(dir.join("alpha.txt"), "alpha content").unwrap();
218 std::fs::create_dir_all(dir.join("sub/deep")).unwrap();
219 std::fs::create_dir(dir.join("empty-dir")).unwrap();
220 std::fs::write(dir.join("sub/beta.txt"), "beta").unwrap();
221 std::fs::write(dir.join("sub/deep/gamma.bin"), (0u8..=255).collect::<Vec<_>>()).unwrap();
222 (tmp, dir)
223 }
224
225 fn build_to_mem(fmt: ArchiveFormat, dir: &Path) -> Vec<u8> {
226 let mut out: Vec<u8> = Vec::new();
227 build(fmt, dir, "top", &mut out).unwrap();
228 out
229 }
230
231 #[test]
232 fn format_parsing() {
233 assert_eq!(ArchiveFormat::parse("zip"), Some(ArchiveFormat::Zip));
234 assert_eq!(ArchiveFormat::parse("tar"), Some(ArchiveFormat::Tar));
235 assert_eq!(ArchiveFormat::parse("tar.gz"), Some(ArchiveFormat::TarGz));
236 assert_eq!(ArchiveFormat::parse("tgz"), Some(ArchiveFormat::TarGz));
237 assert_eq!(ArchiveFormat::parse("tar.zst"), Some(ArchiveFormat::TarZst));
238 assert_eq!(ArchiveFormat::parse("tzst"), Some(ArchiveFormat::TarZst));
239 for bad in ["", "ZIP", "gzip", "rar", "7z", "tar.bz2", "tar.xz", "tar.zstx", "tar.gz "] {
240 assert_eq!(ArchiveFormat::parse(bad), None, "{bad:?}");
241 }
242 }
243
244 #[test]
245 fn format_metadata() {
246 assert_eq!(ArchiveFormat::Zip.extension(), "zip");
247 assert_eq!(ArchiveFormat::Zip.mime(), "application/zip");
248 assert_eq!(ArchiveFormat::Tar.extension(), "tar");
249 assert_eq!(ArchiveFormat::Tar.mime(), "application/x-tar");
250 assert_eq!(ArchiveFormat::TarGz.extension(), "tar.gz");
251 assert_eq!(ArchiveFormat::TarGz.mime(), "application/gzip");
252 assert_eq!(ArchiveFormat::TarZst.extension(), "tar.zst");
253 assert_eq!(ArchiveFormat::TarZst.mime(), "application/zstd");
254 }
255
256 /// Read a tar stream into a name → content map (files only).
257 fn tar_map<R: std::io::Read>(r: R) -> BTreeMap<String, Vec<u8>> {
258 let mut map = BTreeMap::new();
259 for entry in tar::Archive::new(r).entries().unwrap() {
260 let mut e = entry.unwrap();
261 if !e.header().entry_type().is_file() {
262 continue;
263 }
264 let name = e.path().unwrap().to_string_lossy().into_owned();
265 let mut buf = Vec::new();
266 e.read_to_end(&mut buf).unwrap();
267 map.insert(name, buf);
268 }
269 map
270 }
271 fn expected_map() -> BTreeMap<String, Vec<u8>> {
272 let mut m = BTreeMap::new();
273 m.insert("top/alpha.txt".to_string(), b"alpha content".to_vec());
274 m.insert("top/sub/beta.txt".to_string(), b"beta".to_vec());
275 m.insert(
276 "top/sub/deep/gamma.bin".to_string(),
277 (0u8..=255).collect(),
278 );
279 m
280 }
281
282 #[test]
283 fn zip_round_trip() {
284 let (_tmp, dir) = sample_dir();
285 let bytes = build_to_mem(ArchiveFormat::Zip, &dir);
286 let mut zip = zip::ZipArchive::new(std::io::Cursor::new(bytes)).unwrap();
287
288 let mut map = BTreeMap::new();
289 for i in 0..zip.len() {
290 let mut f = zip.by_index(i).unwrap();
291 let name = f.name().unwrap().to_string();
292 if name.ends_with('/') {
293 continue; // directory entry
294 }
295 let mut buf = Vec::new();
296 f.read_to_end(&mut buf).unwrap();
297 map.insert(name, buf);
298 }
299 assert_eq!(map, expected_map());
300
301 // Directory entries are present and the order is deterministic.
302 let names: Vec<String> = zip
303 .file_names()
304 .map(|n| n.unwrap().to_string())
305 .collect();
306 let has = |n: &str| names.iter().any(|x| x == n);
307 assert!(has("top/"));
308 assert!(has("top/sub/"));
309 assert!(has("top/sub/deep/"));
310 assert!(has("top/empty-dir/"));
311 let mut sorted = names.clone();
312 sorted.sort_unstable();
313 assert_eq!(names, sorted);
314 }
315
316 #[test]
317 fn tar_round_trip() {
318 let (_tmp, dir) = sample_dir();
319 let bytes = build_to_mem(ArchiveFormat::Tar, &dir);
320 let map = tar_map(std::io::Cursor::new(bytes));
321 assert_eq!(map, expected_map());
322 }
323
324 #[test]
325 fn tar_gz_round_trip() {
326 let (_tmp, dir) = sample_dir();
327 let bytes = build_to_mem(ArchiveFormat::TarGz, &dir);
328 let gz = flate2::read::GzDecoder::new(std::io::Cursor::new(bytes));
329 let map = tar_map(gz);
330 assert_eq!(map, expected_map());
331 }
332
333 #[test]
334 fn tar_zst_round_trip() {
335 let (_tmp, dir) = sample_dir();
336 let bytes = build_to_mem(ArchiveFormat::TarZst, &dir);
337 let dec = zstd::stream::read::Decoder::new(std::io::Cursor::new(bytes)).unwrap();
338 let map = tar_map(dec);
339 assert_eq!(map, expected_map());
340 }
341
342 #[test]
343 fn walk_is_sorted_case_insensitively() {
344 let tmp = tempfile::tempdir().unwrap();
345 let dir = tmp.path();
346 for name in ["Zeta", "alpha", "Beta", "a.txt", "B.txt"] {
347 if name.ends_with(".txt") {
348 std::fs::write(dir.join(name), name).unwrap();
349 } else {
350 std::fs::create_dir(dir.join(name)).unwrap();
351 }
352 }
353 let mut order = Vec::new();
354 walk(dir, "top", &mut |name, _p, _is_dir| {
355 order.push(name.to_string());
356 Ok(())
357 })
358 .unwrap();
359 assert_eq!(
360 order,
361 vec![
362 "top",
363 "top/a.txt",
364 "top/alpha",
365 "top/B.txt",
366 "top/Beta",
367 "top/Zeta"
368 ]
369 );
370 }
371
372 #[test]
373 fn build_fails_on_missing_dir() {
374 let mut out = Vec::new();
375 let r = build(
376 ArchiveFormat::Zip,
377 Path::new("/nonexistent-filebrowser-ng-test-dir"),
378 "top",
379 &mut out,
380 );
381 assert!(r.is_err());
382 // The tar path fails too.
383 let mut out = Vec::new();
384 let r = build(
385 ArchiveFormat::Tar,
386 Path::new("/nonexistent-filebrowser-ng-test-dir"),
387 "top",
388 &mut out,
389 );
390 assert!(r.is_err());
391 }
392}
393