dovenest

dovenest/server/src/archive.rs
archive.rs
⎇
Raw
1//! Streaming archive builders (zip / tar / tar.gz / tar.zst).
2//!
3//! Everything writes directly into an `impl std::io::Write` sink — no temp
4//! files, no full in-memory buffering. The sink is typically a channel that
5//! feeds the HTTP response body, so bytes reach the client while the tree is
6//! still being walked.
7
8use std::io::{self, Write};
9use std::path::{Path, PathBuf};
10
11/// The archive formats offered for folder downloads.
12#[derive(Clone, Copy, PartialEq, Eq, Debug)]
13pub enum ArchiveFormat {
14 Zip,
15 Tar,
16 TarGz,
17 TarZst,
18}
19
20impl ArchiveFormat {
21 /// Parse the `format` query parameter. Unknown values are rejected.
22 pub fn parse(s: &str) -> Option<Self> {
23 match s {
24 "zip" => Some(Self::Zip),
25 "tar" => Some(Self::Tar),
26 "tar.gz" | "tgz" => Some(Self::TarGz),
27 "tar.zst" | "tzst" => Some(Self::TarZst),
28 _ => None,
29 }
30 }
31
32 /// The file-name suffix for the produced archive.
33 pub fn extension(self) -> &'static str {
34 match self {
35 Self::Zip => "zip",
36 Self::Tar => "tar",
37 Self::TarGz => "tar.gz",
38 Self::TarZst => "tar.zst",
39 }
40 }
41
42 /// MIME type for the produced archive.
43 pub fn mime(self) -> &'static str {
44 match self {
45 Self::Zip => "application/zip",
46 Self::Tar => "application/x-tar",
47 Self::TarGz => "application/gzip",
48 Self::TarZst => "application/zstd",
49 }
50 }
51}
52
53/// Build `dir` (top-level entry named `top_name`) as `format`, streaming into
54/// `sink`. Blocking — call from `spawn_blocking`.
55pub fn build(
56 format: ArchiveFormat,
57 dir: &Path,
58 top_name: &str,
59 sink: impl Write,
60) -> io::Result<()> {
61 match format {
62 ArchiveFormat::Tar => build_tar(dir, top_name, sink).map(|_| ()),
63 ArchiveFormat::TarGz => {
64 let enc = flate2::write::GzEncoder::new(sink, flate2::Compression::default());
65 build_tar(dir, top_name, enc)?.finish().map(|_| ())
66 }
67 ArchiveFormat::TarZst => {
68 let enc = zstd::stream::write::Encoder::new(sink, 3)?;
69 build_tar(dir, top_name, enc)?.finish().map(|_| ())
70 }
71 ArchiveFormat::Zip => build_zip(dir, top_name, sink),
72 }
73}
74
75fn mtime_secs(p: &Path) -> u64 {
76 std::fs::metadata(p)
77 .ok()
78 .and_then(|m| crate::fs::mtime_secs(&m))
79 .unwrap_or(0)
80 .max(0) as u64
81}
82
83/// Cycle guard: a symlink loop would otherwise recurse forever. Real trees
84/// this deep are not worth archiving, so deeper levels are dropped.
85const MAX_DEPTH: usize = 64;
86
87/// Depth-first walk. Invokes `f(entry_name, abs_path, is_dir)` for the
88/// directory itself and every descendant. Directory entry names carry no
89/// trailing slash; each format appends it as needed. Deterministic order
90/// (case-insensitive name) so archives are reproducible.
91fn walk<F: FnMut(&str, &Path, bool) -> io::Result<()>>(
92 abs_dir: &Path,
93 entry_prefix: &str,
94 f: &mut F,
95) -> io::Result<()> {
96 // The canonical top directory is the containment boundary for the whole
97 // walk. Unlike browse and upload, nothing else re-checks it here.
98 let base = abs_dir.canonicalize()?;
99 walk_in(&base, &base, entry_prefix, 0, f)
100}
101
102fn walk_in<F: FnMut(&str, &Path, bool) -> io::Result<()>>(
103 base: &Path,
104 abs_dir: &Path,
105 entry_prefix: &str,
106 depth: usize,
107 f: &mut F,
108) -> io::Result<()> {
109 f(entry_prefix, abs_dir, true)?;
110 if depth >= MAX_DEPTH {
111 tracing::warn!(path = %abs_dir.display(), "archive: depth limit reached, subtree skipped");
112 return Ok(());
113 }
114 let rd = std::fs::read_dir(abs_dir)?;
115 let mut children: Vec<(String, PathBuf, bool)> = Vec::new();
116 for e in rd.flatten() {
117 let name = e.file_name().to_string_lossy().into_owned();
118 // Resolve symlinks: a link inside the tree may point outside it, and
119 // its contents must not end up in the archive. One bad entry is
120 // skipped instead of failing the whole download.
121 let Ok(p) = e.path().canonicalize() else {
122 tracing::warn!(path = %e.path().display(), "archive: unreadable entry skipped");
123 continue;
124 };
125 if !crate::fs::is_within_or_eq(base, &p) {
126 tracing::warn!(path = %e.path().display(), "archive: entry outside the archive root skipped");
127 continue;
128 }
129 let is_dir = p.is_dir();
130 children.push((name, p, is_dir));
131 }
132 children.sort_by_key(|c| c.0.to_lowercase());
133 for (name, p, is_dir) in children {
134 let child = format!("{entry_prefix}/{name}");
135 if is_dir {
136 walk_in(base, &p, &child, depth + 1, f)?;
137 } else {
138 f(&child, &p, false)?;
139 }
140 }
141 Ok(())
142}
143
144// ---------------------------------------------------------------------------
145// tar
146// ---------------------------------------------------------------------------
147
148fn tar_add<W: Write>(
149 tar: &mut tar::Builder<W>,
150 entry: &str,
151 abs: &Path,
152 is_dir: bool,
153) -> io::Result<()> {
154 let mut header = tar::Header::new_gnu();
155 let meta = std::fs::metadata(abs)?;
156 header.set_mode(if is_dir { 0o755 } else { 0o644 });
157 header.set_mtime(mtime_secs(abs));
158 let name = if is_dir {
159 format!("{entry}/")
160 } else {
161 entry.to_string()
162 };
163 if is_dir {
164 header.set_entry_type(tar::EntryType::Directory);
165 header.set_size(0);
166 tar.append_data(&mut header, name, io::empty())?;
167 } else {
168 header.set_entry_type(tar::EntryType::Regular);
169 header.set_size(meta.len());
170 let f = std::fs::File::open(abs)?;
171 tar.append_data(&mut header, name, f)?;
172 }
173 Ok(())
174}
175
176/// Write the tar stream into `sink` and hand `sink` back, so a caller that
177/// wrapped it in a compressor can finish that compressor.
178fn build_tar<W: Write>(dir: &Path, top: &str, sink: W) -> io::Result<W> {
179 let mut tar = tar::Builder::new(sink);
180 let mut add = |entry: &str, abs: &Path, is_dir: bool| -> io::Result<()> {
181 tar_add(&mut tar, entry, abs, is_dir)
182 };
183 walk(dir, top, &mut add)?;
184 tar.finish()?;
185 tar.into_inner()
186}
187
188// ---------------------------------------------------------------------------
189// zip
190// ---------------------------------------------------------------------------
191
192fn build_zip<W: Write>(dir: &Path, top: &str, sink: W) -> io::Result<()> {
193 // Streaming mode: no `Seek` needed, entries use data descriptors.
194 let mut zip = zip::write::ZipWriter::new_stream(sink);
195 let mut add = |entry: &str, abs: &Path, is_dir: bool| -> io::Result<()> {
196 let opts = zip::write::SimpleFileOptions::default();
197 let name = if is_dir {
198 format!("{entry}/")
199 } else {
200 entry.to_string()
201 };
202 if is_dir {
203 zip.add_directory(&name, opts)?;
204 } else {
205 zip.start_file(&name, opts)?;
206 let mut f = std::fs::File::open(abs)?;
207 io::copy(&mut f, &mut zip)?;
208 }
209 Ok(())
210 };
211 walk(dir, top, &mut add)?;
212 zip.finish()?;
213 Ok(())
214}
215
216// ---------------------------------------------------------------------------
217// Tests
218// ---------------------------------------------------------------------------
219
220#[cfg(test)]
221mod tests {
222 use super::*;
223 use std::collections::BTreeMap;
224 use std::io::Read as _;
225
226 fn sample_dir() -> (tempfile::TempDir, PathBuf) {
227 let tmp = tempfile::tempdir().unwrap();
228 let dir = tmp.path().to_path_buf();
229 std::fs::write(dir.join("alpha.txt"), "alpha content").unwrap();
230 std::fs::create_dir_all(dir.join("sub/deep")).unwrap();
231 std::fs::create_dir(dir.join("empty-dir")).unwrap();
232 std::fs::write(dir.join("sub/beta.txt"), "beta").unwrap();
233 std::fs::write(
234 dir.join("sub/deep/gamma.bin"),
235 (0u8..=255).collect::<Vec<_>>(),
236 )
237 .unwrap();
238 (tmp, dir)
239 }
240
241 fn build_to_mem(fmt: ArchiveFormat, dir: &Path) -> Vec<u8> {
242 let mut out: Vec<u8> = Vec::new();
243 build(fmt, dir, "top", &mut out).unwrap();
244 out
245 }
246
247 #[test]
248 fn format_parsing() {
249 assert_eq!(ArchiveFormat::parse("zip"), Some(ArchiveFormat::Zip));
250 assert_eq!(ArchiveFormat::parse("tar"), Some(ArchiveFormat::Tar));
251 assert_eq!(ArchiveFormat::parse("tar.gz"), Some(ArchiveFormat::TarGz));
252 assert_eq!(ArchiveFormat::parse("tgz"), Some(ArchiveFormat::TarGz));
253 assert_eq!(ArchiveFormat::parse("tar.zst"), Some(ArchiveFormat::TarZst));
254 assert_eq!(ArchiveFormat::parse("tzst"), Some(ArchiveFormat::TarZst));
255 for bad in [
256 "", "ZIP", "gzip", "rar", "7z", "tar.bz2", "tar.xz", "tar.zstx", "tar.gz ",
257 ] {
258 assert_eq!(ArchiveFormat::parse(bad), None, "{bad:?}");
259 }
260 }
261
262 #[test]
263 fn format_metadata() {
264 assert_eq!(ArchiveFormat::Zip.extension(), "zip");
265 assert_eq!(ArchiveFormat::Zip.mime(), "application/zip");
266 assert_eq!(ArchiveFormat::Tar.extension(), "tar");
267 assert_eq!(ArchiveFormat::Tar.mime(), "application/x-tar");
268 assert_eq!(ArchiveFormat::TarGz.extension(), "tar.gz");
269 assert_eq!(ArchiveFormat::TarGz.mime(), "application/gzip");
270 assert_eq!(ArchiveFormat::TarZst.extension(), "tar.zst");
271 assert_eq!(ArchiveFormat::TarZst.mime(), "application/zstd");
272 }
273
274 /// Read a tar stream into a name → content map (files only).
275 fn tar_map<R: std::io::Read>(r: R) -> BTreeMap<String, Vec<u8>> {
276 let mut map = BTreeMap::new();
277 for entry in tar::Archive::new(r).entries().unwrap() {
278 let mut e = entry.unwrap();
279 if !e.header().entry_type().is_file() {
280 continue;
281 }
282 let name = e.path().unwrap().to_string_lossy().into_owned();
283 let mut buf = Vec::new();
284 e.read_to_end(&mut buf).unwrap();
285 map.insert(name, buf);
286 }
287 map
288 }
289 fn expected_map() -> BTreeMap<String, Vec<u8>> {
290 let mut m = BTreeMap::new();
291 m.insert("top/alpha.txt".to_string(), b"alpha content".to_vec());
292 m.insert("top/sub/beta.txt".to_string(), b"beta".to_vec());
293 m.insert("top/sub/deep/gamma.bin".to_string(), (0u8..=255).collect());
294 m
295 }
296
297 #[test]
298 fn zip_round_trip() {
299 let (_tmp, dir) = sample_dir();
300 let bytes = build_to_mem(ArchiveFormat::Zip, &dir);
301 let mut zip = zip::ZipArchive::new(std::io::Cursor::new(bytes)).unwrap();
302
303 let mut map = BTreeMap::new();
304 for i in 0..zip.len() {
305 let mut f = zip.by_index(i).unwrap();
306 let name = f.name().unwrap().to_string();
307 if name.ends_with('/') {
308 continue; // directory entry
309 }
310 let mut buf = Vec::new();
311 f.read_to_end(&mut buf).unwrap();
312 map.insert(name, buf);
313 }
314 assert_eq!(map, expected_map());
315
316 // Directory entries are present and the order is deterministic.
317 let names: Vec<String> = zip.file_names().map(|n| n.unwrap().to_string()).collect();
318 let has = |n: &str| names.iter().any(|x| x == n);
319 assert!(has("top/"));
320 assert!(has("top/sub/"));
321 assert!(has("top/sub/deep/"));
322 assert!(has("top/empty-dir/"));
323 let mut sorted = names.clone();
324 sorted.sort_unstable();
325 assert_eq!(names, sorted);
326 }
327
328 #[test]
329 fn tar_round_trip() {
330 let (_tmp, dir) = sample_dir();
331 let bytes = build_to_mem(ArchiveFormat::Tar, &dir);
332 let map = tar_map(std::io::Cursor::new(bytes));
333 assert_eq!(map, expected_map());
334 }
335
336 #[test]
337 fn tar_gz_round_trip() {
338 let (_tmp, dir) = sample_dir();
339 let bytes = build_to_mem(ArchiveFormat::TarGz, &dir);
340 let gz = flate2::read::GzDecoder::new(std::io::Cursor::new(bytes));
341 let map = tar_map(gz);
342 assert_eq!(map, expected_map());
343 }
344
345 #[test]
346 fn tar_zst_round_trip() {
347 let (_tmp, dir) = sample_dir();
348 let bytes = build_to_mem(ArchiveFormat::TarZst, &dir);
349 let dec = zstd::stream::read::Decoder::new(std::io::Cursor::new(bytes)).unwrap();
350 let map = tar_map(dec);
351 assert_eq!(map, expected_map());
352 }
353
354 #[test]
355 fn walk_is_sorted_case_insensitively() {
356 let tmp = tempfile::tempdir().unwrap();
357 let dir = tmp.path();
358 for name in ["Zeta", "alpha", "Beta", "a.txt", "B.txt"] {
359 if name.ends_with(".txt") {
360 std::fs::write(dir.join(name), name).unwrap();
361 } else {
362 std::fs::create_dir(dir.join(name)).unwrap();
363 }
364 }
365 let mut order = Vec::new();
366 walk(dir, "top", &mut |name, _p, _is_dir| {
367 order.push(name.to_string());
368 Ok(())
369 })
370 .unwrap();
371 assert_eq!(
372 order,
373 vec![
374 "top",
375 "top/a.txt",
376 "top/alpha",
377 "top/B.txt",
378 "top/Beta",
379 "top/Zeta"
380 ]
381 );
382 }
383
384 #[test]
385 fn build_fails_on_missing_dir() {
386 let mut out = Vec::new();
387 let r = build(
388 ArchiveFormat::Zip,
389 Path::new("/nonexistent-filebrowser-ng-test-dir"),
390 "top",
391 &mut out,
392 );
393 assert!(r.is_err());
394 // The tar path fails too.
395 let mut out = Vec::new();
396 let r = build(
397 ArchiveFormat::Tar,
398 Path::new("/nonexistent-filebrowser-ng-test-dir"),
399 "top",
400 &mut out,
401 );
402 assert!(r.is_err());
403 }
404}
405