dovenest

dovenest/server/src/archive.rs
archive.rs
⎇
Raw
1//! Streaming archive builders (zip / tar / tar.gz / tar.zst).
2//!
3//! Everything writes directly into an `impl std::io::Write` sink — no temp
4//! files, no full in-memory buffering. The sink is typically a channel that
5//! feeds the HTTP response body, so bytes reach the client while the tree is
6//! still being walked.
7
8use std::io::{self, Write};
9use std::path::{Path, PathBuf};
10use std::time::UNIX_EPOCH;
11
12/// The archive formats offered for folder downloads.
13#[derive(Clone, Copy, PartialEq, Eq, Debug)]
14pub enum ArchiveFormat {
15 Zip,
16 Tar,
17 TarGz,
18 TarZst,
19}
20
21impl ArchiveFormat {
22 /// Parse the `format` query parameter. Unknown values are rejected.
23 pub fn parse(s: &str) -> Option<Self> {
24 match s {
25 "zip" => Some(Self::Zip),
26 "tar" => Some(Self::Tar),
27 "tar.gz" | "tgz" => Some(Self::TarGz),
28 "tar.zst" | "tzst" => Some(Self::TarZst),
29 _ => None,
30 }
31 }
32
33 /// The file-name suffix for the produced archive.
34 pub fn extension(self) -> &'static str {
35 match self {
36 Self::Zip => "zip",
37 Self::Tar => "tar",
38 Self::TarGz => "tar.gz",
39 Self::TarZst => "tar.zst",
40 }
41 }
42
43 /// MIME type for the produced archive.
44 pub fn mime(self) -> &'static str {
45 match self {
46 Self::Zip => "application/zip",
47 Self::Tar => "application/x-tar",
48 Self::TarGz => "application/gzip",
49 Self::TarZst => "application/zstd",
50 }
51 }
52}
53
54/// Build `dir` (top-level entry named `top_name`) as `format`, streaming into
55/// `sink`. Blocking — call from `spawn_blocking`.
56pub fn build(
57 format: ArchiveFormat,
58 dir: &Path,
59 top_name: &str,
60 sink: impl Write,
61) -> io::Result<()> {
62 match format {
63 ArchiveFormat::Tar => build_tar(dir, top_name, sink).map(|_| ()),
64 ArchiveFormat::TarGz => {
65 let enc = flate2::write::GzEncoder::new(sink, flate2::Compression::default());
66 build_tar(dir, top_name, enc)?.finish().map(|_| ())
67 }
68 ArchiveFormat::TarZst => {
69 let enc = zstd::stream::write::Encoder::new(sink, 3)?;
70 build_tar(dir, top_name, enc)?.finish().map(|_| ())
71 }
72 ArchiveFormat::Zip => build_zip(dir, top_name, sink),
73 }
74}
75
76fn mtime_secs(p: &Path) -> u64 {
77 std::fs::metadata(p)
78 .and_then(|m| m.modified())
79 .ok()
80 .and_then(|t| t.duration_since(UNIX_EPOCH).ok())
81 .map(|d| d.as_secs())
82 .unwrap_or(0)
83}
84
85/// Cycle guard: a symlink loop would otherwise recurse forever. Real trees
86/// this deep are not worth archiving, so deeper levels are dropped.
87const MAX_DEPTH: usize = 64;
88
89/// Depth-first walk. Invokes `f(entry_name, abs_path, is_dir)` for the
90/// directory itself and every descendant. Directory entry names carry no
91/// trailing slash; each format appends it as needed. Deterministic order
92/// (case-insensitive name) so archives are reproducible.
93fn walk<F: FnMut(&str, &Path, bool) -> io::Result<()>>(
94 abs_dir: &Path,
95 entry_prefix: &str,
96 f: &mut F,
97) -> io::Result<()> {
98 // The canonical top directory is the containment boundary for the whole
99 // walk. Unlike browse and upload, nothing else re-checks it here.
100 let base = abs_dir.canonicalize()?;
101 walk_in(&base, &base, entry_prefix, 0, f)
102}
103
104fn walk_in<F: FnMut(&str, &Path, bool) -> io::Result<()>>(
105 base: &Path,
106 abs_dir: &Path,
107 entry_prefix: &str,
108 depth: usize,
109 f: &mut F,
110) -> io::Result<()> {
111 f(entry_prefix, abs_dir, true)?;
112 if depth >= MAX_DEPTH {
113 tracing::warn!(path = %abs_dir.display(), "archive: depth limit reached, subtree skipped");
114 return Ok(());
115 }
116 let rd = std::fs::read_dir(abs_dir)?;
117 let mut children: Vec<(String, PathBuf, bool)> = Vec::new();
118 for e in rd.flatten() {
119 let name = e.file_name().to_string_lossy().into_owned();
120 // Resolve symlinks: a link inside the tree may point outside it, and
121 // its contents must not end up in the archive. One bad entry is
122 // skipped instead of failing the whole download.
123 let Ok(p) = e.path().canonicalize() else {
124 tracing::warn!(path = %e.path().display(), "archive: unreadable entry skipped");
125 continue;
126 };
127 if !crate::fs::is_within_or_eq(base, &p) {
128 tracing::warn!(path = %e.path().display(), "archive: entry outside the archive root skipped");
129 continue;
130 }
131 let is_dir = p.is_dir();
132 children.push((name, p, is_dir));
133 }
134 children.sort_by_key(|c| c.0.to_lowercase());
135 for (name, p, is_dir) in children {
136 let child = format!("{entry_prefix}/{name}");
137 if is_dir {
138 walk_in(base, &p, &child, depth + 1, f)?;
139 } else {
140 f(&child, &p, false)?;
141 }
142 }
143 Ok(())
144}
145
146// ---------------------------------------------------------------------------
147// tar
148// ---------------------------------------------------------------------------
149
150fn tar_add<W: Write>(
151 tar: &mut tar::Builder<W>,
152 entry: &str,
153 abs: &Path,
154 is_dir: bool,
155) -> io::Result<()> {
156 let mut header = tar::Header::new_gnu();
157 let meta = std::fs::metadata(abs)?;
158 header.set_mode(if is_dir { 0o755 } else { 0o644 });
159 header.set_mtime(mtime_secs(abs));
160 let name = if is_dir {
161 format!("{entry}/")
162 } else {
163 entry.to_string()
164 };
165 if is_dir {
166 header.set_entry_type(tar::EntryType::Directory);
167 header.set_size(0);
168 tar.append_data(&mut header, name, io::empty())?;
169 } else {
170 header.set_entry_type(tar::EntryType::Regular);
171 header.set_size(meta.len());
172 let f = std::fs::File::open(abs)?;
173 tar.append_data(&mut header, name, f)?;
174 }
175 Ok(())
176}
177
178/// Write the tar stream into `sink` and hand `sink` back, so a caller that
179/// wrapped it in a compressor can finish that compressor.
180fn build_tar<W: Write>(dir: &Path, top: &str, sink: W) -> io::Result<W> {
181 let mut tar = tar::Builder::new(sink);
182 let mut add = |entry: &str, abs: &Path, is_dir: bool| -> io::Result<()> {
183 tar_add(&mut tar, entry, abs, is_dir)
184 };
185 walk(dir, top, &mut add)?;
186 tar.finish()?;
187 tar.into_inner()
188}
189
190// ---------------------------------------------------------------------------
191// zip
192// ---------------------------------------------------------------------------
193
194fn build_zip<W: Write>(dir: &Path, top: &str, sink: W) -> io::Result<()> {
195 // Streaming mode: no `Seek` needed, entries use data descriptors.
196 let mut zip = zip::write::ZipWriter::new_stream(sink);
197 let mut add = |entry: &str, abs: &Path, is_dir: bool| -> io::Result<()> {
198 let opts = zip::write::SimpleFileOptions::default();
199 let name = if is_dir {
200 format!("{entry}/")
201 } else {
202 entry.to_string()
203 };
204 if is_dir {
205 zip.add_directory(&name, opts)?;
206 } else {
207 zip.start_file(&name, opts)?;
208 let mut f = std::fs::File::open(abs)?;
209 io::copy(&mut f, &mut zip)?;
210 }
211 Ok(())
212 };
213 walk(dir, top, &mut add)?;
214 zip.finish()?;
215 Ok(())
216}
217
218// ---------------------------------------------------------------------------
219// Tests
220// ---------------------------------------------------------------------------
221
222#[cfg(test)]
223mod tests {
224 use super::*;
225 use std::collections::BTreeMap;
226 use std::io::Read as _;
227
228 fn sample_dir() -> (tempfile::TempDir, PathBuf) {
229 let tmp = tempfile::tempdir().unwrap();
230 let dir = tmp.path().to_path_buf();
231 std::fs::write(dir.join("alpha.txt"), "alpha content").unwrap();
232 std::fs::create_dir_all(dir.join("sub/deep")).unwrap();
233 std::fs::create_dir(dir.join("empty-dir")).unwrap();
234 std::fs::write(dir.join("sub/beta.txt"), "beta").unwrap();
235 std::fs::write(
236 dir.join("sub/deep/gamma.bin"),
237 (0u8..=255).collect::<Vec<_>>(),
238 )
239 .unwrap();
240 (tmp, dir)
241 }
242
243 fn build_to_mem(fmt: ArchiveFormat, dir: &Path) -> Vec<u8> {
244 let mut out: Vec<u8> = Vec::new();
245 build(fmt, dir, "top", &mut out).unwrap();
246 out
247 }
248
249 #[test]
250 fn format_parsing() {
251 assert_eq!(ArchiveFormat::parse("zip"), Some(ArchiveFormat::Zip));
252 assert_eq!(ArchiveFormat::parse("tar"), Some(ArchiveFormat::Tar));
253 assert_eq!(ArchiveFormat::parse("tar.gz"), Some(ArchiveFormat::TarGz));
254 assert_eq!(ArchiveFormat::parse("tgz"), Some(ArchiveFormat::TarGz));
255 assert_eq!(ArchiveFormat::parse("tar.zst"), Some(ArchiveFormat::TarZst));
256 assert_eq!(ArchiveFormat::parse("tzst"), Some(ArchiveFormat::TarZst));
257 for bad in [
258 "", "ZIP", "gzip", "rar", "7z", "tar.bz2", "tar.xz", "tar.zstx", "tar.gz ",
259 ] {
260 assert_eq!(ArchiveFormat::parse(bad), None, "{bad:?}");
261 }
262 }
263
264 #[test]
265 fn format_metadata() {
266 assert_eq!(ArchiveFormat::Zip.extension(), "zip");
267 assert_eq!(ArchiveFormat::Zip.mime(), "application/zip");
268 assert_eq!(ArchiveFormat::Tar.extension(), "tar");
269 assert_eq!(ArchiveFormat::Tar.mime(), "application/x-tar");
270 assert_eq!(ArchiveFormat::TarGz.extension(), "tar.gz");
271 assert_eq!(ArchiveFormat::TarGz.mime(), "application/gzip");
272 assert_eq!(ArchiveFormat::TarZst.extension(), "tar.zst");
273 assert_eq!(ArchiveFormat::TarZst.mime(), "application/zstd");
274 }
275
276 /// Read a tar stream into a name → content map (files only).
277 fn tar_map<R: std::io::Read>(r: R) -> BTreeMap<String, Vec<u8>> {
278 let mut map = BTreeMap::new();
279 for entry in tar::Archive::new(r).entries().unwrap() {
280 let mut e = entry.unwrap();
281 if !e.header().entry_type().is_file() {
282 continue;
283 }
284 let name = e.path().unwrap().to_string_lossy().into_owned();
285 let mut buf = Vec::new();
286 e.read_to_end(&mut buf).unwrap();
287 map.insert(name, buf);
288 }
289 map
290 }
291 fn expected_map() -> BTreeMap<String, Vec<u8>> {
292 let mut m = BTreeMap::new();
293 m.insert("top/alpha.txt".to_string(), b"alpha content".to_vec());
294 m.insert("top/sub/beta.txt".to_string(), b"beta".to_vec());
295 m.insert("top/sub/deep/gamma.bin".to_string(), (0u8..=255).collect());
296 m
297 }
298
299 #[test]
300 fn zip_round_trip() {
301 let (_tmp, dir) = sample_dir();
302 let bytes = build_to_mem(ArchiveFormat::Zip, &dir);
303 let mut zip = zip::ZipArchive::new(std::io::Cursor::new(bytes)).unwrap();
304
305 let mut map = BTreeMap::new();
306 for i in 0..zip.len() {
307 let mut f = zip.by_index(i).unwrap();
308 let name = f.name().unwrap().to_string();
309 if name.ends_with('/') {
310 continue; // directory entry
311 }
312 let mut buf = Vec::new();
313 f.read_to_end(&mut buf).unwrap();
314 map.insert(name, buf);
315 }
316 assert_eq!(map, expected_map());
317
318 // Directory entries are present and the order is deterministic.
319 let names: Vec<String> = zip.file_names().map(|n| n.unwrap().to_string()).collect();
320 let has = |n: &str| names.iter().any(|x| x == n);
321 assert!(has("top/"));
322 assert!(has("top/sub/"));
323 assert!(has("top/sub/deep/"));
324 assert!(has("top/empty-dir/"));
325 let mut sorted = names.clone();
326 sorted.sort_unstable();
327 assert_eq!(names, sorted);
328 }
329
330 #[test]
331 fn tar_round_trip() {
332 let (_tmp, dir) = sample_dir();
333 let bytes = build_to_mem(ArchiveFormat::Tar, &dir);
334 let map = tar_map(std::io::Cursor::new(bytes));
335 assert_eq!(map, expected_map());
336 }
337
338 #[test]
339 fn tar_gz_round_trip() {
340 let (_tmp, dir) = sample_dir();
341 let bytes = build_to_mem(ArchiveFormat::TarGz, &dir);
342 let gz = flate2::read::GzDecoder::new(std::io::Cursor::new(bytes));
343 let map = tar_map(gz);
344 assert_eq!(map, expected_map());
345 }
346
347 #[test]
348 fn tar_zst_round_trip() {
349 let (_tmp, dir) = sample_dir();
350 let bytes = build_to_mem(ArchiveFormat::TarZst, &dir);
351 let dec = zstd::stream::read::Decoder::new(std::io::Cursor::new(bytes)).unwrap();
352 let map = tar_map(dec);
353 assert_eq!(map, expected_map());
354 }
355
356 #[test]
357 fn walk_is_sorted_case_insensitively() {
358 let tmp = tempfile::tempdir().unwrap();
359 let dir = tmp.path();
360 for name in ["Zeta", "alpha", "Beta", "a.txt", "B.txt"] {
361 if name.ends_with(".txt") {
362 std::fs::write(dir.join(name), name).unwrap();
363 } else {
364 std::fs::create_dir(dir.join(name)).unwrap();
365 }
366 }
367 let mut order = Vec::new();
368 walk(dir, "top", &mut |name, _p, _is_dir| {
369 order.push(name.to_string());
370 Ok(())
371 })
372 .unwrap();
373 assert_eq!(
374 order,
375 vec![
376 "top",
377 "top/a.txt",
378 "top/alpha",
379 "top/B.txt",
380 "top/Beta",
381 "top/Zeta"
382 ]
383 );
384 }
385
386 #[test]
387 fn build_fails_on_missing_dir() {
388 let mut out = Vec::new();
389 let r = build(
390 ArchiveFormat::Zip,
391 Path::new("/nonexistent-filebrowser-ng-test-dir"),
392 "top",
393 &mut out,
394 );
395 assert!(r.is_err());
396 // The tar path fails too.
397 let mut out = Vec::new();
398 let r = build(
399 ArchiveFormat::Tar,
400 Path::new("/nonexistent-filebrowser-ng-test-dir"),
401 "top",
402 &mut out,
403 );
404 assert!(r.is_err());
405 }
406}
407