//! Streaming archive builders (zip / tar / tar.gz / tar.zst). //! //! Everything writes directly into an `impl std::io::Write` sink — no temp //! files, no full in-memory buffering. The sink is typically a channel that //! feeds the HTTP response body, so bytes reach the client while the tree is //! still being walked. use std::io::{self, Write}; use std::path::{Path, PathBuf}; /// The archive formats offered for folder downloads. #[derive(Clone, Copy, PartialEq, Eq, Debug)] pub enum ArchiveFormat { Zip, Tar, TarGz, TarZst, } impl ArchiveFormat { /// Parse the `format` query parameter. Unknown values are rejected. pub fn parse(s: &str) -> Option { match s { "zip" => Some(Self::Zip), "tar" => Some(Self::Tar), "tar.gz" | "tgz" => Some(Self::TarGz), "tar.zst" | "tzst" => Some(Self::TarZst), _ => None, } } /// The file-name suffix for the produced archive. pub fn extension(self) -> &'static str { match self { Self::Zip => "zip", Self::Tar => "tar", Self::TarGz => "tar.gz", Self::TarZst => "tar.zst", } } /// MIME type for the produced archive. pub fn mime(self) -> &'static str { match self { Self::Zip => "application/zip", Self::Tar => "application/x-tar", Self::TarGz => "application/gzip", Self::TarZst => "application/zstd", } } } /// Build `dir` (top-level entry named `top_name`) as `format`, streaming into /// `sink`. Blocking — call from `spawn_blocking`. pub fn build( format: ArchiveFormat, dir: &Path, top_name: &str, sink: impl Write, ) -> io::Result<()> { match format { ArchiveFormat::Tar => build_tar(dir, top_name, sink).map(|_| ()), ArchiveFormat::TarGz => { // Level 1: the archive is streamed, so throughput beats ratio. let enc = flate2::write::GzEncoder::new(sink, flate2::Compression::new(1)); build_tar(dir, top_name, enc)?.finish().map(|_| ()) } ArchiveFormat::TarZst => { let enc = zstd::stream::write::Encoder::new(sink, 3)?; build_tar(dir, top_name, enc)?.finish().map(|_| ()) } ArchiveFormat::Zip => build_zip(dir, top_name, sink), } } fn mtime_secs(m: &std::fs::Metadata) -> u64 { crate::fs::mtime_secs(m).unwrap_or(0).max(0) as u64 } /// Cycle guard: a symlink loop would otherwise recurse forever. Real trees /// this deep are not worth archiving, so deeper levels are dropped. const MAX_DEPTH: usize = 64; /// Depth-first walk. Invokes `f(entry_name, abs_path, is_dir)` for the /// directory itself and every descendant. Directory entry names carry no /// trailing slash; each format appends it as needed. Deterministic order /// (case-insensitive name) so archives are reproducible. fn walk io::Result<()>>( abs_dir: &Path, entry_prefix: &str, f: &mut F, ) -> io::Result<()> { // The canonical top directory is the containment boundary for the whole // walk. Unlike browse and upload, nothing else re-checks it here. let base = abs_dir.canonicalize()?; walk_in(&base, &base, entry_prefix, 0, f) } fn walk_in io::Result<()>>( base: &Path, abs_dir: &Path, entry_prefix: &str, depth: usize, f: &mut F, ) -> io::Result<()> { f(entry_prefix, abs_dir, true)?; if depth >= MAX_DEPTH { tracing::warn!(path = %abs_dir.display(), "archive: depth limit reached, subtree skipped"); return Ok(()); } let rd = std::fs::read_dir(abs_dir)?; let mut children: Vec<(String, PathBuf, bool)> = Vec::new(); for e in rd.flatten() { let name = e.file_name().to_string_lossy().into_owned(); // Resolve symlinks: a link inside the tree may point outside it, and // its contents must not end up in the archive. One bad entry is // skipped instead of failing the whole download. let Ok(p) = e.path().canonicalize() else { tracing::warn!(path = %e.path().display(), "archive: unreadable entry skipped"); continue; }; if !crate::fs::is_within_or_eq(base, &p) { tracing::warn!(path = %e.path().display(), "archive: entry outside the archive root skipped"); continue; } let is_dir = p.is_dir(); children.push((name, p, is_dir)); } children.sort_by_key(|c| c.0.to_lowercase()); for (name, p, is_dir) in children { let child = format!("{entry_prefix}/{name}"); if is_dir { walk_in(base, &p, &child, depth + 1, f)?; } else { f(&child, &p, false)?; } } Ok(()) } // --------------------------------------------------------------------------- // tar // --------------------------------------------------------------------------- fn tar_add( tar: &mut tar::Builder, entry: &str, abs: &Path, is_dir: bool, ) -> io::Result<()> { let mut header = tar::Header::new_gnu(); let meta = std::fs::metadata(abs)?; header.set_mode(if is_dir { 0o755 } else { 0o644 }); header.set_mtime(mtime_secs(&meta)); if is_dir { header.set_entry_type(tar::EntryType::Directory); header.set_size(0); tar.append_data(&mut header, format!("{entry}/"), io::empty()) } else { header.set_entry_type(tar::EntryType::Regular); header.set_size(meta.len()); tar.append_data(&mut header, entry, std::fs::File::open(abs)?) } } /// Write the tar stream into `sink` and hand `sink` back, so a caller that /// wrapped it in a compressor can finish that compressor. fn build_tar(dir: &Path, top: &str, sink: W) -> io::Result { let mut tar = tar::Builder::new(sink); walk(dir, top, &mut |entry: &str, abs: &Path, is_dir: bool| { tar_add(&mut tar, entry, abs, is_dir) })?; tar.finish()?; tar.into_inner() } // --------------------------------------------------------------------------- // zip // --------------------------------------------------------------------------- fn build_zip(dir: &Path, top: &str, sink: W) -> io::Result<()> { // Streaming mode: no `Seek` needed, entries use data descriptors. let mut zip = zip::write::ZipWriter::new_stream(sink); let mut add = |entry: &str, abs: &Path, is_dir: bool| -> io::Result<()> { let opts = zip::write::SimpleFileOptions::default(); let name = if is_dir { format!("{entry}/") } else { entry.to_string() }; if is_dir { zip.add_directory(&name, opts)?; } else { zip.start_file(&name, opts)?; let mut f = std::fs::File::open(abs)?; io::copy(&mut f, &mut zip)?; } Ok(()) }; walk(dir, top, &mut add)?; zip.finish()?; Ok(()) } // --------------------------------------------------------------------------- // Tests // --------------------------------------------------------------------------- #[cfg(test)] mod tests { use super::*; use std::collections::BTreeMap; use std::io::Read as _; fn sample_dir() -> (tempfile::TempDir, PathBuf) { let tmp = tempfile::tempdir().unwrap(); let dir = tmp.path().to_path_buf(); std::fs::write(dir.join("alpha.txt"), "alpha content").unwrap(); std::fs::create_dir_all(dir.join("sub/deep")).unwrap(); std::fs::create_dir(dir.join("empty-dir")).unwrap(); std::fs::write(dir.join("sub/beta.txt"), "beta").unwrap(); std::fs::write( dir.join("sub/deep/gamma.bin"), (0u8..=255).collect::>(), ) .unwrap(); (tmp, dir) } fn build_to_mem(fmt: ArchiveFormat, dir: &Path) -> Vec { let mut out: Vec = Vec::new(); build(fmt, dir, "top", &mut out).unwrap(); out } #[test] fn format_parsing() { assert_eq!(ArchiveFormat::parse("zip"), Some(ArchiveFormat::Zip)); assert_eq!(ArchiveFormat::parse("tar"), Some(ArchiveFormat::Tar)); assert_eq!(ArchiveFormat::parse("tar.gz"), Some(ArchiveFormat::TarGz)); assert_eq!(ArchiveFormat::parse("tgz"), Some(ArchiveFormat::TarGz)); assert_eq!(ArchiveFormat::parse("tar.zst"), Some(ArchiveFormat::TarZst)); assert_eq!(ArchiveFormat::parse("tzst"), Some(ArchiveFormat::TarZst)); for bad in [ "", "ZIP", "gzip", "rar", "7z", "tar.bz2", "tar.xz", "tar.zstx", "tar.gz ", ] { assert_eq!(ArchiveFormat::parse(bad), None, "{bad:?}"); } } #[test] fn format_metadata() { assert_eq!(ArchiveFormat::Zip.extension(), "zip"); assert_eq!(ArchiveFormat::Zip.mime(), "application/zip"); assert_eq!(ArchiveFormat::Tar.extension(), "tar"); assert_eq!(ArchiveFormat::Tar.mime(), "application/x-tar"); assert_eq!(ArchiveFormat::TarGz.extension(), "tar.gz"); assert_eq!(ArchiveFormat::TarGz.mime(), "application/gzip"); assert_eq!(ArchiveFormat::TarZst.extension(), "tar.zst"); assert_eq!(ArchiveFormat::TarZst.mime(), "application/zstd"); } /// Read a tar stream into a name → content map (files only). fn tar_map(r: R) -> BTreeMap> { let mut map = BTreeMap::new(); for entry in tar::Archive::new(r).entries().unwrap() { let mut e = entry.unwrap(); if !e.header().entry_type().is_file() { continue; } let name = e.path().unwrap().to_string_lossy().into_owned(); let mut buf = Vec::new(); e.read_to_end(&mut buf).unwrap(); map.insert(name, buf); } map } fn expected_map() -> BTreeMap> { let mut m = BTreeMap::new(); m.insert("top/alpha.txt".to_string(), b"alpha content".to_vec()); m.insert("top/sub/beta.txt".to_string(), b"beta".to_vec()); m.insert("top/sub/deep/gamma.bin".to_string(), (0u8..=255).collect()); m } #[test] fn zip_round_trip() { let (_tmp, dir) = sample_dir(); let bytes = build_to_mem(ArchiveFormat::Zip, &dir); let mut zip = zip::ZipArchive::new(std::io::Cursor::new(bytes)).unwrap(); let mut map = BTreeMap::new(); for i in 0..zip.len() { let mut f = zip.by_index(i).unwrap(); let name = f.name().unwrap().to_string(); if name.ends_with('/') { continue; // directory entry } let mut buf = Vec::new(); f.read_to_end(&mut buf).unwrap(); map.insert(name, buf); } assert_eq!(map, expected_map()); // Directory entries are present and the order is deterministic. let names: Vec = zip.file_names().map(|n| n.unwrap().to_string()).collect(); let has = |n: &str| names.iter().any(|x| x == n); assert!(has("top/")); assert!(has("top/sub/")); assert!(has("top/sub/deep/")); assert!(has("top/empty-dir/")); let mut sorted = names.clone(); sorted.sort_unstable(); assert_eq!(names, sorted); } #[test] fn tar_round_trip() { let (_tmp, dir) = sample_dir(); let bytes = build_to_mem(ArchiveFormat::Tar, &dir); let map = tar_map(std::io::Cursor::new(bytes)); assert_eq!(map, expected_map()); } #[test] fn tar_gz_round_trip() { let (_tmp, dir) = sample_dir(); let bytes = build_to_mem(ArchiveFormat::TarGz, &dir); let gz = flate2::read::GzDecoder::new(std::io::Cursor::new(bytes)); let map = tar_map(gz); assert_eq!(map, expected_map()); } #[test] fn tar_zst_round_trip() { let (_tmp, dir) = sample_dir(); let bytes = build_to_mem(ArchiveFormat::TarZst, &dir); let dec = zstd::stream::read::Decoder::new(std::io::Cursor::new(bytes)).unwrap(); let map = tar_map(dec); assert_eq!(map, expected_map()); } #[test] fn walk_is_sorted_case_insensitively() { let tmp = tempfile::tempdir().unwrap(); let dir = tmp.path(); for name in ["Zeta", "alpha", "Beta", "a.txt", "B.txt"] { if name.ends_with(".txt") { std::fs::write(dir.join(name), name).unwrap(); } else { std::fs::create_dir(dir.join(name)).unwrap(); } } let mut order = Vec::new(); walk(dir, "top", &mut |name, _p, _is_dir| { order.push(name.to_string()); Ok(()) }) .unwrap(); assert_eq!( order, vec![ "top", "top/a.txt", "top/alpha", "top/B.txt", "top/Beta", "top/Zeta" ] ); } #[test] fn build_fails_on_missing_dir() { let mut out = Vec::new(); let r = build( ArchiveFormat::Zip, Path::new("/nonexistent-filebrowser-ng-test-dir"), "top", &mut out, ); assert!(r.is_err()); // The tar path fails too. let mut out = Vec::new(); let r = build( ArchiveFormat::Tar, Path::new("/nonexistent-filebrowser-ng-test-dir"), "top", &mut out, ); assert!(r.is_err()); } }