mirror of
https://github.com/GraphiteEditor/Graphite.git
synced 2026-09-15 14:18:04 +08:00
Add document-container crate (#4191)
* Add document-container crate: container backends and archive codecs * Address PR review: path/prefix split, safe size casts, OPFS stream aborts * Address PR review round 2: mmap read check, UTF8 entry names, prefix normalization * Make MmappedBytes::new fallible so mmap reads can't silently degrade * Address PR review round 3: backend contract uniformity (symlinks, remove, list) * Apply symlink-component check to FolderBackend listing paths * Address PR review: idempotent OPFS delete logging, tar default-features, shared entry-size cap * Fix validate_path doc: dotfiles pass, CurDir/ParentDir rejected * Omit symlink entries from FolderBackend listings for consistency with resolve * Preserve zip I/O errors and reject non-canonical paths in validate_path * Extend archive apis to return the archive writer * Rename document/document-container directory to document/container * Add archive format sniffing and deserialize_auto * Drop temporal hedge from checked_entry_size comment * Tighten verbose doc comments in document-container * Coalesce consecutive same-path OPFS appends to avoid O(n^2) file copies * Review * Update document-container for the deserialize/store_non_blocking renames --------- Co-authored-by: Timon <me@timon.zip>
This commit is contained in:
committed by
Keavon Chambers
parent
08c6d02e5b
commit
6fe1af3afe
48
document/container/Cargo.toml
Normal file
48
document/container/Cargo.toml
Normal file
@@ -0,0 +1,48 @@
|
||||
[package]
|
||||
name = "document-container"
|
||||
description = "Container abstraction for the on-disk side of the .gdd document format"
|
||||
edition.workspace = true
|
||||
version.workspace = true
|
||||
license.workspace = true
|
||||
authors.workspace = true
|
||||
|
||||
[features]
|
||||
default = []
|
||||
zip = ["dep:zip"]
|
||||
xz = ["dep:lzma-rust2", "dep:tar"]
|
||||
|
||||
[dependencies]
|
||||
thiserror = "2.0"
|
||||
log = { workspace = true }
|
||||
zip = { workspace = true, optional = true, features = ["deflate-flate2-zlib-rs"], default-features = false}
|
||||
lzma-rust2 = { workspace = true, optional = true }
|
||||
tar = { version = "0.4", optional = true, default-features = false }
|
||||
|
||||
[target.'cfg(not(target_family = "wasm"))'.dependencies]
|
||||
mmap-io = { workspace = true }
|
||||
|
||||
[target.'cfg(target_family = "wasm")'.dependencies]
|
||||
web-sys = { workspace = true, features = [
|
||||
"Navigator",
|
||||
"DomException",
|
||||
"Window",
|
||||
"StorageManager",
|
||||
"FileSystemCreateWritableOptions",
|
||||
"FileSystemDirectoryHandle",
|
||||
"FileSystemFileHandle",
|
||||
"FileSystemGetFileOptions",
|
||||
"FileSystemGetDirectoryOptions",
|
||||
"FileSystemHandle",
|
||||
"FileSystemHandleKind",
|
||||
"FileSystemWritableFileStream",
|
||||
"WritableStream",
|
||||
"Blob",
|
||||
] }
|
||||
js-sys = { workspace = true }
|
||||
wasm-bindgen = { workspace = true }
|
||||
wasm-bindgen-futures = { workspace = true }
|
||||
futures = { workspace = true }
|
||||
|
||||
[dev-dependencies]
|
||||
tempfile = "3"
|
||||
futures = { workspace = true }
|
||||
97
document/container/src/archive.rs
Normal file
97
document/container/src/archive.rs
Normal file
@@ -0,0 +1,97 @@
|
||||
//! Archive codecs (zip, xz).
|
||||
//!
|
||||
//! Each codec streams entries in both directions: writers wrap an `io::Write` sink, and
|
||||
//! `deserialize` reads from any `io::Read + Seek` source and streams entries into any [`Container`].
|
||||
|
||||
#[cfg(any(feature = "zip", feature = "xz"))]
|
||||
use crate::ContainerError;
|
||||
use crate::{Container, Result};
|
||||
use std::io::{Read, Seek, Write};
|
||||
|
||||
/// Hard cap on the total decompressed size a codec will materialize from one archive.
|
||||
/// Defends against decompression bombs at the cost of refusing legitimately huge archives.
|
||||
#[cfg(any(feature = "zip", feature = "xz"))]
|
||||
pub(crate) const MAX_DECOMPRESSED_SIZE: u64 = 4 * 1024 * 1024 * 1024; // 4GB
|
||||
|
||||
/// Fold one entry's declared `size` into the running `total` and return it as a `usize` for `write_sized`.
|
||||
/// Both codecs route entries through here so the decompression-bomb cap and 32-bit-safe conversion live in
|
||||
/// one place. `write_sized` pre-allocates the declared size, so an over-large one is rejected before that.
|
||||
#[cfg(any(feature = "zip", feature = "xz"))]
|
||||
pub(crate) fn checked_entry_size(total: &mut u64, size: u64) -> Result<usize> {
|
||||
*total = total.saturating_add(size);
|
||||
if *total > MAX_DECOMPRESSED_SIZE {
|
||||
return Err(ContainerError::SizeLimitExceeded {
|
||||
declared: *total,
|
||||
limit: MAX_DECOMPRESSED_SIZE,
|
||||
});
|
||||
}
|
||||
|
||||
// `usize` is 32-bit on wasm, so convert fallibly to rule out a silent truncation into a smaller allocation.
|
||||
usize::try_from(size).map_err(|_| ContainerError::SizeLimitExceeded {
|
||||
declared: size,
|
||||
limit: usize::MAX as u64,
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(feature = "zip")]
|
||||
mod zip;
|
||||
#[cfg(feature = "zip")]
|
||||
pub use zip::{Zip, ZipWriter};
|
||||
|
||||
#[cfg(feature = "xz")]
|
||||
mod xz;
|
||||
#[cfg(feature = "xz")]
|
||||
pub use xz::{Xz, XzWriter};
|
||||
|
||||
/// Streaming archive codec. The associated `Writer` type wraps a `Write + Seek` sink (zip needs
|
||||
/// `Seek` for the central directory; xz doesn't but `Seek` is free on file-like sinks) and
|
||||
/// accepts entries one at a time. `finish` flushes the codec's trailer and consumes the wrapper.
|
||||
pub trait Archive {
|
||||
type Writer<W: Write + Seek>: ArchiveWriter
|
||||
where
|
||||
W: Write + Seek;
|
||||
|
||||
fn writer<W: Write + Seek>(output: W) -> Result<Self::Writer<W>>;
|
||||
|
||||
/// Read entries from `source` and write each into `dest`, streaming so neither the full
|
||||
/// archive nor the full container ever sits in memory at once.
|
||||
fn open<R: Read + Seek, C: Container>(source: R, dest: &mut C) -> Result<()>;
|
||||
}
|
||||
|
||||
pub trait ArchiveWriter {
|
||||
fn write_entry(&mut self, path: &str, bytes: &[u8]) -> Result<()>;
|
||||
fn finish(self) -> Result<()>;
|
||||
}
|
||||
|
||||
/// Archive container formats distinguishable by their leading magic bytes.
|
||||
#[derive(Copy, Clone, Debug, PartialEq, Eq)]
|
||||
pub enum ArchiveFormat {
|
||||
Xz,
|
||||
Zip,
|
||||
}
|
||||
|
||||
impl ArchiveFormat {
|
||||
/// Sniff the format from the leading magic bytes: xz streams start with `FD 37 7A 58 5A 00`,
|
||||
/// zip archives with `50 4B 03 04` (`PK\x03\x04`). Returns `None` for anything else.
|
||||
pub fn detect(bytes: &[u8]) -> Option<Self> {
|
||||
if bytes.starts_with(&[0xFD, 0x37, 0x7A, 0x58, 0x5A, 0x00]) {
|
||||
Some(Self::Xz)
|
||||
} else if bytes.starts_with(&[0x50, 0x4B, 0x03, 0x04]) {
|
||||
Some(Self::Zip)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Deserialize an archive into `dest`, auto-detecting the format from `bytes`' magic header.
|
||||
/// Errors if the bytes are neither a recognized xz nor zip archive.
|
||||
#[cfg(all(feature = "xz", feature = "zip"))]
|
||||
pub fn open_auto<C: Container>(bytes: &[u8], dest: &mut C) -> Result<()> {
|
||||
let source = std::io::Cursor::new(bytes);
|
||||
match ArchiveFormat::detect(bytes) {
|
||||
Some(ArchiveFormat::Xz) => Xz::open(source, dest),
|
||||
Some(ArchiveFormat::Zip) => Zip::open(source, dest),
|
||||
None => Err(ContainerError::Codec("unrecognized archive format (not xz or zip)".into())),
|
||||
}
|
||||
}
|
||||
97
document/container/src/archive/xz.rs
Normal file
97
document/container/src/archive/xz.rs
Normal file
@@ -0,0 +1,97 @@
|
||||
//! Xz-compressed tarball archive codec.
|
||||
|
||||
use crate::archive::{Archive, ArchiveWriter, MAX_DECOMPRESSED_SIZE, checked_entry_size};
|
||||
use crate::{Container, ContainerError, Result, validate_path};
|
||||
use lzma_rust2::{XzOptions, XzReader, XzWriter as InnerXzWriter};
|
||||
use std::io::{Read, Seek, Write};
|
||||
|
||||
pub struct Xz;
|
||||
|
||||
/// xz-tar writer. Held as an `Option` so `finish` can take ownership and unwind the layered
|
||||
/// writers in the right order: drop the tar builder first to flush its trailer, then finish xz.
|
||||
pub struct XzWriter<W: Write + Seek> {
|
||||
tar: Option<tar::Builder<InnerXzWriter<W>>>,
|
||||
}
|
||||
|
||||
impl Archive for Xz {
|
||||
type Writer<W: Write + Seek> = XzWriter<W>;
|
||||
|
||||
fn writer<W: Write + Seek>(output: W) -> Result<Self::Writer<W>> {
|
||||
let xz_writer = InnerXzWriter::new(output, XzOptions::default()).map_err(lzma_err)?;
|
||||
Ok(XzWriter {
|
||||
tar: Some(tar::Builder::new(xz_writer)),
|
||||
})
|
||||
}
|
||||
|
||||
fn open<R: Read + Seek, C: Container>(source: R, dest: &mut C) -> Result<()> {
|
||||
// `take` bounds how many bytes we decompress from the xz stream, but each tar entry's declared
|
||||
// size is fed to `write_sized`, which pre-allocates from it before reading. Cap the cumulative
|
||||
// declared size too so a header claiming a huge size can't trigger a giant allocation up front.
|
||||
let xz_reader = XzReader::new(source, false);
|
||||
let bounded = xz_reader.take(MAX_DECOMPRESSED_SIZE);
|
||||
|
||||
let mut tar_reader = tar::Archive::new(bounded);
|
||||
let mut total_size = 0u64;
|
||||
|
||||
for entry in tar_reader.entries()? {
|
||||
let mut entry = entry?;
|
||||
if entry.header().entry_type() != tar::EntryType::Regular {
|
||||
continue;
|
||||
}
|
||||
// Reject non-UTF8 entry names rather than lossily rewriting them, so the path we store matches
|
||||
// the archive exactly.
|
||||
let path = entry.path()?;
|
||||
let path = path.to_str().ok_or_else(|| ContainerError::Codec(format!("tar: non-UTF8 entry name {path:?}")))?.to_owned();
|
||||
validate_path(&path)?;
|
||||
|
||||
let size = checked_entry_size(&mut total_size, entry.size())?;
|
||||
|
||||
dest.write_sized(&path, size, &mut |buffer| {
|
||||
entry.read_exact(buffer).map_err(ContainerError::Io)?;
|
||||
Ok(())
|
||||
})?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl<W: Write + Seek> ArchiveWriter for XzWriter<W> {
|
||||
fn write_entry(&mut self, path: &str, bytes: &[u8]) -> Result<()> {
|
||||
validate_path(path)?;
|
||||
let tar = self.tar.as_mut().ok_or_else(|| ContainerError::Codec("XzWriter already finished".into()))?;
|
||||
let mut header = tar::Header::new_gnu();
|
||||
header.set_path(path).map_err(|error| ContainerError::Codec(format!("tar: invalid path {path}: {error}")))?;
|
||||
header.set_size(bytes.len() as u64);
|
||||
header.set_mode(0o644);
|
||||
header.set_cksum();
|
||||
tar.append(&header, bytes)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn finish(mut self) -> Result<()> {
|
||||
self.finish_inner()?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl<W: Write + Seek> XzWriter<W> {
|
||||
/// Finish the archive and return the underlying sink, for in-memory archives where the caller
|
||||
/// wants the written bytes (e.g. `Cursor<Vec<u8>>`) back.
|
||||
pub fn finish_into(mut self) -> Result<W> {
|
||||
self.finish_inner()
|
||||
}
|
||||
|
||||
/// Unwind the layered writers in order (flush the tar trailer, then finish xz) and hand back the
|
||||
/// innermost sink. Shared by `finish` and `finish_into`.
|
||||
fn finish_inner(&mut self) -> Result<W> {
|
||||
let mut tar = self.tar.take().ok_or_else(|| ContainerError::Codec("XzWriter already finished".into()))?;
|
||||
tar.finish()?;
|
||||
let xz_writer = tar.into_inner()?;
|
||||
xz_writer.finish().map_err(lzma_err)
|
||||
}
|
||||
}
|
||||
|
||||
fn lzma_err(error: std::io::Error) -> ContainerError {
|
||||
ContainerError::Codec(format!("lzma: {error}"))
|
||||
}
|
||||
83
document/container/src/archive/zip.rs
Normal file
83
document/container/src/archive/zip.rs
Normal file
@@ -0,0 +1,83 @@
|
||||
//! Zip archive codec.
|
||||
|
||||
use crate::archive::{Archive, ArchiveWriter, checked_entry_size};
|
||||
use crate::{Container, ContainerError, Result, validate_path};
|
||||
use std::io::{Read, Seek, Write};
|
||||
|
||||
use zip::ZipArchive;
|
||||
use zip::write::{SimpleFileOptions, ZipWriter as InnerZipWriter};
|
||||
|
||||
pub struct Zip;
|
||||
|
||||
pub struct ZipWriter<W: Write + Seek> {
|
||||
inner: InnerZipWriter<W>,
|
||||
options: SimpleFileOptions,
|
||||
}
|
||||
|
||||
impl Archive for Zip {
|
||||
type Writer<W: Write + Seek> = ZipWriter<W>;
|
||||
|
||||
fn writer<W: Write + Seek>(output: W) -> Result<Self::Writer<W>> {
|
||||
Ok(ZipWriter {
|
||||
inner: InnerZipWriter::new(output),
|
||||
options: SimpleFileOptions::default().compression_method(zip::CompressionMethod::Deflated),
|
||||
})
|
||||
}
|
||||
|
||||
fn open<R: Read + Seek, C: Container>(source: R, dest: &mut C) -> Result<()> {
|
||||
let mut archive = ZipArchive::new(source).map_err(zip_err)?;
|
||||
|
||||
// Zip headers declare each entry's uncompressed size up front; `checked_entry_size` caps the running
|
||||
// total so a malicious archive can't exhaust memory or disk before any bytes are read.
|
||||
let mut total_size = 0u64;
|
||||
|
||||
for index in 0..archive.len() {
|
||||
let mut entry = archive.by_index(index).map_err(zip_err)?;
|
||||
if !entry.is_file() {
|
||||
continue;
|
||||
}
|
||||
let name = entry.name().to_string();
|
||||
validate_path(&name)?;
|
||||
|
||||
let size = checked_entry_size(&mut total_size, entry.size())?;
|
||||
|
||||
dest.write_sized(&name, size, &mut |buffer| {
|
||||
entry.read_exact(buffer).map_err(ContainerError::Io)?;
|
||||
Ok(())
|
||||
})?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl<W: Write + Seek> ArchiveWriter for ZipWriter<W> {
|
||||
fn write_entry(&mut self, path: &str, bytes: &[u8]) -> Result<()> {
|
||||
validate_path(path)?;
|
||||
self.inner.start_file(path, self.options).map_err(zip_err)?;
|
||||
self.inner.write_all(bytes)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn finish(self) -> Result<()> {
|
||||
self.inner.finish().map_err(zip_err)?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl<W: Write + Seek> ZipWriter<W> {
|
||||
/// Finish the archive and return the underlying sink, for in-memory archives where the caller
|
||||
/// wants the written bytes (e.g. `Cursor<Vec<u8>>`) back.
|
||||
pub fn finish_into(self) -> Result<W> {
|
||||
self.inner.finish().map_err(zip_err)
|
||||
}
|
||||
}
|
||||
|
||||
fn zip_err(error: zip::result::ZipError) -> ContainerError {
|
||||
// Preserve a real I/O failure (disk full, etc.) as a structured `Io` so callers can tell it apart from a
|
||||
// corrupt-archive `Codec` error; only the genuinely archive-format errors collapse into `Codec`.
|
||||
match error {
|
||||
zip::result::ZipError::Io(io) => ContainerError::Io(io),
|
||||
other => ContainerError::Codec(format!("zip: {other}")),
|
||||
}
|
||||
}
|
||||
9
document/container/src/backends.rs
Normal file
9
document/container/src/backends.rs
Normal file
@@ -0,0 +1,9 @@
|
||||
//! Container backend implementations.
|
||||
|
||||
pub mod memory;
|
||||
|
||||
#[cfg(not(target_family = "wasm"))]
|
||||
pub mod folder;
|
||||
|
||||
#[cfg(target_family = "wasm")]
|
||||
pub mod opfs;
|
||||
191
document/container/src/backends/folder.rs
Normal file
191
document/container/src/backends/folder.rs
Normal file
@@ -0,0 +1,191 @@
|
||||
//! Loose-folder backend.
|
||||
|
||||
use crate::{ByteHolder, Container, ContainerError, MmappedBytes, Result, validate_path, validate_prefix};
|
||||
use mmap_io::mmap::{MemoryMappedFile, MmapMode};
|
||||
use std::fs::{self, OpenOptions};
|
||||
use std::io::Write;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
pub struct FolderBackend {
|
||||
root: PathBuf,
|
||||
}
|
||||
|
||||
impl FolderBackend {
|
||||
/// Open an existing folder. Errors if `root` is not a directory.
|
||||
pub fn open(root: impl Into<PathBuf>) -> Result<Self> {
|
||||
let root = root.into();
|
||||
if !root.is_dir() {
|
||||
return Err(ContainerError::NotFound(root.display().to_string()));
|
||||
}
|
||||
Ok(Self { root })
|
||||
}
|
||||
|
||||
/// Create the folder if it does not exist, then open it.
|
||||
pub fn create(root: impl Into<PathBuf>) -> Result<Self> {
|
||||
let root = root.into();
|
||||
fs::create_dir_all(&root)?;
|
||||
Ok(Self { root })
|
||||
}
|
||||
|
||||
pub fn root(&self) -> &std::path::Path {
|
||||
&self.root
|
||||
}
|
||||
|
||||
fn resolve(&self, path: &str) -> Result<PathBuf> {
|
||||
validate_path(path)?;
|
||||
self.reject_symlinked_components(path)?;
|
||||
Ok(self.root.join(path))
|
||||
}
|
||||
|
||||
/// Reject any existing component along `root/relative` that is a symlink. `validate_path`/`validate_prefix`
|
||||
/// block `..` and absolute paths, but a symlink stored under the root could still point outside it, so every
|
||||
/// path that gets joined onto the root must pass through here before it is opened or traversed.
|
||||
fn reject_symlinked_components(&self, relative: &str) -> Result<()> {
|
||||
let mut partial = self.root.clone();
|
||||
for component in Path::new(relative).components() {
|
||||
partial.push(component);
|
||||
if let Ok(metadata) = fs::symlink_metadata(&partial)
|
||||
&& metadata.file_type().is_symlink()
|
||||
{
|
||||
return Err(ContainerError::InvalidPath(relative.to_string()));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn list_filtered(&self, prefix: &str, want_files: bool) -> Result<Vec<String>> {
|
||||
validate_prefix(prefix)?;
|
||||
let base = if prefix.is_empty() || prefix == "." {
|
||||
self.root.clone()
|
||||
} else {
|
||||
self.reject_symlinked_components(prefix)?;
|
||||
self.root.join(prefix)
|
||||
};
|
||||
|
||||
// A missing prefix has no entries; a prefix that names a file is a misuse.
|
||||
if base.is_file() {
|
||||
return Err(ContainerError::NotADirectory(prefix.to_string()));
|
||||
}
|
||||
if !base.is_dir() {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
|
||||
let mut results = Vec::new();
|
||||
for entry in fs::read_dir(&base)? {
|
||||
let entry = entry?;
|
||||
|
||||
// `file_type` does not follow symlinks, unlike `is_file`/`is_dir`. Skip symlink entries so a
|
||||
// listing never advertises a path that `resolve` would then reject as a container escape.
|
||||
let Ok(file_type) = entry.file_type() else { continue };
|
||||
let matches = if want_files { file_type.is_file() } else { file_type.is_dir() };
|
||||
if !matches {
|
||||
continue;
|
||||
}
|
||||
|
||||
let path = entry.path();
|
||||
let relative = path.strip_prefix(&self.root).map_err(|_| ContainerError::Backend("path escaped root".into()))?;
|
||||
results.push(relative.to_string_lossy().replace('\\', "/"));
|
||||
}
|
||||
Ok(results)
|
||||
}
|
||||
}
|
||||
|
||||
impl Container for FolderBackend {
|
||||
fn read(&self, path: &str) -> Result<ByteHolder> {
|
||||
let full = self.resolve(path)?;
|
||||
if !full.is_file() {
|
||||
return Err(ContainerError::NotFound(path.to_string()));
|
||||
}
|
||||
|
||||
// Mmapping a zero-length file is platform-dependent and often fails, so serve empty files as owned
|
||||
// bytes and reserve mmap for files that actually have content.
|
||||
if fs::metadata(&full).map(|metadata| metadata.len() == 0).unwrap_or(false) {
|
||||
return Ok(ByteHolder::Owned(Vec::new()));
|
||||
}
|
||||
|
||||
Ok(ByteHolder::Mmapped(MmappedBytes::new(open_mmap(&full)?)?))
|
||||
}
|
||||
|
||||
fn write(&self, path: &str, bytes: &[u8]) -> Result<()> {
|
||||
let full = self.resolve(path)?;
|
||||
if let Some(parent) = full.parent() {
|
||||
fs::create_dir_all(parent)?;
|
||||
}
|
||||
fs::write(&full, bytes)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn append(&self, path: &str, bytes: &[u8]) -> Result<()> {
|
||||
let full = self.resolve(path)?;
|
||||
if let Some(parent) = full.parent() {
|
||||
fs::create_dir_all(parent)?;
|
||||
}
|
||||
let mut file = OpenOptions::new().create(true).append(true).open(&full)?;
|
||||
file.write_all(bytes)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn write_sized(&self, path: &str, size: usize, fill: &mut dyn FnMut(&mut [u8]) -> Result<()>) -> Result<()> {
|
||||
if size == 0 {
|
||||
return self.write(path, &[]);
|
||||
}
|
||||
|
||||
let full = self.resolve(path)?;
|
||||
if let Some(parent) = full.parent() {
|
||||
fs::create_dir_all(parent)?;
|
||||
}
|
||||
|
||||
let file = MemoryMappedFile::create_rw(&full, size as u64).map_err(|error| ContainerError::Backend(format!("create_rw {full:?} failed: {error}")))?;
|
||||
|
||||
let result = {
|
||||
let mut slice = file
|
||||
.as_slice_mut(0, size as u64)
|
||||
.map_err(|error| ContainerError::Backend(format!("as_slice_mut {full:?} failed: {error}")))?;
|
||||
fill(slice.as_mut())?;
|
||||
drop(slice);
|
||||
file.flush().map_err(|error| ContainerError::Backend(format!("flush {full:?} failed: {error}")))
|
||||
};
|
||||
|
||||
// `create_rw` materializes the full-size file before `fill` runs, so remove it on failure rather
|
||||
// than leave a zeroed or half-written remnant.
|
||||
if result.is_err() {
|
||||
drop(file);
|
||||
let _ = fs::remove_file(&full);
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
fn list(&self, prefix: &str) -> Result<Vec<String>> {
|
||||
self.list_filtered(prefix, true)
|
||||
}
|
||||
|
||||
fn list_dirs(&self, prefix: &str) -> Result<Vec<String>> {
|
||||
self.list_filtered(prefix, false)
|
||||
}
|
||||
|
||||
fn exists(&self, path: &str) -> bool {
|
||||
match self.resolve(path) {
|
||||
Ok(full) => full.is_file(),
|
||||
Err(_) => false,
|
||||
}
|
||||
}
|
||||
|
||||
fn remove(&self, path: &str) -> Result<()> {
|
||||
let full = self.resolve(path)?;
|
||||
// Idempotent: a missing file is not an error. Directories are left alone.
|
||||
if full.is_file() {
|
||||
fs::remove_file(full)?;
|
||||
}
|
||||
// TODO: decide if we should remove empty parent directories
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Open a memory-mapped read-only view of `path`, trying huge pages first. Callers must ensure `path`
|
||||
/// is non-empty, since mmapping a zero-length file is platform-dependent.
|
||||
fn open_mmap(path: &Path) -> Result<MemoryMappedFile> {
|
||||
match MemoryMappedFile::builder(path).mode(MmapMode::ReadOnly).huge_pages(true).open() {
|
||||
Ok(file) => Ok(file),
|
||||
Err(_) => MemoryMappedFile::open_ro(path).map_err(|error| ContainerError::Backend(format!("mmap of {path:?} failed: {error}"))),
|
||||
}
|
||||
}
|
||||
89
document/container/src/backends/memory.rs
Normal file
89
document/container/src/backends/memory.rs
Normal file
@@ -0,0 +1,89 @@
|
||||
//! In-memory backend. Useful for tests and as the deserialize target for archive codecs.
|
||||
|
||||
use crate::{ByteHolder, Container, ContainerError, Result, validate_path, validate_prefix, with_trailing_slash};
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::sync::Mutex;
|
||||
|
||||
#[derive(Default)]
|
||||
pub struct MemoryBackend {
|
||||
files: Mutex<HashMap<String, Vec<u8>>>,
|
||||
}
|
||||
|
||||
impl MemoryBackend {
|
||||
pub fn new() -> Self {
|
||||
Self::default()
|
||||
}
|
||||
}
|
||||
|
||||
impl Container for MemoryBackend {
|
||||
fn read(&self, path: &str) -> Result<ByteHolder> {
|
||||
validate_path(path)?;
|
||||
self.files
|
||||
.lock()
|
||||
.unwrap()
|
||||
.get(path)
|
||||
.map(|bytes| ByteHolder::Owned(bytes.clone()))
|
||||
.ok_or_else(|| ContainerError::NotFound(path.to_string()))
|
||||
}
|
||||
|
||||
fn write(&self, path: &str, bytes: &[u8]) -> Result<()> {
|
||||
validate_path(path)?;
|
||||
self.files.lock().unwrap().insert(path.to_string(), bytes.to_vec());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn append(&self, path: &str, bytes: &[u8]) -> Result<()> {
|
||||
validate_path(path)?;
|
||||
self.files.lock().unwrap().entry(path.to_string()).or_default().extend_from_slice(bytes);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn list(&self, prefix: &str) -> Result<Vec<String>> {
|
||||
validate_prefix(prefix)?;
|
||||
let files = self.files.lock().unwrap();
|
||||
// A prefix that is itself a stored key names a file, not a directory.
|
||||
if files.contains_key(prefix) {
|
||||
return Err(ContainerError::NotADirectory(prefix.to_string()));
|
||||
}
|
||||
|
||||
let normalized = with_trailing_slash(prefix);
|
||||
let results = files.keys().filter(|path| path.starts_with(&normalized) && !path[normalized.len()..].contains('/')).cloned().collect();
|
||||
Ok(results)
|
||||
}
|
||||
|
||||
fn list_dirs(&self, prefix: &str) -> Result<Vec<String>> {
|
||||
validate_prefix(prefix)?;
|
||||
let files = self.files.lock().unwrap();
|
||||
if files.contains_key(prefix) {
|
||||
return Err(ContainerError::NotADirectory(prefix.to_string()));
|
||||
}
|
||||
|
||||
let normalized = with_trailing_slash(prefix);
|
||||
let mut seen = HashSet::new();
|
||||
let mut results = Vec::new();
|
||||
for path in files.keys() {
|
||||
if !path.starts_with(&normalized) {
|
||||
continue;
|
||||
}
|
||||
let remainder = &path[normalized.len()..];
|
||||
if let Some((segment, _)) = remainder.split_once('/') {
|
||||
let dir = format!("{normalized}{segment}");
|
||||
if seen.insert(dir.clone()) {
|
||||
results.push(dir);
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(results)
|
||||
}
|
||||
|
||||
fn exists(&self, path: &str) -> bool {
|
||||
validate_path(path).is_ok() && self.files.lock().unwrap().contains_key(path)
|
||||
}
|
||||
|
||||
fn remove(&self, path: &str) -> Result<()> {
|
||||
validate_path(path)?;
|
||||
// Idempotent: removing a missing path is not an error.
|
||||
self.files.lock().unwrap().remove(path);
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
475
document/container/src/backends/opfs.rs
Normal file
475
document/container/src/backends/opfs.rs
Normal file
@@ -0,0 +1,475 @@
|
||||
//! OPFS (Origin Private File System) backend for browser wasm.
|
||||
|
||||
use crate::{AsyncContainer, ByteHolder, ContainerError, Result, validate_path, validate_prefix, with_trailing_slash};
|
||||
use futures::channel::oneshot;
|
||||
use js_sys::Uint8Array;
|
||||
use std::collections::{HashSet, VecDeque};
|
||||
use std::sync::{Arc, Mutex};
|
||||
use wasm_bindgen::JsCast;
|
||||
use wasm_bindgen::prelude::*;
|
||||
use wasm_bindgen_futures::{JsFuture, spawn_local};
|
||||
use web_sys::{
|
||||
Blob, DomException, FileSystemCreateWritableOptions, FileSystemDirectoryHandle, FileSystemFileHandle, FileSystemGetDirectoryOptions, FileSystemGetFileOptions, FileSystemWritableFileStream,
|
||||
WritableStream,
|
||||
};
|
||||
|
||||
enum Mutation {
|
||||
Write {
|
||||
path: String,
|
||||
bytes: Vec<u8>,
|
||||
},
|
||||
Append {
|
||||
path: String,
|
||||
bytes: Vec<u8>,
|
||||
},
|
||||
Delete {
|
||||
path: String,
|
||||
},
|
||||
/// In-band flush barrier. The worker fulfills the sender once it dequeues this, which (by FIFO
|
||||
/// order) signals that every mutation enqueued before it has been applied to disk.
|
||||
Barrier(oneshot::Sender<()>),
|
||||
}
|
||||
|
||||
struct Inner {
|
||||
directory: FileSystemDirectoryHandle,
|
||||
/// Paths believed to be on disk, letting `exists_non_blocking` answer without OPFS's missing sync API.
|
||||
/// Optimistic: a path is inserted when its write is queued and kept even if that write later fails, so it
|
||||
/// can briefly over-report. Rebuilt from real disk state on the next `open`.
|
||||
on_disk: HashSet<String>,
|
||||
queue: VecDeque<Mutation>,
|
||||
worker_active: bool,
|
||||
}
|
||||
|
||||
pub struct OpfsBackend {
|
||||
inner: Arc<Mutex<Inner>>,
|
||||
}
|
||||
|
||||
// Safety: only built for browser wasm where JS handles never leave the main thread.
|
||||
unsafe impl Send for OpfsBackend {}
|
||||
unsafe impl Sync for OpfsBackend {}
|
||||
|
||||
impl OpfsBackend {
|
||||
/// Open (or create) `directory_name` under the OPFS root.
|
||||
pub async fn open(directory_name: &str) -> Result<Self> {
|
||||
let directory = open_directory(directory_name).await.map_err(js_err)?;
|
||||
let on_disk = enumerate_paths(&directory, "").await.map_err(js_err)?;
|
||||
Ok(Self {
|
||||
inner: Arc::new(Mutex::new(Inner {
|
||||
directory,
|
||||
on_disk,
|
||||
queue: VecDeque::new(),
|
||||
worker_active: false,
|
||||
})),
|
||||
})
|
||||
}
|
||||
|
||||
fn directory(&self) -> FileSystemDirectoryHandle {
|
||||
self.inner.lock().unwrap().directory.clone()
|
||||
}
|
||||
|
||||
/// Wait until all queued non-blocking mutations have hit disk, so awaited reads observe them. Plants a
|
||||
/// barrier at the back of the queue and awaits the worker reaching it; by FIFO order, everything ahead is
|
||||
/// then applied. Draining stays the worker's job so it remains the sole mutator.
|
||||
async fn flush(&self) {
|
||||
let (sender, receiver) = oneshot::channel();
|
||||
{
|
||||
let mut guard = self.inner.lock().unwrap();
|
||||
guard.queue.push_back(Mutation::Barrier(sender));
|
||||
kick_worker(&self.inner, &mut guard);
|
||||
}
|
||||
// The sender is only dropped without sending if the worker is torn down mid-drain; either way
|
||||
// there is nothing left to wait for, so a receive error is treated as "already flushed".
|
||||
let _ = receiver.await;
|
||||
}
|
||||
}
|
||||
|
||||
impl AsyncContainer for OpfsBackend {
|
||||
async fn read(&self, path: &str) -> Result<ByteHolder> {
|
||||
validate_path(path)?;
|
||||
|
||||
// Non-blocking writes only land on disk once the queue drains, so flush first and then treat
|
||||
// disk as authoritative. Draining (rather than folding the queue against an awaited base read)
|
||||
// avoids racing the background worker, which could otherwise double-apply a queued append.
|
||||
self.flush().await;
|
||||
|
||||
let bytes = read_file(&self.directory(), path).await.map_err(js_err)?;
|
||||
Ok(ByteHolder::Owned(bytes))
|
||||
}
|
||||
|
||||
async fn write(&self, path: &str, bytes: &[u8]) -> Result<()> {
|
||||
validate_path(path)?;
|
||||
// Flush first so this awaited write lands after any non-blocking mutation already queued for the
|
||||
// same path, then apply directly so the real disk error still propagates to the caller.
|
||||
self.flush().await;
|
||||
write_file(&self.directory(), path, bytes).await.map_err(js_err)?;
|
||||
self.inner.lock().unwrap().on_disk.insert(path.to_string());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn append(&self, path: &str, bytes: &[u8]) -> Result<()> {
|
||||
validate_path(path)?;
|
||||
self.flush().await;
|
||||
append_file(&self.directory(), path, bytes).await.map_err(js_err)?;
|
||||
self.inner.lock().unwrap().on_disk.insert(path.to_string());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn write_sized(&self, path: &str, size: usize, fill: &mut dyn FnMut(&mut [u8]) -> Result<()>) -> Result<()> {
|
||||
validate_path(path)?;
|
||||
let mut buffer = vec![0; size];
|
||||
fill(&mut buffer)?;
|
||||
self.flush().await;
|
||||
write_file(&self.directory(), path, &buffer).await.map_err(js_err)?;
|
||||
self.inner.lock().unwrap().on_disk.insert(path.to_string());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn list(&self, prefix: &str) -> Result<Vec<String>> {
|
||||
validate_prefix(prefix)?;
|
||||
self.flush().await;
|
||||
list_entries(&self.directory(), prefix, EntryKind::File).await.map_err(|error| list_error(prefix, error))
|
||||
}
|
||||
|
||||
async fn list_dirs(&self, prefix: &str) -> Result<Vec<String>> {
|
||||
validate_prefix(prefix)?;
|
||||
self.flush().await;
|
||||
list_entries(&self.directory(), prefix, EntryKind::Directory).await.map_err(|error| list_error(prefix, error))
|
||||
}
|
||||
|
||||
async fn exists(&self, path: &str) -> bool {
|
||||
if validate_path(path).is_err() {
|
||||
return false;
|
||||
}
|
||||
self.flush().await;
|
||||
file_exists(&self.directory(), path).await
|
||||
}
|
||||
|
||||
async fn remove(&self, path: &str) -> Result<()> {
|
||||
validate_path(path)?;
|
||||
self.flush().await;
|
||||
// Idempotent: a missing entry is not an error, matching the other backends.
|
||||
if let Err(error) = remove_file(&self.directory(), path).await
|
||||
&& !is_not_found(&error)
|
||||
{
|
||||
return Err(js_err(error));
|
||||
}
|
||||
self.inner.lock().unwrap().on_disk.remove(path);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn write_non_blocking(&self, path: &str, bytes: &[u8]) -> Result<()> {
|
||||
validate_path(path)?;
|
||||
let mut guard = self.inner.lock().unwrap();
|
||||
guard.on_disk.insert(path.to_string());
|
||||
guard.queue.push_back(Mutation::Write {
|
||||
path: path.to_string(),
|
||||
bytes: bytes.to_vec(),
|
||||
});
|
||||
kick_worker(&self.inner, &mut guard);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn append_non_blocking(&self, path: &str, bytes: &[u8]) -> Result<()> {
|
||||
validate_path(path)?;
|
||||
let mut guard = self.inner.lock().unwrap();
|
||||
guard.on_disk.insert(path.to_string());
|
||||
guard.queue.push_back(Mutation::Append {
|
||||
path: path.to_string(),
|
||||
bytes: bytes.to_vec(),
|
||||
});
|
||||
kick_worker(&self.inner, &mut guard);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn remove_non_blocking(&self, path: &str) -> Result<()> {
|
||||
validate_path(path)?;
|
||||
let mut guard = self.inner.lock().unwrap();
|
||||
guard.on_disk.remove(path);
|
||||
guard.queue.push_back(Mutation::Delete { path: path.to_string() });
|
||||
kick_worker(&self.inner, &mut guard);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn exists_non_blocking(&self, path: &str) -> bool {
|
||||
validate_path(path).is_ok() && self.inner.lock().unwrap().on_disk.contains(path)
|
||||
}
|
||||
}
|
||||
|
||||
fn kick_worker(inner: &Arc<Mutex<Inner>>, guard: &mut Inner) {
|
||||
if guard.worker_active {
|
||||
return;
|
||||
}
|
||||
guard.worker_active = true;
|
||||
let inner = inner.clone();
|
||||
spawn_local(drain_queue(inner));
|
||||
}
|
||||
|
||||
/// Apply every queued mutation to disk, in FIFO order, until the queue is empty, then mark the worker
|
||||
/// idle. Spawned once via [`kick_worker`] and runs as the sole mutator; the awaited read paths wait on
|
||||
/// a [`Mutation::Barrier`] rather than draining themselves, so there is never more than one drainer.
|
||||
///
|
||||
/// Consecutive appends to the same path are coalesced into one combined append. Each `createWritable`
|
||||
/// on OPFS copies the whole existing file, so appending N frames one at a time is O(N^2) in bytes
|
||||
/// copied (every per-op autosave queues one append per hot op). Concatenating a run into a single
|
||||
/// append collapses that to one file copy, while staying byte-identical to applying them in order.
|
||||
async fn drain_queue(inner: Arc<Mutex<Inner>>) {
|
||||
loop {
|
||||
// Pop the front mutation, coalescing a run of same-path appends behind it into one batch.
|
||||
let (directory, mutation) = {
|
||||
let mut guard = inner.lock().unwrap();
|
||||
let Some(mut mutation) = guard.queue.pop_front() else {
|
||||
guard.worker_active = false;
|
||||
return;
|
||||
};
|
||||
|
||||
if let Mutation::Append { path, bytes } = &mut mutation {
|
||||
while let Some(Mutation::Append { path: next_path, .. }) = guard.queue.front()
|
||||
&& next_path == path
|
||||
{
|
||||
let Some(Mutation::Append { bytes: next_bytes, .. }) = guard.queue.pop_front() else {
|
||||
unreachable!()
|
||||
};
|
||||
bytes.extend_from_slice(&next_bytes);
|
||||
}
|
||||
}
|
||||
|
||||
(guard.directory.clone(), mutation)
|
||||
};
|
||||
|
||||
match mutation {
|
||||
Mutation::Write { path, bytes } => {
|
||||
if let Err(error) = write_file(&directory, &path, &bytes).await {
|
||||
log::error!("OPFS background write for {path} failed: {error:?}");
|
||||
}
|
||||
}
|
||||
Mutation::Append { path, bytes } => {
|
||||
if let Err(error) = append_file(&directory, &path, &bytes).await {
|
||||
log::error!("OPFS background append for {path} failed: {error:?}");
|
||||
}
|
||||
}
|
||||
Mutation::Delete { path } => {
|
||||
// Removal is idempotent, so a missing entry is the expected outcome of a redundant delete, not an error.
|
||||
if let Err(error) = remove_file(&directory, &path).await
|
||||
&& !is_not_found(&error)
|
||||
{
|
||||
log::error!("OPFS background delete for {path} failed: {error:?}");
|
||||
}
|
||||
}
|
||||
// A receive error on the waiter side just means the reader gave up; nothing to apply.
|
||||
Mutation::Barrier(sender) => {
|
||||
let _ = sender.send(());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn js_err(error: JsValue) -> ContainerError {
|
||||
ContainerError::Backend(format!("{error:?}"))
|
||||
}
|
||||
|
||||
/// Resolve `directory_path` (a `/`-separated relative path) under the OPFS root, creating each
|
||||
/// segment. OPFS rejects directory names containing `/`, so a multi-segment path like
|
||||
/// `documents/<id>` must be descended one segment at a time rather than passed whole.
|
||||
async fn open_directory(directory_path: &str) -> std::result::Result<FileSystemDirectoryHandle, JsValue> {
|
||||
let storage = web_sys::window().ok_or_else(|| JsValue::from_str("no window"))?.navigator().storage();
|
||||
let mut current: FileSystemDirectoryHandle = JsFuture::from(storage.get_directory()).await?.dyn_into()?;
|
||||
|
||||
for segment in directory_path.split('/').filter(|segment| !segment.is_empty()) {
|
||||
let options = FileSystemGetDirectoryOptions::new();
|
||||
options.set_create(true);
|
||||
current = JsFuture::from(current.get_directory_handle_with_options(segment, &options)).await?.dyn_into()?;
|
||||
}
|
||||
|
||||
Ok(current)
|
||||
}
|
||||
|
||||
/// Descend the `/`-separated path against `root` and return the directory handle plus the final segment.
|
||||
async fn descend<'a>(root: &FileSystemDirectoryHandle, relative: &'a str, create_dirs: bool) -> std::result::Result<(FileSystemDirectoryHandle, &'a str), JsValue> {
|
||||
let mut current = root.clone();
|
||||
let mut segments = relative.split('/').filter(|s| !s.is_empty()).collect::<Vec<_>>();
|
||||
let file = segments.pop().ok_or_else(|| JsValue::from_str("empty path"))?;
|
||||
|
||||
for segment in segments {
|
||||
let options = FileSystemGetDirectoryOptions::new();
|
||||
options.set_create(create_dirs);
|
||||
current = JsFuture::from(current.get_directory_handle_with_options(segment, &options)).await?.dyn_into()?;
|
||||
}
|
||||
Ok((current, file))
|
||||
}
|
||||
|
||||
async fn read_file(root: &FileSystemDirectoryHandle, path: &str) -> std::result::Result<Vec<u8>, JsValue> {
|
||||
let (directory, name) = descend(root, path, false).await?;
|
||||
|
||||
let handle: FileSystemFileHandle = JsFuture::from(directory.get_file_handle(name)).await?.dyn_into()?;
|
||||
let file_value = JsFuture::from(handle.get_file()).await?;
|
||||
let blob: Blob = file_value.dyn_into()?;
|
||||
let buffer = JsFuture::from(blob.array_buffer()).await?;
|
||||
Ok(Uint8Array::new(&buffer).to_vec())
|
||||
}
|
||||
|
||||
async fn write_file(root: &FileSystemDirectoryHandle, path: &str, bytes: &[u8]) -> std::result::Result<(), JsValue> {
|
||||
let (directory, name) = descend(root, path, true).await?;
|
||||
|
||||
let options = FileSystemGetFileOptions::new();
|
||||
options.set_create(true);
|
||||
|
||||
let handle: FileSystemFileHandle = JsFuture::from(directory.get_file_handle_with_options(name, &options)).await?.dyn_into()?;
|
||||
let writable: FileSystemWritableFileStream = JsFuture::from(handle.create_writable()).await?.dyn_into()?;
|
||||
let stream: WritableStream = writable.clone().unchecked_into();
|
||||
|
||||
// Wrap in an async block so any failure, including a synchronous JS throw from `write_with_js_u8_array`,
|
||||
// aborts the stream before returning instead of leaving a dangling locked writable.
|
||||
let write = async {
|
||||
let array = Uint8Array::from(bytes);
|
||||
JsFuture::from(writable.write_with_js_u8_array(&array)?).await?;
|
||||
Ok::<(), JsValue>(())
|
||||
}
|
||||
.await;
|
||||
|
||||
if let Err(error) = write {
|
||||
let _ = JsFuture::from(stream.abort()).await;
|
||||
return Err(error);
|
||||
}
|
||||
|
||||
JsFuture::from(stream.close()).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn append_file(root: &FileSystemDirectoryHandle, path: &str, bytes: &[u8]) -> std::result::Result<(), JsValue> {
|
||||
let (directory, name) = descend(root, path, true).await?;
|
||||
|
||||
let file_options = FileSystemGetFileOptions::new();
|
||||
file_options.set_create(true);
|
||||
let handle: FileSystemFileHandle = JsFuture::from(directory.get_file_handle_with_options(name, &file_options)).await?.dyn_into()?;
|
||||
|
||||
// Determine the current end-of-file so we can seek there before writing.
|
||||
let file_value = JsFuture::from(handle.get_file()).await?;
|
||||
let blob: Blob = file_value.dyn_into()?;
|
||||
let offset = blob.size();
|
||||
|
||||
// `keepExistingData: true` preserves bytes outside the written range; without it OPFS truncates to the written length.
|
||||
let writable_options = FileSystemCreateWritableOptions::new();
|
||||
writable_options.set_keep_existing_data(true);
|
||||
let writable: FileSystemWritableFileStream = JsFuture::from(handle.create_writable_with_options(&writable_options)).await?.dyn_into()?;
|
||||
let stream: WritableStream = writable.clone().unchecked_into();
|
||||
|
||||
// Wrap in an async block so a synchronous JS throw from `seek_with_f64`/`write_with_js_u8_array`
|
||||
// aborts the stream instead of bypassing the abort via `?`.
|
||||
let seek_and_write = async {
|
||||
JsFuture::from(writable.seek_with_f64(offset)?).await?;
|
||||
let array = Uint8Array::from(bytes);
|
||||
JsFuture::from(writable.write_with_js_u8_array(&array)?).await?;
|
||||
Ok::<(), JsValue>(())
|
||||
}
|
||||
.await;
|
||||
|
||||
if let Err(error) = seek_and_write {
|
||||
let _ = JsFuture::from(stream.abort()).await;
|
||||
return Err(error);
|
||||
}
|
||||
|
||||
JsFuture::from(stream.close()).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn remove_file(root: &FileSystemDirectoryHandle, path: &str) -> std::result::Result<(), JsValue> {
|
||||
let (directory, name) = descend(root, path, false).await?;
|
||||
JsFuture::from(directory.remove_entry(name)).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn file_exists(root: &FileSystemDirectoryHandle, path: &str) -> bool {
|
||||
let Ok((directory, name)) = descend(root, path, false).await else {
|
||||
return false;
|
||||
};
|
||||
JsFuture::from(directory.get_file_handle(name)).await.is_ok()
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
enum EntryKind {
|
||||
File,
|
||||
Directory,
|
||||
}
|
||||
|
||||
async fn list_entries(root: &FileSystemDirectoryHandle, prefix: &str, want: EntryKind) -> std::result::Result<Vec<String>, JsValue> {
|
||||
// `.` and the empty string both name the container root.
|
||||
let prefix = if prefix == "." { "" } else { prefix };
|
||||
|
||||
let directory = if prefix.is_empty() {
|
||||
root.clone()
|
||||
} else {
|
||||
let mut current = root.clone();
|
||||
for segment in prefix.split('/').filter(|s| !s.is_empty()) {
|
||||
let options = FileSystemGetDirectoryOptions::new();
|
||||
options.set_create(false);
|
||||
current = match JsFuture::from(current.get_directory_handle_with_options(segment, &options)).await {
|
||||
Ok(value) => value.dyn_into()?,
|
||||
Err(error) if is_not_found(&error) => return Ok(Vec::new()),
|
||||
Err(error) => return Err(error),
|
||||
};
|
||||
}
|
||||
current
|
||||
};
|
||||
|
||||
let entries = directory.entries();
|
||||
let iterator: js_sys::AsyncIterator = entries.unchecked_into();
|
||||
let mut results = Vec::new();
|
||||
let prefix_with_slash = with_trailing_slash(prefix);
|
||||
|
||||
let want_kind = match want {
|
||||
EntryKind::File => web_sys::FileSystemHandleKind::File,
|
||||
EntryKind::Directory => web_sys::FileSystemHandleKind::Directory,
|
||||
};
|
||||
|
||||
loop {
|
||||
let next: js_sys::IteratorNext = JsFuture::from(iterator.next()?).await?.unchecked_into();
|
||||
if next.done() {
|
||||
break;
|
||||
}
|
||||
let pair: js_sys::Array = next.value().unchecked_into();
|
||||
let Some(name) = pair.get(0).as_string() else { continue };
|
||||
let handle: web_sys::FileSystemHandle = pair.get(1).unchecked_into();
|
||||
if handle.kind() != want_kind {
|
||||
continue;
|
||||
}
|
||||
results.push(format!("{prefix_with_slash}{name}"));
|
||||
}
|
||||
Ok(results)
|
||||
}
|
||||
|
||||
/// Walk every file under `prefix` (recursively) and collect their full container paths.
|
||||
/// Used at open time to populate the in-memory tracking set.
|
||||
async fn enumerate_paths(root: &FileSystemDirectoryHandle, prefix: &str) -> std::result::Result<HashSet<String>, JsValue> {
|
||||
let mut paths = HashSet::new();
|
||||
let mut to_visit = vec![prefix.to_string()];
|
||||
|
||||
while let Some(current_prefix) = to_visit.pop() {
|
||||
for file in list_entries(root, ¤t_prefix, EntryKind::File).await? {
|
||||
paths.insert(file);
|
||||
}
|
||||
for dir in list_entries(root, ¤t_prefix, EntryKind::Directory).await? {
|
||||
to_visit.push(dir);
|
||||
}
|
||||
}
|
||||
|
||||
Ok(paths)
|
||||
}
|
||||
|
||||
fn is_not_found(error: &JsValue) -> bool {
|
||||
error.dyn_ref::<DomException>().is_some_and(|error| error.name() == "NotFoundError")
|
||||
}
|
||||
|
||||
/// OPFS raises `TypeMismatchError` when a path segment used as a directory is actually a file.
|
||||
fn is_type_mismatch(error: &JsValue) -> bool {
|
||||
error.dyn_ref::<DomException>().is_some_and(|error| error.name() == "TypeMismatchError")
|
||||
}
|
||||
|
||||
/// Map a listing error: a prefix that names a file becomes [`ContainerError::NotADirectory`], anything
|
||||
/// else passes through as a backend error.
|
||||
fn list_error(prefix: &str, error: JsValue) -> ContainerError {
|
||||
if is_type_mismatch(&error) {
|
||||
ContainerError::NotADirectory(prefix.to_string())
|
||||
} else {
|
||||
js_err(error)
|
||||
}
|
||||
}
|
||||
500
document/container/src/lib.rs
Normal file
500
document/container/src/lib.rs
Normal file
@@ -0,0 +1,500 @@
|
||||
//! Container abstraction for the on-disk side of the `.gdd` document format.
|
||||
//!
|
||||
//! A [`Container`] is a virtual filesystem of named byte payloads.
|
||||
//! Backends include a loose folder, an in-memory map, and an OPFS-backed wasm store.
|
||||
//! Archive codecs ([`archive::Zip`], [`archive::Xz`]) round-trip a container's contents
|
||||
//! through a compressed byte stream.
|
||||
//!
|
||||
//! Reads return a [`ByteHolder`], an ownership-carrying handle whose variant depends on
|
||||
//! how the backend produced the bytes (mmap region, owned vector, external file mmap).
|
||||
//! [`AsyncContainer`] mirrors [`Container`] for inherently async backends; every sync
|
||||
//! [`Container`] is reachable from async code via a blanket impl.
|
||||
|
||||
pub mod archive;
|
||||
pub mod backends;
|
||||
|
||||
pub enum ByteHolder {
|
||||
/// Bytes synthesized in memory (decompressed from an archive, produced by serialization).
|
||||
/// The only variant available on `target_family = "wasm"`.
|
||||
Owned(Vec<u8>),
|
||||
/// Bytes mmap'd from a file inside the container.
|
||||
#[cfg(not(target_family = "wasm"))]
|
||||
Mmapped(MmappedBytes),
|
||||
/// Bytes mmap'd from a file outside the container (e.g. a linked resource).
|
||||
#[cfg(not(target_family = "wasm"))]
|
||||
External { path: std::path::PathBuf, bytes: MmappedBytes },
|
||||
}
|
||||
|
||||
impl ByteHolder {
|
||||
pub fn as_slice(&self) -> &[u8] {
|
||||
match self {
|
||||
ByteHolder::Owned(bytes) => bytes,
|
||||
#[cfg(not(target_family = "wasm"))]
|
||||
ByteHolder::Mmapped(bytes) => bytes.as_ref(),
|
||||
#[cfg(not(target_family = "wasm"))]
|
||||
ByteHolder::External { bytes, .. } => bytes.as_ref(),
|
||||
}
|
||||
}
|
||||
|
||||
/// If the bytes are backed by a real filesystem path, return it. Enables consumers to
|
||||
/// short-circuit byte copies with `fs::copy` (CoW on supported filesystems).
|
||||
#[cfg(not(target_family = "wasm"))]
|
||||
pub fn source_path(&self) -> Option<&std::path::Path> {
|
||||
match self {
|
||||
ByteHolder::Owned(_) => None,
|
||||
ByteHolder::Mmapped(bytes) => Some(bytes.path()),
|
||||
ByteHolder::External { path, .. } => Some(path),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(target_family = "wasm")]
|
||||
pub fn source_path(&self) -> Option<&std::path::Path> {
|
||||
None
|
||||
}
|
||||
|
||||
/// Open an external file and produce a [`ByteHolder::External`] backed by mmap.
|
||||
#[cfg(not(target_family = "wasm"))]
|
||||
pub fn open_external(path: impl Into<std::path::PathBuf>) -> Result<Self> {
|
||||
let path = path.into();
|
||||
let file = mmap_io::mmap::MemoryMappedFile::open_ro(&path).map_err(|error| ContainerError::Backend(format!("mmap of {path:?} failed: {error}")))?;
|
||||
let bytes = MmappedBytes::new(file)?;
|
||||
Ok(ByteHolder::External { path, bytes })
|
||||
}
|
||||
}
|
||||
|
||||
impl AsRef<[u8]> for ByteHolder {
|
||||
fn as_ref(&self) -> &[u8] {
|
||||
self.as_slice()
|
||||
}
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for ByteHolder {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
match self {
|
||||
ByteHolder::Owned(bytes) => f.debug_tuple("Owned").field(&format_args!("{} bytes", bytes.len())).finish(),
|
||||
#[cfg(not(target_family = "wasm"))]
|
||||
ByteHolder::Mmapped(bytes) => f.debug_tuple("Mmapped").field(&format_args!("{} bytes", bytes.as_ref().len())).finish(),
|
||||
#[cfg(not(target_family = "wasm"))]
|
||||
ByteHolder::External { path, bytes } => f.debug_struct("External").field("path", path).field("len", &bytes.as_ref().len()).finish(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Owning wrapper around a memory-mapped file that exposes the mapped region as `&[u8]`.
|
||||
#[cfg(not(target_family = "wasm"))]
|
||||
pub struct MmappedBytes(mmap_io::mmap::MemoryMappedFile);
|
||||
|
||||
#[cfg(not(target_family = "wasm"))]
|
||||
impl MmappedBytes {
|
||||
/// Wrap a mapped file, probing that its region is sliceable so the failure surfaces here rather than
|
||||
/// later degrading to the `&[]` fallback in [`AsRef::as_ref`], which cannot return an error.
|
||||
pub fn new(file: mmap_io::mmap::MemoryMappedFile) -> Result<Self> {
|
||||
let len = file.len();
|
||||
file.as_slice(0, len)
|
||||
.map_err(|error| ContainerError::Backend(format!("mmap slice of {:?} failed: {error}", file.path())))?;
|
||||
Ok(Self(file))
|
||||
}
|
||||
|
||||
pub fn path(&self) -> &std::path::Path {
|
||||
self.0.path()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(not(target_family = "wasm"))]
|
||||
impl AsRef<[u8]> for MmappedBytes {
|
||||
fn as_ref(&self) -> &[u8] {
|
||||
let len = self.0.len();
|
||||
match self.0.as_slice(0, len) {
|
||||
Ok(slice) => slice,
|
||||
Err(error) => {
|
||||
log::error!("Failed to obtain mmap slice: {error}");
|
||||
&[]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum ContainerError {
|
||||
#[error("path not found: {0}")]
|
||||
NotFound(String),
|
||||
|
||||
#[error("invalid path: {0}")]
|
||||
InvalidPath(String),
|
||||
|
||||
#[error("not a directory: {0}")]
|
||||
NotADirectory(String),
|
||||
|
||||
#[error("I/O error: {0}")]
|
||||
Io(#[from] std::io::Error),
|
||||
|
||||
/// An archive declared more decompressed data (`declared` bytes) than the codec is willing to
|
||||
/// materialize (`limit` bytes).
|
||||
#[error("declared decompressed size {declared} exceeds the {limit}-byte limit")]
|
||||
SizeLimitExceeded { declared: u64, limit: u64 },
|
||||
|
||||
/// Failure inside an archive codec (zip, lzma, tar). Wraps a foreign error as text since those
|
||||
/// error types don't share a common Rust trait we can chain through.
|
||||
#[error("codec error: {0}")]
|
||||
Codec(String),
|
||||
|
||||
/// Failure inside a storage backend (mmap, OPFS/JS). Wraps a foreign error as text for the same reason.
|
||||
#[error("backend error: {0}")]
|
||||
Backend(String),
|
||||
}
|
||||
|
||||
pub type Result<T> = std::result::Result<T, ContainerError>;
|
||||
|
||||
/// Normalize a listing prefix to end with a trailing slash (unless it names the container root).
|
||||
/// The root (empty string or `.`) normalizes to the empty string, so backends concatenate child paths
|
||||
/// as `prefix/child` without a double, missing, or `./`-rooted slash.
|
||||
pub(crate) fn with_trailing_slash(prefix: &str) -> String {
|
||||
if prefix.is_empty() || prefix == "." {
|
||||
String::new()
|
||||
} else if prefix.ends_with('/') {
|
||||
prefix.to_string()
|
||||
} else {
|
||||
format!("{prefix}/")
|
||||
}
|
||||
}
|
||||
|
||||
/// Validate that `path` names a container-safe file: relative, no `.`/`..` segments, no backslashes,
|
||||
/// no redundant separators (`a//b`, `a/b/`). Dotfile names like `.gitignore` are fine. Requiring a canonical
|
||||
/// form gives a file one identity across backends, some of which key on the raw string rather than path
|
||||
/// components. Used by backends to block container escapes and by archive codecs on untrusted entry names.
|
||||
/// For listing prefixes, which may name the container root, use [`validate_prefix`] instead.
|
||||
pub fn validate_path(path: &str) -> Result<()> {
|
||||
let invalid = || ContainerError::InvalidPath(path.to_string());
|
||||
|
||||
if path.is_empty() || path.contains('\\') || path.starts_with('/') {
|
||||
return Err(invalid());
|
||||
}
|
||||
|
||||
// `Path::components` silently folds away `//`, trailing `/`, and interior `.` segments, but backends key
|
||||
// on the raw string, so a non-canonical path would resolve to one file on a path-joining backend yet a
|
||||
// different identity on a string-keyed backend. Reject the redundant segments the component loop below
|
||||
// can't see (it never observes a folded-away `CurDir`/empty segment).
|
||||
if path.split('/').any(|segment| segment.is_empty() || segment == ".") {
|
||||
return Err(invalid());
|
||||
}
|
||||
|
||||
// Reject Windows drive-letter prefixes (`C:foo`, `C:/foo`) that platform-agnostic Path doesn't recognize as absolute on Linux.
|
||||
let bytes = path.as_bytes();
|
||||
if bytes.len() >= 2 && bytes[1] == b':' && bytes[0].is_ascii_alphabetic() {
|
||||
return Err(invalid());
|
||||
}
|
||||
|
||||
for component in std::path::Path::new(path).components() {
|
||||
use std::path::Component;
|
||||
match component {
|
||||
Component::Normal(_) => {}
|
||||
Component::CurDir | Component::ParentDir | Component::Prefix(_) | Component::RootDir => return Err(invalid()),
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Validate a listing prefix. Same rules as [`validate_path`], except the container root is also a valid
|
||||
/// prefix, named by either the empty string or `.`. Backends pass this to `list`/`list_dirs`.
|
||||
pub fn validate_prefix(prefix: &str) -> Result<()> {
|
||||
if prefix.is_empty() || prefix == "." {
|
||||
return Ok(());
|
||||
}
|
||||
validate_path(prefix)
|
||||
}
|
||||
|
||||
/// Synchronous virtual filesystem of named byte payloads.
|
||||
pub trait Container {
|
||||
/// Read the contents of `path` into a [`ByteHolder`].
|
||||
fn read(&self, path: &str) -> Result<ByteHolder>;
|
||||
|
||||
/// Write `bytes` at `path`, creating intermediate directories as needed.
|
||||
fn write(&self, path: &str, bytes: &[u8]) -> Result<()>;
|
||||
|
||||
/// Append `bytes` to the file at `path`, creating it (and any intermediate directories)
|
||||
/// if it does not yet exist. Equivalent to `write` on a fresh path.
|
||||
fn append(&self, path: &str, bytes: &[u8]) -> Result<()>;
|
||||
|
||||
/// Write `size` bytes whose contents are produced by `fill`.
|
||||
/// The default implementation allocates and forwards to [`Container::write`];
|
||||
/// backends that can mmap a writable region may override to fill in place.
|
||||
fn write_sized(&self, path: &str, size: usize, fill: &mut dyn FnMut(&mut [u8]) -> Result<()>) -> Result<()> {
|
||||
let mut buffer = vec![0; size];
|
||||
fill(&mut buffer)?;
|
||||
self.write(path, &buffer)
|
||||
}
|
||||
|
||||
/// List file entries directly under `prefix` (non-recursive).
|
||||
/// Returned paths include the `prefix` (e.g. `list("resources")` returns
|
||||
/// `["resources/abc123", ...]`). A missing prefix yields an empty list; a prefix that names a
|
||||
/// file (not a directory) is an error.
|
||||
fn list(&self, prefix: &str) -> Result<Vec<String>>;
|
||||
|
||||
/// List subdirectory entries directly under `prefix` (non-recursive).
|
||||
/// Returned names include the `prefix`, without a trailing slash
|
||||
/// (e.g. `list_dirs("")` returns `["resources"]`). Same missing/file-prefix semantics as [`Container::list`].
|
||||
fn list_dirs(&self, prefix: &str) -> Result<Vec<String>>;
|
||||
|
||||
/// Whether a file exists at `path`. Directories return `false`.
|
||||
fn exists(&self, path: &str) -> bool;
|
||||
|
||||
/// Remove the file at `path`. Idempotent: removing a missing path succeeds. Directories are never
|
||||
/// removed (they exist only implicitly as parents of files).
|
||||
fn remove(&self, path: &str) -> Result<()>;
|
||||
}
|
||||
|
||||
/// Asynchronous virtual filesystem of named byte payloads. Mirrors [`Container`].
|
||||
///
|
||||
/// The returned futures are intentionally not `Send`: native uses `block_on` at the save seam
|
||||
/// and wasm is single-threaded, so neither needs cross-thread futures. Revisit if we ever want
|
||||
/// to run container I/O on a thread pool.
|
||||
#[expect(async_fn_in_trait, reason = "see trait docs — Send is not required")]
|
||||
pub trait AsyncContainer {
|
||||
async fn read(&self, path: &str) -> Result<ByteHolder>;
|
||||
|
||||
async fn write(&self, path: &str, bytes: &[u8]) -> Result<()>;
|
||||
|
||||
async fn append(&self, path: &str, bytes: &[u8]) -> Result<()>;
|
||||
|
||||
async fn write_sized(&self, path: &str, size: usize, fill: &mut dyn FnMut(&mut [u8]) -> Result<()>) -> Result<()>;
|
||||
|
||||
async fn list(&self, prefix: &str) -> Result<Vec<String>>;
|
||||
|
||||
async fn list_dirs(&self, prefix: &str) -> Result<Vec<String>>;
|
||||
|
||||
async fn exists(&self, path: &str) -> bool;
|
||||
|
||||
async fn remove(&self, path: &str) -> Result<()>;
|
||||
|
||||
/// Synchronous write. On backends with sync I/O (folder, memory) the write completes durably
|
||||
/// before return and reports real errors. On OPFS the write is enqueued onto a background task
|
||||
/// and `Ok` is returned eagerly; a later failure is logged.
|
||||
fn write_non_blocking(&self, path: &str, bytes: &[u8]) -> Result<()>;
|
||||
|
||||
/// Synchronous append. Same eager-enqueue semantics on OPFS as [`write_non_blocking`](Self::write_non_blocking);
|
||||
/// queued appends preserve order relative to earlier queued writes/appends.
|
||||
fn append_non_blocking(&self, path: &str, bytes: &[u8]) -> Result<()>;
|
||||
|
||||
/// Synchronous remove. Same semantics as [`write_non_blocking`](Self::write_non_blocking).
|
||||
fn remove_non_blocking(&self, path: &str) -> Result<()>;
|
||||
|
||||
/// Non-blocking existence check. On OPFS this reads from an in-memory tracking set populated by
|
||||
/// the sync write/remove paths, since the underlying OPFS existence API is async only.
|
||||
fn exists_non_blocking(&self, path: &str) -> bool;
|
||||
}
|
||||
|
||||
impl<C: Container + ?Sized> AsyncContainer for C {
|
||||
async fn read(&self, path: &str) -> Result<ByteHolder> {
|
||||
Container::read(self, path)
|
||||
}
|
||||
|
||||
async fn write(&self, path: &str, bytes: &[u8]) -> Result<()> {
|
||||
Container::write(self, path, bytes)
|
||||
}
|
||||
|
||||
async fn append(&self, path: &str, bytes: &[u8]) -> Result<()> {
|
||||
Container::append(self, path, bytes)
|
||||
}
|
||||
|
||||
async fn write_sized(&self, path: &str, size: usize, fill: &mut dyn FnMut(&mut [u8]) -> Result<()>) -> Result<()> {
|
||||
Container::write_sized(self, path, size, fill)
|
||||
}
|
||||
|
||||
async fn list(&self, prefix: &str) -> Result<Vec<String>> {
|
||||
Container::list(self, prefix)
|
||||
}
|
||||
|
||||
async fn list_dirs(&self, prefix: &str) -> Result<Vec<String>> {
|
||||
Container::list_dirs(self, prefix)
|
||||
}
|
||||
|
||||
async fn exists(&self, path: &str) -> bool {
|
||||
Container::exists(self, path)
|
||||
}
|
||||
|
||||
async fn remove(&self, path: &str) -> Result<()> {
|
||||
Container::remove(self, path)
|
||||
}
|
||||
|
||||
fn write_non_blocking(&self, path: &str, bytes: &[u8]) -> Result<()> {
|
||||
Container::write(self, path, bytes)
|
||||
}
|
||||
|
||||
fn append_non_blocking(&self, path: &str, bytes: &[u8]) -> Result<()> {
|
||||
Container::append(self, path, bytes)
|
||||
}
|
||||
|
||||
fn remove_non_blocking(&self, path: &str) -> Result<()> {
|
||||
Container::remove(self, path)
|
||||
}
|
||||
|
||||
fn exists_non_blocking(&self, path: &str) -> bool {
|
||||
Container::exists(self, path)
|
||||
}
|
||||
}
|
||||
|
||||
/// Type-erased container that dispatches to one of the in-tree backends.
|
||||
///
|
||||
/// `AsyncContainer::read` returns `impl Future`, so `dyn AsyncContainer` is not object-safe.
|
||||
/// `AnyContainer` is the workaround: `Gdd` holds one of these by value, and the `AsyncContainer`
|
||||
/// impl forwards to the active variant.
|
||||
pub enum AnyContainer {
|
||||
Memory(backends::memory::MemoryBackend),
|
||||
#[cfg(not(target_family = "wasm"))]
|
||||
Folder(backends::folder::FolderBackend),
|
||||
#[cfg(target_family = "wasm")]
|
||||
Opfs(backends::opfs::OpfsBackend),
|
||||
}
|
||||
|
||||
impl AsyncContainer for AnyContainer {
|
||||
async fn read(&self, path: &str) -> Result<ByteHolder> {
|
||||
match self {
|
||||
Self::Memory(backend) => AsyncContainer::read(backend, path).await,
|
||||
#[cfg(not(target_family = "wasm"))]
|
||||
Self::Folder(backend) => AsyncContainer::read(backend, path).await,
|
||||
#[cfg(target_family = "wasm")]
|
||||
Self::Opfs(backend) => AsyncContainer::read(backend, path).await,
|
||||
}
|
||||
}
|
||||
|
||||
async fn write(&self, path: &str, bytes: &[u8]) -> Result<()> {
|
||||
match self {
|
||||
Self::Memory(backend) => AsyncContainer::write(backend, path, bytes).await,
|
||||
#[cfg(not(target_family = "wasm"))]
|
||||
Self::Folder(backend) => AsyncContainer::write(backend, path, bytes).await,
|
||||
#[cfg(target_family = "wasm")]
|
||||
Self::Opfs(backend) => AsyncContainer::write(backend, path, bytes).await,
|
||||
}
|
||||
}
|
||||
|
||||
async fn append(&self, path: &str, bytes: &[u8]) -> Result<()> {
|
||||
match self {
|
||||
Self::Memory(backend) => AsyncContainer::append(backend, path, bytes).await,
|
||||
#[cfg(not(target_family = "wasm"))]
|
||||
Self::Folder(backend) => AsyncContainer::append(backend, path, bytes).await,
|
||||
#[cfg(target_family = "wasm")]
|
||||
Self::Opfs(backend) => AsyncContainer::append(backend, path, bytes).await,
|
||||
}
|
||||
}
|
||||
|
||||
async fn write_sized(&self, path: &str, size: usize, fill: &mut dyn FnMut(&mut [u8]) -> Result<()>) -> Result<()> {
|
||||
match self {
|
||||
Self::Memory(backend) => AsyncContainer::write_sized(backend, path, size, fill).await,
|
||||
#[cfg(not(target_family = "wasm"))]
|
||||
Self::Folder(backend) => AsyncContainer::write_sized(backend, path, size, fill).await,
|
||||
#[cfg(target_family = "wasm")]
|
||||
Self::Opfs(backend) => AsyncContainer::write_sized(backend, path, size, fill).await,
|
||||
}
|
||||
}
|
||||
|
||||
async fn list(&self, prefix: &str) -> Result<Vec<String>> {
|
||||
match self {
|
||||
Self::Memory(backend) => AsyncContainer::list(backend, prefix).await,
|
||||
#[cfg(not(target_family = "wasm"))]
|
||||
Self::Folder(backend) => AsyncContainer::list(backend, prefix).await,
|
||||
#[cfg(target_family = "wasm")]
|
||||
Self::Opfs(backend) => AsyncContainer::list(backend, prefix).await,
|
||||
}
|
||||
}
|
||||
|
||||
async fn list_dirs(&self, prefix: &str) -> Result<Vec<String>> {
|
||||
match self {
|
||||
Self::Memory(backend) => AsyncContainer::list_dirs(backend, prefix).await,
|
||||
#[cfg(not(target_family = "wasm"))]
|
||||
Self::Folder(backend) => AsyncContainer::list_dirs(backend, prefix).await,
|
||||
#[cfg(target_family = "wasm")]
|
||||
Self::Opfs(backend) => AsyncContainer::list_dirs(backend, prefix).await,
|
||||
}
|
||||
}
|
||||
|
||||
async fn exists(&self, path: &str) -> bool {
|
||||
match self {
|
||||
Self::Memory(backend) => AsyncContainer::exists(backend, path).await,
|
||||
#[cfg(not(target_family = "wasm"))]
|
||||
Self::Folder(backend) => AsyncContainer::exists(backend, path).await,
|
||||
#[cfg(target_family = "wasm")]
|
||||
Self::Opfs(backend) => AsyncContainer::exists(backend, path).await,
|
||||
}
|
||||
}
|
||||
|
||||
async fn remove(&self, path: &str) -> Result<()> {
|
||||
match self {
|
||||
Self::Memory(backend) => AsyncContainer::remove(backend, path).await,
|
||||
#[cfg(not(target_family = "wasm"))]
|
||||
Self::Folder(backend) => AsyncContainer::remove(backend, path).await,
|
||||
#[cfg(target_family = "wasm")]
|
||||
Self::Opfs(backend) => AsyncContainer::remove(backend, path).await,
|
||||
}
|
||||
}
|
||||
|
||||
fn write_non_blocking(&self, path: &str, bytes: &[u8]) -> Result<()> {
|
||||
match self {
|
||||
Self::Memory(backend) => AsyncContainer::write_non_blocking(backend, path, bytes),
|
||||
#[cfg(not(target_family = "wasm"))]
|
||||
Self::Folder(backend) => AsyncContainer::write_non_blocking(backend, path, bytes),
|
||||
#[cfg(target_family = "wasm")]
|
||||
Self::Opfs(backend) => AsyncContainer::write_non_blocking(backend, path, bytes),
|
||||
}
|
||||
}
|
||||
|
||||
fn append_non_blocking(&self, path: &str, bytes: &[u8]) -> Result<()> {
|
||||
match self {
|
||||
Self::Memory(backend) => AsyncContainer::append_non_blocking(backend, path, bytes),
|
||||
#[cfg(not(target_family = "wasm"))]
|
||||
Self::Folder(backend) => AsyncContainer::append_non_blocking(backend, path, bytes),
|
||||
#[cfg(target_family = "wasm")]
|
||||
Self::Opfs(backend) => AsyncContainer::append_non_blocking(backend, path, bytes),
|
||||
}
|
||||
}
|
||||
|
||||
fn remove_non_blocking(&self, path: &str) -> Result<()> {
|
||||
match self {
|
||||
Self::Memory(backend) => AsyncContainer::remove_non_blocking(backend, path),
|
||||
#[cfg(not(target_family = "wasm"))]
|
||||
Self::Folder(backend) => AsyncContainer::remove_non_blocking(backend, path),
|
||||
#[cfg(target_family = "wasm")]
|
||||
Self::Opfs(backend) => AsyncContainer::remove_non_blocking(backend, path),
|
||||
}
|
||||
}
|
||||
|
||||
fn exists_non_blocking(&self, path: &str) -> bool {
|
||||
match self {
|
||||
Self::Memory(backend) => AsyncContainer::exists_non_blocking(backend, path),
|
||||
#[cfg(not(target_family = "wasm"))]
|
||||
Self::Folder(backend) => AsyncContainer::exists_non_blocking(backend, path),
|
||||
#[cfg(target_family = "wasm")]
|
||||
Self::Opfs(backend) => AsyncContainer::exists_non_blocking(backend, path),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{ContainerError, validate_path, validate_prefix};
|
||||
|
||||
#[test]
|
||||
fn validate_path_accepts_well_formed() {
|
||||
// `..` and `.` are only rejected as whole path components; as substrings of a name they are fine.
|
||||
for ok in ["manifest.json", "resources/abc", "a/b/c.bin", "my..file.txt", "..hidden", ".gitignore"] {
|
||||
assert!(validate_path(ok).is_ok(), "{ok:?} should be accepted");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_path_rejects_unsafe() {
|
||||
for bad in ["", "../escape", "a/../b", ".", "./leading", "/abs", "back\\slash", "C:/win", "\\\\?\\unc"] {
|
||||
let result = validate_path(bad);
|
||||
assert!(matches!(result, Err(ContainerError::InvalidPath(_))), "{bad:?} should be rejected, got {result:?}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_prefix_accepts_root_tokens() {
|
||||
// The container root is a valid listing prefix, named by either the empty string or `.`.
|
||||
for root in ["", ".", "resources", "a/b"] {
|
||||
assert!(validate_prefix(root).is_ok(), "{root:?} should be accepted as a prefix");
|
||||
}
|
||||
// Unsafe prefixes are still rejected, same as paths.
|
||||
assert!(validate_prefix("../escape").is_err());
|
||||
}
|
||||
}
|
||||
81
document/container/tests/archive.rs
Normal file
81
document/container/tests/archive.rs
Normal file
@@ -0,0 +1,81 @@
|
||||
#![cfg(any(feature = "zip", feature = "xz"))]
|
||||
|
||||
use document_container::Container;
|
||||
use document_container::archive::{Archive, ArchiveWriter};
|
||||
use document_container::backends::folder::FolderBackend;
|
||||
use document_container::backends::memory::MemoryBackend;
|
||||
|
||||
fn entries() -> Vec<(&'static str, &'static [u8])> {
|
||||
vec![
|
||||
("manifest.json", br#"{"format":"gdd"}"#),
|
||||
("document.json", b"{\"registry\":\"...\"}"),
|
||||
("history.jsonl", b"{\"rev\":1}\n{\"rev\":2}\n"),
|
||||
("resources/abc123", &[0xDE, 0xAD, 0xBE, 0xEF]),
|
||||
("resources/xyz789", b"another resource"),
|
||||
]
|
||||
}
|
||||
|
||||
fn assert_round_trip(restored: &MemoryBackend) {
|
||||
assert_eq!(restored.read("manifest.json").unwrap().as_slice(), br#"{"format":"gdd"}"#);
|
||||
assert_eq!(restored.read("document.json").unwrap().as_slice(), b"{\"registry\":\"...\"}");
|
||||
assert_eq!(restored.read("history.jsonl").unwrap().as_slice(), b"{\"rev\":1}\n{\"rev\":2}\n");
|
||||
assert_eq!(restored.read("resources/abc123").unwrap().as_slice(), &[0xDE, 0xAD, 0xBE, 0xEF]);
|
||||
assert_eq!(restored.read("resources/xyz789").unwrap().as_slice(), b"another resource");
|
||||
}
|
||||
|
||||
#[cfg(feature = "zip")]
|
||||
#[test]
|
||||
fn zip_round_trip() {
|
||||
use document_container::archive::Zip;
|
||||
use std::io::Cursor;
|
||||
|
||||
let mut buffer = Cursor::new(Vec::new());
|
||||
let mut writer = Zip::writer(&mut buffer).unwrap();
|
||||
for (path, bytes) in entries() {
|
||||
writer.write_entry(path, bytes).unwrap();
|
||||
}
|
||||
writer.finish().unwrap();
|
||||
|
||||
let mut restored = MemoryBackend::new();
|
||||
<Zip as Archive>::open(Cursor::new(buffer.get_ref()), &mut restored).unwrap();
|
||||
assert_round_trip(&restored);
|
||||
}
|
||||
|
||||
#[cfg(feature = "zip")]
|
||||
#[test]
|
||||
fn zip_deserialize_streams_into_folder_backend() {
|
||||
use document_container::archive::Zip;
|
||||
use std::io::Cursor;
|
||||
|
||||
let mut buffer = Cursor::new(Vec::new());
|
||||
let mut writer = Zip::writer(&mut buffer).unwrap();
|
||||
for (path, bytes) in entries() {
|
||||
writer.write_entry(path, bytes).unwrap();
|
||||
}
|
||||
writer.finish().unwrap();
|
||||
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let mut restored = FolderBackend::create(dir.path()).unwrap();
|
||||
<Zip as Archive>::open(Cursor::new(buffer.get_ref()), &mut restored).unwrap();
|
||||
|
||||
assert_eq!(restored.read("manifest.json").unwrap().as_slice(), br#"{"format":"gdd"}"#);
|
||||
assert_eq!(restored.read("resources/abc123").unwrap().as_slice(), &[0xDE, 0xAD, 0xBE, 0xEF]);
|
||||
}
|
||||
|
||||
#[cfg(feature = "xz")]
|
||||
#[test]
|
||||
fn xz_round_trip() {
|
||||
use document_container::archive::Xz;
|
||||
use std::io::Cursor;
|
||||
|
||||
let mut buffer = Cursor::new(Vec::new());
|
||||
let mut writer = Xz::writer(&mut buffer).unwrap();
|
||||
for (path, bytes) in entries() {
|
||||
writer.write_entry(path, bytes).unwrap();
|
||||
}
|
||||
writer.finish().unwrap();
|
||||
|
||||
let mut restored = MemoryBackend::new();
|
||||
<Xz as Archive>::open(Cursor::new(buffer.get_ref()), &mut restored).unwrap();
|
||||
assert_round_trip(&restored);
|
||||
}
|
||||
178
document/container/tests/round_trip.rs
Normal file
178
document/container/tests/round_trip.rs
Normal file
@@ -0,0 +1,178 @@
|
||||
use document_container::backends::folder::FolderBackend;
|
||||
use document_container::backends::memory::MemoryBackend;
|
||||
use document_container::{AnyContainer, Container, ContainerError};
|
||||
|
||||
fn run_round_trip<C: Container>(container: C) {
|
||||
container.write("manifest.json", br#"{"format":"gdd"}"#).unwrap();
|
||||
container.write("resources/abc123", &[0xDE, 0xAD, 0xBE, 0xEF]).unwrap();
|
||||
container.write("resources/xyz789", b"another resource").unwrap();
|
||||
|
||||
assert!(container.exists("manifest.json"));
|
||||
assert!(container.exists("resources/abc123"));
|
||||
assert!(!container.exists("does-not-exist"));
|
||||
|
||||
let manifest = container.read("manifest.json").unwrap();
|
||||
assert_eq!(manifest.as_slice(), br#"{"format":"gdd"}"#);
|
||||
|
||||
let blob = container.read("resources/abc123").unwrap();
|
||||
assert_eq!(blob.as_slice(), &[0xDE, 0xAD, 0xBE, 0xEF]);
|
||||
|
||||
let top_level = container.list("").unwrap();
|
||||
assert!(top_level.iter().any(|p| p == "manifest.json"));
|
||||
assert!(
|
||||
top_level.iter().all(|p| !p.starts_with("resources/")),
|
||||
"list(\"\") must not descend into subdirectories, got {top_level:?}"
|
||||
);
|
||||
|
||||
let mut resources = container.list("resources").unwrap();
|
||||
resources.sort();
|
||||
assert_eq!(resources, vec!["resources/abc123".to_string(), "resources/xyz789".to_string()]);
|
||||
|
||||
container.remove("resources/abc123").unwrap();
|
||||
assert!(!container.exists("resources/abc123"));
|
||||
assert!(matches!(container.read("resources/abc123"), Err(ContainerError::NotFound(_))));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn memory_backend_round_trip() {
|
||||
run_round_trip(MemoryBackend::new());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn folder_backend_round_trip() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let backend = FolderBackend::create(dir.path()).unwrap();
|
||||
run_round_trip(backend);
|
||||
}
|
||||
|
||||
fn run_list_and_remove_semantics<C: Container>(container: C) {
|
||||
container.write("dir/file", b"x").unwrap();
|
||||
|
||||
// Removing a missing path is idempotent.
|
||||
container.remove("dir/missing").unwrap();
|
||||
container.remove("dir/file").unwrap();
|
||||
container.remove("dir/file").unwrap();
|
||||
|
||||
// Listing a missing prefix yields an empty list.
|
||||
assert_eq!(container.list("nonexistent").unwrap(), Vec::<String>::new());
|
||||
|
||||
// Listing a prefix that names a file is an error.
|
||||
container.write("manifest.json", b"{}").unwrap();
|
||||
assert!(matches!(container.list("manifest.json"), Err(ContainerError::NotADirectory(_))));
|
||||
assert!(matches!(container.list_dirs("manifest.json"), Err(ContainerError::NotADirectory(_))));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn memory_backend_list_and_remove_semantics() {
|
||||
run_list_and_remove_semantics(MemoryBackend::new());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn folder_backend_list_and_remove_semantics() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
run_list_and_remove_semantics(FolderBackend::create(dir.path()).unwrap());
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[cfg(unix)]
|
||||
fn folder_backend_rejects_symlink_escape() {
|
||||
let outside = tempfile::tempdir().unwrap();
|
||||
std::fs::write(outside.path().join("secret"), b"sensitive").unwrap();
|
||||
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let backend = FolderBackend::create(dir.path()).unwrap();
|
||||
// A symlink planted inside the root pointing outside it must not be traversable.
|
||||
std::os::unix::fs::symlink(outside.path(), dir.path().join("link")).unwrap();
|
||||
|
||||
let result = backend.read("link/secret");
|
||||
assert!(matches!(result, Err(ContainerError::InvalidPath(_))), "symlink escape should be rejected, got {result:?}");
|
||||
|
||||
// Listing must reject the symlinked prefix too, not just reads, so it can't leak entry names from outside the root.
|
||||
assert!(matches!(backend.list("link"), Err(ContainerError::InvalidPath(_))), "list through symlink should be rejected");
|
||||
assert!(matches!(backend.list_dirs("link"), Err(ContainerError::InvalidPath(_))), "list_dirs through symlink should be rejected");
|
||||
|
||||
// A symlink entry directly under root must not appear in listings, since `resolve` would reject reading it.
|
||||
std::os::unix::fs::symlink(outside.path().join("secret"), dir.path().join("file_link")).unwrap();
|
||||
std::fs::write(dir.path().join("real"), b"ok").unwrap();
|
||||
assert_eq!(backend.list("").unwrap(), vec!["real".to_string()], "symlink entries should be omitted from file listings");
|
||||
assert!(backend.list_dirs("").unwrap().is_empty(), "symlink-to-dir entries should be omitted from dir listings");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn folder_backend_rejects_path_traversal() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let backend = FolderBackend::create(dir.path()).unwrap();
|
||||
for bad in ["../escape", "subdir/../escape", "/abs", "back\\slash", "a//b", "trailing/", "a/./b"] {
|
||||
let result = backend.write(bad, b"nope");
|
||||
assert!(matches!(result, Err(ContainerError::InvalidPath(_))), "expected InvalidPath for {bad:?}, got {result:?}");
|
||||
}
|
||||
}
|
||||
|
||||
fn run_append<C: Container>(container: C) {
|
||||
// Appending to a non-existent path creates it — same semantics as `OpenOptions::append().create(true)`.
|
||||
container.append("history.jsonl", b"{\"op\":1}\n").unwrap();
|
||||
container.append("history.jsonl", b"{\"op\":2}\n").unwrap();
|
||||
container.append("history.jsonl", b"{\"op\":3}\n").unwrap();
|
||||
|
||||
let log = container.read("history.jsonl").unwrap();
|
||||
assert_eq!(log.as_slice(), b"{\"op\":1}\n{\"op\":2}\n{\"op\":3}\n");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn memory_backend_append() {
|
||||
run_append(MemoryBackend::new());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn folder_backend_append() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let backend = FolderBackend::create(dir.path()).unwrap();
|
||||
run_append(backend);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn any_container_dispatches_to_active_variant() {
|
||||
use document_container::AsyncContainer;
|
||||
|
||||
let container = AnyContainer::Memory(MemoryBackend::new());
|
||||
|
||||
futures::executor::block_on(async {
|
||||
container.write("manifest.json", br#"{"format":"gdd"}"#).await.unwrap();
|
||||
container.append("history.jsonl", b"frame-1\n").await.unwrap();
|
||||
container.append("history.jsonl", b"frame-2\n").await.unwrap();
|
||||
|
||||
assert!(container.exists("manifest.json").await);
|
||||
let manifest = container.read("manifest.json").await.unwrap();
|
||||
assert_eq!(manifest.as_slice(), br#"{"format":"gdd"}"#);
|
||||
|
||||
let history = container.read("history.jsonl").await.unwrap();
|
||||
assert_eq!(history.as_slice(), b"frame-1\nframe-2\n");
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn folder_backend_reads_empty_file() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let backend = FolderBackend::create(dir.path()).unwrap();
|
||||
|
||||
backend.write("empty.bin", &[]).unwrap();
|
||||
let read_back = backend.read("empty.bin").unwrap();
|
||||
assert_eq!(read_back.as_slice(), &[] as &[u8]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn folder_backend_write_sized_fills_via_mmap() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let backend = FolderBackend::create(dir.path()).unwrap();
|
||||
|
||||
let payload = b"hello world";
|
||||
backend
|
||||
.write_sized("resources/sized", payload.len(), &mut |buffer| {
|
||||
buffer.copy_from_slice(payload);
|
||||
Ok(())
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
let read_back = backend.read("resources/sized").unwrap();
|
||||
assert_eq!(read_back.as_slice(), payload);
|
||||
}
|
||||
Reference in New Issue
Block a user