Add document-container crate (#4191)

* Add document-container crate: container backends and archive codecs

* Address PR review: path/prefix split, safe size casts, OPFS stream aborts

* Address PR review round 2: mmap read check, UTF8 entry names, prefix normalization

* Make MmappedBytes::new fallible so mmap reads can't silently degrade

* Address PR review round 3: backend contract uniformity (symlinks, remove, list)

* Apply symlink-component check to FolderBackend listing paths

* Address PR review: idempotent OPFS delete logging, tar default-features, shared entry-size cap

* Fix validate_path doc: dotfiles pass, CurDir/ParentDir rejected

* Omit symlink entries from FolderBackend listings for consistency with resolve

* Preserve zip I/O errors and reject non-canonical paths in validate_path

* Extend archive apis to return the archive writer

* Rename document/document-container directory to document/container

* Add archive format sniffing and deserialize_auto

* Drop temporal hedge from checked_entry_size comment

* Tighten verbose doc comments in document-container

* Coalesce consecutive same-path OPFS appends to avoid O(n^2) file copies

* Review

* Update document-container for the deserialize/store_non_blocking renames

---------

Co-authored-by: Timon <me@timon.zip>

document/container/tests/archive.rs

@@ -0,0 +1,81 @@
+#![cfg(any(feature = "zip", feature = "xz"))]
+
+use document_container::Container;
+use document_container::archive::{Archive, ArchiveWriter};
+use document_container::backends::folder::FolderBackend;
+use document_container::backends::memory::MemoryBackend;
+
+fn entries() -> Vec<(&'static str, &'static [u8])> {
+	vec![
+		("manifest.json", br#"{"format":"gdd"}"#),
+		("document.json", b"{\"registry\":\"...\"}"),
+		("history.jsonl", b"{\"rev\":1}\n{\"rev\":2}\n"),
+		("resources/abc123", &[0xDE, 0xAD, 0xBE, 0xEF]),
+		("resources/xyz789", b"another resource"),
+	]
+}
+
+fn assert_round_trip(restored: &MemoryBackend) {
+	assert_eq!(restored.read("manifest.json").unwrap().as_slice(), br#"{"format":"gdd"}"#);
+	assert_eq!(restored.read("document.json").unwrap().as_slice(), b"{\"registry\":\"...\"}");
+	assert_eq!(restored.read("history.jsonl").unwrap().as_slice(), b"{\"rev\":1}\n{\"rev\":2}\n");
+	assert_eq!(restored.read("resources/abc123").unwrap().as_slice(), &[0xDE, 0xAD, 0xBE, 0xEF]);
+	assert_eq!(restored.read("resources/xyz789").unwrap().as_slice(), b"another resource");
+}
+
+#[cfg(feature = "zip")]
+#[test]
+fn zip_round_trip() {
+	use document_container::archive::Zip;
+	use std::io::Cursor;
+
+	let mut buffer = Cursor::new(Vec::new());
+	let mut writer = Zip::writer(&mut buffer).unwrap();
+	for (path, bytes) in entries() {
+		writer.write_entry(path, bytes).unwrap();
+	}
+	writer.finish().unwrap();
+
+	let mut restored = MemoryBackend::new();
+	<Zip as Archive>::open(Cursor::new(buffer.get_ref()), &mut restored).unwrap();
+	assert_round_trip(&restored);
+}
+
+#[cfg(feature = "zip")]
+#[test]
+fn zip_deserialize_streams_into_folder_backend() {
+	use document_container::archive::Zip;
+	use std::io::Cursor;
+
+	let mut buffer = Cursor::new(Vec::new());
+	let mut writer = Zip::writer(&mut buffer).unwrap();
+	for (path, bytes) in entries() {
+		writer.write_entry(path, bytes).unwrap();
+	}
+	writer.finish().unwrap();
+
+	let dir = tempfile::tempdir().unwrap();
+	let mut restored = FolderBackend::create(dir.path()).unwrap();
+	<Zip as Archive>::open(Cursor::new(buffer.get_ref()), &mut restored).unwrap();
+
+	assert_eq!(restored.read("manifest.json").unwrap().as_slice(), br#"{"format":"gdd"}"#);
+	assert_eq!(restored.read("resources/abc123").unwrap().as_slice(), &[0xDE, 0xAD, 0xBE, 0xEF]);
+}
+
+#[cfg(feature = "xz")]
+#[test]
+fn xz_round_trip() {
+	use document_container::archive::Xz;
+	use std::io::Cursor;
+
+	let mut buffer = Cursor::new(Vec::new());
+	let mut writer = Xz::writer(&mut buffer).unwrap();
+	for (path, bytes) in entries() {
+		writer.write_entry(path, bytes).unwrap();
+	}
+	writer.finish().unwrap();
+
+	let mut restored = MemoryBackend::new();
+	<Xz as Archive>::open(Cursor::new(buffer.get_ref()), &mut restored).unwrap();
+	assert_round_trip(&restored);
+}

document/container/tests/round_trip.rs

@@ -0,0 +1,178 @@
+use document_container::backends::folder::FolderBackend;
+use document_container::backends::memory::MemoryBackend;
+use document_container::{AnyContainer, Container, ContainerError};
+
+fn run_round_trip<C: Container>(container: C) {
+	container.write("manifest.json", br#"{"format":"gdd"}"#).unwrap();
+	container.write("resources/abc123", &[0xDE, 0xAD, 0xBE, 0xEF]).unwrap();
+	container.write("resources/xyz789", b"another resource").unwrap();
+
+	assert!(container.exists("manifest.json"));
+	assert!(container.exists("resources/abc123"));
+	assert!(!container.exists("does-not-exist"));
+
+	let manifest = container.read("manifest.json").unwrap();
+	assert_eq!(manifest.as_slice(), br#"{"format":"gdd"}"#);
+
+	let blob = container.read("resources/abc123").unwrap();
+	assert_eq!(blob.as_slice(), &[0xDE, 0xAD, 0xBE, 0xEF]);
+
+	let top_level = container.list("").unwrap();
+	assert!(top_level.iter().any(|p| p == "manifest.json"));
+	assert!(
+		top_level.iter().all(|p| !p.starts_with("resources/")),
+		"list(\"\") must not descend into subdirectories, got {top_level:?}"
+	);
+
+	let mut resources = container.list("resources").unwrap();
+	resources.sort();
+	assert_eq!(resources, vec!["resources/abc123".to_string(), "resources/xyz789".to_string()]);
+
+	container.remove("resources/abc123").unwrap();
+	assert!(!container.exists("resources/abc123"));
+	assert!(matches!(container.read("resources/abc123"), Err(ContainerError::NotFound(_))));
+}
+
+#[test]
+fn memory_backend_round_trip() {
+	run_round_trip(MemoryBackend::new());
+}
+
+#[test]
+fn folder_backend_round_trip() {
+	let dir = tempfile::tempdir().unwrap();
+	let backend = FolderBackend::create(dir.path()).unwrap();
+	run_round_trip(backend);
+}
+
+fn run_list_and_remove_semantics<C: Container>(container: C) {
+	container.write("dir/file", b"x").unwrap();
+
+	// Removing a missing path is idempotent.
+	container.remove("dir/missing").unwrap();
+	container.remove("dir/file").unwrap();
+	container.remove("dir/file").unwrap();
+
+	// Listing a missing prefix yields an empty list.
+	assert_eq!(container.list("nonexistent").unwrap(), Vec::<String>::new());
+
+	// Listing a prefix that names a file is an error.
+	container.write("manifest.json", b"{}").unwrap();
+	assert!(matches!(container.list("manifest.json"), Err(ContainerError::NotADirectory(_))));
+	assert!(matches!(container.list_dirs("manifest.json"), Err(ContainerError::NotADirectory(_))));
+}
+
+#[test]
+fn memory_backend_list_and_remove_semantics() {
+	run_list_and_remove_semantics(MemoryBackend::new());
+}
+
+#[test]
+fn folder_backend_list_and_remove_semantics() {
+	let dir = tempfile::tempdir().unwrap();
+	run_list_and_remove_semantics(FolderBackend::create(dir.path()).unwrap());
+}
+
+#[test]
+#[cfg(unix)]
+fn folder_backend_rejects_symlink_escape() {
+	let outside = tempfile::tempdir().unwrap();
+	std::fs::write(outside.path().join("secret"), b"sensitive").unwrap();
+
+	let dir = tempfile::tempdir().unwrap();
+	let backend = FolderBackend::create(dir.path()).unwrap();
+	// A symlink planted inside the root pointing outside it must not be traversable.
+	std::os::unix::fs::symlink(outside.path(), dir.path().join("link")).unwrap();
+
+	let result = backend.read("link/secret");
+	assert!(matches!(result, Err(ContainerError::InvalidPath(_))), "symlink escape should be rejected, got {result:?}");
+
+	// Listing must reject the symlinked prefix too, not just reads, so it can't leak entry names from outside the root.
+	assert!(matches!(backend.list("link"), Err(ContainerError::InvalidPath(_))), "list through symlink should be rejected");
+	assert!(matches!(backend.list_dirs("link"), Err(ContainerError::InvalidPath(_))), "list_dirs through symlink should be rejected");
+
+	// A symlink entry directly under root must not appear in listings, since `resolve` would reject reading it.
+	std::os::unix::fs::symlink(outside.path().join("secret"), dir.path().join("file_link")).unwrap();
+	std::fs::write(dir.path().join("real"), b"ok").unwrap();
+	assert_eq!(backend.list("").unwrap(), vec!["real".to_string()], "symlink entries should be omitted from file listings");
+	assert!(backend.list_dirs("").unwrap().is_empty(), "symlink-to-dir entries should be omitted from dir listings");
+}
+
+#[test]
+fn folder_backend_rejects_path_traversal() {
+	let dir = tempfile::tempdir().unwrap();
+	let backend = FolderBackend::create(dir.path()).unwrap();
+	for bad in ["../escape", "subdir/../escape", "/abs", "back\\slash", "a//b", "trailing/", "a/./b"] {
+		let result = backend.write(bad, b"nope");
+		assert!(matches!(result, Err(ContainerError::InvalidPath(_))), "expected InvalidPath for {bad:?}, got {result:?}");
+	}
+}
+
+fn run_append<C: Container>(container: C) {
+	// Appending to a non-existent path creates it — same semantics as `OpenOptions::append().create(true)`.
+	container.append("history.jsonl", b"{\"op\":1}\n").unwrap();
+	container.append("history.jsonl", b"{\"op\":2}\n").unwrap();
+	container.append("history.jsonl", b"{\"op\":3}\n").unwrap();
+
+	let log = container.read("history.jsonl").unwrap();
+	assert_eq!(log.as_slice(), b"{\"op\":1}\n{\"op\":2}\n{\"op\":3}\n");
+}
+
+#[test]
+fn memory_backend_append() {
+	run_append(MemoryBackend::new());
+}
+
+#[test]
+fn folder_backend_append() {
+	let dir = tempfile::tempdir().unwrap();
+	let backend = FolderBackend::create(dir.path()).unwrap();
+	run_append(backend);
+}
+
+#[test]
+fn any_container_dispatches_to_active_variant() {
+	use document_container::AsyncContainer;
+
+	let container = AnyContainer::Memory(MemoryBackend::new());
+
+	futures::executor::block_on(async {
+		container.write("manifest.json", br#"{"format":"gdd"}"#).await.unwrap();
+		container.append("history.jsonl", b"frame-1\n").await.unwrap();
+		container.append("history.jsonl", b"frame-2\n").await.unwrap();
+
+		assert!(container.exists("manifest.json").await);
+		let manifest = container.read("manifest.json").await.unwrap();
+		assert_eq!(manifest.as_slice(), br#"{"format":"gdd"}"#);
+
+		let history = container.read("history.jsonl").await.unwrap();
+		assert_eq!(history.as_slice(), b"frame-1\nframe-2\n");
+	});
+}
+
+#[test]
+fn folder_backend_reads_empty_file() {
+	let dir = tempfile::tempdir().unwrap();
+	let backend = FolderBackend::create(dir.path()).unwrap();
+
+	backend.write("empty.bin", &[]).unwrap();
+	let read_back = backend.read("empty.bin").unwrap();
+	assert_eq!(read_back.as_slice(), &[] as &[u8]);
+}
+
+#[test]
+fn folder_backend_write_sized_fills_via_mmap() {
+	let dir = tempfile::tempdir().unwrap();
+	let backend = FolderBackend::create(dir.path()).unwrap();
+
+	let payload = b"hello world";
+	backend
+		.write_sized("resources/sized", payload.len(), &mut |buffer| {
+			buffer.copy_from_slice(payload);
+			Ok(())
+		})
+		.unwrap();
+
+	let read_back = backend.read("resources/sized").unwrap();
+	assert_eq!(read_back.as_slice(), payload);
+}