Add document-format crate (#4234)

* Add document-format crate

* Adapt document-format to merged graph-storage and consolidate errors

* Fix resource write ordering, codec error masking, and export docs

* Remove document-format RFC; tracked in a dedicated PR

* Restructure document-format into gdd/persist/export/resource modules

Split the 963-line lib.rs by responsibility: persist path into persist.rs, resource I/O into resource.rs, export engine into export.rs. Rename the GddV1 layout struct to GddV1Layout and add a GddV1 = Gdd<GddV1Layout> alias.

* Feature-split document-format and graph-storage for minimal builds

* Address PR review: O(1) history-delta lookup and no-default-features test build

* Address PR review: propagate apply_hot_op persist error and assert resource hash

* Make decompression from archive streaming

* Share archive export body between export and export_to_bytes

* Review cleanup

* Export documents with un-retired hot ops losslessly

* Persist last_broadcast_rev in session.json, drop unused last_retired_at

* Move peer_id from manifest to session.json

* Rename gesture to interaction in document-format storage layer

document/container/src/archive.rs

@@ -3,9 +3,11 @@
 //! Each codec streams entries in both directions: writers wrap an `io::Write` sink, and
 //! `deserialize` reads from any `io::Read + Seek` source and streams entries into any [`Container`].
 
+#[cfg(any(feature = "xz", feature = "zip"))]
+use crate::AsyncContainer;
 #[cfg(any(feature = "zip", feature = "xz"))]
 use crate::ContainerError;
-use crate::{Container, Result};
+use crate::Result;
 use std::io::{Read, Seek, Write};
 
 /// Hard cap on the total decompressed size a codec will materialize from one archive.
@@ -55,12 +57,23 @@ pub trait Archive {
 
 	/// Read entries from `source` and write each into `dest`, streaming so neither the full
 	/// archive nor the full container ever sits in memory at once.
-	fn open<R: Read + Seek, C: Container>(source: R, dest: &mut C) -> Result<()>;
+	fn open<R: Read + Seek, C: AsyncContainer>(source: R, dest: &mut C) -> Result<()>;
 }
 
-pub trait ArchiveWriter {
+pub trait ArchiveWriter: Sized {
+	type Sink;
+
 	fn write_entry(&mut self, path: &str, bytes: &[u8]) -> Result<()>;
-	fn finish(self) -> Result<()>;
+
+	/// Finish the archive and return the underlying sink, for in-memory archives where the caller
+	/// wants the written bytes (e.g. `Cursor<Vec<u8>>`) back.
+	fn finish_into(self) -> Result<Self::Sink>;
+
+	/// Finish the archive, discarding the sink. For file-backed archives the bytes are already on disk.
+	fn finish(self) -> Result<()> {
+		self.finish_into()?;
+		Ok(())
+	}
 }
 
 /// Archive container formats distinguishable by their leading magic bytes.
@@ -86,12 +99,16 @@ impl ArchiveFormat {
 
 /// Deserialize an archive into `dest`, auto-detecting the format from `bytes`' magic header.
 /// Errors if the bytes are neither a recognized xz nor zip archive.
-#[cfg(all(feature = "xz", feature = "zip"))]
-pub fn open_auto<C: Container>(bytes: &[u8], dest: &mut C) -> Result<()> {
+#[cfg(any(feature = "xz", feature = "zip"))]
+pub fn open_auto<C: AsyncContainer>(bytes: &[u8], dest: &mut C) -> Result<()> {
 	let source = std::io::Cursor::new(bytes);
 	match ArchiveFormat::detect(bytes) {
+		#[cfg(feature = "xz")]
 		Some(ArchiveFormat::Xz) => Xz::open(source, dest),
+		#[cfg(feature = "zip")]
 		Some(ArchiveFormat::Zip) => Zip::open(source, dest),
-		None => Err(ContainerError::Codec("unrecognized archive format (not xz or zip)".into())),
+		#[allow(unreachable_patterns)]
+		Some(format) => Err(ContainerError::Codec(format!("tried to open {format:?}, but the binary was compiled without the feature enabled  "))),
+		None => Err(ContainerError::Codec("unrecognized archive format (not xz or zip) ".into())),
 	}
 }

document/container/src/archive/xz.rs

@@ -1,7 +1,7 @@
 //! Xz-compressed tarball archive codec.
 
 use crate::archive::{Archive, ArchiveWriter, MAX_DECOMPRESSED_SIZE, checked_entry_size};
-use crate::{Container, ContainerError, Result, validate_path};
+use crate::{AsyncContainer, ContainerError, Result, validate_path};
 use lzma_rust2::{XzOptions, XzReader, XzWriter as InnerXzWriter};
 use std::io::{Read, Seek, Write};
 
@@ -23,7 +23,7 @@ impl Archive for Xz {
 		})
 	}
 
-	fn open<R: Read + Seek, C: Container>(source: R, dest: &mut C) -> Result<()> {
+	fn open<R: Read + Seek, C: AsyncContainer>(source: R, dest: &mut C) -> Result<()> {
 		// `take` bounds how many bytes we decompress from the xz stream, but each tar entry's declared
 		// size is fed to `write_sized`, which pre-allocates from it before reading. Cap the cumulative
 		// declared size too so a header claiming a huge size can't trigger a giant allocation up front.
@@ -46,7 +46,7 @@ impl Archive for Xz {
 
 			let size = checked_entry_size(&mut total_size, entry.size())?;
 
-			dest.write_sized(&path, size, &mut |buffer| {
+			dest.write_sized_non_blocking(&path, size, &mut |buffer| {
 				entry.read_exact(buffer).map_err(ContainerError::Io)?;
 				Ok(())
 			})?;
@@ -57,6 +57,8 @@ impl Archive for Xz {
 }
 
 impl<W: Write + Seek> ArchiveWriter for XzWriter<W> {
+	type Sink = W;
+
 	fn write_entry(&mut self, path: &str, bytes: &[u8]) -> Result<()> {
 		validate_path(path)?;
 		let tar = self.tar.as_mut().ok_or_else(|| ContainerError::Codec("XzWriter already finished".into()))?;
@@ -69,21 +71,14 @@ impl<W: Write + Seek> ArchiveWriter for XzWriter<W> {
 		Ok(())
 	}
 
-	fn finish(mut self) -> Result<()> {
-		self.finish_inner()?;
-		Ok(())
+	fn finish_into(mut self) -> Result<W> {
+		self.finish_inner()
 	}
 }
 
 impl<W: Write + Seek> XzWriter<W> {
-	/// Finish the archive and return the underlying sink, for in-memory archives where the caller
-	/// wants the written bytes (e.g. `Cursor<Vec<u8>>`) back.
-	pub fn finish_into(mut self) -> Result<W> {
-		self.finish_inner()
-	}
-
 	/// Unwind the layered writers in order (flush the tar trailer, then finish xz) and hand back the
-	/// innermost sink. Shared by `finish` and `finish_into`.
+	/// innermost sink.
 	fn finish_inner(&mut self) -> Result<W> {
 		let mut tar = self.tar.take().ok_or_else(|| ContainerError::Codec("XzWriter already finished".into()))?;
 		tar.finish()?;

document/container/src/archive/zip.rs

@@ -1,7 +1,7 @@
 //! Zip archive codec.
 
 use crate::archive::{Archive, ArchiveWriter, checked_entry_size};
-use crate::{Container, ContainerError, Result, validate_path};
+use crate::{AsyncContainer, ContainerError, Result, validate_path};
 use std::io::{Read, Seek, Write};
 
 use zip::ZipArchive;
@@ -24,7 +24,7 @@ impl Archive for Zip {
 		})
 	}
 
-	fn open<R: Read + Seek, C: Container>(source: R, dest: &mut C) -> Result<()> {
+	fn open<R: Read + Seek, C: AsyncContainer>(source: R, dest: &mut C) -> Result<()> {
 		let mut archive = ZipArchive::new(source).map_err(zip_err)?;
 
 		// Zip headers declare each entry's uncompressed size up front; `checked_entry_size` caps the running
@@ -41,7 +41,7 @@ impl Archive for Zip {
 
 			let size = checked_entry_size(&mut total_size, entry.size())?;
 
-			dest.write_sized(&name, size, &mut |buffer| {
+			dest.write_sized_non_blocking(&name, size, &mut |buffer| {
 				entry.read_exact(buffer).map_err(ContainerError::Io)?;
 				Ok(())
 			})?;
@@ -52,6 +52,8 @@ impl Archive for Zip {
 }
 
 impl<W: Write + Seek> ArchiveWriter for ZipWriter<W> {
+	type Sink = W;
+
 	fn write_entry(&mut self, path: &str, bytes: &[u8]) -> Result<()> {
 		validate_path(path)?;
 		self.inner.start_file(path, self.options).map_err(zip_err)?;
@@ -59,16 +61,7 @@ impl<W: Write + Seek> ArchiveWriter for ZipWriter<W> {
 		Ok(())
 	}
 
-	fn finish(self) -> Result<()> {
-		self.inner.finish().map_err(zip_err)?;
-		Ok(())
-	}
-}
-
-impl<W: Write + Seek> ZipWriter<W> {
-	/// Finish the archive and return the underlying sink, for in-memory archives where the caller
-	/// wants the written bytes (e.g. `Cursor<Vec<u8>>`) back.
-	pub fn finish_into(self) -> Result<W> {
+	fn finish_into(self) -> Result<W> {
 		self.inner.finish().map_err(zip_err)
 	}
 }

document/container/src/lib.rs

@@ -271,6 +271,14 @@ pub trait AsyncContainer {
 	/// and `Ok` is returned eagerly; a later failure is logged.
 	fn write_non_blocking(&self, path: &str, bytes: &[u8]) -> Result<()>;
 
+	/// Synchronous write. Same eager-enqueue semantics on OPFS as [`write_non_blocking`](Self::write_non_blocking);
+	/// queued appends preserve order relative to earlier queued writes/appends.
+	fn write_sized_non_blocking(&self, path: &str, size: usize, fill: &mut dyn FnMut(&mut [u8]) -> Result<()>) -> Result<()> {
+		let mut buf = vec![0u8; size];
+		fill(&mut buf)?;
+		self.write_non_blocking(path, &buf)
+	}
+
 	/// Synchronous append. Same eager-enqueue semantics on OPFS as [`write_non_blocking`](Self::write_non_blocking);
 	/// queued appends preserve order relative to earlier queued writes/appends.
 	fn append_non_blocking(&self, path: &str, bytes: &[u8]) -> Result<()>;
@@ -320,6 +328,10 @@ impl<C: Container + ?Sized> AsyncContainer for C {
 		Container::write(self, path, bytes)
 	}
 
+	fn write_sized_non_blocking(&self, path: &str, size: usize, fill: &mut dyn FnMut(&mut [u8]) -> Result<()>) -> Result<()> {
+		Container::write_sized(self, path, size, fill)
+	}
+
 	fn append_non_blocking(&self, path: &str, bytes: &[u8]) -> Result<()> {
 		Container::append(self, path, bytes)
 	}