mirror of
https://github.com/GraphiteEditor/Graphite.git
synced 2026-09-15 14:18:04 +08:00
More consistent document crate names (#4323)
* More consistent document crate names * Fix fmt * Fix ASCII art diagrams * rename document-graph to document-graph-storage
This commit is contained in:
36
document/format/Cargo.toml
Normal file
36
document/format/Cargo.toml
Normal file
@@ -0,0 +1,36 @@
|
||||
[package]
|
||||
name = "document-format"
|
||||
description = "Typed handle for the .gdd document format, sitting over document-graph-storage and document-container"
|
||||
edition.workspace = true
|
||||
version.workspace = true
|
||||
license.workspace = true
|
||||
authors.workspace = true
|
||||
|
||||
[features]
|
||||
# Runtime bridge: the editor <-> storage conversion methods (stage/commit from a `NodeNetwork`,
|
||||
# `network_ids`, `declarations`). Off lets a standalone migration tool build without `graph-craft`
|
||||
# or `core-types`. Forwards to `document-graph-storage/conversion`.
|
||||
conversion = ["document-graph-storage/conversion", "dep:graph-craft", "dep:core-types"]
|
||||
# Compressed-archive export/open. Each forwards to the matching `document-container` feature and
|
||||
# gates that format's `ExportFormat` arm and sink. The folder/memory codec-free core needs neither.
|
||||
zip = ["document-container/zip"]
|
||||
xz = ["document-container/xz"]
|
||||
default = ["conversion", "zip", "xz"]
|
||||
|
||||
[dependencies]
|
||||
document-container = { workspace = true }
|
||||
document-graph-storage = { workspace = true, default-features = false }
|
||||
graph-craft = { workspace = true, optional = true }
|
||||
graphene-resource = { workspace = true }
|
||||
core-types = { workspace = true, optional = true }
|
||||
serde = { workspace = true }
|
||||
serde_json = { workspace = true }
|
||||
rmp-serde = { workspace = true }
|
||||
futures = { workspace = true }
|
||||
thiserror = "2.0"
|
||||
log = { workspace = true }
|
||||
|
||||
[dev-dependencies]
|
||||
futures = { workspace = true }
|
||||
graphene-resource = { workspace = true }
|
||||
tempfile = "3"
|
||||
327
document/format/src/codec.rs
Normal file
327
document/format/src/codec.rs
Normal file
@@ -0,0 +1,327 @@
|
||||
//! Codec for a stream of values. Single-value writes are just streams of length one.
|
||||
|
||||
use serde::{Deserialize, Serialize, de::DeserializeOwned};
|
||||
|
||||
#[derive(Copy, Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub enum Codec {
|
||||
/// A single JSON document. `append` to a non-empty buffer errors.
|
||||
Json,
|
||||
/// Newline-delimited compact JSON, one value per line.
|
||||
JsonLines,
|
||||
/// A single MessagePack blob. `append` to a non-empty buffer errors.
|
||||
MessagePack,
|
||||
/// Length-prefixed MessagePack frames: `[u32 big-endian length][MessagePack bytes]` per value.
|
||||
MessagePackFrames,
|
||||
}
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum CodecError {
|
||||
#[error("MessagePack encode error: {0}")]
|
||||
MessagePackEncode(#[from] rmp_serde::encode::Error),
|
||||
#[error("MessagePack decode error: {0}")]
|
||||
MessagePackDecode(#[from] rmp_serde::decode::Error),
|
||||
#[error("JSON error: {0}")]
|
||||
Json(#[from] serde_json::Error),
|
||||
#[error("frame length {0} exceeds u32")]
|
||||
FrameTooLarge(usize),
|
||||
#[error("frame length prefix truncated: need 4 bytes, have {0}")]
|
||||
TruncatedLengthPrefix(usize),
|
||||
#[error("declared frame length {declared} exceeds remaining buffer ({remaining} bytes)")]
|
||||
TruncatedFrame { declared: usize, remaining: usize },
|
||||
#[error("single-value codec cannot append to a non-empty buffer")]
|
||||
SingleValueAlreadyWritten,
|
||||
#[error("expected at least one value, got none")]
|
||||
Empty,
|
||||
#[error("expected exactly one value, got more")]
|
||||
ExpectedSingle,
|
||||
}
|
||||
|
||||
impl Codec {
|
||||
pub fn extension(self) -> &'static str {
|
||||
match self {
|
||||
Codec::Json => "json",
|
||||
Codec::JsonLines => "jsonl",
|
||||
Codec::MessagePack => "bin",
|
||||
Codec::MessagePackFrames => "frames",
|
||||
}
|
||||
}
|
||||
|
||||
/// Append one value to `output` in this codec's framing.
|
||||
/// Single-value codecs error if `output` is non-empty.
|
||||
pub fn append<T: Serialize>(self, output: &mut Vec<u8>, value: &T) -> Result<(), CodecError> {
|
||||
match self {
|
||||
Codec::Json => {
|
||||
if !output.is_empty() {
|
||||
return Err(CodecError::SingleValueAlreadyWritten);
|
||||
}
|
||||
serde_json::to_writer_pretty(output, value)?;
|
||||
Ok(())
|
||||
}
|
||||
Codec::JsonLines => {
|
||||
serde_json::to_writer(&mut *output, value)?;
|
||||
output.push(b'\n');
|
||||
Ok(())
|
||||
}
|
||||
Codec::MessagePack => {
|
||||
if !output.is_empty() {
|
||||
return Err(CodecError::SingleValueAlreadyWritten);
|
||||
}
|
||||
rmp_serde::encode::write(output, value)?;
|
||||
Ok(())
|
||||
}
|
||||
Codec::MessagePackFrames => {
|
||||
let payload = rmp_serde::to_vec(value)?;
|
||||
let length = u32::try_from(payload.len()).map_err(|_| CodecError::FrameTooLarge(payload.len()))?;
|
||||
output.extend_from_slice(&length.to_be_bytes());
|
||||
output.extend_from_slice(&payload);
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Iterate values from `bytes`. Single-value codecs yield exactly one item;
|
||||
/// stream codecs yield however many were written.
|
||||
pub fn iter<'a, T: DeserializeOwned + 'a>(self, bytes: &'a [u8]) -> Box<dyn Iterator<Item = Result<T, CodecError>> + 'a> {
|
||||
match self {
|
||||
Codec::Json => {
|
||||
let single = serde_json::from_slice::<T>(bytes).map_err(CodecError::from);
|
||||
Box::new(std::iter::once(single))
|
||||
}
|
||||
Codec::JsonLines => Box::new(JsonLineIter {
|
||||
remaining: bytes,
|
||||
_marker: std::marker::PhantomData,
|
||||
}),
|
||||
Codec::MessagePack => {
|
||||
let single = rmp_serde::from_slice::<T>(bytes).map_err(CodecError::from);
|
||||
Box::new(std::iter::once(single))
|
||||
}
|
||||
Codec::MessagePackFrames => Box::new(MessagePackFrameIter {
|
||||
remaining: bytes,
|
||||
_marker: std::marker::PhantomData,
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
/// Serialize a single value into a fresh buffer.
|
||||
pub fn write_single<T: Serialize>(self, value: &T) -> Result<Vec<u8>, CodecError> {
|
||||
let mut output = Vec::new();
|
||||
self.append(&mut output, value)?;
|
||||
Ok(output)
|
||||
}
|
||||
|
||||
/// Deserialize the single value in `bytes`. Errors if zero or more than one value is present.
|
||||
pub fn read_single<T: DeserializeOwned>(self, bytes: &[u8]) -> Result<T, CodecError> {
|
||||
let mut iter = self.iter::<T>(bytes);
|
||||
let first = iter.next().ok_or(CodecError::Empty)??;
|
||||
match iter.next() {
|
||||
// A trailing decode error is more informative than `ExpectedSingle`, so surface it.
|
||||
Some(Err(error)) => return Err(error),
|
||||
Some(Ok(_)) => return Err(CodecError::ExpectedSingle),
|
||||
None => {}
|
||||
}
|
||||
Ok(first)
|
||||
}
|
||||
}
|
||||
|
||||
struct JsonLineIter<'a, T> {
|
||||
remaining: &'a [u8],
|
||||
_marker: std::marker::PhantomData<fn() -> T>,
|
||||
}
|
||||
|
||||
impl<T: DeserializeOwned> Iterator for JsonLineIter<'_, T> {
|
||||
type Item = Result<T, CodecError>;
|
||||
|
||||
fn next(&mut self) -> Option<Self::Item> {
|
||||
loop {
|
||||
if self.remaining.is_empty() {
|
||||
return None;
|
||||
}
|
||||
|
||||
let (line, tail) = match self.remaining.iter().position(|&byte| byte == b'\n') {
|
||||
Some(index) => (&self.remaining[..index], &self.remaining[index + 1..]),
|
||||
None => (self.remaining, &[][..]),
|
||||
};
|
||||
self.remaining = tail;
|
||||
|
||||
let trimmed = trim_ascii(line);
|
||||
if trimmed.is_empty() {
|
||||
continue;
|
||||
}
|
||||
|
||||
return Some(serde_json::from_slice(trimmed).map_err(CodecError::from));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct MessagePackFrameIter<'a, T> {
|
||||
remaining: &'a [u8],
|
||||
_marker: std::marker::PhantomData<fn() -> T>,
|
||||
}
|
||||
|
||||
impl<T: DeserializeOwned> Iterator for MessagePackFrameIter<'_, T> {
|
||||
type Item = Result<T, CodecError>;
|
||||
|
||||
fn next(&mut self) -> Option<Self::Item> {
|
||||
if self.remaining.is_empty() {
|
||||
return None;
|
||||
}
|
||||
|
||||
let buffer = std::mem::take(&mut self.remaining);
|
||||
|
||||
let Some((length_bytes, tail)) = buffer.split_first_chunk::<4>() else {
|
||||
return Some(Err(CodecError::TruncatedLengthPrefix(buffer.len())));
|
||||
};
|
||||
let length = u32::from_be_bytes(*length_bytes) as usize;
|
||||
|
||||
if tail.len() < length {
|
||||
return Some(Err(CodecError::TruncatedFrame {
|
||||
declared: length,
|
||||
remaining: tail.len(),
|
||||
}));
|
||||
}
|
||||
|
||||
let (frame, after) = tail.split_at(length);
|
||||
self.remaining = after;
|
||||
|
||||
Some(rmp_serde::from_slice(frame).map_err(CodecError::from))
|
||||
}
|
||||
}
|
||||
|
||||
fn trim_ascii(bytes: &[u8]) -> &[u8] {
|
||||
let start = bytes.iter().position(|byte| !byte.is_ascii_whitespace()).unwrap_or(bytes.len());
|
||||
let end = bytes.iter().rposition(|byte| !byte.is_ascii_whitespace()).map(|index| index + 1).unwrap_or(start);
|
||||
&bytes[start..end]
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
#[derive(Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
struct Frame {
|
||||
id: u32,
|
||||
label: String,
|
||||
}
|
||||
|
||||
fn frames() -> [Frame; 3] {
|
||||
[Frame { id: 1, label: "alpha".into() }, Frame { id: 2, label: "beta".into() }, Frame { id: 3, label: "gamma".into() }]
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn json_round_trip_single() {
|
||||
let frame = Frame { id: 7, label: "solo".into() };
|
||||
let bytes = Codec::Json.write_single(&frame).unwrap();
|
||||
let decoded: Frame = Codec::Json.read_single(&bytes).unwrap();
|
||||
assert_eq!(decoded, frame);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn json_append_to_non_empty_errors() {
|
||||
let mut buffer = b"already here".to_vec();
|
||||
let result = Codec::Json.append(&mut buffer, &Frame { id: 1, label: "x".into() });
|
||||
assert!(matches!(result, Err(CodecError::SingleValueAlreadyWritten)), "got {result:?}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn message_pack_round_trip_single() {
|
||||
let frame = Frame { id: 99, label: "blob".into() };
|
||||
let bytes = Codec::MessagePack.write_single(&frame).unwrap();
|
||||
let decoded: Frame = Codec::MessagePack.read_single(&bytes).unwrap();
|
||||
assert_eq!(decoded, frame);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn message_pack_append_to_non_empty_errors() {
|
||||
let mut buffer = vec![0xAB];
|
||||
let result = Codec::MessagePack.append(&mut buffer, &Frame { id: 1, label: "x".into() });
|
||||
assert!(matches!(result, Err(CodecError::SingleValueAlreadyWritten)), "got {result:?}");
|
||||
}
|
||||
|
||||
/// A type-erased `serde_json::Value` round-trips through the binary codec: the property postcard
|
||||
/// could not satisfy (it raises `WontImplement` on self-describing values), which is why the
|
||||
/// resource/attribute deltas that carry `serde_json::Value` bodies need a self-describing codec.
|
||||
#[test]
|
||||
fn message_pack_round_trips_serde_json_value() {
|
||||
let value = serde_json::json!({ "kind": "embedded", "priority": 1.5, "tags": ["a", "b"] });
|
||||
let bytes = Codec::MessagePack.write_single(&value).unwrap();
|
||||
let decoded: serde_json::Value = Codec::MessagePack.read_single(&bytes).unwrap();
|
||||
assert_eq!(decoded, value);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn json_lines_round_trip_and_skip_blanks() {
|
||||
let frames = [Frame { id: 1, label: "alpha".into() }, Frame { id: 2, label: "beta".into() }];
|
||||
|
||||
let mut buffer = Vec::new();
|
||||
Codec::JsonLines.append(&mut buffer, &frames[0]).unwrap();
|
||||
buffer.extend_from_slice(b" \n\n");
|
||||
Codec::JsonLines.append(&mut buffer, &frames[1]).unwrap();
|
||||
|
||||
let decoded: Vec<Frame> = Codec::JsonLines.iter(&buffer).collect::<Result<_, _>>().unwrap();
|
||||
assert_eq!(decoded, frames);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn message_pack_frames_round_trip() {
|
||||
let frames = frames();
|
||||
let mut buffer = Vec::new();
|
||||
for frame in &frames {
|
||||
Codec::MessagePackFrames.append(&mut buffer, frame).unwrap();
|
||||
}
|
||||
let decoded: Vec<Frame> = Codec::MessagePackFrames.iter(&buffer).collect::<Result<_, _>>().unwrap();
|
||||
assert_eq!(decoded, frames);
|
||||
}
|
||||
|
||||
/// A crash mid-append leaves a torn final frame. The length prefix lets us detect that
|
||||
/// deterministically (declared length exceeds the bytes that actually made it to disk) rather
|
||||
/// than decoding a partial value into a plausible-but-wrong one.
|
||||
#[test]
|
||||
fn message_pack_frames_detect_truncation() {
|
||||
let mut buffer = Vec::new();
|
||||
Codec::MessagePackFrames.append(&mut buffer, &Frame { id: 7, label: "ok".into() }).unwrap();
|
||||
buffer.truncate(buffer.len() - 1);
|
||||
let last = Codec::MessagePackFrames.iter::<Frame>(&buffer).last().unwrap();
|
||||
assert!(matches!(last, Err(CodecError::TruncatedFrame { .. })), "got {last:?}");
|
||||
}
|
||||
|
||||
/// A buffer whose first record's length prefix itself is incomplete (fewer than 4 bytes) is
|
||||
/// reported as a truncated prefix rather than mis-read as a zero-length frame.
|
||||
#[test]
|
||||
fn message_pack_frames_detect_truncated_length_prefix() {
|
||||
let buffer = vec![0x00, 0x00];
|
||||
let last = Codec::MessagePackFrames.iter::<Frame>(&buffer).last().unwrap();
|
||||
assert!(matches!(last, Err(CodecError::TruncatedLengthPrefix(2))), "got {last:?}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn write_single_then_read_with_iter_yields_one() {
|
||||
let frame = Frame { id: 5, label: "one".into() };
|
||||
for codec in [Codec::Json, Codec::JsonLines, Codec::MessagePack, Codec::MessagePackFrames] {
|
||||
let bytes = codec.write_single(&frame).unwrap();
|
||||
let collected: Vec<Frame> = codec.iter(&bytes).collect::<Result<_, _>>().unwrap();
|
||||
assert_eq!(collected, vec![Frame { id: 5, label: "one".into() }], "codec {codec:?}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn read_single_rejects_multi_value_stream() {
|
||||
let mut buffer = Vec::new();
|
||||
Codec::JsonLines.append(&mut buffer, &Frame { id: 1, label: "a".into() }).unwrap();
|
||||
Codec::JsonLines.append(&mut buffer, &Frame { id: 2, label: "b".into() }).unwrap();
|
||||
let result: Result<Frame, _> = Codec::JsonLines.read_single(&buffer);
|
||||
assert!(matches!(result, Err(CodecError::ExpectedSingle)), "got {result:?}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extensions_are_distinct() {
|
||||
let exts = [
|
||||
Codec::Json.extension(),
|
||||
Codec::JsonLines.extension(),
|
||||
Codec::MessagePack.extension(),
|
||||
Codec::MessagePackFrames.extension(),
|
||||
];
|
||||
let unique: std::collections::HashSet<_> = exts.iter().collect();
|
||||
assert_eq!(unique.len(), exts.len(), "extensions collide: {exts:?}");
|
||||
}
|
||||
}
|
||||
50
document/format/src/error.rs
Normal file
50
document/format/src/error.rs
Normal file
@@ -0,0 +1,50 @@
|
||||
//! Unified error type for the `document-format` crate.
|
||||
//!
|
||||
//! Every fallible [`crate::Gdd`] method returns [`Result<T>`]. Variants are grouped by failure
|
||||
//! domain (container I/O, codec, CRDT, format validation, export)
|
||||
|
||||
use document_container::ContainerError;
|
||||
#[cfg(feature = "conversion")]
|
||||
use document_graph_storage::CommitError;
|
||||
use document_graph_storage::CrdtError;
|
||||
use graphene_resource::ResourceHash;
|
||||
|
||||
use crate::codec::CodecError;
|
||||
use crate::io::ReadError;
|
||||
|
||||
/// Crate-wide result alias.
|
||||
pub type Result<T> = std::result::Result<T, Error>;
|
||||
|
||||
/// Anything that can go wrong reading, mutating, or exporting a `.gdd` document. Per the format's
|
||||
/// load-time policy, any unexpected condition is a hard error rather than a silent fallback.
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum Error {
|
||||
/// Working-copy container I/O failed (read, write, or path validation).
|
||||
#[error("container error: {0}")]
|
||||
Container(#[from] ContainerError),
|
||||
/// A typed payload could not be located or read from the container.
|
||||
#[error("read error: {0}")]
|
||||
Read(#[from] ReadError),
|
||||
/// A payload failed to encode or decode in its recorded codec.
|
||||
#[error("codec error: {0}")]
|
||||
Codec(#[from] CodecError),
|
||||
/// A CRDT operation was rejected while replaying a hot op or moving the undo/redo cursor.
|
||||
#[error("CRDT error: {0}")]
|
||||
Crdt(#[from] CrdtError),
|
||||
/// Staging a runtime snapshot into the session failed (conversion or CRDT apply).
|
||||
#[cfg(feature = "conversion")]
|
||||
#[error("commit error: {0}")]
|
||||
Commit(#[from] CommitError),
|
||||
/// The manifest's `format` field is not the `.gdd` magic, so this is not a `.gdd` document.
|
||||
#[error("not a .gdd document (manifest format = {found:?}, expected {expected:?})")]
|
||||
WrongFormat { found: String, expected: &'static str },
|
||||
/// The manifest declares a format version newer than this build can open.
|
||||
#[error("unsupported format version: found {found}, max supported {max_supported}")]
|
||||
UnsupportedVersion { found: u32, max_supported: u32 },
|
||||
/// The requested export options are incoherent (e.g. neither registry nor history included).
|
||||
#[error("invalid export options: {0}")]
|
||||
InvalidExportOptions(&'static str),
|
||||
/// An export marked a resource for embedding but its bytes were absent from the byte store.
|
||||
#[error("embedded resource {0} missing from the byte store")]
|
||||
MissingResource(ResourceHash),
|
||||
}
|
||||
310
document/format/src/export.rs
Normal file
310
document/format/src/export.rs
Normal file
@@ -0,0 +1,310 @@
|
||||
//! Export: walking the working copy through an archive codec, keeping payloads as-is.
|
||||
//!
|
||||
//! [`ExportFormat`] / [`ExportOptions`] are the public settings; the [`Gdd`] export methods drive a
|
||||
//! [`ExportSink`] (folder / zip / xz) through the manifest → registry → history → resources sequence.
|
||||
|
||||
#[cfg(not(target_family = "wasm"))]
|
||||
use std::path::Path;
|
||||
|
||||
use graphene_resource::{LoadResource, Resource, ResourceHash};
|
||||
|
||||
use crate::error::Error;
|
||||
use crate::layout::Layout;
|
||||
use crate::session_state::SessionState;
|
||||
use crate::{Gdd, MANIFEST_CODEC, io};
|
||||
|
||||
/// Export wrapping. Payloads keep the working copy's recorded per-payload codecs (see
|
||||
/// [`crate::manifest::PayloadCodecs`]); export does not re-encode.
|
||||
#[derive(Copy, Clone, Debug)]
|
||||
pub enum ExportFormat {
|
||||
/// Copy the working copy to a destination folder.
|
||||
Folder,
|
||||
/// Wrap as a `.gdd.zip` archive.
|
||||
#[cfg(feature = "zip")]
|
||||
Zip,
|
||||
/// Wrap as a `.gdd.tar.xz` archive (whole-archive xz via `lzma-rust2`).
|
||||
#[cfg(feature = "xz")]
|
||||
Xz,
|
||||
}
|
||||
|
||||
#[derive(Copy, Clone, Debug)]
|
||||
pub struct ExportOptions {
|
||||
/// Whether to include the registry snapshot. `false` produces a history-only export, useful
|
||||
/// for VCS workflows where the diffable `history.jsonl` is the interesting payload and the
|
||||
/// registry would rewrite whole-file on every retirement. Consumers replay history from an
|
||||
/// empty registry.
|
||||
pub include_registry: bool,
|
||||
/// Whether to include history + hot-log. `false` produces a flat snapshot (registry only),
|
||||
/// useful for sharing without revealing edit history and for cutting file size.
|
||||
pub include_history: bool,
|
||||
/// Materialize every non-`DataSource::Embedded` resource into `resources/<hash>` for portability.
|
||||
/// Does not mutate the in-memory `Gdd`.
|
||||
pub embed_all_resources: bool,
|
||||
}
|
||||
|
||||
impl ExportOptions {
|
||||
/// Returns an error description if the combination is incoherent.
|
||||
pub fn validate(&self) -> Result<(), &'static str> {
|
||||
if !self.include_registry && !self.include_history {
|
||||
return Err("export must include at least one of: registry, history");
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for ExportOptions {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
include_registry: true,
|
||||
include_history: true,
|
||||
embed_all_resources: false,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<L: Layout> Gdd<L> {
|
||||
/// Stream the working copy to `dest` as a folder/zip/xz archive, keeping payload codecs as-is.
|
||||
/// Does not mutate `self` and does not buffer the export. Native-only (writes a filesystem path).
|
||||
///
|
||||
/// # Errors
|
||||
/// [`Error::InvalidExportOptions`] for incoherent options, [`Error::MissingResource`] if an
|
||||
/// embedded resource's bytes are absent from `byte_store`.
|
||||
#[cfg(not(target_family = "wasm"))]
|
||||
pub async fn export(&self, dest: &Path, format: ExportFormat, options: ExportOptions, byte_store: &dyn LoadResource, legacy_document: Option<&[u8]>) -> Result<(), Error> {
|
||||
options.validate().map_err(Error::InvalidExportOptions)?;
|
||||
|
||||
match format {
|
||||
ExportFormat::Folder => {
|
||||
let mut folder = document_container::backends::folder::FolderBackend::create(dest)?;
|
||||
let mut sink = FolderSink { folder: &mut folder };
|
||||
self.stream_entries(options, byte_store, &mut sink).await?;
|
||||
if let Some(legacy) = legacy_document {
|
||||
sink.write_entry(self.layout.legacy_path(), legacy)?;
|
||||
}
|
||||
}
|
||||
#[cfg(feature = "zip")]
|
||||
ExportFormat::Zip => {
|
||||
let file = std::fs::File::create(dest).map_err(document_container::ContainerError::Io)?;
|
||||
self.export_archive::<document_container::archive::Zip, _>(file, options, byte_store, legacy_document).await?;
|
||||
}
|
||||
#[cfg(feature = "xz")]
|
||||
ExportFormat::Xz => {
|
||||
let file = std::fs::File::create(dest).map_err(document_container::ContainerError::Io)?;
|
||||
self.export_archive::<document_container::archive::Xz, _>(file, options, byte_store, legacy_document).await?;
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// In-memory variant of [`export`](Self::export) returning the archive bytes. Available on every
|
||||
/// target (no `std::fs`) but buffers the whole archive. `legacy_document` is embedded verbatim at
|
||||
/// [`Layout::legacy_path`]. `ExportFormat::Folder` has no single-file form and is rejected.
|
||||
#[cfg(any(feature = "zip", feature = "xz"))]
|
||||
pub async fn export_to_bytes(&self, format: ExportFormat, options: ExportOptions, byte_store: &dyn LoadResource, legacy_document: Option<&[u8]>) -> Result<Vec<u8>, Error> {
|
||||
options.validate().map_err(Error::InvalidExportOptions)?;
|
||||
|
||||
let cursor = std::io::Cursor::new(Vec::new());
|
||||
let buffer = match format {
|
||||
ExportFormat::Folder => return Err(Error::InvalidExportOptions("folder export has no single-file byte form")),
|
||||
#[cfg(feature = "zip")]
|
||||
ExportFormat::Zip => self.export_archive::<document_container::archive::Zip, _>(cursor, options, byte_store, legacy_document).await?,
|
||||
#[cfg(feature = "xz")]
|
||||
ExportFormat::Xz => self.export_archive::<document_container::archive::Xz, _>(cursor, options, byte_store, legacy_document).await?,
|
||||
};
|
||||
|
||||
Ok(buffer.into_inner())
|
||||
}
|
||||
|
||||
/// Stream entries into a fresh `A` archive over `output`, append the optional legacy blob, then
|
||||
/// finalize and hand back the inner sink. The shared body of both archive export paths.
|
||||
#[cfg(any(feature = "zip", feature = "xz"))]
|
||||
async fn export_archive<A, W>(&self, output: W, options: ExportOptions, byte_store: &dyn LoadResource, legacy_document: Option<&[u8]>) -> Result<W, Error>
|
||||
where
|
||||
A: document_container::archive::Archive,
|
||||
W: std::io::Write + std::io::Seek + Send,
|
||||
A::Writer<W>: ExportSink + document_container::archive::ArchiveWriter<Sink = W>,
|
||||
{
|
||||
use document_container::archive::ArchiveWriter;
|
||||
|
||||
let mut writer = A::writer(output)?;
|
||||
self.stream_entries(options, byte_store, &mut writer).await?;
|
||||
if let Some(legacy) = legacy_document {
|
||||
ExportSink::write_entry(&mut writer, self.layout.legacy_path(), legacy)?;
|
||||
}
|
||||
Ok(writer.finish_into()?)
|
||||
}
|
||||
|
||||
/// Drive a sink through manifest → session → registry → history → resources, one entry at a time,
|
||||
/// keeping each payload's recorded codec.
|
||||
async fn stream_entries(&self, options: ExportOptions, byte_store: &dyn LoadResource, sink: &mut dyn ExportSink) -> Result<(), Error> {
|
||||
use document_container::AsyncContainer;
|
||||
|
||||
let codecs = self.manifest.codecs;
|
||||
sink.write_entry(&io::path_for(self.layout.manifest_basename(), MANIFEST_CODEC), &MANIFEST_CODEC.write_single(&self.manifest)?)?;
|
||||
|
||||
// Carry the per-peer cursor + view settings so a `.gdd` reopened elsewhere restores the viewport.
|
||||
let session_state = SessionState {
|
||||
peer_id: self.session.peer(),
|
||||
head_rev: self.session.head_rev(),
|
||||
last_broadcast_rev: self.session.last_broadcast_rev(),
|
||||
redo_stack: self.session.redo_stack().to_vec(),
|
||||
next_node_counter: self.session.next_node_counter(),
|
||||
view_settings: self.view_settings.clone(),
|
||||
network_view_settings: self.network_view_settings.clone(),
|
||||
};
|
||||
sink.write_entry(&io::path_for(self.layout.session_basename(), codecs.session), &codecs.session.write_single(&session_state)?)?;
|
||||
|
||||
let working_copy_hashes: std::collections::HashSet<ResourceHash> = self.resource_hashes().await?.into_iter().collect();
|
||||
|
||||
// Resources to embed as bytes: every `Embedded` entry, plus link-only ones when
|
||||
// `embed_all_resources`. Bytes already in the working copy are written by the copy-through pass
|
||||
// below, so only the gap is loaded from the byte store here.
|
||||
let mut export_session = self.session.clone();
|
||||
let mut hashes_from_store: Vec<ResourceHash> = Vec::new();
|
||||
let mut links_to_promote: Vec<document_graph_storage::ResourceId> = Vec::new();
|
||||
for (id, entry) in &export_session.registry().resources {
|
||||
let Some(hash) = entry.hash else { continue };
|
||||
let embed = entry.has_embedded_source() || options.embed_all_resources;
|
||||
if !embed {
|
||||
continue;
|
||||
}
|
||||
if !entry.has_embedded_source() {
|
||||
links_to_promote.push(*id);
|
||||
}
|
||||
if !working_copy_hashes.contains(&hash) {
|
||||
hashes_from_store.push(hash);
|
||||
}
|
||||
}
|
||||
|
||||
hashes_from_store.sort_unstable();
|
||||
hashes_from_store.dedup();
|
||||
|
||||
// Fail fast if an embedded resource is missing, then promote link-only sources on the clone so
|
||||
// the exported registry and history stay consistent. The live `Gdd` is untouched.
|
||||
let mut embedded_bytes: Vec<(ResourceHash, Resource)> = Vec::new();
|
||||
for hash in hashes_from_store {
|
||||
let Some(resource) = byte_store.load(hash).await else {
|
||||
return Err(Error::MissingResource(hash));
|
||||
};
|
||||
embedded_bytes.push((hash, resource));
|
||||
}
|
||||
export_session.embed_resource_sources(links_to_promote)?;
|
||||
|
||||
if options.include_registry {
|
||||
// With history, the persisted snapshot is the retired registry and the hot log layers on top
|
||||
// (mirrors `Session::load`); without history it must be the full working registry, since
|
||||
// `bootstrap_from_registry` reconstructs the whole document from it alone.
|
||||
let snapshot = if options.include_history { export_session.retired_registry() } else { export_session.registry() };
|
||||
sink.write_entry(&io::path_for(self.layout.registry_basename(), codecs.registry), &codecs.registry.write_single(snapshot)?)?;
|
||||
}
|
||||
|
||||
if options.include_history {
|
||||
let mut buffer = Vec::new();
|
||||
for delta in export_session.history() {
|
||||
codecs.history.append(&mut buffer, delta)?;
|
||||
}
|
||||
if !buffer.is_empty() {
|
||||
sink.write_entry(&io::path_for(self.layout.history_basename(), codecs.history), &buffer)?;
|
||||
}
|
||||
|
||||
// Carry the un-retired hot ops alongside history so a document exported mid-interaction (e.g. a save
|
||||
// during a tool drag) isn't shipped with its pending edits dropped. `open` replays them on top of
|
||||
// the retired snapshot, same as the working copy does.
|
||||
let mut hot_buffer = Vec::new();
|
||||
for hot_op in export_session.hot_log() {
|
||||
codecs.hot_log.append(&mut hot_buffer, hot_op)?;
|
||||
}
|
||||
if !hot_buffer.is_empty() {
|
||||
sink.write_entry(&io::path_for(self.layout.hot_log_basename(), codecs.hot_log), &hot_buffer)?;
|
||||
}
|
||||
}
|
||||
|
||||
// Copy bytes the working copy already holds, tracking covered hashes so the embed pass below
|
||||
// doesn't re-emit them.
|
||||
let mut emitted = std::collections::HashSet::new();
|
||||
let resources_dir = self.layout.resources_dir();
|
||||
if self.working.list_dirs("").await?.iter().any(|d| d == resources_dir) {
|
||||
let prefix = format!("{resources_dir}/");
|
||||
for path in self.working.list(resources_dir).await? {
|
||||
if let Some(hash) = path.strip_prefix(&prefix).and_then(|name| name.parse::<ResourceHash>().ok()) {
|
||||
emitted.insert(hash);
|
||||
}
|
||||
let holder = self.working.read(&path).await?;
|
||||
|
||||
// Native `External` (mmap'd) holders copy CoW via a source path; others write bytes.
|
||||
#[cfg(not(target_family = "wasm"))]
|
||||
match holder.source_path() {
|
||||
Some(src_path) => sink.write_entry_from_path(&path, src_path)?,
|
||||
None => sink.write_entry(&path, holder.as_slice())?,
|
||||
}
|
||||
#[cfg(target_family = "wasm")]
|
||||
sink.write_entry(&path, holder.as_slice())?;
|
||||
}
|
||||
}
|
||||
|
||||
for (hash, resource) in &embedded_bytes {
|
||||
if emitted.insert(*hash) {
|
||||
sink.write_entry(&self.layout.resource_path(hash), resource.as_ref())?;
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Sink an export streams entries into, so one async loop drives folder/zip/xz writes. Archive sinks
|
||||
/// work on every target; the folder sink and `write_entry_from_path` are native-only. `Send` because
|
||||
/// `stream_entries` holds `&mut dyn ExportSink` across `.await`s.
|
||||
pub(crate) trait ExportSink: Send {
|
||||
fn write_entry(&mut self, path: &str, bytes: &[u8]) -> Result<(), Error>;
|
||||
|
||||
/// Copy a file from disk into the sink. Default reads it into memory; the folder sink overrides to
|
||||
/// `fs::copy` (CoW). Native-only: only reachable for an `External` (mmap'd) holder.
|
||||
#[cfg(not(target_family = "wasm"))]
|
||||
fn write_entry_from_path(&mut self, path: &str, src: &std::path::Path) -> Result<(), Error> {
|
||||
let bytes = std::fs::read(src).map_err(document_container::ContainerError::Io)?;
|
||||
self.write_entry(path, &bytes)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(not(target_family = "wasm"))]
|
||||
struct FolderSink<'a> {
|
||||
folder: &'a mut document_container::backends::folder::FolderBackend,
|
||||
}
|
||||
|
||||
#[cfg(not(target_family = "wasm"))]
|
||||
impl ExportSink for FolderSink<'_> {
|
||||
fn write_entry(&mut self, path: &str, bytes: &[u8]) -> Result<(), Error> {
|
||||
document_container::Container::write(self.folder, path, bytes)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn write_entry_from_path(&mut self, path: &str, src: &std::path::Path) -> Result<(), Error> {
|
||||
document_container::validate_path(path)?;
|
||||
let dest = self.folder.root().join(path);
|
||||
if let Some(parent) = dest.parent() {
|
||||
std::fs::create_dir_all(parent).map_err(document_container::ContainerError::Io)?;
|
||||
}
|
||||
std::fs::copy(src, &dest).map_err(document_container::ContainerError::Io)?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "zip")]
|
||||
impl<W: std::io::Write + std::io::Seek + Send> ExportSink for document_container::archive::ZipWriter<W> {
|
||||
fn write_entry(&mut self, path: &str, bytes: &[u8]) -> Result<(), Error> {
|
||||
use document_container::archive::ArchiveWriter;
|
||||
ArchiveWriter::write_entry(self, path, bytes)?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "xz")]
|
||||
impl<W: std::io::Write + std::io::Seek + Send> ExportSink for document_container::archive::XzWriter<W> {
|
||||
fn write_entry(&mut self, path: &str, bytes: &[u8]) -> Result<(), Error> {
|
||||
use document_container::archive::ArchiveWriter;
|
||||
ArchiveWriter::write_entry(self, path, bytes)?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
60
document/format/src/io.rs
Normal file
60
document/format/src/io.rs
Normal file
@@ -0,0 +1,60 @@
|
||||
//! Bridge between [`crate::Codec`] and [`document_container::AnyContainer`]. Each payload's codec
|
||||
//! is known up front (the manifest is always JSON; every other payload's codec is recorded in the
|
||||
//! manifest), so reads and writes address a fixed `{basename}.{ext}` path without probing.
|
||||
|
||||
use document_container::{AnyContainer, AsyncContainer};
|
||||
use serde::Serialize;
|
||||
use serde::de::DeserializeOwned;
|
||||
|
||||
use crate::{Codec, CodecError};
|
||||
|
||||
/// Compose a container path from `basename` and `codec.extension()`.
|
||||
pub fn path_for(basename: &str, codec: Codec) -> String {
|
||||
format!("{basename}.{}", codec.extension())
|
||||
}
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum ReadError {
|
||||
#[error("file not found for basename {basename:?} with codec {codec:?}")]
|
||||
NotFound { basename: String, codec: Codec },
|
||||
#[error("container error: {0}")]
|
||||
Container(#[from] document_container::ContainerError),
|
||||
#[error("codec error: {0}")]
|
||||
Codec(#[from] CodecError),
|
||||
}
|
||||
|
||||
/// Read `{basename}.{ext}` and decode the single value it contains.
|
||||
pub async fn read_single<T: DeserializeOwned>(container: &AnyContainer, basename: &str, codec: Codec) -> Result<T, ReadError> {
|
||||
let bytes = read_bytes(container, basename, codec).await?;
|
||||
Ok(codec.read_single::<T>(bytes.as_slice())?)
|
||||
}
|
||||
|
||||
/// Same as [`read_single`] but yields every value when `codec` is a stream codec.
|
||||
pub async fn iter<T: DeserializeOwned>(container: &AnyContainer, basename: &str, codec: Codec) -> Result<Vec<T>, ReadError> {
|
||||
let bytes = read_bytes(container, basename, codec).await?;
|
||||
Ok(codec.iter::<T>(bytes.as_slice()).collect::<Result<Vec<_>, _>>()?)
|
||||
}
|
||||
|
||||
/// Whether `{basename}.{ext}` exists for the given codec.
|
||||
pub async fn exists(container: &AnyContainer, basename: &str, codec: Codec) -> bool {
|
||||
container.exists(&path_for(basename, codec)).await
|
||||
}
|
||||
|
||||
/// Encode `value` with `codec` and write to `{basename}.{ext}`. Synchronous: the write goes through
|
||||
/// the container's sync write surface (durable on folder/memory, enqueued on OPFS).
|
||||
pub fn write_single<T: Serialize>(container: &AnyContainer, basename: &str, codec: Codec, value: &T) -> Result<(), crate::Error> {
|
||||
let bytes = codec.write_single(value)?;
|
||||
container.write_non_blocking(&path_for(basename, codec), &bytes)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn read_bytes(container: &AnyContainer, basename: &str, codec: Codec) -> Result<document_container::ByteHolder, ReadError> {
|
||||
let path = path_for(basename, codec);
|
||||
if !container.exists(&path).await {
|
||||
return Err(ReadError::NotFound {
|
||||
basename: basename.to_string(),
|
||||
codec,
|
||||
});
|
||||
}
|
||||
Ok(container.read(&path).await?)
|
||||
}
|
||||
51
document/format/src/layout.rs
Normal file
51
document/format/src/layout.rs
Normal file
@@ -0,0 +1,51 @@
|
||||
//! Path layout for a `.gdd` working copy.
|
||||
//!
|
||||
//! Layout owns basenames only — the codec choice for each payload is a runtime parameter at the
|
||||
//! read/write call site. Working-copy creation, exports, and migrations may all hit the same
|
||||
//! basename with different codecs.
|
||||
|
||||
use graphene_resource::ResourceHash;
|
||||
|
||||
pub trait Layout {
|
||||
fn manifest_basename(&self) -> &str;
|
||||
fn session_basename(&self) -> &str;
|
||||
fn registry_basename(&self) -> &str;
|
||||
fn history_basename(&self) -> &str;
|
||||
fn hot_log_basename(&self) -> &str;
|
||||
fn resources_dir(&self) -> &str;
|
||||
fn resource_path(&self, hash: &ResourceHash) -> String;
|
||||
/// The embedded legacy `.graphite` document, stored verbatim during the dual-write soak so the
|
||||
/// new format can be validated against (and recovered from) the old one. Dropped once `.gdd`
|
||||
/// becomes the sole source of truth.
|
||||
fn legacy_path(&self) -> &str;
|
||||
}
|
||||
|
||||
#[derive(Copy, Clone, Debug, Default)]
|
||||
pub struct GddV1Layout;
|
||||
|
||||
impl Layout for GddV1Layout {
|
||||
fn manifest_basename(&self) -> &str {
|
||||
"manifest"
|
||||
}
|
||||
fn session_basename(&self) -> &str {
|
||||
"session"
|
||||
}
|
||||
fn registry_basename(&self) -> &str {
|
||||
"registry"
|
||||
}
|
||||
fn history_basename(&self) -> &str {
|
||||
"history"
|
||||
}
|
||||
fn hot_log_basename(&self) -> &str {
|
||||
"hot-log"
|
||||
}
|
||||
fn resources_dir(&self) -> &str {
|
||||
"resources"
|
||||
}
|
||||
fn resource_path(&self, hash: &ResourceHash) -> String {
|
||||
format!("{}/{hash}", self.resources_dir())
|
||||
}
|
||||
fn legacy_path(&self) -> &str {
|
||||
"legacy.graphite"
|
||||
}
|
||||
}
|
||||
344
document/format/src/lib.rs
Normal file
344
document/format/src/lib.rs
Normal file
@@ -0,0 +1,344 @@
|
||||
//! Typed handle for `.gdd` documents.
|
||||
//!
|
||||
//! [`Gdd`] owns a [`document_graph_storage::Session`] plus a working-copy [`document_container::AnyContainer`].
|
||||
//! Mutations flow through `Gdd` to keep the session and the on-disk working copy mirrored.
|
||||
//! Export is a separate, explicit operation — see [`export::ExportFormat`].
|
||||
//!
|
||||
//! See the "On-disk container" section of `node-graph/rfcs/document-format.md` for the format spec.
|
||||
|
||||
use std::sync::Arc;
|
||||
// `Path` and `FolderBackend` are only used by the native-only path-based open/create, so they're
|
||||
// gated off wasm to avoid unused-import warnings.
|
||||
#[cfg(not(target_family = "wasm"))]
|
||||
use std::path::Path;
|
||||
|
||||
#[cfg(not(target_family = "wasm"))]
|
||||
use document_container::backends::folder::FolderBackend;
|
||||
use document_container::{AnyContainer, AsyncContainer, ByteHolder, ContainerError};
|
||||
#[cfg(feature = "conversion")]
|
||||
use document_graph_storage::{CommitError, NodeMetadataSource};
|
||||
use document_graph_storage::{Delta, HotOp, PeerId, Registry, Session};
|
||||
#[cfg(feature = "conversion")]
|
||||
use graphene_resource::LoadResource;
|
||||
use graphene_resource::ResourceHash;
|
||||
|
||||
pub mod codec;
|
||||
pub mod error;
|
||||
pub mod export;
|
||||
pub mod io;
|
||||
pub mod layout;
|
||||
pub mod manifest;
|
||||
pub mod persist;
|
||||
pub mod resource;
|
||||
pub mod session_state;
|
||||
|
||||
pub use codec::{Codec, CodecError};
|
||||
pub use error::Error;
|
||||
pub use export::{ExportFormat, ExportOptions};
|
||||
pub use io::ReadError;
|
||||
pub use layout::{GddV1Layout, Layout};
|
||||
pub use manifest::{Manifest, PayloadCodecs};
|
||||
pub use resource::ResourceProxy;
|
||||
pub use session_state::SessionState;
|
||||
|
||||
/// The default [`Layout`], so callers write `GddV1` for the common `Gdd<GddV1Layout>` handle.
|
||||
pub type GddV1 = Gdd<GddV1Layout>;
|
||||
|
||||
/// The manifest is always JSON: it is the bootstrap file, read before any other payload's codec is
|
||||
/// known, so its own codec cannot itself be configurable.
|
||||
pub const MANIFEST_CODEC: Codec = Codec::Json;
|
||||
|
||||
/// Working-copy codecs. The working copy lives in appdata, not under VCS — these defaults
|
||||
/// optimize for size and write cost. MessagePack is self-describing, so it round-trips the
|
||||
/// type-erased `serde_json::Value` bodies that resource and attribute deltas carry (a non-self-
|
||||
/// describing format like postcard cannot). JSON/JSONL is opt-in via `ExportFormat::Folder` for
|
||||
/// users who want a diffable on-disk representation. Recorded in the manifest at create time and
|
||||
/// read back on open (see [`manifest::PayloadCodecs`]), so the persist path never probes the filesystem.
|
||||
pub const DEFAULT_SESSION_CODEC: Codec = Codec::Json;
|
||||
pub const DEFAULT_REGISTRY_CODEC: Codec = Codec::MessagePack;
|
||||
pub const DEFAULT_HISTORY_CODEC: Codec = Codec::MessagePackFrames;
|
||||
pub const DEFAULT_HOT_LOG_CODEC: Codec = Codec::MessagePackFrames;
|
||||
|
||||
/// Editor-facing handle. Owns the `Session` and the working-copy container; mutations are mirrored
|
||||
/// to disk continuously (every retirement appends to the history file and re-snapshots the registry).
|
||||
///
|
||||
/// The per-edit persist path (`commit_from_runtime`, `apply_hot_op`, `retire`) is synchronous and
|
||||
/// read-free: the manifest is cached in memory (so payload codecs need no disk read), and writes go
|
||||
/// through the container's sync write surface. Only `open` / `create` / `export` are async, since they
|
||||
/// read.
|
||||
/// `Clone` shares the working-copy container (`Arc<AnyContainer>`) so a cloned handle reads and writes
|
||||
/// the *same* on-disk/OPFS working copy — including any writes still queued on the OPFS backend. The
|
||||
/// `Session` is cloned (a snapshot copy); the container is shared.
|
||||
#[derive(Clone)]
|
||||
pub struct Gdd<L: Layout = GddV1Layout> {
|
||||
pub(crate) session: Session,
|
||||
pub(crate) working: Arc<AnyContainer>,
|
||||
pub(crate) layout: L,
|
||||
/// In-memory copy of the manifest, kept authoritative since `Gdd` is its sole writer. Holds the
|
||||
/// per-payload codecs so the persist path never probes the filesystem, keeping it fully read-free
|
||||
/// and synchronous.
|
||||
pub(crate) manifest: Manifest,
|
||||
/// Per-peer view settings (PTZ, rulers, etc.), persisted in `session.json` not the registry, so
|
||||
/// they stay out of the CRDT/history. Opaque to the storage layer; the editor owns the keys/values.
|
||||
pub(crate) view_settings: std::collections::BTreeMap<String, serde_json::Value>,
|
||||
/// Per-network view settings (node-graph nav + previewing), keyed by stable [`NetworkId`]. Same per-peer
|
||||
/// `session.json` treatment as [`view_settings`](Self::view_settings), but scoped per network.
|
||||
pub(crate) network_view_settings: std::collections::BTreeMap<document_graph_storage::NetworkId, std::collections::BTreeMap<String, serde_json::Value>>,
|
||||
}
|
||||
|
||||
/// Native folder-backed convenience constructors. On wasm the editor builds an OPFS-backed
|
||||
/// `AnyContainer` itself and uses [`Gdd::open_in`] / [`Gdd::create_in`] directly.
|
||||
#[cfg(not(target_family = "wasm"))]
|
||||
impl<L: Layout + Default> Gdd<L> {
|
||||
/// Open an existing working copy at `path`. Validates the manifest, materializes the session
|
||||
/// from `registry.bin` (fast path) or by replaying `history.jsonl` (slow path), then applies
|
||||
/// the persisted hot log on top.
|
||||
pub async fn open(path: &Path) -> Result<Self, Error> {
|
||||
let working = AnyContainer::Folder(FolderBackend::open(path)?);
|
||||
let layout = L::default();
|
||||
Self::open_in(working, layout).await
|
||||
}
|
||||
|
||||
/// Create a fresh, empty working copy at `path` bound to `peer`. Writes a default manifest
|
||||
/// and session state; the caller fills in editor metadata via [`Gdd::update_manifest`].
|
||||
pub async fn create(path: &Path, peer: PeerId, document_uuid: u64, editor_version: String, stdlib_version: String) -> Result<Self, Error> {
|
||||
let working = AnyContainer::Folder(FolderBackend::create(path)?);
|
||||
let layout = L::default();
|
||||
Self::create_in(working, layout, peer, document_uuid, editor_version, stdlib_version).await
|
||||
}
|
||||
}
|
||||
|
||||
impl<L: Layout> Gdd<L> {
|
||||
/// Open a `.gdd` from archive bytes (xz or zip, auto-detected) by materializing it into `working`,
|
||||
/// then opening it as a working copy. The archive is deserialized into an in-memory staging backend
|
||||
/// (the archive reader is synchronous), then each entry is written into `working` via the sync
|
||||
/// `write_non_blocking` surface — durable on folder/memory, eagerly enqueued on OPFS. `working` is
|
||||
/// expected to be a fresh per-document container; entries with colliding paths are overwritten.
|
||||
#[cfg(any(feature = "zip", feature = "xz"))]
|
||||
pub async fn open_from_archive(bytes: &[u8], mut working: AnyContainer, layout: L) -> Result<Self, Error> {
|
||||
document_container::archive::open_auto(bytes, &mut working)?;
|
||||
|
||||
Self::open_in(working, layout).await
|
||||
}
|
||||
|
||||
/// Backend-agnostic open. Splits out so tests can supply a [`document_container::backends::memory::MemoryBackend`].
|
||||
///
|
||||
/// # Errors
|
||||
/// [`Error::WrongFormat`] / [`Error::UnsupportedVersion`] if the manifest fails validation, plus
|
||||
/// the usual [`Error::Read`] / [`Error::Codec`] / [`Error::Crdt`] if a payload is malformed.
|
||||
pub async fn open_in(working: AnyContainer, layout: L) -> Result<Self, Error> {
|
||||
let manifest: Manifest = io::read_single(&working, layout.manifest_basename(), MANIFEST_CODEC).await?;
|
||||
validate_manifest(&manifest)?;
|
||||
let codecs = manifest.codecs;
|
||||
|
||||
let session_state: SessionState = match io::exists(&working, layout.session_basename(), codecs.session).await {
|
||||
true => io::read_single(&working, layout.session_basename(), codecs.session).await?,
|
||||
false => SessionState::default(),
|
||||
};
|
||||
|
||||
let has_registry = io::exists(&working, layout.registry_basename(), codecs.registry).await;
|
||||
let has_history = io::exists(&working, layout.history_basename(), codecs.history).await;
|
||||
|
||||
let peer = session_state.peer_id;
|
||||
let mut session = match (has_registry, has_history) {
|
||||
(true, true) => {
|
||||
let registry: Registry = io::read_single(&working, layout.registry_basename(), codecs.registry).await?;
|
||||
let history = load_history(&working, &layout, codecs.history).await?;
|
||||
Session::load(peer, registry, history, session_state.head_rev, session_state.redo_stack, session_state.next_node_counter)
|
||||
}
|
||||
(true, false) => {
|
||||
// Registry-only export: synthesize a history that reproduces this state.
|
||||
let registry: Registry = io::read_single(&working, layout.registry_basename(), codecs.registry).await?;
|
||||
Session::bootstrap_from_registry(peer, registry)?
|
||||
}
|
||||
(false, _) => Session::replay_from_history(peer, load_history(&working, &layout, codecs.history).await?, session_state.next_node_counter)?,
|
||||
};
|
||||
|
||||
// Restore the published frontier (silent/published undo boundary) regardless of which load arm ran.
|
||||
if let Some(rev) = session_state.last_broadcast_rev {
|
||||
session.publish_up_to(rev);
|
||||
}
|
||||
|
||||
replay_hot_log(&working, &layout, codecs.hot_log, &mut session).await?;
|
||||
|
||||
Ok(Self {
|
||||
session,
|
||||
working: Arc::new(working),
|
||||
layout,
|
||||
manifest,
|
||||
view_settings: session_state.view_settings,
|
||||
network_view_settings: session_state.network_view_settings,
|
||||
})
|
||||
}
|
||||
|
||||
/// Backend-agnostic create. Records the working-copy default codecs (see `DEFAULT_*_CODEC`) in
|
||||
/// the manifest and writes each payload with its recorded codec.
|
||||
pub async fn create_in(working: AnyContainer, layout: L, peer: PeerId, document_uuid: u64, editor_version: String, stdlib_version: String) -> Result<Self, Error> {
|
||||
let manifest = Manifest::new(document_uuid, editor_version, stdlib_version);
|
||||
let codecs = manifest.codecs;
|
||||
io::write_single(&working, layout.manifest_basename(), MANIFEST_CODEC, &manifest)?;
|
||||
let session_state = SessionState { peer_id: peer, ..Default::default() };
|
||||
io::write_single(&working, layout.session_basename(), codecs.session, &session_state)?;
|
||||
|
||||
let session = Session::with_peer(peer);
|
||||
io::write_single(&working, layout.registry_basename(), codecs.registry, session.registry())?;
|
||||
|
||||
Ok(Self {
|
||||
session,
|
||||
working: Arc::new(working),
|
||||
layout,
|
||||
manifest,
|
||||
view_settings: std::collections::BTreeMap::new(),
|
||||
network_view_settings: std::collections::BTreeMap::new(),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
fn validate_manifest(manifest: &Manifest) -> Result<(), Error> {
|
||||
if manifest.format != manifest::FORMAT_MAGIC {
|
||||
return Err(Error::WrongFormat {
|
||||
found: manifest.format.clone(),
|
||||
expected: manifest::FORMAT_MAGIC,
|
||||
});
|
||||
}
|
||||
if manifest.format_version > manifest::SUPPORTED_FORMAT_VERSION {
|
||||
return Err(Error::UnsupportedVersion {
|
||||
found: manifest.format_version,
|
||||
max_supported: manifest::SUPPORTED_FORMAT_VERSION,
|
||||
});
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn load_history<L: Layout>(working: &AnyContainer, layout: &L, codec: Codec) -> Result<Vec<Delta>, Error> {
|
||||
if !io::exists(working, layout.history_basename(), codec).await {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
Ok(io::iter::<Delta>(working, layout.history_basename(), codec).await?)
|
||||
}
|
||||
|
||||
async fn replay_hot_log<L: Layout>(working: &AnyContainer, layout: &L, codec: Codec, session: &mut Session) -> Result<(), Error> {
|
||||
if !io::exists(working, layout.hot_log_basename(), codec).await {
|
||||
return Ok(());
|
||||
}
|
||||
for hot_op in io::iter::<HotOp>(working, layout.hot_log_basename(), codec).await? {
|
||||
session.replay_hot_op(hot_op)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
impl<L: Layout> Gdd<L> {
|
||||
pub fn session(&self) -> &Session {
|
||||
&self.session
|
||||
}
|
||||
|
||||
pub fn can_undo(&self) -> bool {
|
||||
self.session.can_undo()
|
||||
}
|
||||
|
||||
pub fn can_redo(&self) -> bool {
|
||||
self.session.can_redo()
|
||||
}
|
||||
|
||||
pub fn registry(&self) -> &Registry {
|
||||
self.session.registry()
|
||||
}
|
||||
|
||||
/// The in-memory manifest. `Gdd` is its sole writer, so this is authoritative without re-reading
|
||||
/// disk.
|
||||
pub fn manifest(&self) -> &Manifest {
|
||||
&self.manifest
|
||||
}
|
||||
|
||||
pub fn layout(&self) -> &L {
|
||||
&self.layout
|
||||
}
|
||||
|
||||
/// The per-peer view settings read from `session.json` (PTZ, rulers, overlays, snapping, collapse).
|
||||
/// Opaque `ui::doc::*` blobs; the editor decodes them. Empty for a fresh document.
|
||||
pub fn view_settings(&self) -> &std::collections::BTreeMap<String, serde_json::Value> {
|
||||
&self.view_settings
|
||||
}
|
||||
|
||||
/// The per-network view settings read from `session.json` (node-graph nav + previewing), keyed by
|
||||
/// [`NetworkId`](document_graph_storage::NetworkId). Opaque `ui::nav::*` / `ui::previewing` blobs the editor decodes.
|
||||
pub fn network_view_settings(&self) -> &std::collections::BTreeMap<document_graph_storage::NetworkId, std::collections::BTreeMap<String, serde_json::Value>> {
|
||||
&self.network_view_settings
|
||||
}
|
||||
|
||||
/// Resolve each runtime `network_path` to its stable [`NetworkId`](document_graph_storage::NetworkId), so the
|
||||
/// editor can key per-network, per-peer view state by a stable id. See [`Session::network_ids`].
|
||||
#[cfg(feature = "conversion")]
|
||||
pub fn network_ids<M: NodeMetadataSource>(
|
||||
&self,
|
||||
network: &graph_craft::document::NodeNetwork,
|
||||
metadata: &M,
|
||||
) -> Result<std::collections::HashMap<Vec<core_types::uuid::NodeId>, document_graph_storage::NetworkId>, CommitError> {
|
||||
self.session.network_ids(network, metadata)
|
||||
}
|
||||
|
||||
/// Every resource hash referenced by the current registry or anywhere in history, so resource GC keeps
|
||||
/// redoable/re-undoable interactions' resources (notably proto-node declaration bytes) alive even when an
|
||||
/// undo has dropped them from the current registry.
|
||||
pub fn all_referenced_resource_hashes(&self) -> std::collections::HashSet<ResourceHash> {
|
||||
self.session.all_referenced_resource_hashes()
|
||||
}
|
||||
|
||||
/// Drop the session and return the working-copy container + layout.
|
||||
/// Intended for test code that needs to reopen against the same container; panics if the container
|
||||
/// is still shared by a `Gdd` clone (tests don't clone before calling this).
|
||||
pub fn into_storage(self) -> (AnyContainer, L) {
|
||||
let working = Arc::try_unwrap(self.working).unwrap_or_else(|_| panic!("into_storage called while the working-copy container is still shared by a Gdd clone"));
|
||||
(working, self.layout)
|
||||
}
|
||||
|
||||
/// Resolve the proto-node declarations referenced by the registry into a [`document_graph_storage::Declarations`]
|
||||
/// map, loading each `ProtoNode`'s bytes from `byte_store` (the global cache in the editor, the
|
||||
/// working-copy container for standalone). Only resources referenced by `Implementation::ProtoNode`
|
||||
/// are visited, so image/font resources are skipped. Cold-path (open / `to_runtime`); async
|
||||
/// because resource loads are.
|
||||
#[cfg(feature = "conversion")]
|
||||
pub async fn declarations(&self, byte_store: &dyn LoadResource) -> document_graph_storage::Declarations {
|
||||
use document_graph_storage::Implementation;
|
||||
|
||||
let registry = self.session.registry();
|
||||
let mut declarations = document_graph_storage::Declarations::new();
|
||||
|
||||
for node in registry.node_instances.values() {
|
||||
let Implementation::ProtoNode(id) = node.implementation() else { continue };
|
||||
if declarations.contains_key(id) {
|
||||
continue;
|
||||
}
|
||||
let Some(hash) = registry.resources.get(id).and_then(|entry| entry.hash) else {
|
||||
log::error!("Declaration resource {id} has no resolved hash; cannot load ProtoNode");
|
||||
continue;
|
||||
};
|
||||
let Some(resource) = byte_store.load(hash).await else {
|
||||
log::error!("Declaration bytes for {id} (hash {hash}) missing from byte store");
|
||||
continue;
|
||||
};
|
||||
match document_graph_storage::decode_declaration(resource.as_ref()) {
|
||||
Ok(proto) => {
|
||||
declarations.insert(*id, proto);
|
||||
}
|
||||
Err(error) => log::error!("Failed to deserialize ProtoNode for {id}: {error}"),
|
||||
}
|
||||
}
|
||||
|
||||
declarations
|
||||
}
|
||||
|
||||
/// Store the legacy `.graphite` document bytes verbatim inside the working copy (dual-write soak).
|
||||
/// Synchronous (hot-path safe via `write_non_blocking`): called at the autosave boundary alongside
|
||||
/// the registry snapshot. The bytes are opaque to `Gdd` — it never deserializes them.
|
||||
// TODO: Add feature gate for legacy embedding
|
||||
pub fn store_legacy_document(&self, bytes: &[u8]) -> Result<(), ContainerError> {
|
||||
self.working.write_non_blocking(self.layout.legacy_path(), bytes)
|
||||
}
|
||||
|
||||
/// Read back the embedded legacy `.graphite` document, if present. The compare-on-open oracle and
|
||||
/// the recovery fallback both go through here. `None` when no legacy blob was ever written.
|
||||
pub async fn read_legacy_document(&self) -> Option<ByteHolder> {
|
||||
self.working.read(self.layout.legacy_path()).await.ok()
|
||||
}
|
||||
}
|
||||
63
document/format/src/manifest.rs
Normal file
63
document/format/src/manifest.rs
Normal file
@@ -0,0 +1,63 @@
|
||||
//! Bootstrap file for a `.gdd` document. Always JSON regardless of payload codec choice.
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::Codec;
|
||||
use crate::{DEFAULT_HISTORY_CODEC, DEFAULT_HOT_LOG_CODEC, DEFAULT_REGISTRY_CODEC, DEFAULT_SESSION_CODEC};
|
||||
|
||||
/// Magic string carried in [`Manifest::format`] to identify a `.gdd` document.
|
||||
pub const FORMAT_MAGIC: &str = "gdd";
|
||||
|
||||
/// Maximum manifest version this build can open. Bumped when manifest layout changes
|
||||
/// in a way that older builds can't safely read.
|
||||
pub const SUPPORTED_FORMAT_VERSION: u32 = 1;
|
||||
|
||||
/// The on-disk codec for each working-copy payload, recorded so reads/writes never have to probe
|
||||
/// the filesystem to discover it. The manifest itself is excluded: it is always JSON, since it must
|
||||
/// be parsed before any other codec is known.
|
||||
#[derive(Clone, Copy, Debug, Serialize, Deserialize)]
|
||||
pub struct PayloadCodecs {
|
||||
pub registry: Codec,
|
||||
pub history: Codec,
|
||||
pub hot_log: Codec,
|
||||
pub session: Codec,
|
||||
}
|
||||
|
||||
impl Default for PayloadCodecs {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
registry: DEFAULT_REGISTRY_CODEC,
|
||||
history: DEFAULT_HISTORY_CODEC,
|
||||
hot_log: DEFAULT_HOT_LOG_CODEC,
|
||||
session: DEFAULT_SESSION_CODEC,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Serialize, Deserialize)]
|
||||
pub struct Manifest {
|
||||
pub format: String,
|
||||
|
||||
pub format_version: u32,
|
||||
pub editor_version: String,
|
||||
pub stdlib_version: String,
|
||||
|
||||
pub document_id: u64,
|
||||
/// Codec used for each non-manifest payload on disk. Authoritative — never inferred from which
|
||||
/// file extension is present.
|
||||
#[serde(default)]
|
||||
pub codecs: PayloadCodecs,
|
||||
}
|
||||
|
||||
impl Manifest {
|
||||
pub fn new(document_id: u64, editor_version: String, stdlib_version: String) -> Self {
|
||||
Self {
|
||||
format: FORMAT_MAGIC.to_string(),
|
||||
format_version: SUPPORTED_FORMAT_VERSION,
|
||||
document_id,
|
||||
editor_version,
|
||||
stdlib_version,
|
||||
codecs: PayloadCodecs::default(),
|
||||
}
|
||||
}
|
||||
}
|
||||
259
document/format/src/persist.rs
Normal file
259
document/format/src/persist.rs
Normal file
@@ -0,0 +1,259 @@
|
||||
//! The per-edit persist path on the [`Gdd`] handle: stage/retire/commit, hot-log and history
|
||||
//! append, registry snapshots, session-state and manifest writes, plus view-settings setters.
|
||||
//! Synchronous and read-free (the manifest is cached on the handle); the container's `*_non_blocking`
|
||||
//! surface absorbs durability.
|
||||
|
||||
use document_container::AsyncContainer;
|
||||
#[cfg(feature = "conversion")]
|
||||
use document_graph_storage::NodeMetadataSource;
|
||||
use document_graph_storage::{HotOp, Rev, TimeStamp};
|
||||
#[cfg(feature = "conversion")]
|
||||
use graphene_resource::ResourceStorage;
|
||||
|
||||
use crate::error::Error;
|
||||
use crate::layout::Layout;
|
||||
use crate::manifest::Manifest;
|
||||
use crate::session_state::SessionState;
|
||||
use crate::{Gdd, MANIFEST_CODEC, io};
|
||||
|
||||
impl<L: Layout> Gdd<L> {
|
||||
/// Move the undo cursor back one commit (silent-zone reflog undo) and persist the new cursor. Returns
|
||||
/// the undone `Rev`. The working registry is rewound in place by the reverse delta, so re-snapshot it
|
||||
/// (alongside `head`) or a reopen would read a `registry.bin` inconsistent with the persisted cursor.
|
||||
pub fn undo(&mut self) -> Result<Rev, Error> {
|
||||
let rev = self.session.undo()?;
|
||||
self.persist_registry_snapshot()?;
|
||||
self.persist_session_state()?;
|
||||
Ok(rev)
|
||||
}
|
||||
|
||||
/// Re-apply the most-recently-undone commit and persist the new cursor and re-snapshotted registry.
|
||||
pub fn redo(&mut self) -> Result<Rev, Error> {
|
||||
let rev = self.session.redo()?;
|
||||
self.persist_registry_snapshot()?;
|
||||
self.persist_session_state()?;
|
||||
Ok(rev)
|
||||
}
|
||||
|
||||
/// Edit the cached manifest and persist it. Always JSON, synchronous.
|
||||
pub fn update_manifest(&mut self, edit: impl FnOnce(&mut Manifest)) -> Result<(), Error> {
|
||||
edit(&mut self.manifest);
|
||||
io::write_single(&self.working, self.layout.manifest_basename(), MANIFEST_CODEC, &self.manifest)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Stage a runtime snapshot as hot ops without retiring: diff the runtime against the working
|
||||
/// registry, append the hot frames (so a crash recovers the work), and persist proto-node
|
||||
/// declaration bytes. The working registry reflects the edit immediately, but nothing enters durable
|
||||
/// retired history until [`retire_pending_interaction`](Self::retire_pending_interaction). Staging on
|
||||
/// every edit while retiring only at interaction boundaries lets several edits coalesce into one retired
|
||||
/// interaction.
|
||||
///
|
||||
/// # Errors
|
||||
/// [`Error::Commit`] if the runtime diff is rejected by the session. On an [`Error::Container`] /
|
||||
/// [`Error::Codec`] from persisting the hot frames, the session has already advanced past what the
|
||||
/// working copy reflects, so the caller should treat the document as needing re-persist.
|
||||
#[cfg(feature = "conversion")]
|
||||
pub fn stage_runtime_snapshot<M: NodeMetadataSource>(
|
||||
&mut self,
|
||||
network: &graph_craft::document::NodeNetwork,
|
||||
metadata: &M,
|
||||
resources: &graphene_resource::ResourceRegistry,
|
||||
byte_store: &dyn ResourceStorage,
|
||||
) -> Result<(), Error> {
|
||||
let (hot_ops, declaration_bytes) = self.session.stage_from_runtime(network, metadata, resources)?;
|
||||
|
||||
for hot_op in &hot_ops {
|
||||
self.append_hot_frame(hot_op)?;
|
||||
}
|
||||
|
||||
// Persist proto-node declaration content to the byte store (the global cache in the editor,
|
||||
// the working-copy container for standalone export). Content-addressed, so re-storing
|
||||
// identical bytes on every commit is an idempotent no-op.
|
||||
for bytes in declaration_bytes.values() {
|
||||
byte_store.store(bytes);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Retire every pending hot op into durable history as a single interaction (marking the batch's last
|
||||
/// delta as the interaction boundary), then re-snapshot the registry. One interaction is one undo unit,
|
||||
/// so the caller invokes this at each undo-step boundary and before any undo/redo. A no-op when there
|
||||
/// are no pending hot ops.
|
||||
pub fn retire_pending_interaction(&mut self) -> Result<Vec<Rev>, Error> {
|
||||
let Some(up_to) = self.session.hot_log().iter().map(|hot_op| hot_op.timestamp).max() else {
|
||||
return Ok(Vec::new());
|
||||
};
|
||||
self.retire_inner(up_to, true)
|
||||
}
|
||||
|
||||
/// Commit a runtime snapshot as one complete interaction: stage it, then immediately retire it into
|
||||
/// durable history. Convenience for callers that produce a whole interaction atomically (tests, and any
|
||||
/// one-shot commit). Equivalent to [`stage_runtime_snapshot`](Self::stage_runtime_snapshot) followed
|
||||
/// by [`retire_pending_interaction`](Self::retire_pending_interaction).
|
||||
#[cfg(feature = "conversion")]
|
||||
pub fn commit_from_runtime<M: NodeMetadataSource>(
|
||||
&mut self,
|
||||
network: &graph_craft::document::NodeNetwork,
|
||||
metadata: &M,
|
||||
resources: &graphene_resource::ResourceRegistry,
|
||||
byte_store: &dyn ResourceStorage,
|
||||
) -> Result<Vec<Rev>, Error> {
|
||||
self.stage_runtime_snapshot(network, metadata, resources, byte_store)?;
|
||||
self.retire_pending_interaction()
|
||||
}
|
||||
|
||||
/// Apply a hot op from the broadcast stream, appending one frame to the hot log.
|
||||
///
|
||||
/// # Errors
|
||||
/// Returns [`Error::Crdt`] if the op is rejected by the session, or an [`Error::Container`] /
|
||||
/// [`Error::Codec`] if persisting the hot frame fails. On a persist failure the session has already
|
||||
/// advanced past what the working copy reflects, so the caller should treat the document as needing
|
||||
/// re-persist (mirrors [`stage_runtime_snapshot`](Self::stage_runtime_snapshot)).
|
||||
pub fn apply_hot_op(&mut self, op: HotOp) -> Result<(), Error> {
|
||||
self.session.apply_hot_op(op.clone())?;
|
||||
self.append_hot_frame(&op)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Persist freshly-staged hot ops and immediately retire them into durable history. Appends each
|
||||
/// hot frame (so a crash before retirement still recovers the work), then retires up to the last
|
||||
/// staged timestamp, which drains exactly these ops and re-snapshots the registry. Returns the
|
||||
/// retired `Rev`s. A no-op when nothing was staged.
|
||||
pub(crate) fn append_and_retire(&mut self, hot_ops: &[HotOp], interaction_end: bool) -> Result<Vec<Rev>, Error> {
|
||||
let Some(last) = hot_ops.last() else { return Ok(Vec::new()) };
|
||||
|
||||
for hot_op in hot_ops {
|
||||
self.append_hot_frame(hot_op)?;
|
||||
}
|
||||
|
||||
self.retire_inner(last.timestamp, interaction_end)
|
||||
}
|
||||
|
||||
/// Encode the history deltas identified by `revs` and append them to the history file. `revs` comes
|
||||
/// from `Session::retire` in append order, which is a valid replay order, so a direct per-rev lookup
|
||||
/// preserves replay order without scanning the whole history.
|
||||
fn append_history_deltas(&mut self, revs: &[Rev]) -> Result<(), Error> {
|
||||
let mut buffer = Vec::new();
|
||||
for &rev in revs {
|
||||
let Some(delta) = self.session.delta(rev) else {
|
||||
log::error!("Retired rev {rev:?} missing from history; skipping its history frame");
|
||||
continue;
|
||||
};
|
||||
self.manifest.codecs.history.append(&mut buffer, delta)?;
|
||||
}
|
||||
self.working.append_non_blocking(&io::path_for(self.layout.history_basename(), self.manifest.codecs.history), &buffer)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Set a local annotation (e.g. a commit message) on an existing retired delta and re-persist it.
|
||||
/// Unlike the per-interaction marker written inline at retire, this targets an already-written delta, so
|
||||
/// the whole history file is rewritten in topological order. O(history) — fine for occasional user
|
||||
/// labeling, not for per-interaction marking (which uses the inline path). No-op if `rev` is unknown.
|
||||
pub fn annotate_delta(&mut self, rev: Rev, key: &str, value: serde_json::Value) -> Result<(), Error> {
|
||||
if self.session.annotate_delta(rev, key, value) {
|
||||
self.rewrite_history()?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Rewrite the entire history file from the in-memory session. `history()` yields deltas in
|
||||
/// topological (append) order, which is a valid replay order, so no separate sort is needed.
|
||||
fn rewrite_history(&mut self) -> Result<(), Error> {
|
||||
let mut buffer = Vec::new();
|
||||
for delta in self.session.history() {
|
||||
self.manifest.codecs.history.append(&mut buffer, delta)?;
|
||||
}
|
||||
self.working.write_non_blocking(&io::path_for(self.layout.history_basename(), self.manifest.codecs.history), &buffer)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn persist_session_state(&mut self) -> Result<(), Error> {
|
||||
let state = SessionState {
|
||||
peer_id: self.session.peer(),
|
||||
head_rev: self.session.head_rev(),
|
||||
last_broadcast_rev: self.session.last_broadcast_rev(),
|
||||
redo_stack: self.session.redo_stack().to_vec(),
|
||||
next_node_counter: self.session.next_node_counter(),
|
||||
view_settings: self.view_settings.clone(),
|
||||
network_view_settings: self.network_view_settings.clone(),
|
||||
};
|
||||
io::write_single(&self.working, self.layout.session_basename(), self.manifest.codecs.session, &state)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Re-snapshot the materialized working registry to `registry.bin`. `Session::load` trusts the stored
|
||||
/// registry to match the persisted `head`, so any cursor move (undo/redo) that rewinds the working
|
||||
/// registry without retiring must re-persist it or a reopen would read a registry inconsistent with
|
||||
/// `head`. Synchronous and hot-path-safe (`write_non_blocking`).
|
||||
fn persist_registry_snapshot(&mut self) -> Result<(), Error> {
|
||||
io::write_single(&self.working, self.layout.registry_basename(), self.manifest.codecs.registry, self.session.registry())?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Replace the per-peer view settings and persist them to `session.json`. Called by the editor when
|
||||
/// the viewport or a document-level toggle changes; never enters the registry, history, or CRDT.
|
||||
pub fn set_view_settings(&mut self, view_settings: std::collections::BTreeMap<String, serde_json::Value>) -> Result<(), Error> {
|
||||
self.view_settings = view_settings;
|
||||
self.persist_session_state()
|
||||
}
|
||||
|
||||
/// Advance the published frontier to `rev` and persist it to `session.json`, so the silent/published
|
||||
/// undo boundary survives a reopen. Called by the (future) broadcast transport as commits are shared.
|
||||
pub fn publish_up_to(&mut self, rev: document_graph_storage::Rev) -> Result<(), Error> {
|
||||
self.session.publish_up_to(rev);
|
||||
self.persist_session_state()
|
||||
}
|
||||
|
||||
/// Replace the per-network view settings and persist them to `session.json`. Per-peer, per-network; never
|
||||
/// enters the registry, history, or CRDT.
|
||||
pub fn set_network_view_settings(
|
||||
&mut self,
|
||||
network_view_settings: std::collections::BTreeMap<document_graph_storage::NetworkId, std::collections::BTreeMap<String, serde_json::Value>>,
|
||||
) -> Result<(), Error> {
|
||||
self.network_view_settings = network_view_settings;
|
||||
self.persist_session_state()
|
||||
}
|
||||
|
||||
fn append_hot_frame(&mut self, op: &HotOp) -> Result<(), Error> {
|
||||
let mut buffer = Vec::new();
|
||||
self.manifest.codecs.hot_log.append(&mut buffer, op)?;
|
||||
self.working.append_non_blocking(&io::path_for(self.layout.hot_log_basename(), self.manifest.codecs.hot_log), &buffer)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Working-copy checkpoint: promote hot ops with timestamp `≤ up_to` into retired deltas,
|
||||
/// append them to the history file, rewrite the hot log with remaining (unretired) ops, and
|
||||
/// re-snapshot the registry. Synchronous.
|
||||
pub fn retire(&mut self, up_to: TimeStamp) -> Result<Vec<Rev>, Error> {
|
||||
self.retire_inner(up_to, false)
|
||||
}
|
||||
|
||||
/// `interaction_end`: mark the batch's last delta as an interaction boundary (one undo unit) before its
|
||||
/// history frame is written, so the marker persists on reopen without a later frame rewrite.
|
||||
fn retire_inner(&mut self, up_to: TimeStamp, interaction_end: bool) -> Result<Vec<Rev>, Error> {
|
||||
let new_revs = self.session.retire(up_to)?;
|
||||
|
||||
// Mark before `append_history_deltas` so the on-disk frame carries the boundary.
|
||||
if interaction_end && let Some(&last) = new_revs.last() {
|
||||
self.session.mark_interaction_end(last);
|
||||
}
|
||||
|
||||
if !new_revs.is_empty() {
|
||||
self.append_history_deltas(&new_revs)?;
|
||||
}
|
||||
|
||||
// Rewrite hot log with whatever survived retirement.
|
||||
let mut hot_buffer = Vec::new();
|
||||
for hot_op in self.session.hot_log() {
|
||||
self.manifest.codecs.hot_log.append(&mut hot_buffer, hot_op)?;
|
||||
}
|
||||
self.working
|
||||
.write_non_blocking(&io::path_for(self.layout.hot_log_basename(), self.manifest.codecs.hot_log), &hot_buffer)?;
|
||||
|
||||
self.persist_registry_snapshot()?;
|
||||
self.persist_session_state()?;
|
||||
|
||||
Ok(new_revs)
|
||||
}
|
||||
}
|
||||
159
document/format/src/resource.rs
Normal file
159
document/format/src/resource.rs
Normal file
@@ -0,0 +1,159 @@
|
||||
//! Resource I/O on the [`Gdd`] handle: the content-addressed byte store that backs raster images,
|
||||
//! fonts, embedded WASM, and proto-node declarations. Registration goes through the session as an
|
||||
//! `AddResource` delta; the bytes live in the working copy's `resources/<hash>` directory.
|
||||
|
||||
#[cfg(not(target_family = "wasm"))]
|
||||
use std::path::Path;
|
||||
use std::sync::Arc;
|
||||
|
||||
use document_container::{AnyContainer, AsyncContainer, ByteHolder, ContainerError};
|
||||
use graphene_resource::ResourceFuture;
|
||||
use graphene_resource::{LoadResource, Resource, ResourceHash, ResourceStorage};
|
||||
|
||||
use crate::Gdd;
|
||||
use crate::error::Error;
|
||||
use crate::layout::Layout;
|
||||
|
||||
impl<L: Layout> Gdd<L> {
|
||||
pub async fn read_resource(&self, hash: &ResourceHash) -> Result<ByteHolder, ContainerError> {
|
||||
self.working.read(&self.layout.resource_path(hash)).await
|
||||
}
|
||||
|
||||
/// Register a resource under `id` and store its bytes. Commits an `AddResource` delta (a single
|
||||
/// `DataSource::Embedded` source resolved to the content hash) through the session so the registry
|
||||
/// records the resource and the entry replicates, then writes the bytes into the working copy's
|
||||
/// content-addressed store. The caller owns `id` allocation.
|
||||
pub fn add_resource(&mut self, id: document_graph_storage::ResourceId, bytes: &[u8]) -> Result<(), Error> {
|
||||
let hash = ResourceHash::from(bytes);
|
||||
|
||||
self.working.write_non_blocking(&self.layout.resource_path(&hash), bytes)?;
|
||||
|
||||
let hot_ops = self.session.stage_embedded_resource(id, hash)?;
|
||||
self.append_and_retire(&hot_ops, false)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Like [`add_resource`](Self::add_resource) but copies the bytes from a filesystem `src` rather
|
||||
/// than buffering them. Folder backends use `fs::copy` (CoW on supported filesystems); other
|
||||
/// backends fall back to read-then-write. Native-only: there is no filesystem source path on wasm.
|
||||
#[cfg(not(target_family = "wasm"))]
|
||||
pub fn add_resource_from_path(&mut self, id: document_graph_storage::ResourceId, hash: ResourceHash, src: &Path) -> Result<(), Error> {
|
||||
let dest_path = self.layout.resource_path(&hash);
|
||||
if let AnyContainer::Folder(folder) = self.working.as_ref() {
|
||||
let full = folder.root().join(&dest_path);
|
||||
if let Some(parent) = full.parent() {
|
||||
std::fs::create_dir_all(parent).map_err(ContainerError::Io)?;
|
||||
}
|
||||
std::fs::copy(src, &full).map_err(ContainerError::Io)?;
|
||||
} else {
|
||||
let bytes = std::fs::read(src).map_err(ContainerError::Io)?;
|
||||
// The folder fast path trusts the caller's hash to avoid reading the file; here we've read
|
||||
// the bytes anyway, so verify the hash matches and flag a content-addressing bug in debug.
|
||||
debug_assert_eq!(hash, ResourceHash::from(bytes.as_slice()), "add_resource_from_path hash does not match the file at {src:?}");
|
||||
self.working.write_non_blocking(&dest_path, &bytes)?;
|
||||
}
|
||||
|
||||
let hot_ops = self.session.stage_embedded_resource(id, hash)?;
|
||||
self.append_and_retire(&hot_ops, false)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn has_resource(&self, hash: &ResourceHash) -> bool {
|
||||
self.working.exists(&self.layout.resource_path(hash)).await
|
||||
}
|
||||
|
||||
pub fn remove_resource(&self, hash: &ResourceHash) -> Result<(), ContainerError> {
|
||||
self.working.remove_non_blocking(&self.layout.resource_path(hash))
|
||||
}
|
||||
|
||||
/// Enumerate every resource currently in the working copy. Paths that don't parse as a
|
||||
/// `ResourceHash` (foreign files dropped into the resources directory) are silently skipped.
|
||||
pub async fn resource_hashes(&self) -> Result<Vec<ResourceHash>, ContainerError> {
|
||||
let dir = self.layout.resources_dir();
|
||||
if !self.working.list_dirs("").await?.iter().any(|d| d == dir) {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
let entries = self.working.list(dir).await?;
|
||||
let prefix = format!("{dir}/");
|
||||
let mut hashes = Vec::with_capacity(entries.len());
|
||||
for entry in entries {
|
||||
let Some(name) = entry.strip_prefix(&prefix) else { continue };
|
||||
if let Ok(hash) = name.parse::<ResourceHash>() {
|
||||
hashes.push(hash);
|
||||
}
|
||||
}
|
||||
Ok(hashes)
|
||||
}
|
||||
|
||||
pub fn resource_proxy(&self) -> ResourceProxy<L>
|
||||
where
|
||||
L: Clone,
|
||||
{
|
||||
ResourceProxy(self.working.clone(), self.layout.clone())
|
||||
}
|
||||
}
|
||||
|
||||
impl<L: Layout + Send + Sync> LoadResource for Gdd<L> {
|
||||
fn load(&self, hash: ResourceHash) -> ResourceFuture<'_> {
|
||||
Box::pin(async move {
|
||||
let bytes = self.working.read(&self.layout.resource_path(&hash)).await.ok()?;
|
||||
Some(Resource::new(bytes))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
pub struct ResourceProxy<T: Layout>(Arc<AnyContainer>, T);
|
||||
|
||||
impl<L: Layout + Send + Sync> LoadResource for ResourceProxy<L> {
|
||||
fn load(&self, hash: ResourceHash) -> ResourceFuture<'_> {
|
||||
Box::pin(async move {
|
||||
let bytes = self.0.read(&self.1.resource_path(&hash)).await.ok()?;
|
||||
Some(Resource::new(bytes))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl<L: Layout + Send + Sync> ResourceStorage for Gdd<L> {
|
||||
fn store(&self, data: &[u8]) -> ResourceHash {
|
||||
let hash = ResourceHash::from(data);
|
||||
if let Err(error) = self.working.write_non_blocking(&self.layout.resource_path(&hash), data) {
|
||||
log::error!("ResourceStorage::store failed for {hash}: {error}");
|
||||
}
|
||||
hash
|
||||
}
|
||||
|
||||
fn contains(&self, hash: &ResourceHash) -> bool {
|
||||
self.working.exists_non_blocking(&self.layout.resource_path(hash))
|
||||
}
|
||||
|
||||
fn garbage_collect(&self, used: &[ResourceHash]) {
|
||||
// `garbage_collect` is synchronous but listing resources is async, so the native path blocks on
|
||||
// it. That's unavailable on wasm (single-threaded; `block_on` would deadlock). The editor never
|
||||
// uses `Gdd` as the runtime `ResourceStorage` on wasm (it GCs the app-global cache instead), so
|
||||
// this is an unreachable configuration there rather than a missing feature.
|
||||
#[cfg(target_family = "wasm")]
|
||||
{
|
||||
let _ = used;
|
||||
log::error!("ResourceStorage::garbage_collect is not supported for Gdd on wasm");
|
||||
}
|
||||
#[cfg(not(target_family = "wasm"))]
|
||||
{
|
||||
let kept: std::collections::HashSet<&ResourceHash> = used.iter().collect();
|
||||
let hashes = match futures::executor::block_on(self.resource_hashes()) {
|
||||
Ok(hashes) => hashes,
|
||||
Err(error) => {
|
||||
log::error!("Failed to list resources during garbage_collect: {error}");
|
||||
return;
|
||||
}
|
||||
};
|
||||
for hash in hashes {
|
||||
if kept.contains(&hash) {
|
||||
continue;
|
||||
}
|
||||
if let Err(error) = self.working.remove_non_blocking(&self.layout.resource_path(&hash)) {
|
||||
log::error!("ResourceStorage::garbage_collect failed to remove {hash}: {error}");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
42
document/format/src/session_state.rs
Normal file
42
document/format/src/session_state.rs
Normal file
@@ -0,0 +1,42 @@
|
||||
//! Persistent cursor state for the local peer. Separate from [`crate::Manifest`] because the
|
||||
//! manifest describes document identity (what this document *is*), while [`SessionState`]
|
||||
//! describes where the local peer's cursor sits inside it.
|
||||
//!
|
||||
//! Lives in `session.json`. Rewritten on retirement.
|
||||
|
||||
use document_graph_storage::{NetworkId, PeerId, Rev};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
#[derive(Clone, Debug, Default, Serialize, Deserialize)]
|
||||
pub struct SessionState {
|
||||
/// This peer's identity for the document, stable per (device, document). Per-peer rather than
|
||||
/// document identity, so it lives with the cursor here, not in the manifest. Used for CRDT
|
||||
/// tiebreaking and minting peer-scoped IDs.
|
||||
#[serde(default)]
|
||||
pub peer_id: PeerId,
|
||||
/// Local-chain cursor. Points at the most recently applied retired delta, or `None` on an empty
|
||||
/// document (no commits yet).
|
||||
#[serde(default)]
|
||||
pub head_rev: Option<Rev>,
|
||||
/// Published frontier: the latest retired commit broadcast to a peer. Commits after it are silently
|
||||
/// rewritable on undo; commits at or before it are published. `None` until broadcast transport lands.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub last_broadcast_rev: Option<Rev>,
|
||||
/// Revs the user has undone past, so redo survives a reopen. (The legacy `VecDeque` redo history
|
||||
/// is not persisted, so within the shadow phase this is strictly more capable than the live editor.)
|
||||
#[serde(default)]
|
||||
pub redo_stack: Vec<Rev>,
|
||||
/// Shared-monotonic counter feeding `Document::next_node_id`. Persisted so reopens don't
|
||||
/// collide on minted IDs.
|
||||
#[serde(default)]
|
||||
pub next_node_counter: u64,
|
||||
/// Per-peer view settings (PTZ, rulers, overlays, snapping, panel collapse). Local to the viewer,
|
||||
/// so kept out of the CRDT/history. Editor owns the keys/values (opaque `ui::doc::*` blobs).
|
||||
#[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
|
||||
pub view_settings: BTreeMap<String, serde_json::Value>,
|
||||
/// Per-network view settings (node-graph nav + previewing), keyed by the stable storage [`NetworkId`].
|
||||
/// Per-peer like [`view_settings`](Self::view_settings); opaque `ui::nav::*` / `ui::previewing` blobs.
|
||||
#[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
|
||||
pub network_view_settings: BTreeMap<NetworkId, BTreeMap<String, serde_json::Value>>,
|
||||
}
|
||||
722
document/format/tests/open_create.rs
Normal file
722
document/format/tests/open_create.rs
Normal file
@@ -0,0 +1,722 @@
|
||||
// Exercises the runtime-conversion bridge and the zip archive round-trip, so it only compiles with
|
||||
// both features. A minimal-dependency build (e.g. `--no-default-features`) skips it entirely.
|
||||
#![cfg(all(feature = "conversion", feature = "zip"))]
|
||||
|
||||
use document_container::AnyContainer;
|
||||
use document_container::backends::memory::MemoryBackend;
|
||||
use document_format::{Codec, Error, GddV1, GddV1Layout, Layout, Manifest, io, manifest};
|
||||
use document_graph_storage::{HotOp, Network, NetworkId, PeerId, ROOT_NETWORK, RegistryDelta, TimeStamp};
|
||||
|
||||
fn empty_container() -> AnyContainer {
|
||||
AnyContainer::Memory(MemoryBackend::new())
|
||||
}
|
||||
|
||||
/// A resource byte store for export calls. Empty unless a test pre-populates it; only consulted when
|
||||
/// `embed_all_resources` is set.
|
||||
fn empty_byte_store() -> graph_craft::application_io::resource::HashMapResourceStorage {
|
||||
graph_craft::application_io::resource::HashMapResourceStorage::new()
|
||||
}
|
||||
|
||||
/// A one-node network referencing `id` via a `TaggedValue::Resource` input. Conversion only snapshots
|
||||
/// resources the network references, so a resource needs a referencing node to survive into storage.
|
||||
fn network_referencing_resource(id: graphene_resource::ResourceId) -> graph_craft::document::NodeNetwork {
|
||||
use graph_craft::ProtoNodeIdentifier;
|
||||
use graph_craft::document::value::TaggedValue;
|
||||
use graph_craft::document::{DocumentNode, DocumentNodeImplementation, NodeId, NodeInput, NodeNetwork};
|
||||
|
||||
NodeNetwork {
|
||||
nodes: [(
|
||||
NodeId(0),
|
||||
DocumentNode {
|
||||
inputs: vec![NodeInput::value(TaggedValue::Resource(id), false)],
|
||||
implementation: DocumentNodeImplementation::ProtoNode(ProtoNodeIdentifier::new("graphene_core::ops::identity::IdentityNode")),
|
||||
..Default::default()
|
||||
},
|
||||
)]
|
||||
.into_iter()
|
||||
.collect(),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn create_in_round_trips_empty_document() {
|
||||
futures::executor::block_on(async {
|
||||
let container = empty_container();
|
||||
|
||||
let created = match GddV1::create_in(container, GddV1Layout, PeerId(7), 0xFEED, "editor-x".into(), "stdlib-x".into()).await {
|
||||
Ok(gdd) => gdd,
|
||||
Err(error) => panic!("create_in failed: {error:?}"),
|
||||
};
|
||||
|
||||
let (working, layout) = created.into_storage();
|
||||
let reopened = match GddV1::open_in(working, layout).await {
|
||||
Ok(gdd) => gdd,
|
||||
Err(error) => panic!("open_in failed: {error:?}"),
|
||||
};
|
||||
|
||||
assert_eq!(reopened.session().peer(), PeerId(7));
|
||||
assert!(reopened.registry().node_instances.is_empty());
|
||||
assert!(reopened.registry().networks.is_empty());
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn open_in_rejects_wrong_format_magic() {
|
||||
futures::executor::block_on(async {
|
||||
let container = empty_container();
|
||||
let layout = GddV1Layout;
|
||||
|
||||
let mut bogus = Manifest::new(0xC0DE, "ed".into(), "std".into());
|
||||
bogus.format = "not-gdd".into();
|
||||
io::write_single(&container, layout.manifest_basename(), Codec::Json, &bogus).unwrap();
|
||||
|
||||
match GddV1::open_in(container, layout).await {
|
||||
Err(Error::WrongFormat { .. }) => {}
|
||||
Ok(_) => panic!("expected WrongFormat, got Ok"),
|
||||
Err(other) => panic!("expected WrongFormat, got {other:?}"),
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn manifest_returns_what_create_in_wrote() {
|
||||
futures::executor::block_on(async {
|
||||
let gdd = GddV1::create_in(empty_container(), GddV1Layout, PeerId(13), 0xC0FFEE, "ed-1.2".into(), "std-0.7".into())
|
||||
.await
|
||||
.unwrap_or_else(|error| panic!("create_in failed: {error:?}"));
|
||||
|
||||
assert_eq!(gdd.session().peer(), PeerId(13));
|
||||
|
||||
let manifest = gdd.manifest();
|
||||
assert_eq!(manifest.document_id, 0xC0FFEE);
|
||||
assert_eq!(manifest.editor_version, "ed-1.2");
|
||||
assert_eq!(manifest.stdlib_version, "std-0.7");
|
||||
assert_eq!(manifest.format, manifest::FORMAT_MAGIC);
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn update_manifest_changes_visible_after_reopen() {
|
||||
futures::executor::block_on(async {
|
||||
let mut gdd = GddV1::create_in(empty_container(), GddV1Layout, PeerId(1), 0xAB, "ed".into(), "std".into())
|
||||
.await
|
||||
.unwrap_or_else(|error| panic!("create_in failed: {error:?}"));
|
||||
|
||||
gdd.update_manifest(|m| m.editor_version = "ed-NEW".into())
|
||||
.unwrap_or_else(|error| panic!("update_manifest failed: {error:?}"));
|
||||
|
||||
let (working, layout) = gdd.into_storage();
|
||||
let reopened = GddV1::open_in(working, layout).await.unwrap_or_else(|error| panic!("open_in failed: {error:?}"));
|
||||
let manifest = reopened.manifest();
|
||||
assert_eq!(manifest.editor_version, "ed-NEW");
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn apply_hot_op_persists_to_hot_log_and_survives_reopen() {
|
||||
futures::executor::block_on(async {
|
||||
let mut gdd = GddV1::create_in(empty_container(), GddV1Layout, PeerId(5), 0xDEAD, "ed".into(), "std".into())
|
||||
.await
|
||||
.unwrap_or_else(|error| panic!("create_in failed: {error:?}"));
|
||||
|
||||
// AddNetwork on the root network. Idempotent at apply, so two hot ops applied in sequence
|
||||
// produces one network in the registry.
|
||||
let hot_op = HotOp {
|
||||
op: RegistryDelta::AddNetwork {
|
||||
id: ROOT_NETWORK,
|
||||
network: Network::default(),
|
||||
},
|
||||
timestamp: TimeStamp { counter: 1, peer: PeerId(5) },
|
||||
};
|
||||
gdd.apply_hot_op(hot_op).unwrap_or_else(|error| panic!("apply_hot_op failed: {error:?}"));
|
||||
|
||||
assert!(gdd.registry().networks.contains_key(&ROOT_NETWORK), "hot op should have created the root network in memory");
|
||||
|
||||
let (working, layout) = gdd.into_storage();
|
||||
let reopened = GddV1::open_in(working, layout).await.unwrap_or_else(|error| panic!("open_in failed: {error:?}"));
|
||||
|
||||
assert!(reopened.registry().networks.contains_key(&ROOT_NETWORK), "hot op should have been replayed from the hot log on reopen");
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn retire_moves_eligible_hot_ops_to_history_and_keeps_rest() {
|
||||
futures::executor::block_on(async {
|
||||
let mut gdd = GddV1::create_in(empty_container(), GddV1Layout, PeerId(5), 0xDEAD, "ed".into(), "std".into())
|
||||
.await
|
||||
.unwrap_or_else(|error| panic!("create_in failed: {error:?}"));
|
||||
|
||||
// Two hot ops: one with low timestamp (will retire), one with high (will stay).
|
||||
let early = HotOp {
|
||||
op: RegistryDelta::AddNetwork {
|
||||
id: ROOT_NETWORK,
|
||||
network: Network::default(),
|
||||
},
|
||||
timestamp: TimeStamp { counter: 1, peer: PeerId(5) },
|
||||
};
|
||||
let late = HotOp {
|
||||
op: RegistryDelta::AddNetwork {
|
||||
id: NetworkId(42),
|
||||
network: Network::default(),
|
||||
},
|
||||
timestamp: TimeStamp { counter: 10, peer: PeerId(5) },
|
||||
};
|
||||
gdd.apply_hot_op(early).unwrap();
|
||||
gdd.apply_hot_op(late).unwrap();
|
||||
assert_eq!(gdd.session().hot_log().len(), 2);
|
||||
|
||||
// Retire only up to timestamp 5 → drains the early op, leaves the late one.
|
||||
let cutoff = TimeStamp { counter: 5, peer: PeerId(5) };
|
||||
gdd.retire(cutoff).unwrap_or_else(|error| panic!("retire failed: {error:?}"));
|
||||
|
||||
assert_eq!(gdd.session().hot_log().len(), 1, "late hot op should still be in hot log");
|
||||
assert_eq!(gdd.session().history().count(), 1, "early hot op should be in retired history");
|
||||
|
||||
// Reopen and confirm survival: hot log has the late op (replayed), history has the early op.
|
||||
let (working, layout) = gdd.into_storage();
|
||||
let reopened = GddV1::open_in(working, layout).await.unwrap_or_else(|error| panic!("open_in failed: {error:?}"));
|
||||
|
||||
assert!(reopened.registry().networks.contains_key(&ROOT_NETWORK), "retired op's effect should be in registry");
|
||||
assert!(reopened.registry().networks.contains_key(&NetworkId(42)), "hot op's effect should be replayed");
|
||||
assert_eq!(reopened.session().history().count(), 1);
|
||||
assert_eq!(reopened.session().hot_log().len(), 1);
|
||||
});
|
||||
}
|
||||
|
||||
/// The published frontier (`last_broadcast_rev`, the silent/published undo boundary) persists in
|
||||
/// `session.json` and is restored on reopen.
|
||||
#[test]
|
||||
fn last_broadcast_rev_persists_across_reopen() {
|
||||
futures::executor::block_on(async {
|
||||
let mut gdd = GddV1::create_in(empty_container(), GddV1Layout, PeerId(5), 0xDEAD, "ed".into(), "std".into())
|
||||
.await
|
||||
.unwrap_or_else(|error| panic!("create_in failed: {error:?}"));
|
||||
|
||||
// Retire one op so there is a real retired rev to mark as published.
|
||||
let op = HotOp {
|
||||
op: RegistryDelta::AddNetwork {
|
||||
id: ROOT_NETWORK,
|
||||
network: Network::default(),
|
||||
},
|
||||
timestamp: TimeStamp { counter: 1, peer: PeerId(5) },
|
||||
};
|
||||
gdd.apply_hot_op(op).unwrap();
|
||||
let retired = gdd.retire(TimeStamp { counter: 1, peer: PeerId(5) }).unwrap_or_else(|error| panic!("retire failed: {error:?}"));
|
||||
let published = *retired.last().expect("one retired rev");
|
||||
|
||||
assert_eq!(gdd.session().last_broadcast_rev(), None, "nothing is published before publish_up_to");
|
||||
gdd.publish_up_to(published).unwrap_or_else(|error| panic!("publish_up_to failed: {error:?}"));
|
||||
assert_eq!(gdd.session().last_broadcast_rev(), Some(published));
|
||||
|
||||
let (working, layout) = gdd.into_storage();
|
||||
let reopened = GddV1::open_in(working, layout).await.unwrap_or_else(|error| panic!("open_in failed: {error:?}"));
|
||||
assert_eq!(reopened.session().last_broadcast_rev(), Some(published), "published frontier should survive reopen");
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn export_folder_round_trips_through_open() {
|
||||
use document_format::{ExportFormat, ExportOptions};
|
||||
|
||||
futures::executor::block_on(async {
|
||||
let gdd = GddV1::create_in(empty_container(), GddV1Layout, PeerId(3), 0xAB, "ed".into(), "std".into())
|
||||
.await
|
||||
.unwrap_or_else(|error| panic!("create_in failed: {error:?}"));
|
||||
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let dest = dir.path().join("export");
|
||||
|
||||
gdd.export(&dest, ExportFormat::Folder, ExportOptions::default(), &empty_byte_store(), None)
|
||||
.await
|
||||
.unwrap_or_else(|error| panic!("export failed: {error:?}"));
|
||||
|
||||
// Payloads keep the working-copy codecs: registry is MessagePack (`.bin`), manifest is JSON.
|
||||
assert!(dest.join("registry.bin").exists());
|
||||
assert!(dest.join("manifest.json").exists());
|
||||
assert!(dest.join("session.json").exists());
|
||||
assert!(!dest.join("hot-log.bin").exists());
|
||||
assert!(!dest.join("hot-log.frames").exists());
|
||||
|
||||
// And the export is itself openable.
|
||||
let reopened = GddV1::open(&dest).await.unwrap_or_else(|error| panic!("open failed: {error:?}"));
|
||||
assert_eq!(reopened.session().peer(), PeerId(3));
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn export_zip_round_trips_via_deserialize() {
|
||||
use document_container::archive::{Archive, Zip};
|
||||
use document_format::{ExportFormat, ExportOptions};
|
||||
|
||||
futures::executor::block_on(async {
|
||||
let gdd = GddV1::create_in(empty_container(), GddV1Layout, PeerId(4), 0xCD, "ed".into(), "std".into())
|
||||
.await
|
||||
.unwrap_or_else(|error| panic!("create_in failed: {error:?}"));
|
||||
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let dest = dir.path().join("doc.gdd.zip");
|
||||
|
||||
gdd.export(&dest, ExportFormat::Zip, ExportOptions::default(), &empty_byte_store(), None)
|
||||
.await
|
||||
.unwrap_or_else(|error| panic!("export failed: {error:?}"));
|
||||
|
||||
let bytes = std::fs::read(&dest).unwrap();
|
||||
let mut restored = document_container::backends::memory::MemoryBackend::new();
|
||||
Zip::open(std::io::Cursor::new(&bytes), &mut restored).unwrap();
|
||||
use document_container::Container;
|
||||
assert!(restored.exists("manifest.json"));
|
||||
assert!(restored.exists("registry.bin"));
|
||||
assert!(restored.exists("session.json"));
|
||||
assert!(!restored.exists("hot-log.frames"));
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn export_rejects_invalid_options() {
|
||||
use document_format::{ExportFormat, ExportOptions};
|
||||
|
||||
futures::executor::block_on(async {
|
||||
let gdd = GddV1::create_in(empty_container(), GddV1Layout, PeerId(1), 0xEF, "ed".into(), "std".into())
|
||||
.await
|
||||
.unwrap_or_else(|error| panic!("create_in failed: {error:?}"));
|
||||
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let dest = dir.path().join("nope");
|
||||
let options = ExportOptions {
|
||||
include_registry: false,
|
||||
include_history: false,
|
||||
embed_all_resources: false,
|
||||
};
|
||||
|
||||
match gdd.export(&dest, ExportFormat::Folder, options, &empty_byte_store(), None).await {
|
||||
Err(Error::InvalidExportOptions(_)) => {}
|
||||
Ok(_) => panic!("expected InvalidOptions, got Ok"),
|
||||
Err(other) => panic!("expected InvalidOptions, got {other:?}"),
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resource_round_trip_add_read_remove() {
|
||||
use graphene_resource::{ResourceHash, ResourceId};
|
||||
|
||||
futures::executor::block_on(async {
|
||||
let mut gdd = GddV1::create_in(empty_container(), GddV1Layout, PeerId(99), 0xCAFE, "ed".into(), "std".into())
|
||||
.await
|
||||
.unwrap_or_else(|error| panic!("create_in failed: {error:?}"));
|
||||
|
||||
let payload = b"deadbeef cafe babe";
|
||||
let hash = ResourceHash::from(&payload[..]);
|
||||
let id = ResourceId::new();
|
||||
|
||||
assert!(!gdd.has_resource(&hash).await);
|
||||
gdd.add_resource(id, payload).unwrap_or_else(|error| panic!("add_resource failed: {error:?}"));
|
||||
assert!(gdd.has_resource(&hash).await);
|
||||
|
||||
let read_back = gdd.read_resource(&hash).await.unwrap();
|
||||
assert_eq!(read_back.as_slice(), payload);
|
||||
|
||||
// The registry records the resource (entry keyed by id, resolved to the content hash).
|
||||
let entry = gdd.registry().resources.get(&id).expect("registry records the added resource");
|
||||
assert_eq!(entry.hash, Some(hash));
|
||||
|
||||
let hashes = gdd.resource_hashes().await.unwrap();
|
||||
assert_eq!(hashes, vec![hash]);
|
||||
|
||||
gdd.remove_resource(&hash).unwrap();
|
||||
assert!(!gdd.has_resource(&hash).await);
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resource_survives_reopen() {
|
||||
use graphene_resource::{ResourceHash, ResourceId};
|
||||
|
||||
futures::executor::block_on(async {
|
||||
let mut gdd = GddV1::create_in(empty_container(), GddV1Layout, PeerId(7), 0xC0DE, "ed".into(), "std".into())
|
||||
.await
|
||||
.unwrap_or_else(|error| panic!("create_in failed: {error:?}"));
|
||||
|
||||
let payload = b"persistent bytes";
|
||||
let hash = ResourceHash::from(&payload[..]);
|
||||
let id = ResourceId::new();
|
||||
gdd.add_resource(id, payload).unwrap();
|
||||
|
||||
let (working, layout) = gdd.into_storage();
|
||||
let reopened = GddV1::open_in(working, layout).await.unwrap_or_else(|error| panic!("open_in failed: {error:?}"));
|
||||
|
||||
assert!(reopened.has_resource(&hash).await);
|
||||
assert_eq!(reopened.read_resource(&hash).await.unwrap().as_slice(), payload);
|
||||
|
||||
// The registry entry replicated through the history file and survives reopen.
|
||||
let entry = reopened.registry().resources.get(&id).expect("reopened registry records the resource");
|
||||
assert_eq!(entry.hash, Some(hash));
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resource_from_path_uses_fs_copy_on_folder_backend() {
|
||||
use document_container::AnyContainer;
|
||||
use document_container::backends::folder::FolderBackend;
|
||||
use graphene_resource::{ResourceHash, ResourceId};
|
||||
|
||||
futures::executor::block_on(async {
|
||||
// Need a folder-backed working copy to exercise the fs::copy path.
|
||||
let working_dir = tempfile::tempdir().unwrap();
|
||||
let working = AnyContainer::Folder(FolderBackend::create(working_dir.path()).unwrap());
|
||||
let mut gdd = GddV1::create_in(working, GddV1Layout, PeerId(1), 0xAB, "ed".into(), "std".into())
|
||||
.await
|
||||
.unwrap_or_else(|error| panic!("create_in failed: {error:?}"));
|
||||
|
||||
// Source file outside the working copy.
|
||||
let payload = b"external resource bytes";
|
||||
let src_dir = tempfile::tempdir().unwrap();
|
||||
let src_path = src_dir.path().join("blob");
|
||||
std::fs::write(&src_path, payload).unwrap();
|
||||
|
||||
let hash = ResourceHash::from(&payload[..]);
|
||||
let id = ResourceId::new();
|
||||
gdd.add_resource_from_path(id, hash, &src_path)
|
||||
.unwrap_or_else(|error| panic!("add_resource_from_path failed: {error:?}"));
|
||||
|
||||
assert!(gdd.has_resource(&hash).await);
|
||||
assert_eq!(gdd.read_resource(&hash).await.unwrap().as_slice(), payload);
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn export_carries_resources() {
|
||||
use document_format::{ExportFormat, ExportOptions};
|
||||
use graphene_resource::{ResourceHash, ResourceId};
|
||||
|
||||
futures::executor::block_on(async {
|
||||
let mut gdd = GddV1::create_in(empty_container(), GddV1Layout, PeerId(2), 0xBC, "ed".into(), "std".into())
|
||||
.await
|
||||
.unwrap_or_else(|error| panic!("create_in failed: {error:?}"));
|
||||
|
||||
let payload = b"exported resource";
|
||||
let hash = ResourceHash::from(&payload[..]);
|
||||
let id = ResourceId::new();
|
||||
gdd.add_resource(id, payload).unwrap();
|
||||
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let dest = dir.path().join("export");
|
||||
gdd.export(&dest, ExportFormat::Folder, ExportOptions::default(), &empty_byte_store(), None).await.unwrap();
|
||||
|
||||
let resource_file = dest.join("resources").join(format!("{hash}"));
|
||||
assert!(resource_file.exists(), "exported resource file should exist at {resource_file:?}");
|
||||
assert_eq!(std::fs::read(&resource_file).unwrap(), payload);
|
||||
});
|
||||
}
|
||||
|
||||
/// `embed_all_resources` makes a link-only resource self-contained: the bytes (which live only in
|
||||
/// the byte store, not the working copy) are written into the export, the exported registry's chain
|
||||
/// gains a leading `Embedded` source ahead of the original `Url`, and the export reopens with both.
|
||||
#[test]
|
||||
fn embed_all_resources_materializes_link_only_resource() {
|
||||
use document_format::{ExportFormat, ExportOptions};
|
||||
use document_graph_storage::NoMetadata;
|
||||
use graph_craft::application_io::resource::ResourceStorage;
|
||||
use graphene_resource::{DataSource, ResourceHash, ResourceId, ResourceRegistry};
|
||||
|
||||
futures::executor::block_on(async {
|
||||
let mut gdd = GddV1::create_in(empty_container(), GddV1Layout, PeerId(8), 0xF00D, "ed".into(), "std".into())
|
||||
.await
|
||||
.unwrap_or_else(|error| panic!("create_in failed: {error:?}"));
|
||||
|
||||
// A resource whose only source is a URL, resolved to a hash. The bytes live solely in the
|
||||
// byte store; the working copy never holds them.
|
||||
let payload = b"bytes behind a url";
|
||||
let hash = ResourceHash::from(&payload[..]);
|
||||
let byte_store = empty_byte_store();
|
||||
byte_store.store(payload);
|
||||
|
||||
let mut resources = ResourceRegistry::new();
|
||||
let id = ResourceId::new();
|
||||
resources.push_source_back(&id, DataSource::Url("https://example.com/r.bin".parse().unwrap()));
|
||||
resources.resolve(&id, hash);
|
||||
|
||||
gdd.commit_from_runtime(&network_referencing_resource(id), &NoMetadata, &resources, &byte_store)
|
||||
.unwrap_or_else(|error| panic!("commit_from_runtime failed: {error:?}"));
|
||||
|
||||
// The working copy holds no resource bytes (URL source, nothing embedded yet).
|
||||
assert!(!gdd.has_resource(&hash).await);
|
||||
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let dest = dir.path().join("embedded");
|
||||
gdd.export(
|
||||
&dest,
|
||||
ExportFormat::Folder,
|
||||
ExportOptions {
|
||||
embed_all_resources: true,
|
||||
..Default::default()
|
||||
},
|
||||
&byte_store,
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.unwrap_or_else(|error| panic!("export failed: {error:?}"));
|
||||
|
||||
// Bytes materialized into the export.
|
||||
let resource_file = dest.join("resources").join(format!("{hash}"));
|
||||
assert!(resource_file.exists(), "embedded resource bytes should be written to {resource_file:?}");
|
||||
assert_eq!(std::fs::read(&resource_file).unwrap(), payload);
|
||||
|
||||
// Reopen the export: the registry chain now leads with Embedded, keeping the URL as fallback,
|
||||
// and the bytes are resolvable from the export itself with no byte store.
|
||||
let reopened = GddV1::open(&dest).await.unwrap_or_else(|error| panic!("open export failed: {error:?}"));
|
||||
assert!(reopened.has_resource(&hash).await, "embedded bytes should be resolvable from the export");
|
||||
|
||||
let entry = reopened.registry().resources.get(&id).expect("resource entry survived export");
|
||||
assert_eq!(entry.hash, Some(hash));
|
||||
let embedded = serde_json::to_value(DataSource::Embedded).unwrap();
|
||||
let url = serde_json::to_value(DataSource::Url("https://example.com/r.bin".parse().unwrap())).unwrap();
|
||||
let chain: Vec<_> = entry.sources.iter().map(|(_, value)| value.source.clone()).collect();
|
||||
assert_eq!(chain, vec![embedded, url], "Embedded leads the chain, URL kept as fallback");
|
||||
});
|
||||
}
|
||||
|
||||
/// A plain export (no `embed_all_resources`) still materializes the bytes of an already-`Embedded`
|
||||
/// resource, pulling from the byte store when the working copy doesn't hold them (the editor case
|
||||
/// where bytes live in the app-global cache, not the per-document working copy).
|
||||
#[test]
|
||||
fn export_materializes_embedded_resource_from_byte_store() {
|
||||
use document_format::{ExportFormat, ExportOptions};
|
||||
use document_graph_storage::NoMetadata;
|
||||
use graph_craft::application_io::resource::ResourceStorage;
|
||||
use graphene_resource::{DataSource, ResourceHash, ResourceId, ResourceRegistry};
|
||||
|
||||
futures::executor::block_on(async {
|
||||
let mut gdd = GddV1::create_in(empty_container(), GddV1Layout, PeerId(9), 0xBEEF, "ed".into(), "std".into())
|
||||
.await
|
||||
.unwrap_or_else(|error| panic!("create_in failed: {error:?}"));
|
||||
|
||||
// An Embedded resource whose bytes live only in the byte store, not the working copy.
|
||||
let payload = b"embedded bytes in the cache";
|
||||
let hash = ResourceHash::from(&payload[..]);
|
||||
let byte_store = empty_byte_store();
|
||||
byte_store.store(payload);
|
||||
|
||||
let mut resources = ResourceRegistry::new();
|
||||
let id = ResourceId::new();
|
||||
resources.push_source_back(&id, DataSource::Embedded);
|
||||
resources.resolve(&id, hash);
|
||||
gdd.commit_from_runtime(&network_referencing_resource(id), &NoMetadata, &resources, &byte_store)
|
||||
.unwrap_or_else(|error| panic!("commit_from_runtime failed: {error:?}"));
|
||||
|
||||
assert!(!gdd.has_resource(&hash).await, "bytes should not be in the working copy");
|
||||
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let dest = dir.path().join("plain");
|
||||
// Default options: embed_all_resources is false.
|
||||
gdd.export(&dest, ExportFormat::Folder, ExportOptions::default(), &byte_store, None)
|
||||
.await
|
||||
.unwrap_or_else(|error| panic!("export failed: {error:?}"));
|
||||
|
||||
let resource_file = dest.join("resources").join(format!("{hash}"));
|
||||
assert!(resource_file.exists(), "embedded resource bytes should be pulled from the store into {resource_file:?}");
|
||||
assert_eq!(std::fs::read(&resource_file).unwrap(), payload);
|
||||
});
|
||||
}
|
||||
|
||||
/// A document with un-retired hot ops (e.g. a freshly converted document saved without an interaction
|
||||
/// boundary, or a save mid-drag) must export losslessly: the hot log travels alongside history and the
|
||||
/// reopened document reflects the staged edits. Regression for the converted-artwork "no history"
|
||||
/// export, which previously failed at `embed_resource_sources` and fell back to the legacy blob.
|
||||
#[test]
|
||||
fn export_round_trips_unretired_hot_ops() {
|
||||
use document_graph_storage::NoMetadata;
|
||||
use graph_craft::application_io::resource::ResourceStorage;
|
||||
use graphene_resource::{DataSource, ResourceId, ResourceRegistry};
|
||||
|
||||
futures::executor::block_on(async {
|
||||
let mut gdd = GddV1::create_in(empty_container(), GddV1Layout, PeerId(3), 0xF15E, "ed".into(), "std".into())
|
||||
.await
|
||||
.unwrap_or_else(|error| panic!("create_in failed: {error:?}"));
|
||||
|
||||
let payload = b"hot resource bytes";
|
||||
let hash = graphene_resource::ResourceHash::from(&payload[..]);
|
||||
let byte_store = empty_byte_store();
|
||||
byte_store.store(payload);
|
||||
|
||||
let mut resources = ResourceRegistry::new();
|
||||
let id = ResourceId::new();
|
||||
resources.push_source_back(&id, DataSource::Embedded);
|
||||
resources.resolve(&id, hash);
|
||||
|
||||
// Stage into the working copy without retiring, leaving the conversion in the hot log.
|
||||
gdd.stage_runtime_snapshot(&network_referencing_resource(id), &NoMetadata, &resources, &byte_store)
|
||||
.unwrap_or_else(|error| panic!("stage_runtime_snapshot failed: {error:?}"));
|
||||
assert!(!gdd.session().hot_log().is_empty(), "staging without retiring should leave hot ops");
|
||||
assert!(gdd.registry().resources.contains_key(&id), "working registry should reflect the staged resource");
|
||||
|
||||
// Export to an archive and reopen it from scratch: the staged (hot) state must survive.
|
||||
let archive = gdd
|
||||
.export_to_bytes(document_format::ExportFormat::Zip, document_format::ExportOptions::default(), &byte_store, None)
|
||||
.await
|
||||
.unwrap_or_else(|error| panic!("export_to_bytes failed: {error:?}"));
|
||||
|
||||
let reopened = GddV1::open_from_archive(&archive, empty_container(), GddV1Layout)
|
||||
.await
|
||||
.unwrap_or_else(|error| panic!("open_from_archive failed: {error:?}"));
|
||||
|
||||
assert!(reopened.registry().resources.contains_key(&id), "the staged resource must survive export and reopen");
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn open_in_rejects_future_format_version() {
|
||||
futures::executor::block_on(async {
|
||||
let container = empty_container();
|
||||
let layout = GddV1Layout;
|
||||
|
||||
let mut future_version = Manifest::new(0xC0DE, "ed".into(), "std".into());
|
||||
future_version.format_version = manifest::SUPPORTED_FORMAT_VERSION + 1;
|
||||
io::write_single(&container, layout.manifest_basename(), Codec::Json, &future_version).unwrap();
|
||||
|
||||
match GddV1::open_in(container, layout).await {
|
||||
Err(Error::UnsupportedVersion { .. }) => {}
|
||||
Ok(_) => panic!("expected UnsupportedVersion, got Ok"),
|
||||
Err(other) => panic!("expected UnsupportedVersion, got {other:?}"),
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn create_in_records_default_codecs_in_manifest() {
|
||||
futures::executor::block_on(async {
|
||||
let gdd = GddV1::create_in(empty_container(), GddV1Layout, PeerId(1), 0xAB, "ed".into(), "std".into())
|
||||
.await
|
||||
.unwrap_or_else(|error| panic!("create_in failed: {error:?}"));
|
||||
|
||||
let codecs = gdd.manifest().codecs;
|
||||
assert_eq!(codecs.registry, Codec::MessagePack);
|
||||
assert_eq!(codecs.history, Codec::MessagePackFrames);
|
||||
assert_eq!(codecs.hot_log, Codec::MessagePackFrames);
|
||||
assert_eq!(codecs.session, Codec::Json);
|
||||
});
|
||||
}
|
||||
|
||||
/// The `RegisterPeer` op auto-emitted on the first commit rides the hot-op pipeline through
|
||||
/// persistence and retirement, so the `peer_users` mapping survives a reopen.
|
||||
#[test]
|
||||
fn first_commit_registers_peer_and_survives_reopen() {
|
||||
use document_graph_storage::{NoMetadata, UserId};
|
||||
use graph_craft::application_io::resource::HashMapResourceStorage;
|
||||
use graph_craft::document::{DocumentNode, DocumentNodeImplementation, NodeInput, NodeNetwork};
|
||||
use graph_craft::{ProtoNodeIdentifier, concrete};
|
||||
use graphene_resource::ResourceRegistry;
|
||||
|
||||
futures::executor::block_on(async {
|
||||
let mut gdd = GddV1::create_in(empty_container(), GddV1Layout, PeerId(21), 0xAB, "ed".into(), "std".into())
|
||||
.await
|
||||
.unwrap_or_else(|error| panic!("create_in failed: {error:?}"));
|
||||
|
||||
let network = NodeNetwork {
|
||||
exports: vec![NodeInput::node(core_types::uuid::NodeId(0), 0)],
|
||||
nodes: [(
|
||||
core_types::uuid::NodeId(0),
|
||||
DocumentNode {
|
||||
inputs: vec![NodeInput::import(concrete!(u32), 0)],
|
||||
implementation: DocumentNodeImplementation::ProtoNode(ProtoNodeIdentifier::new("graphene_core::ops::identity::IdentityNode")),
|
||||
..Default::default()
|
||||
},
|
||||
)]
|
||||
.into_iter()
|
||||
.collect(),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
gdd.commit_from_runtime(&network, &NoMetadata, &ResourceRegistry::new(), &HashMapResourceStorage::new())
|
||||
.unwrap_or_else(|error| panic!("commit_from_runtime failed: {error:?}"));
|
||||
assert_eq!(gdd.registry().peer_users.get(&PeerId(21)), Some(&UserId(21)), "first commit registers the peer");
|
||||
|
||||
let (working, layout) = gdd.into_storage();
|
||||
let reopened = GddV1::open_in(working, layout).await.unwrap_or_else(|error| panic!("open_in failed: {error:?}"));
|
||||
assert_eq!(reopened.registry().peer_users.get(&PeerId(21)), Some(&UserId(21)), "registration survives reopen");
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn persist_path_writes_at_manifest_declared_codec_paths() {
|
||||
// The manifest declares the on-disk codec for each payload; the persist path must write at the
|
||||
// extension that codec implies, and reopen (which reads the codec from the manifest) must find them.
|
||||
futures::executor::block_on(async {
|
||||
use document_container::AsyncContainer;
|
||||
|
||||
let mut gdd = GddV1::create_in(empty_container(), GddV1Layout, PeerId(5), 0xDEAD, "ed".into(), "std".into())
|
||||
.await
|
||||
.unwrap_or_else(|error| panic!("create_in failed: {error:?}"));
|
||||
|
||||
let hot_op = HotOp {
|
||||
op: RegistryDelta::AddNetwork {
|
||||
id: ROOT_NETWORK,
|
||||
network: Network::default(),
|
||||
},
|
||||
timestamp: TimeStamp { counter: 1, peer: PeerId(5) },
|
||||
};
|
||||
gdd.apply_hot_op(hot_op).unwrap_or_else(|error| panic!("apply_hot_op failed: {error:?}"));
|
||||
|
||||
let (working, layout) = gdd.into_storage();
|
||||
// Defaults: hot log is MessagePackFrames (.frames), manifest is always JSON.
|
||||
assert!(working.exists(&io::path_for(layout.hot_log_basename(), Codec::MessagePackFrames)).await);
|
||||
assert!(working.exists(&io::path_for(layout.manifest_basename(), Codec::Json)).await);
|
||||
|
||||
let reopened = GddV1::open_in(working, layout).await.unwrap_or_else(|error| panic!("open_in failed: {error:?}"));
|
||||
assert!(reopened.registry().networks.contains_key(&ROOT_NETWORK));
|
||||
});
|
||||
}
|
||||
|
||||
/// Complete declaration round-trip through the byte store: committing a runtime network with a
|
||||
/// proto-node persists its `ProtoNode` content into a `ResourceStorage`, and resolving declarations
|
||||
/// back through that store reconstructs the proto-node identifier in `to_runtime`. This is the
|
||||
/// editor-shaped path (declaration bytes live in the resource store, not the Gdd container).
|
||||
#[test]
|
||||
fn declarations_round_trip_through_byte_store() {
|
||||
use document_graph_storage::NoMetadata;
|
||||
use graph_craft::application_io::resource::HashMapResourceStorage;
|
||||
use graph_craft::document::{DocumentNode, DocumentNodeImplementation, NodeInput, NodeNetwork};
|
||||
use graph_craft::{ProtoNodeIdentifier, concrete};
|
||||
use graphene_resource::ResourceRegistry;
|
||||
|
||||
const PROTO: &str = "graphene_core::ops::identity::IdentityNode";
|
||||
|
||||
futures::executor::block_on(async {
|
||||
let network = NodeNetwork {
|
||||
exports: vec![NodeInput::node(core_types::uuid::NodeId(0), 0)],
|
||||
nodes: [(
|
||||
core_types::uuid::NodeId(0),
|
||||
DocumentNode {
|
||||
inputs: vec![NodeInput::import(concrete!(u32), 0)],
|
||||
implementation: DocumentNodeImplementation::ProtoNode(ProtoNodeIdentifier::new(PROTO)),
|
||||
..Default::default()
|
||||
},
|
||||
)]
|
||||
.into_iter()
|
||||
.collect(),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let mut gdd = GddV1::create_in(empty_container(), GddV1Layout, PeerId(1), 0xAB, "ed".into(), "std".into())
|
||||
.await
|
||||
.unwrap_or_else(|error| panic!("create_in failed: {error:?}"));
|
||||
|
||||
// Commit: declaration bytes flow into the byte store, not the Gdd container.
|
||||
let byte_store = HashMapResourceStorage::new();
|
||||
gdd.commit_from_runtime(&network, &NoMetadata, &ResourceRegistry::new(), &byte_store)
|
||||
.unwrap_or_else(|error| panic!("commit_from_runtime failed: {error:?}"));
|
||||
|
||||
// Resolve declarations back through the store and convert to a runtime network.
|
||||
let declarations = gdd.declarations(&byte_store).await;
|
||||
assert_eq!(declarations.len(), 1, "expected one proto-node declaration resolved from the byte store");
|
||||
|
||||
let (converted, _entries) = gdd.registry().to_runtime_with_metadata(&declarations).unwrap_or_else(|error| panic!("to_runtime failed: {error:?}"));
|
||||
|
||||
let node = converted.nodes.values().next().expect("converted network has the node");
|
||||
match &node.implementation {
|
||||
DocumentNodeImplementation::ProtoNode(identifier) => assert_eq!(identifier.as_str(), PROTO, "proto-node identifier survived the byte-store round-trip"),
|
||||
other => panic!("expected a ProtoNode implementation, got {other:?}"),
|
||||
}
|
||||
});
|
||||
}
|
||||
Reference in New Issue
Block a user