use std::collections::HashMap; use core_types::Context; use core_types::uuid::NodeId as RuntimeNodeId; use graph_craft::concrete; use graph_craft::document::value::TaggedValue; use graph_craft::document::{DocumentNode, DocumentNodeImplementation, NodeInput as GraphCraftNodeInput, NodeNetwork}; use serde::Serialize; use crate::attr::*; use crate::metadata_source::{NoMetadata, NodeMetadataSource}; use crate::{AttributesWrite, ExportSlot, Implementation, InputSlot, Network, NetworkId, Node, NodeId, NodeInput, PeerId, ProtoNode, ROOT_NETWORK, Registry, ResourceHash, ResourceId, TimeStamp}; fn map_serialization_error(key: &str) -> impl FnOnce(serde_json::Error) -> ConversionError + '_ { move |e| ConversionError::SerializationError(format!("{key}: {e:?}")) } /// Path to a node, used to mint stable global IDs by hashing. /// /// Hashing uses blake3 truncated to 64 bits with the document's `PeerId` mixed in, so two peers /// converting runtime states that happen to share local IDs (e.g. both editors seeded the same /// UUID RNG) still produce distinct global IDs. Determinism: same `(peer, path, local_id)` always /// yields the same global ID, so a peer re-converting its own runtime state preserves IDs. #[derive(Clone, Debug, PartialEq, Eq, Serialize)] struct NodePath { path: Vec<(RuntimeNodeId, NetworkId)>, local_id: RuntimeNodeId, } impl NodePath { fn root(node_id: RuntimeNodeId) -> Self { Self { path: vec![], local_id: node_id } } fn nested(parent_path: &NodePath, parent_node_id: RuntimeNodeId, network_id: NetworkId, local_id: RuntimeNodeId) -> Self { let mut path = parent_path.path.clone(); path.push((parent_node_id, network_id)); Self { path, local_id } } fn to_global_id(&self, peer: PeerId) -> NodeId { let bytes = rmp_serde::to_vec(&(peer, self)).expect("NodePath must serialize"); let digest = blake3::hash(&bytes); let mut truncated = [0u8; 8]; truncated.copy_from_slice(&digest.as_bytes()[..8]); NodeId(u64::from_le_bytes(truncated)) } /// Stable id of the network owned by the node at this path, derived purely from the (structural) /// path and peer so it reproduces across `to_runtime` -> `from_runtime` round trips rather than /// depending on traversal order. A domain tag keeps it from colliding with this node's own /// `to_global_id`. The root network is `ROOT_NETWORK` and never goes through here. fn owned_network_id(&self, peer: PeerId) -> NetworkId { let bytes = rmp_serde::to_vec(&("network", peer, self)).expect("NodePath must serialize"); let digest = blake3::hash(&bytes); let mut truncated = [0u8; 8]; truncated.copy_from_slice(&digest.as_bytes()[..8]); NetworkId(u64::from_le_bytes(truncated)) } } #[derive(Debug, thiserror::Error)] pub enum ConversionError { #[error("Failed to serialize value: {0}")] SerializationError(String), #[error("Unsupported node implementation type")] UnsupportedImplementation, #[error("Invalid network structure: {0}")] InvalidNetwork(String), #[error("Index {0} exceeds the storage format's u32 range")] IndexOverflow(usize), } /// Graph-only conversion (no editor metadata). Use [`Registry::from_runtime_with_metadata`] for /// editor round-trips. impl TryFrom<&NodeNetwork> for Registry { type Error = ConversionError; /// Test/utility entry point: scopes IDs under `PeerId(0)`. Real editor conversions go through /// `from_runtime_with_metadata` and pass the document's actual peer. fn try_from(node_network: &NodeNetwork) -> Result { Registry::from_runtime_with_metadata(node_network, &NoMetadata, &graphene_resource::ResourceRegistry::new(), PeerId(0)) } } /// Proto-node declaration bytes extracted during conversion, keyed by content hash, for the caller /// to persist into its byte store. pub type DeclarationBytes = HashMap>; /// A `from_runtime` conversion result: the reference-only [`Registry`] plus the proto-node /// declaration *bytes* it extracted, keyed by content hash. `document-graph-storage` doesn't own a byte /// store, so the caller (the `Gdd`) persists these into its content store; the registry only holds /// the `ResourceId`/`ResourceHash` references. pub struct RuntimeConversion { pub registry: Registry, pub declaration_bytes: DeclarationBytes, /// Each network's runtime `metadata_path` mapped to its stable storage `NetworkId`, for associating /// per-network, per-peer view state (`session.json`) without re-deriving ids. pub network_ids: HashMap, NetworkId>, } impl RuntimeConversion { /// Rebuild the [`Declarations`](crate::Declarations) map (`ResourceId` → [`ProtoNode`]) from the /// extracted bytes, for callers that keep the bytes in hand instead of routing them through a /// byte store (tests, the round-trip CLI). Editor/`Gdd` paths persist the bytes and resolve via /// their byte store instead. pub fn declarations(&self) -> Result { self.declaration_bytes .iter() .map(|(hash, bytes)| { let proto = decode_declaration(bytes).map_err(|error| ConversionError::SerializationError(format!("declaration {hash}: {error}")))?; Ok((ResourceId::from_hash(hash), proto)) }) .collect() } } /// Encode a [`ProtoNode`] declaration to its content-addressed bytes: through a self-describing /// `serde_json::Value` (so serde aliases keep working and the on-disk shape stays migratable), then /// rmp-serialized (which encodes the intermediate `Value` compactly). Paired with [`decode_declaration`]. pub fn encode_declaration(proto: &ProtoNode) -> Result, String> { let value = serde_json::to_value(proto).map_err(|error| error.to_string())?; rmp_serde::to_vec(&value).map_err(|error| error.to_string()) } /// Decode a [`ProtoNode`] declaration from the bytes [`encode_declaration`] produced. pub fn decode_declaration(bytes: &[u8]) -> Result { let value: serde_json::Value = rmp_serde::from_slice(bytes).map_err(|error| error.to_string())?; serde_json::from_value(value).map_err(|error| error.to_string()) } impl Registry { /// Convenience wrapper returning only the registry (declaration bytes discarded). For callers /// that don't persist a byte store — e.g. the graph-only `TryFrom` and value-comparison tests. pub fn from_runtime_with_metadata(node_network: &NodeNetwork, metadata: &M, resources: &graphene_resource::ResourceRegistry, peer: PeerId) -> Result { Ok(Self::convert_from_runtime(node_network, metadata, resources, peer)?.registry) } /// Full conversion: returns the registry and the extracted declaration bytes for the caller to /// persist. See [`RuntimeConversion`]. pub fn convert_from_runtime( node_network: &NodeNetwork, metadata: &M, resources: &graphene_resource::ResourceRegistry, peer: PeerId, ) -> Result { let mut registry = Registry::default(); let mut ctx = ConversionContext { declaration_ids: HashMap::new(), declaration_bytes: HashMap::new(), network_ids: HashMap::new(), metadata, peer, }; convert_network(node_network, ROOT_NETWORK, None, &[], &mut registry, &mut ctx)?; // Only snapshot resources the network actually references. The runtime resource cache also keeps // resources alive across undo (so legacy redo can restore them), so it can contain orphans whose // node was removed by an undo. Snapshotting those would re-introduce an `AddResource` on the next // diff and let an undone resource resurface as a phantom edit. Declaration resources are added // separately by `convert_network` and are always referenced, so they're unaffected by this filter. let referenced = collect_referenced_resources(node_network); convert_resources(resources, &referenced, peer, &mut registry)?; Ok(RuntimeConversion { registry, declaration_bytes: ctx.declaration_bytes, network_ids: ctx.network_ids, }) } } /// Snapshot the runtime [`ResourceRegistry`](graphene_resource::ResourceRegistry) into the storage /// [`ResourceStore`](crate::ResourceStore). Each source's chain position becomes a fractional /// [`Priority`](crate::Priority) (index-as-priority preserves order); the `DataSource` body is /// stored type-erased as `serde_json::Value` so its on-disk shape can migrate freely. All /// timestamps are `ORIGIN`, since this is a bootstrap snapshot, not an edit. fn convert_resources(resources: &graphene_resource::ResourceRegistry, referenced: &std::collections::HashSet, peer: PeerId, registry: &mut Registry) -> Result<(), ConversionError> { for id in resources.ids() { if !referenced.contains(&id) { continue; } let Some(info) = resources.info(&id) else { continue }; let mut entry = crate::ResourceEntry { hash: info.hash.copied(), hash_timestamp: TimeStamp::ORIGIN, ..Default::default() }; for (position, source) in info.sources.iter().enumerate() { let key = crate::SourceKey { priority: crate::Priority::new(position as f64).expect("enumerate index is finite"), peer, }; let body = serde_json::to_value(source).map_err(|error| ConversionError::SerializationError(error.to_string()))?; entry.set_source( key, crate::SourceValue { source: body, timestamp: TimeStamp::ORIGIN, }, ); } registry.resources.insert(id, entry); } Ok(()) } /// Collect the `ResourceId`s referenced by `TaggedValue::Resource` inputs anywhere in the network /// (recursively through nested networks). These are the resources the document actually uses; the /// runtime cache may hold more (history-retained orphans) that shouldn't be snapshotted into storage. fn collect_referenced_resources(network: &NodeNetwork) -> std::collections::HashSet { let mut referenced = std::collections::HashSet::new(); collect_referenced_resources_inner(network, &mut referenced); referenced } fn collect_referenced_resources_inner(network: &NodeNetwork, referenced: &mut std::collections::HashSet) { for export in &network.exports { collect_input_resource(export, referenced); } for node in network.nodes.values() { for input in &node.inputs { collect_input_resource(input, referenced); } if let DocumentNodeImplementation::Network(nested) = &node.implementation { collect_referenced_resources_inner(nested, referenced); } } } fn collect_input_resource(input: &GraphCraftNodeInput, referenced: &mut std::collections::HashSet) { if let GraphCraftNodeInput::Value { tagged_value, .. } = input && let TaggedValue::Resource(id) = &**tagged_value { referenced.insert(*id); } } /// Register a proto-node declaration as a content-addressed resource: a single `DataSource::Embedded` /// source resolved to `hash`. The bytes themselves are persisted by the caller's byte store. fn register_declaration_resource(registry: &mut Registry, id: ResourceId, hash: ResourceHash, peer: PeerId) { registry.resources.insert(id, crate::ResourceEntry::embedded(hash, peer, TimeStamp::ORIGIN)); } struct ConversionContext<'m, M: NodeMetadataSource + ?Sized> { /// Cache from proto-node identifier to its derived `ResourceId`, so repeated proto-nodes reuse /// one id without re-serializing. (Identical content hashes to the same id anyway; this just /// skips the work.) declaration_ids: HashMap, /// Extracted declaration content keyed by hash, handed back for the caller's byte store. declaration_bytes: DeclarationBytes, /// Maps each network's runtime `metadata_path` to its stable storage `NetworkId`, so the caller can /// associate per-network, per-peer view state (in `session.json`) with networks without re-deriving ids. network_ids: HashMap, NetworkId>, metadata: &'m M, peer: PeerId, } fn convert_network( node_network: &NodeNetwork, network_id: NetworkId, parent_path: Option<&NodePath>, metadata_path: &[RuntimeNodeId], registry: &mut Registry, ctx: &mut ConversionContext<'_, M>, ) -> Result<(), ConversionError> { for (runtime_node_id, doc_node) in &node_network.nodes { let node_path = child_path(parent_path, network_id, *runtime_node_id); let global_id = node_path.to_global_id(ctx.peer); let location = NodeLocation { network_id, parent_path, metadata_path, runtime_node_id: *runtime_node_id, }; let mut node = convert_node(doc_node, location, registry, ctx)?; node.attributes.set(node::ORIGINAL_NODE_ID, serde_json::json!(runtime_node_id.0), TimeStamp::ORIGIN); registry.node_instances.insert(global_id, node); } let exports = node_network .exports .iter() .map(|export| { Ok(ExportSlot { target: Some(convert_input(export, parent_path, network_id, ctx.peer)?), timestamp: TimeStamp::ORIGIN, }) }) .collect::, ConversionError>>()?; let mut attributes = crate::Attributes::new(); write_ui_network_attributes(&mut attributes, ctx.metadata, metadata_path, TimeStamp::ORIGIN)?; write_scope_injections(&mut attributes, node_network, parent_path, network_id, ctx.peer, TimeStamp::ORIGIN)?; registry.networks.insert(network_id, Network { exports, attributes }); ctx.network_ids.insert(metadata_path.to_vec(), network_id); Ok(()) } /// Serialize a network's `scope_injections` onto its attributes as one whole-map LWW blob, remapping /// each runtime-local node reference to its stable storage global ID so the reference survives a /// round trip even if runtime IDs are later reshuffled. fn write_scope_injections( attributes: &mut crate::Attributes, node_network: &NodeNetwork, parent_path: Option<&NodePath>, network_id: NetworkId, peer: PeerId, timestamp: TimeStamp, ) -> Result<(), ConversionError> { if node_network.scope_injections.is_empty() { return Ok(()); } let stored: HashMap = node_network .scope_injections .iter() .map(|(key, (runtime_id, ty))| { let storage_id = child_path(parent_path, network_id, *runtime_id).to_global_id(peer); (key.clone(), (storage_id, ty.clone())) }) .collect(); attributes .set_serialized(network::SCOPE_INJECTIONS, &stored, timestamp) .map_err(map_serialization_error(network::SCOPE_INJECTIONS)) } fn child_path(parent_path: Option<&NodePath>, network_id: NetworkId, local_id: RuntimeNodeId) -> NodePath { match parent_path { None => NodePath::root(local_id), Some(parent) => NodePath::nested(parent, parent.local_id, network_id, local_id), } } /// Where a node sits in both the storage tree (`network_id`, `parent_path`) and the runtime tree /// (`metadata_path`, `runtime_node_id`). `metadata_path` is the chain of runtime IDs from the root /// down to (but not including) this node. struct NodeLocation<'a> { network_id: NetworkId, parent_path: Option<&'a NodePath>, metadata_path: &'a [RuntimeNodeId], runtime_node_id: RuntimeNodeId, } fn convert_node(doc_node: &DocumentNode, location: NodeLocation<'_>, registry: &mut Registry, ctx: &mut ConversionContext<'_, M>) -> Result { let NodeLocation { network_id, parent_path, metadata_path, runtime_node_id, } = location; let node_path = child_path(parent_path, network_id, runtime_node_id); let timestamp = TimeStamp::ORIGIN; let mut inputs = Vec::with_capacity(doc_node.inputs.len()); for (input_index, input) in doc_node.inputs.iter().enumerate() { let mut input_attrs = convert_input_attributes(input)?; write_ui_input_attributes(&mut input_attrs, ctx.metadata, metadata_path, runtime_node_id, input_index, timestamp)?; inputs.push(InputSlot { input: convert_input(input, parent_path, network_id, ctx.peer)?, timestamp, attributes: input_attrs, }); } // For nested networks, append this node onto the metadata path. let mut extended_path = Vec::new(); let child_metadata_path = if matches!(doc_node.implementation, DocumentNodeImplementation::Network(_)) { extended_path.extend_from_slice(metadata_path); extended_path.push(runtime_node_id); extended_path.as_slice() } else { metadata_path }; let implementation = convert_implementation(&doc_node.implementation, &node_path, child_metadata_path, registry, ctx)?; // Defaults match `DocumentNode::default()`; `to_runtime` rehydrates absent keys from the same defaults. let mut attributes = crate::Attributes::new(); attributes .set_if_not_default(node::CALL_ARGUMENT, &doc_node.call_argument, &concrete!(Context), timestamp) .map_err(map_serialization_error(node::CALL_ARGUMENT))?; attributes .set_if_not_default(node::VISIBLE, &doc_node.visible, &true, timestamp) .map_err(map_serialization_error(node::VISIBLE))?; attributes .set_if_not_default(node::SKIP_DEDUPLICATION, &doc_node.skip_deduplication, &false, timestamp) .map_err(map_serialization_error(node::SKIP_DEDUPLICATION))?; write_ui_attributes(&mut attributes, ctx.metadata, metadata_path, runtime_node_id, timestamp)?; Ok(Node { implementation, inputs, attributes, network: network_id, }) } fn write_ui_attributes( attributes: &mut crate::Attributes, metadata: &M, metadata_path: &[RuntimeNodeId], runtime_node_id: RuntimeNodeId, timestamp: TimeStamp, ) -> Result<(), ConversionError> { if let Some(position) = metadata.position(metadata_path, runtime_node_id) { attributes .set_serialized(node::ui::POSITION, &position, timestamp) .map_err(map_serialization_error(node::ui::POSITION))?; } // Bool flags are only emitted when true; absence reads as false. for (key, value) in [ (node::ui::IS_LAYER, metadata.is_layer(metadata_path, runtime_node_id)), (node::ui::LOCKED, metadata.locked(metadata_path, runtime_node_id)), (node::ui::PINNED, metadata.pinned(metadata_path, runtime_node_id)), ] { if value { attributes.set(key, serde_json::Value::Bool(true), timestamp); } } if let Some(name) = metadata.display_name(metadata_path, runtime_node_id) && !name.is_empty() { attributes.set(node::ui::DISPLAY_NAME, serde_json::Value::String(name.to_string()), timestamp); } // One whole-vec attribute; per-slot LWW would be overkill for rename-on-output. let output_names = metadata.output_names(metadata_path, runtime_node_id); if !output_names.is_empty() { attributes .set_serialized(node::ui::OUTPUT_NAMES, &output_names, timestamp) .map_err(map_serialization_error(node::ui::OUTPUT_NAMES))?; } Ok(()) } fn write_ui_network_attributes(attributes: &mut crate::Attributes, metadata: &M, network_path: &[RuntimeNodeId], timestamp: TimeStamp) -> Result<(), ConversionError> { if let Some(reference) = metadata.reference(network_path) { attributes.set(node::ui::REFERENCE, serde_json::Value::String(reference.to_string()), timestamp); } Ok(()) } /// Empty strings (the runtime's "unset" sentinel) and absent values are both skipped. /// `input_data` entries each get their own `ui::input_data::` attribute for per-key LWW. fn write_ui_input_attributes( attributes: &mut crate::Attributes, metadata: &M, metadata_path: &[RuntimeNodeId], runtime_node_id: RuntimeNodeId, input_index: usize, timestamp: TimeStamp, ) -> Result<(), ConversionError> { let non_empty_string = |key: &'static str, value: Option<&str>, attributes: &mut crate::Attributes| { if let Some(value) = value.filter(|s| !s.is_empty()) { attributes.set(key, serde_json::Value::String(value.to_string()), timestamp); } }; non_empty_string(node::input::ui::NAME, metadata.input_name(metadata_path, runtime_node_id, input_index), attributes); non_empty_string(node::input::ui::DESCRIPTION, metadata.input_description(metadata_path, runtime_node_id, input_index), attributes); non_empty_string(node::input::ui::WIDGET_OVERRIDE, metadata.widget_override(metadata_path, runtime_node_id, input_index), attributes); for (sub_key, value) in metadata.input_data(metadata_path, runtime_node_id, input_index) { attributes.set(&format!("{prefix}{sub_key}", prefix = node::input::ui::DATA_PREFIX), value, timestamp); } Ok(()) } fn convert_input(input: &GraphCraftNodeInput, parent_path: Option<&NodePath>, network_id: NetworkId, peer: PeerId) -> Result { Ok(match input { GraphCraftNodeInput::Node { node_id, output_index } => NodeInput::Node { id: child_path(parent_path, network_id, *node_id).to_global_id(peer), index: (*output_index).try_into().map_err(|_| ConversionError::IndexOverflow(*output_index))?, }, GraphCraftNodeInput::Value { tagged_value, exposed } => { let value = serde_json::to_value(&**tagged_value).map_err(|e| ConversionError::SerializationError(format!("{e:?}")))?; NodeInput::Value { value, exposed: *exposed } } GraphCraftNodeInput::Scope(s) => NodeInput::Scope(s.clone()), GraphCraftNodeInput::Import { import_index, .. } => NodeInput::Import { index: (*import_index).try_into().map_err(|_| ConversionError::IndexOverflow(*import_index))?, }, GraphCraftNodeInput::Reflection(_) => NodeInput::Reflection, // GPU-specific; not modeled in the Registry format. GraphCraftNodeInput::Inline(_) => return Err(ConversionError::UnsupportedImplementation), }) } fn convert_input_attributes(input: &GraphCraftNodeInput) -> Result { let mut attributes = crate::Attributes::new(); let timestamp = TimeStamp::ORIGIN; match input { GraphCraftNodeInput::Import { import_type, .. } => { attributes .set_serialized(node::input::IMPORT_TYPE, import_type, timestamp) .map_err(map_serialization_error(node::input::IMPORT_TYPE))?; } GraphCraftNodeInput::Reflection(metadata) => { attributes .set_serialized(node::REFLECTION_METADATA, metadata, timestamp) .map_err(map_serialization_error(node::REFLECTION_METADATA))?; } _ => {} } Ok(attributes) } fn convert_implementation( implementation: &DocumentNodeImplementation, current_node_path: &NodePath, child_metadata_path: &[RuntimeNodeId], registry: &mut Registry, ctx: &mut ConversionContext<'_, M>, ) -> Result { Ok(match implementation { DocumentNodeImplementation::ProtoNode(identifier) => { let identifier_str = identifier.as_str().to_string(); // Reuse a previously-converted proto-node's id; identical content hashes to the same id // anyway, so this only skips re-serializing. if let Some(id) = ctx.declaration_ids.get(&identifier_str) { return Ok(Implementation::ProtoNode(*id)); } let proto = ProtoNode { identifier: identifier_str.clone(), attributes: Default::default(), }; // Content-address the declaration: serialize, hash, derive a deterministic id. let bytes = encode_declaration(&proto).map_err(|error| ConversionError::SerializationError(format!("proto-node {identifier_str}: {error}")))?; let hash = ResourceHash::from(bytes.as_slice()); let id = ResourceId::from_hash(&hash); register_declaration_resource(registry, id, hash, ctx.peer); ctx.declaration_bytes.insert(hash, bytes); ctx.declaration_ids.insert(identifier_str, id); Implementation::ProtoNode(id) } DocumentNodeImplementation::Network(nested_network) => { // Stable, traversal-order-independent id derived from the owning node's path, so a // `to_runtime` -> `from_runtime` round trip reproduces the same `NetworkId` (and thus the // same node-path hashes underneath it). let nested_network_id = current_node_path.owned_network_id(ctx.peer); convert_network(nested_network, nested_network_id, Some(current_node_path), child_metadata_path, registry, ctx)?; Implementation::Network(nested_network_id) } // TODO: Support Extract in the Registry format. DocumentNodeImplementation::Extract => return Err(ConversionError::UnsupportedImplementation), }) }