Add support for opening GDD documents from the CLI (#4262)

* Wire document-format into graphene-cli

* Support loading both legacy and new graphite files in the cli

* Address PR review: propagate CLI errors and use dyn resolver

* Fix CLI network wrap + preprocessor ordering

* Cleanup preprocessor

* Remove unused import

node-graph/graph-craft/src/lib.rs

@@ -7,6 +7,7 @@ pub use core_types::{ProtoNodeIdentifier, Type, TypeDescriptor, concrete, descri
 
 pub mod application_io;
 pub mod document;
+pub use document::{DocumentNode, NodeNetwork};
 pub mod graphene_compiler;
 pub mod proto;
 #[cfg(feature = "loading")]

node-graph/graphene-cli/Cargo.toml

@@ -17,6 +17,8 @@ graphene-std = { workspace = true }
 interpreted-executor = { workspace = true }
 graph-craft = { workspace = true, features = ["loading"] }
 preprocessor = { workspace = true }
+document-format = { workspace = true, features = ["zip", "xz"] }
+document-container = { workspace = true, features = ["zip", "xz"] }
 
 # Workspace dependencies
 log = { workspace = true }

node-graph/graphene-cli/src/main.rs

@@ -1,6 +1,9 @@
 mod export;
 
 use clap::{Args, Parser, Subcommand};
+use document_container::AnyContainer;
+use document_container::backends::memory::MemoryBackend;
+use document_format::{GddV1, GddV1Layout};
 use fern::colors::{Color, ColoredLevelConfig};
 use futures::executor::block_on;
 use graph_craft::application_io::EditorPreferences;
@@ -84,6 +87,11 @@ enum Command {
 		duration: Option<f64>,
 	},
 	ListNodeIdentifiers,
+
+	/// Extract embedded legacy .graphite file from the new .gdd file
+	ExtractLegacyDoc {
+		document: PathBuf,
+	},
 }
 
 #[derive(Debug, Args)]
@@ -104,6 +112,7 @@ async fn main() -> Result<(), Box<dyn Error>> {
 	let document_path = match app.command {
 		Command::Compile { ref document, .. } => document,
 		Command::Export { ref document, .. } => document,
+		Command::ExtractLegacyDoc { ref document } => document,
 		Command::ListNodeIdentifiers => {
 			let mut nodes: Vec<_> = graphene_std::registry::NODE_METADATA.lock().unwrap().keys().cloned().collect();
 			nodes.sort_by_key(|x| x.as_str().to_string());
@@ -114,10 +123,51 @@ async fn main() -> Result<(), Box<dyn Error>> {
 		}
 	};
 
-	let document_string = std::fs::read_to_string(document_path).expect("Failed to read document");
+	// Load the document by extension: `.gdd` opens the new archive format, anything else is treated as a
+	// legacy `.graphite` document. The legacy path has no `Gdd`, so resources fall back to the default registry.
+	let is_gdd = document_path.extension().is_some_and(|extension| extension.eq_ignore_ascii_case("gdd"));
+
+	let gdd = if is_gdd {
+		let archive = std::fs::read(document_path).map_err(|error| format!("Failed to read document {}: {error}", document_path.display()))?;
+		let container = AnyContainer::Memory(MemoryBackend::new());
+		let gdd = document_format::Gdd::open_from_archive(archive.as_ref(), container, GddV1Layout)
+			.await
+			.map_err(|error| format!("Failed to open document: {error}"))?;
+		Some(gdd)
+	} else {
+		None
+	};
+
+	if let Command::ExtractLegacyDoc { ref document } = app.command {
+		let Some(gdd) = &gdd else { return Err("ExtractLegacyDoc requires a .gdd document".into()) };
+		let Some(legacy_doc) = gdd.read_legacy_document().await else {
+			return Err("gdd file did not contain a legacy .graphite document".into());
+		};
+		let mut new_path = document.clone();
+		new_path.set_extension("graphite");
+		std::fs::write(&new_path, legacy_doc).map_err(|error| format!("Failed to write .graphite file: {error}"))?;
+		eprintln!("Saved file to {}", new_path.to_string_lossy());
+		return Ok(());
+	}
+
+	// Build the runtime network: from the `.gdd` registry, or by loading a legacy `.graphite` document.
+	let node_network = match &gdd {
+		Some(gdd) => {
+			let declarations = gdd.declarations(gdd).await;
+			let (node_network, _metadata) = gdd.registry().to_runtime_with_metadata(&declarations)?;
+			node_network
+		}
+		None => {
+			let document_string = std::fs::read_to_string(document_path).map_err(|error| format!("Failed to read document {}: {error}", document_path.display()))?;
+			load_network(&document_string)
+		}
+	};
 
 	log::info!("Creating GPU context");
-	let application_io = block_on(PlatformApplicationIo::new());
+	let mut application_io = PlatformApplicationIo::new().await;
+	if let Some(gdd) = &gdd {
+		application_io.inject_resource_proxy(Box::new(gdd.resource_proxy()));
+	}
 
 	// Convert application_io to Arc first
 	let application_io_arc = Arc::new(application_io);
@@ -137,8 +187,7 @@ async fn main() -> Result<(), Box<dyn Error>> {
 		node_graph_message_sender: Box::new(UpdateLogger {}),
 		editor_preferences: Box::new(preferences),
 	});
-
-	let proto_graph = compile_graph(document_string, editor_api)?;
+	let proto_graph = compile_graph(node_network, editor_api, gdd.as_ref())?;
 
 	match app.command {
 		Command::Compile { print_proto, .. } => {
@@ -218,37 +267,22 @@ fn init_logging(log_level: u8) {
 		.unwrap();
 }
 
-// Migrations are done in the editor which is unfortunately not available here.
-// TODO: remove this and share migrations between the editor and the CLI.
-fn fix_nodes(network: &mut NodeNetwork) {
-	for node in network.nodes.values_mut() {
-		match &mut node.implementation {
-			// Recursively fix
-			DocumentNodeImplementation::Network(network) => fix_nodes(network),
-			// This replicates the migration from the editor linked:
-			// https://github.com/GraphiteEditor/Graphite/blob/d68f91ccca69e90e6d2df78d544d36cd1aaf348e/editor/src/messages/portfolio/portfolio_message_handler.rs#L535
-			// Since the CLI doesn't have the document node definitions, a less robust method of just patching the inputs is used.
-			DocumentNodeImplementation::ProtoNode(proto_node_identifier)
-				if (proto_node_identifier.as_str().starts_with("graphene_core::ConstructLayerNode") || proto_node_identifier.as_str().starts_with("graphene_core::AddArtboardNode"))
-					&& node.inputs.len() < 3 =>
-			{
-				node.inputs.push(NodeInput::Reflection(DocumentNodeMetadata::DocumentNodePath));
-			}
-			_ => {}
-		}
-	}
-}
-fn compile_graph(document_string: String, editor_api: Arc<PlatformEditorApi>) -> Result<ProtoNetwork, Box<dyn Error>> {
-	let mut network = load_network(&document_string);
-	fix_nodes(&mut network);
-
-	let mut wrapped_network = wrap_network_in_scope(network, editor_api);
-
+fn compile_graph(network: NodeNetwork, editor_api: Arc<PlatformEditorApi>, gdd: Option<&GddV1>) -> Result<ProtoNetwork, Box<dyn Error>> {
 	let preprocessor = preprocessor::Preprocessor::new();
-	preprocessor.preprocess(&mut wrapped_network, &ResourceRegistry::default()).expect("Failed to expand network"); // TODO: actually load the resources from the document
+
+	let mut network = wrap_network_in_scope(network, editor_api);
+
+	// A `.gdd` resolves resource hashes from its registry; a legacy `.graphite` has no resource store, so it
+	// preprocesses against an empty registry (matching the pre-`.gdd` CLI behavior).
+	match gdd {
+		Some(gdd) => preprocessor
+			.preprocess(&mut network, &|resource_id| gdd.registry().resources.get(&resource_id).and_then(|r| r.hash))
+			.expect("Failed to expand network"),
+		None => { preprocessor.preprocess(&mut network, &|_| None) }.expect("Failed to expand network"),
+	}
 
 	let compiler = Compiler {};
-	compiler.compile_single(wrapped_network).map_err(|x| x.into())
+	compiler.compile_single(network).map_err(|x| x.into())
 }
 
 fn create_executor(proto_network: ProtoNetwork) -> Result<DynamicExecutor, Box<dyn Error>> {

node-graph/interpreted-executor/benches/benchmark_util.rs

@@ -4,7 +4,6 @@ use futures::executor::block_on;
 use graph_craft::proto::ProtoNetwork;
 use graph_craft::util::{DEMO_ART, compile, load_from_name};
 use graphene_std::application_io::EditorApi;
-use graphene_std::application_io::resource::ResourceRegistry;
 use interpreted_executor::dynamic_executor::DynamicExecutor;
 use interpreted_executor::util::wrap_network_in_scope;
 
@@ -13,7 +12,7 @@ pub fn setup_network(name: &str) -> (DynamicExecutor, ProtoNetwork) {
 	let editor_api = std::sync::Arc::new(EditorApi::default());
 	let mut network = wrap_network_in_scope(network, editor_api);
 	let preprocessor = preprocessor::Preprocessor::new();
-	preprocessor.preprocess(&mut network, &ResourceRegistry::default()).unwrap();
+	preprocessor.preprocess(&mut network, &|_| None).unwrap();
 	let proto_network = compile(network);
 	let executor = block_on(DynamicExecutor::new(proto_network.clone())).unwrap();
 	(executor, proto_network)

node-graph/preprocessor/src/lib.rs

@@ -2,11 +2,12 @@
 extern crate log;
 
 use graph_craft::Type;
-use graph_craft::application_io::resource::{ResourceId, ResourceRegistry};
+use graph_craft::application_io::resource::ResourceId;
 use graph_craft::document::value::*;
 use graph_craft::document::*;
 use graph_craft::proto::RegistryValueSource;
 use graph_craft::{ProtoNodeIdentifier, concrete};
+use graphene_std::platform_application_io::ResourceHash;
 use graphene_std::registry::*;
 use graphene_std::*;
 use std::collections::{HashMap, HashSet};
@@ -19,9 +20,9 @@ pub struct Preprocessor {
 }
 
 impl Preprocessor {
-	pub fn preprocess(&self, network: &mut NodeNetwork, resources: &ResourceRegistry) -> Result<(), PreprocessorError> {
+	pub fn preprocess(&self, network: &mut NodeNetwork, resolve_resource: &dyn Fn(ResourceId) -> Option<ResourceHash>) -> Result<(), PreprocessorError> {
 		self.insert_inject_scopes(network);
-		self.replace_resource_inputs(network, resources)?;
+		self.replace_resource_inputs(network, resolve_resource)?;
 		self.expand_network(network);
 		Ok(())
 	}
@@ -41,13 +42,13 @@ impl Preprocessor {
 	}
 
 	/// Replace every `TaggedValue::Resource(hash)` input with a reference to a freshly inserted `resource` proto node.
-	fn replace_resource_inputs(&self, network: &mut NodeNetwork, resources: &ResourceRegistry) -> Result<(), PreprocessorError> {
+	fn replace_resource_inputs(&self, network: &mut NodeNetwork, resolve_resource: &dyn Fn(ResourceId) -> Option<ResourceHash>) -> Result<(), PreprocessorError> {
 		let mut hash_to_node_id: HashMap<graph_craft::application_io::resource::ResourceHash, NodeId> = HashMap::new();
 		let mut new_resource_nodes: Vec<(NodeId, DocumentNode)> = Vec::new();
 
 		for node in network.nodes.values_mut() {
 			if let DocumentNodeImplementation::Network(nested) = &mut node.implementation {
-				self.replace_resource_inputs(nested, resources)?;
+				self.replace_resource_inputs(nested, resolve_resource)?;
 				continue;
 			}
 
@@ -59,7 +60,7 @@ impl Preprocessor {
 				let NodeInput::Value { tagged_value, .. } = input else { continue };
 				let TaggedValue::Resource(resource_id) = **tagged_value else { continue };
 
-				let Some(hash) = resources.hash(&resource_id) else {
+				let Some(hash) = resolve_resource(resource_id) else {
 					return Err(PreprocessorError::ResourceNotFound(resource_id));
 				};