Add manually-runnable benchmarks for runtime profiling (#2005)

* Split benches into two files

* Implement executor update bench

* Restructure benchmarks

* Unify usages of wrap network in scope

* Remove unused imports

* Fix oom bug

* Remove bounding box impl

node-graph/graph-craft/Cargo.toml

@@ -11,8 +11,7 @@ dealloc_nodes = ["graphene-core/dealloc_nodes"]
 wgpu = []
 tokio = ["dep:tokio"]
 wayland = []
-criterion = []
-iai = []
+loading = ["serde_json", "serde"]
 
 [dependencies]
 # Local dependencies
@@ -40,6 +39,7 @@ wgpu-executor = { workspace = true }
 # Optional workspace dependencies
 serde = { workspace = true, optional = true }
 tokio = { workspace = true, optional = true }
+serde_json = { workspace = true, optional = true }
 
 # Workspace dependencies
 [target.'cfg(target_arch = "wasm32")'.dependencies]
@@ -53,19 +53,17 @@ winit = { workspace = true }
 
 [dev-dependencies]
 # Workspace dependencies
-serde_json = { workspace = true }
-graph-craft = { workspace = true, features = ["serde"] }
+graph-craft = { workspace = true, features = ["loading"] }
 
 # Required dependencies
 criterion = { version = "0.5", features = ["html_reports"]}
-glob = "0.3"
 iai-callgrind = { version = "0.12.3"}
 
 # Benchmarks
 [[bench]]
-name = "compile_demo_art"
+name = "compile_demo_art_criterion"
 harness = false
 
-# [[bench]]
-# name = "exec_demo_art"
-# harness = false
+[[bench]]
+name = "compile_demo_art_iai"
+harness = false

node-graph/graph-craft/benches/compile_demo_art.rs

@@ -1,64 +0,0 @@
-use graph_craft::document::NodeNetwork;
-#[cfg(any(feature = "criterion", feature = "iai"))]
-use graph_craft::graphene_compiler::Compiler;
-#[cfg(any(feature = "criterion", feature = "iai"))]
-use graph_craft::proto::ProtoNetwork;
-
-#[cfg(feature = "criterion")]
-use criterion::{black_box, criterion_group, criterion_main, Criterion};
-#[cfg(all(not(feature = "criterion"), feature = "iai"))]
-use iai_callgrind::{black_box, library_benchmark, library_benchmark_group, main};
-
-#[cfg(any(feature = "criterion", feature = "iai"))]
-fn load_network(document_string: &str) -> NodeNetwork {
-	let document: serde_json::Value = serde_json::from_str(document_string).expect("Failed to parse document");
-	serde_json::from_value::<NodeNetwork>(document["network_interface"]["network"].clone()).expect("Failed to parse document")
-}
-
-#[cfg(any(feature = "criterion", feature = "iai"))]
-fn compile(network: NodeNetwork) -> ProtoNetwork {
-	let compiler = Compiler {};
-	compiler.compile_single(network).unwrap()
-}
-#[cfg(all(not(feature = "criterion"), feature = "iai"))]
-fn load_from_name(name: &str) -> NodeNetwork {
-	let content = std::fs::read(&format!("../../demo-artwork/{name}.graphite")).expect("failed to read file");
-	let network = load_network(std::str::from_utf8(&content).unwrap());
-	let content = std::str::from_utf8(&content).unwrap();
-	black_box(compile(black_box(network)));
-	load_network(content)
-}
-#[cfg(feature = "criterion")]
-fn compile_to_proto(c: &mut Criterion) {
-	let artworks = glob::glob("../../demo-artwork/*.graphite").expect("failed to read glob pattern");
-	for path in artworks {
-		let Ok(path) = path else { continue };
-		let name = path.file_stem().unwrap().to_str().unwrap();
-		let content = std::fs::read(&path).expect("failed to read file");
-		let network = load_network(std::str::from_utf8(&content).unwrap());
-		c.bench_function(name, |b| b.iter_batched(|| network.clone(), |network| compile(black_box(network)), criterion::BatchSize::SmallInput));
-	}
-}
-
-#[cfg_attr(all(feature = "iai", not(feature = "criterion")), library_benchmark)]
-#[cfg_attr(all(feature = "iai", not(feature="criterion")), benches::with_setup(args = ["isometric-fountain", "painted-dreams", "procedural-string-lights", "red-dress", "valley-of-spires"], setup = load_from_name))]
-// Note that this can not be disabled with a `#[cfg(...)]` because this causes a compile error.
-// Therefore negated condition is used in `#[cfg_attr(...)]` with the attribute `cfg(any())` that is always false.
-pub fn iai_compile_to_proto(_input: NodeNetwork) {
-	#[cfg(all(feature = "iai", not(feature = "criterion")))]
-	black_box(compile(_input));
-}
-
-#[cfg(feature = "criterion")]
-criterion_group!(benches, compile_to_proto);
-#[cfg(feature = "criterion")]
-criterion_main!(benches);
-#[cfg(all(not(feature = "criterion"), feature = "iai"))]
-library_benchmark_group!(name = compile_group; benchmarks = iai_compile_to_proto);
-
-#[cfg(all(not(feature = "criterion"), feature = "iai"))]
-main!(library_benchmark_groups = compile_group);
-
-// An empty main function so the crate compiles with no features enabled.
-#[cfg(all(not(feature = "criterion"), not(feature = "iai")))]
-fn main() {}

node-graph/graph-craft/benches/compile_demo_art_criterion.rs

@@ -0,0 +1,14 @@
+use criterion::{black_box, criterion_group, criterion_main, Criterion};
+use graph_craft::util::DEMO_ART;
+fn compile_to_proto(c: &mut Criterion) {
+	use graph_craft::util::{compile, load_from_name};
+	let mut c = c.benchmark_group("Compile Network cold");
+
+	for name in DEMO_ART {
+		let network = load_from_name(name);
+		c.bench_function(name, |b| b.iter_batched(|| network.clone(), |network| compile(black_box(network)), criterion::BatchSize::SmallInput));
+	}
+}
+
+criterion_group!(benches, compile_to_proto);
+criterion_main!(benches);

node-graph/graph-craft/benches/compile_demo_art_iai.rs

@@ -0,0 +1,13 @@
+use graph_craft::document::NodeNetwork;
+use graph_craft::util::*;
+use iai_callgrind::{black_box, library_benchmark, library_benchmark_group, main};
+
+#[library_benchmark]
+#[benches::with_setup(args = ["isometric-fountain", "painted-dreams", "procedural-string-lights", "red-dress", "valley-of-spires"], setup = load_from_name)]
+pub fn compile_to_proto(_input: NodeNetwork) {
+	black_box(compile(_input));
+}
+
+library_benchmark_group!(name = compile_group; benchmarks = compile_to_proto);
+
+main!(library_benchmark_groups = compile_group);

node-graph/graph-craft/src/document.rs

@@ -26,20 +26,23 @@ fn merge_ids(a: NodeId, b: NodeId) -> NodeId {
 
 /// Utility function for providing a default boolean value to serde.
 #[inline(always)]
+#[cfg(feature = "serde")]
 fn return_true() -> bool {
 	true
 }
 
 // TODO: Eventually remove this (probably starting late 2024)
-#[derive(Debug, serde::Deserialize)]
-#[serde(untagged)]
+#[derive(Debug)]
+#[cfg_attr(feature = "serde", derive(serde::Deserialize))]
+#[cfg_attr(feature = "serde", serde(untagged))]
 enum NodeInputVersions {
 	OldNodeInput(OldNodeInput),
 	NodeInput(NodeInput),
 }
 
 // TODO: Eventually remove this (probably starting late 2024)
-#[derive(Debug, serde::Deserialize)]
+#[derive(Debug)]
+#[cfg_attr(feature = "serde", derive(serde::Deserialize))]
 pub enum OldNodeInput {
 	/// A reference to another node in the same network from which this node can receive its input.
 	Node { node_id: NodeId, output_index: usize, lambda: bool },
@@ -56,11 +59,12 @@ pub enum OldNodeInput {
 }
 
 // TODO: Eventually remove this (probably starting late 2024)
-use serde::Deserialize;
+#[cfg(feature = "serde")]
 fn deserialize_inputs<'de, D>(deserializer: D) -> Result<Vec<NodeInput>, D::Error>
 where
 	D: serde::Deserializer<'de>,
 {
+	use serde::Deserialize;
 	let input_versions = Vec::<NodeInputVersions>::deserialize(deserializer)?;
 
 	let inputs = input_versions
@@ -95,7 +99,7 @@ pub struct DocumentNode {
 	///
 	/// In the root network, it is resolved when evaluating the borrow tree.
 	/// Ensure the click target in the encapsulating network is updated when the inputs cause the node shape to change (currently only when exposing/hiding an input) by using network.update_click_target(node_id).
-	#[serde(deserialize_with = "deserialize_inputs")]
+	#[cfg_attr(feature = "serde", serde(deserialize_with = "deserialize_inputs"))]
 	pub inputs: Vec<NodeInput>,
 	/// Manual composition is a way to override the default composition flow of one node into another.
 	///
@@ -184,15 +188,15 @@ pub struct DocumentNode {
 	// A nested document network or a proto-node identifier.
 	pub implementation: DocumentNodeImplementation,
 	/// Represents the eye icon for hiding/showing the node in the graph UI. When hidden, a node gets replaced with an identity node during the graph flattening step.
-	#[serde(default = "return_true")]
+	#[cfg_attr(feature = "serde", serde(default = "return_true"))]
 	pub visible: bool,
 	/// When two different proto nodes hash to the same value (e.g. two value nodes each containing `2_u32` or two multiply nodes that have the same node IDs as input), the duplicates are removed.
 	/// See [`crate::proto::ProtoNetwork::generate_stable_node_ids`] for details.
 	/// However sometimes this is not desirable, for example in the case of a [`graphene_core::memo::MonitorNode`] that needs to be accessed outside of the graph.
-	#[serde(default)]
+	#[cfg_attr(feature = "serde", serde(default))]
 	pub skip_deduplication: bool,
 	/// The path to this node and its inputs and outputs as of when [`NodeNetwork::generate_node_paths`] was called.
-	#[serde(skip)]
+	#[cfg_attr(feature = "serde", serde(skip))]
 	pub original_location: OriginalLocation,
 }
 
@@ -480,7 +484,7 @@ pub enum OldDocumentNodeImplementation {
 	/// This describes a (document) node implemented as a proto node.
 	///
 	/// A proto node identifier which can be found in `node_registry.rs`.
-	#[serde(alias = "Unresolved")] // TODO: Eventually remove this alias (probably starting late 2024)
+	#[cfg_attr(feature = "serde", serde(alias = "Unresolved"))] // TODO: Eventually remove this alias (probably starting late 2024)
 	ProtoNode(ProtoNodeIdentifier),
 	/// The Extract variant is a tag which tells the compilation process to do something special. It invokes language-level functionality built for use by the ExtractNode to enable metaprogramming.
 	/// When the ExtractNode is compiled, it gets replaced by a value node containing a representation of the source code for the function/lambda of the document node that's fed into the ExtractNode
@@ -515,7 +519,7 @@ pub enum DocumentNodeImplementation {
 	/// This describes a (document) node implemented as a proto node.
 	///
 	/// A proto node identifier which can be found in `node_registry.rs`.
-	#[serde(alias = "Unresolved")] // TODO: Eventually remove this alias (probably starting late 2024)
+	#[cfg_attr(feature = "serde", serde(alias = "Unresolved"))] // TODO: Eventually remove this alias (probably starting late 2024)
 	ProtoNode(ProtoNodeIdentifier),
 	/// The Extract variant is a tag which tells the compilation process to do something special. It invokes language-level functionality built for use by the ExtractNode to enable metaprogramming.
 	/// When the ExtractNode is compiled, it gets replaced by a value node containing a representation of the source code for the function/lambda of the document node that's fed into the ExtractNode
@@ -573,25 +577,29 @@ impl DocumentNodeImplementation {
 }
 
 // TODO: Eventually remove this (probably starting late 2024)
-#[derive(Debug, serde::Deserialize)]
-#[serde(untagged)]
+#[derive(Debug)]
+#[cfg_attr(feature = "serde", derive(serde::Deserialize))]
+#[cfg_attr(feature = "serde", serde(untagged))]
 pub enum NodeExportVersions {
 	OldNodeInput(NodeOutput),
 	NodeInput(NodeInput),
 }
 
 // TODO: Eventually remove this (probably starting late 2024)
-#[derive(Debug, serde::Deserialize)]
+#[derive(Debug)]
+#[cfg_attr(feature = "serde", derive(serde::Deserialize))]
 pub struct NodeOutput {
 	pub node_id: NodeId,
 	pub node_output_index: usize,
 }
 
 // TODO: Eventually remove this (probably starting late 2024)
+#[cfg(feature = "serde")]
 fn deserialize_exports<'de, D>(deserializer: D) -> Result<Vec<NodeInput>, D::Error>
 where
 	D: serde::Deserializer<'de>,
 {
+	use serde::Deserialize;
 	let node_input_versions = Vec::<NodeExportVersions>::deserialize(deserializer)?;
 
 	// Convert Vec<NodeOutput> to Vec<NodeInput>
@@ -617,11 +625,11 @@ where
 pub struct OldDocumentNode {
 	/// A name chosen by the user for this instance of the node. Empty indicates no given name, in which case the node definition's name is displayed to the user in italics.
 	///  Ensure the click target in the encapsulating network is updated when this is modified by using network.update_click_target(node_id).
-	#[serde(default)]
+	#[cfg_attr(feature = "serde", serde(default))]
 	pub alias: String,
 	// TODO: Replace this name with a reference to the [`DocumentNodeDefinition`] node definition to use the name from there instead.
 	/// The name of the node definition, as originally set by [`DocumentNodeDefinition`], used to display in the UI and to display the appropriate properties.
-	#[serde(deserialize_with = "migrate_layer_to_merge")]
+	#[cfg_attr(feature = "serde", serde(deserialize_with = "migrate_layer_to_merge"))]
 	pub name: String,
 	/// The inputs to a node, which are either:
 	/// - From other nodes within this graph [`NodeInput::Node`],
@@ -630,34 +638,34 @@ pub struct OldDocumentNode {
 	///
 	/// In the root network, it is resolved when evaluating the borrow tree.
 	/// Ensure the click target in the encapsulating network is updated when the inputs cause the node shape to change (currently only when exposing/hiding an input) by using network.update_click_target(node_id).
-	#[serde(deserialize_with = "deserialize_inputs")]
+	#[cfg_attr(feature = "serde", serde(deserialize_with = "deserialize_inputs"))]
 	pub inputs: Vec<NodeInput>,
 	pub manual_composition: Option<Type>,
 	// TODO: Remove once this references its definition instead (see above TODO).
 	/// Indicates to the UI if a primary output should be drawn for this node.
 	/// True for most nodes, but the Split Channels node is an example of a node that has multiple secondary outputs but no primary output.
-	#[serde(default = "return_true")]
+	#[cfg_attr(feature = "serde", serde(default = "return_true"))]
 	pub has_primary_output: bool,
 	// A nested document network or a proto-node identifier.
 	pub implementation: OldDocumentNodeImplementation,
 	/// User chosen state for displaying this as a left-to-right node or bottom-to-top layer. Ensure the click target in the encapsulating network is updated when the node changes to a layer by using network.update_click_target(node_id).
-	#[serde(default)]
+	#[cfg_attr(feature = "serde", serde(default))]
 	pub is_layer: bool,
 	/// Represents the eye icon for hiding/showing the node in the graph UI. When hidden, a node gets replaced with an identity node during the graph flattening step.
-	#[serde(default = "return_true")]
+	#[cfg_attr(feature = "serde", serde(default = "return_true"))]
 	pub visible: bool,
 	/// Represents the lock icon for locking/unlocking the node in the graph UI. When locked, a node cannot be moved in the graph UI.
-	#[serde(default)]
+	#[cfg_attr(feature = "serde", serde(default))]
 	pub locked: bool,
 	/// Metadata about the node including its position in the graph UI. Ensure the click target in the encapsulating network is updated when the node moves by using network.update_click_target(node_id).
 	pub metadata: OldDocumentNodeMetadata,
 	/// When two different proto nodes hash to the same value (e.g. two value nodes each containing `2_u32` or two multiply nodes that have the same node IDs as input), the duplicates are removed.
 	/// See [`crate::proto::ProtoNetwork::generate_stable_node_ids`] for details.
 	/// However sometimes this is not desirable, for example in the case of a [`graphene_core::memo::MonitorNode`] that needs to be accessed outside of the graph.
-	#[serde(default)]
+	#[cfg_attr(feature = "serde", serde(default))]
 	pub skip_deduplication: bool,
 	/// The path to this node and its inputs and outputs as of when [`NodeNetwork::generate_node_paths`] was called.
-	#[serde(skip)]
+	#[cfg_attr(feature = "serde", serde(skip))]
 	pub original_location: OriginalLocation,
 }
 
@@ -696,27 +704,28 @@ pub enum OldPreviewing {
 pub struct OldNodeNetwork {
 	/// The list of data outputs that are exported from this network to the parent network.
 	/// Each export is a reference to a node within this network, paired with its output index, that is the source of the network's exported data.
-	#[serde(alias = "outputs", deserialize_with = "deserialize_exports")] // TODO: Eventually remove this alias (probably starting late 2024)
+	#[cfg_attr(feature = "serde", serde(alias = "outputs", deserialize_with = "deserialize_exports"))] // TODO: Eventually remove this alias (probably starting late 2024)
 	pub exports: Vec<NodeInput>,
 	/// The list of all nodes in this network.
-	//#[serde(serialize_with = "graphene_core::vector::serialize_hashmap", deserialize_with = "graphene_core::vector::deserialize_hashmap")]
+	//cfg_attr(feature = "serde", #[cfg_attr(feature = "serde", serde(serialize_with = "graphene_core::vector::serialize_hashmap", deserialize_with = "graphene_core::vector::deserialize_hashmap")))]
 	pub nodes: HashMap<NodeId, OldDocumentNode>,
 	/// Indicates whether the network is currently rendered with a particular node that is previewed, and if so, which connection should be restored when the preview ends.
-	#[serde(default)]
+	#[cfg_attr(feature = "serde", serde(default))]
 	pub previewing: OldPreviewing,
 	/// Temporary fields to store metadata for "Import"/"Export" UI-only nodes, eventually will be replaced with lines leading to edges
-	#[serde(default = "default_import_metadata")]
+	#[cfg_attr(feature = "serde", serde(default = "default_import_metadata"))]
 	pub imports_metadata: (NodeId, IVec2),
-	#[serde(default = "default_export_metadata")]
+	#[cfg_attr(feature = "serde", serde(default = "default_export_metadata"))]
 	pub exports_metadata: (NodeId, IVec2),
 
 	/// A network may expose nodes as constants which can by used by other nodes using a `NodeInput::Scope(key)`.
-	#[serde(default)]
-	//#[serde(serialize_with = "graphene_core::vector::serialize_hashmap", deserialize_with = "graphene_core::vector::deserialize_hashmap")]
+	#[cfg_attr(feature = "serde", serde(default))]
+	//cfg_attr(feature = "serde", #[cfg_attr(feature = "serde", serde(serialize_with = "graphene_core::vector::serialize_hashmap", deserialize_with = "graphene_core::vector::deserialize_hashmap")))]
 	pub scope_injections: HashMap<String, (NodeId, Type)>,
 }
 
 // TODO: Eventually remove this (probably starting late 2024)
+#[cfg(feature = "serde")]
 fn migrate_layer_to_merge<'de, D: serde::Deserializer<'de>>(deserializer: D) -> Result<String, D::Error> {
 	let mut s: String = serde::Deserialize::deserialize(deserializer)?;
 	if s == "Layer" {
@@ -739,16 +748,22 @@ fn default_export_metadata() -> (NodeId, IVec2) {
 pub struct NodeNetwork {
 	/// The list of data outputs that are exported from this network to the parent network.
 	/// Each export is a reference to a node within this network, paired with its output index, that is the source of the network's exported data.
-	#[serde(alias = "outputs", deserialize_with = "deserialize_exports")] // TODO: Eventually remove this alias (probably starting late 2024)
+	#[cfg_attr(feature = "serde", serde(alias = "outputs", deserialize_with = "deserialize_exports"))] // TODO: Eventually remove this alias (probably starting late 2024)
 	pub exports: Vec<NodeInput>,
 	/// TODO: Instead of storing import types in each NodeInput::Network connection, the types are stored here. This is similar to how types need to be defined for parameters when creating a function in Rust.
 	// pub import_types: Vec<Type>,
 	/// The list of all nodes in this network.
-	#[serde(serialize_with = "graphene_core::vector::serialize_hashmap", deserialize_with = "graphene_core::vector::deserialize_hashmap")]
+	#[cfg_attr(
+		feature = "serde",
+		serde(serialize_with = "graphene_core::vector::serialize_hashmap", deserialize_with = "graphene_core::vector::deserialize_hashmap")
+	)]
 	pub nodes: FxHashMap<NodeId, DocumentNode>,
 	/// A network may expose nodes as constants which can by used by other nodes using a `NodeInput::Scope(key)`.
-	#[serde(default)]
-	#[serde(serialize_with = "graphene_core::vector::serialize_hashmap", deserialize_with = "graphene_core::vector::deserialize_hashmap")]
+	#[cfg_attr(feature = "serde", serde(default))]
+	#[cfg_attr(
+		feature = "serde",
+		serde(serialize_with = "graphene_core::vector::serialize_hashmap", deserialize_with = "graphene_core::vector::deserialize_hashmap")
+	)]
 	pub scope_injections: FxHashMap<String, (NodeId, Type)>,
 }
 

node-graph/graph-craft/src/document/value.rs

@@ -30,7 +30,7 @@ macro_rules! tagged_value {
 			$( $(#[$meta] ) *$identifier( $ty ), )*
 			RenderOutput(RenderOutput),
 			SurfaceFrame(graphene_core::SurfaceFrame),
-			#[serde(skip)]
+			#[cfg_attr(feature = "serde", serde(skip))]
 			EditorApi(Arc<WasmEditorApi>)
 		}
 
@@ -117,7 +117,7 @@ tagged_value! {
 	String(String),
 	U32(u32),
 	U64(u64),
-	#[serde(alias = "F32")] // TODO: Eventually remove this alias (probably starting late 2024)
+	#[cfg_attr(feature = "serde", serde(alias = "F32"))] // TODO: Eventually remove this alias (probably starting late 2024)
 	F64(f64),
 	Bool(bool),
 	UVec2(UVec2),
@@ -139,7 +139,7 @@ tagged_value! {
 	Fill(graphene_core::vector::style::Fill),
 	Stroke(graphene_core::vector::style::Stroke),
 	F64Array4([f64; 4]),
-	#[serde(alias = "VecF32")] // TODO: Eventually remove this alias (probably starting late 2024)
+	#[cfg_attr(feature = "serde", serde(alias = "VecF32"))] // TODO: Eventually remove this alias (probably starting late 2024)
 	VecF64(Vec<f64>),
 	VecU64(Vec<u64>),
 	NodePath(Vec<NodeId>),
@@ -159,10 +159,10 @@ tagged_value! {
 	FillChoice(graphene_core::vector::style::FillChoice),
 	Gradient(graphene_core::vector::style::Gradient),
 	GradientType(graphene_core::vector::style::GradientType),
-	#[serde(alias = "GradientPositions")] // TODO: Eventually remove this alias (probably starting late 2024)
+	#[cfg_attr(feature = "serde", serde(alias = "GradientPositions"))] // TODO: Eventually remove this alias (probably starting late 2024)
 	GradientStops(graphene_core::vector::style::GradientStops),
 	OptionalColor(Option<graphene_core::raster::color::Color>),
-	#[serde(alias = "ManipulatorGroupIds")] // TODO: Eventually remove this alias (probably starting late 2024)
+	#[cfg_attr(feature = "serde", serde(alias = "ManipulatorGroupIds"))] // TODO: Eventually remove this alias (probably starting late 2024)
 	PointIds(Vec<graphene_core::vector::PointId>),
 	Font(graphene_core::text::Font),
 	BrushStrokes(Vec<graphene_core::vector::brush_stroke::BrushStroke>),

node-graph/graph-craft/src/lib.rs

@@ -12,3 +12,6 @@ pub mod graphene_compiler;
 pub mod imaginate_input;
 
 pub mod wasm_application_io;
+
+#[cfg(feature = "loading")]
+pub mod util;

node-graph/graph-craft/src/util.rs

@@ -0,0 +1,21 @@
+use crate::document::NodeNetwork;
+use crate::graphene_compiler::Compiler;
+use crate::proto::ProtoNetwork;
+
+pub fn load_network(document_string: &str) -> NodeNetwork {
+	let document: serde_json::Value = serde_json::from_str(document_string).expect("Failed to parse document");
+	serde_json::from_value::<NodeNetwork>(document["network_interface"]["network"].clone()).expect("Failed to parse document")
+}
+
+pub fn compile(network: NodeNetwork) -> ProtoNetwork {
+	let compiler = Compiler {};
+	compiler.compile_single(network).unwrap()
+}
+
+pub fn load_from_name(name: &str) -> NodeNetwork {
+	let content = std::fs::read(format!("../../demo-artwork/{name}.graphite")).expect("failed to read file");
+	let content = std::str::from_utf8(&content).unwrap();
+	load_network(content)
+}
+
+pub static DEMO_ART: [&str; 5] = ["painted-dreams", "red-dress", "valley-of-spires", "isometric-fountain", "procedural-string-lights"];

node-graph/graphene-cli/Cargo.toml

@@ -32,7 +32,7 @@ serde = { workspace = true }
 serde_json = { workspace = true }
 bezier-rs = { workspace = true }
 glam = { workspace = true }
-graph-craft = { workspace = true }
+graph-craft = { workspace = true, features = ["loading"] }
 dyn-any = { workspace = true }
 graphene-core = { workspace = true }
 futures = { workspace = true }

node-graph/graphene-cli/src/main.rs

@@ -1,16 +1,15 @@
-use graph_craft::document::value::TaggedValue;
+use graph_craft::document::*;
 use graph_craft::graphene_compiler::{Compiler, Executor};
+use graph_craft::util::load_network;
 use graph_craft::wasm_application_io::EditorPreferences;
-use graph_craft::{concrete, ProtoNodeIdentifier};
-use graph_craft::{document::*, generic};
 use graphene_core::application_io::{ApplicationIo, NodeGraphUpdateSender};
 use graphene_core::text::FontCache;
-use graphene_std::transform::Footprint;
 use graphene_std::wasm_application_io::{WasmApplicationIo, WasmEditorApi};
 use interpreted_executor::dynamic_executor::DynamicExecutor;
 
 use fern::colors::{Color, ColoredLevelConfig};
 use futures::executor::block_on;
+use interpreted_executor::util::wrap_network_in_scope;
 use std::{error::Error, sync::Arc};
 
 struct UpdateLogger {}
@@ -49,6 +48,7 @@ async fn main() -> Result<(), Box<dyn Error>> {
 		node_graph_message_sender: Box::new(UpdateLogger {}),
 		editor_preferences: Box::new(EditorPreferences::default()),
 	});
+
 	let executor = create_executor(document_string, editor_api)?;
 	let render_config = graphene_core::application_io::RenderConfig::default();
 
@@ -100,8 +100,7 @@ fn fix_nodes(network: &mut NodeNetwork) {
 }
 
 fn create_executor(document_string: String, editor_api: Arc<WasmEditorApi>) -> Result<DynamicExecutor, Box<dyn Error>> {
-	let document: serde_json::Value = serde_json::from_str(&document_string).expect("Failed to parse document");
-	let mut network = serde_json::from_value::<NodeNetwork>(document["network_interface"]["network"].clone()).expect("Failed to parse document");
+	let mut network = load_network(&document_string);
 	fix_nodes(&mut network);
 
 	let wrapped_network = wrap_network_in_scope(network.clone(), editor_api);
@@ -111,79 +110,6 @@ fn create_executor(document_string: String, editor_api: Arc<WasmEditorApi>) -> R
 	Ok(executor)
 }
 
-// TODO: this is copy pasta from the editor (and does get out of sync)
-pub fn wrap_network_in_scope(mut network: NodeNetwork, editor_api: Arc<WasmEditorApi>) -> NodeNetwork {
-	network.generate_node_paths(&[]);
-
-	let inner_network = DocumentNode {
-		implementation: DocumentNodeImplementation::Network(network),
-		inputs: vec![],
-		..Default::default()
-	};
-
-	// TODO: Replace with "Output" definition?
-	// let render_node = resolve_document_node_type("Output")
-	// 	.expect("Output node type not found")
-	// 	.node_template_input_override(vec![Some(NodeInput::node(NodeId(1), 0)), Some(NodeInput::node(NodeId(0), 1))])
-	// 	.document_node;
-
-	let render_node = graph_craft::document::DocumentNode {
-		inputs: vec![NodeInput::node(NodeId(0), 0), NodeInput::node(NodeId(2), 0)],
-		implementation: graph_craft::document::DocumentNodeImplementation::Network(NodeNetwork {
-			exports: vec![NodeInput::node(NodeId(2), 0)],
-			nodes: [
-				DocumentNode {
-					inputs: vec![NodeInput::scope("editor-api")],
-					manual_composition: Some(concrete!(())),
-					implementation: DocumentNodeImplementation::ProtoNode(ProtoNodeIdentifier::new("wgpu_executor::CreateGpuSurfaceNode")),
-					skip_deduplication: true,
-					..Default::default()
-				},
-				DocumentNode {
-					manual_composition: Some(concrete!(())),
-					inputs: vec![NodeInput::node(NodeId(0), 0)],
-					implementation: DocumentNodeImplementation::ProtoNode(ProtoNodeIdentifier::new("graphene_core::memo::MemoNode")),
-					..Default::default()
-				},
-				// TODO: Add conversion step
-				DocumentNode {
-					manual_composition: Some(concrete!(graphene_std::application_io::RenderConfig)),
-					inputs: vec![
-						NodeInput::scope("editor-api"),
-						NodeInput::network(graphene_core::Type::Fn(Box::new(concrete!(Footprint)), Box::new(generic!(T))), 0),
-						NodeInput::node(NodeId(1), 0),
-					],
-					implementation: DocumentNodeImplementation::ProtoNode(ProtoNodeIdentifier::new("graphene_std::wasm_application_io::RenderNode")),
-					..Default::default()
-				},
-			]
-			.into_iter()
-			.enumerate()
-			.map(|(id, node)| (NodeId(id as u64), node))
-			.collect(),
-			..Default::default()
-		}),
-		..Default::default()
-	};
-
-	// wrap the inner network in a scope
-	let nodes = vec![
-		inner_network,
-		render_node,
-		DocumentNode {
-			implementation: DocumentNodeImplementation::proto("graphene_core::ops::IdentityNode"),
-			inputs: vec![NodeInput::value(TaggedValue::EditorApi(editor_api), false)],
-			..Default::default()
-		},
-	];
-
-	NodeNetwork {
-		exports: vec![NodeInput::node(NodeId(1), 0)],
-		nodes: nodes.into_iter().enumerate().map(|(id, node)| (NodeId(id as u64), node)).collect(),
-		scope_injections: [("editor-api".to_string(), (NodeId(2), concrete!(&WasmEditorApi)))].into_iter().collect(),
-	}
-}
-
 // #[cfg(test)]
 // mod test {
 // 	use super::*;

node-graph/interpreted-executor/Cargo.toml

@@ -28,3 +28,26 @@ once_cell = { workspace = true }
 
 # Optional workspace dependencies
 serde = { workspace = true, optional = true }
+
+[dev-dependencies]
+# Workspace dependencies
+graph-craft = { workspace = true, features = ["loading"] }
+
+# Required dependencies
+criterion = { version = "0.5", features = ["html_reports"]}
+glob = "0.3"
+iai-callgrind = { version = "0.12.3"}
+
+# Benchmarks
+[[bench]]
+name = "update_executor"
+harness = false
+
+[[bench]]
+name = "run_once"
+harness = false
+
+[[bench]]
+name = "run_cached"
+harness = false
+

node-graph/interpreted-executor/benches/benchmark_util.rs

@@ -0,0 +1,23 @@
+use criterion::{measurement::Measurement, BenchmarkGroup};
+use futures::executor::block_on;
+use graph_craft::{
+	proto::ProtoNetwork,
+	util::{compile, load_from_name, DEMO_ART},
+};
+use interpreted_executor::dynamic_executor::DynamicExecutor;
+
+pub fn setup_network(name: &str) -> (DynamicExecutor, ProtoNetwork) {
+	let network = load_from_name(name);
+	let proto_network = compile(network);
+	let executor = block_on(DynamicExecutor::new(proto_network.clone())).unwrap();
+	(executor, proto_network)
+}
+
+pub fn bench_for_each_demo<M: Measurement, F>(group: &mut BenchmarkGroup<M>, f: F)
+where
+	F: Fn(&str, &mut BenchmarkGroup<M>),
+{
+	for name in DEMO_ART {
+		f(name, group);
+	}
+}

node-graph/interpreted-executor/benches/run_cached.rs

@@ -0,0 +1,20 @@
+use criterion::{criterion_group, criterion_main, Criterion};
+use graph_craft::graphene_compiler::Executor;
+use graphene_std::transform::Footprint;
+
+mod benchmark_util;
+use benchmark_util::{bench_for_each_demo, setup_network};
+
+fn subsequent_evaluations(c: &mut Criterion) {
+	let mut group = c.benchmark_group("Subsequent Evaluations");
+	let footprint = Footprint::default();
+	bench_for_each_demo(&mut group, |name, g| {
+		let (executor, _) = setup_network(name);
+		futures::executor::block_on((&executor).execute(criterion::black_box(footprint))).unwrap();
+		g.bench_function(name, |b| b.iter(|| futures::executor::block_on((&executor).execute(criterion::black_box(footprint)))));
+	});
+	group.finish();
+}
+
+criterion_group!(benches, subsequent_evaluations);
+criterion_main!(benches);

node-graph/interpreted-executor/benches/run_demo_art_criterion.rs

@@ -0,0 +1,50 @@
+use criterion::{black_box, criterion_group, criterion_main, measurement::Measurement, BenchmarkGroup, Criterion};
+use graph_craft::{
+	graphene_compiler::Executor,
+	proto::ProtoNetwork,
+	util::{compile, load_from_name, DEMO_ART},
+};
+use graphene_std::transform::Footprint;
+use interpreted_executor::dynamic_executor::DynamicExecutor;
+
+fn update_executor<M: Measurement>(name: &str, c: &mut BenchmarkGroup<M>) {
+	let network = load_from_name(name);
+	let proto_network = compile(network);
+	let empty = ProtoNetwork::default();
+
+	let executor = futures::executor::block_on(DynamicExecutor::new(empty)).unwrap();
+
+	c.bench_function(name, |b| {
+		b.iter_batched(
+			|| (executor.clone(), proto_network.clone()),
+			|(mut executor, network)| futures::executor::block_on(executor.update(black_box(network))),
+			criterion::BatchSize::SmallInput,
+		)
+	});
+}
+
+fn update_executor_demo(c: &mut Criterion) {
+	let mut g = c.benchmark_group("Update Executor");
+	for name in DEMO_ART {
+		update_executor(name, &mut g);
+	}
+}
+
+fn run_once<M: Measurement>(name: &str, c: &mut BenchmarkGroup<M>) {
+	let network = load_from_name(name);
+	let proto_network = compile(network);
+
+	let executor = futures::executor::block_on(DynamicExecutor::new(proto_network)).unwrap();
+	let footprint = Footprint::default();
+
+	c.bench_function(name, |b| b.iter(|| futures::executor::block_on((&executor).execute(footprint))));
+}
+fn run_once_demo(c: &mut Criterion) {
+	let mut g = c.benchmark_group("Run Once no render");
+	for name in DEMO_ART {
+		run_once(name, &mut g);
+	}
+}
+
+criterion_group!(benches, update_executor_demo, run_once_demo);
+criterion_main!(benches);

node-graph/interpreted-executor/benches/run_once.rs

@@ -0,0 +1,24 @@
+use criterion::{criterion_group, criterion_main, Criterion};
+use graph_craft::graphene_compiler::Executor;
+use graphene_std::transform::Footprint;
+
+mod benchmark_util;
+use benchmark_util::{bench_for_each_demo, setup_network};
+
+fn run_once(c: &mut Criterion) {
+	let mut group = c.benchmark_group("Run Once");
+	let footprint = Footprint::default();
+	bench_for_each_demo(&mut group, |name, g| {
+		g.bench_function(name, |b| {
+			b.iter_batched(
+				|| setup_network(name),
+				|(executor, _)| futures::executor::block_on((&executor).execute(criterion::black_box(footprint))),
+				criterion::BatchSize::SmallInput,
+			)
+		});
+	});
+	group.finish();
+}
+
+criterion_group!(benches, run_once);
+criterion_main!(benches);

node-graph/interpreted-executor/benches/update_executor.rs

@@ -0,0 +1,28 @@
+use criterion::{criterion_group, criterion_main, Criterion};
+use graph_craft::proto::ProtoNetwork;
+use interpreted_executor::dynamic_executor::DynamicExecutor;
+
+mod benchmark_util;
+use benchmark_util::{bench_for_each_demo, setup_network};
+
+fn update_executor(c: &mut Criterion) {
+	let mut group = c.benchmark_group("Update Executor");
+	bench_for_each_demo(&mut group, |name, g| {
+		g.bench_function(name, |b| {
+			b.iter_batched(
+				|| {
+					let (_, proto_network) = setup_network(name);
+					let empty = ProtoNetwork::default();
+					let executor = futures::executor::block_on(DynamicExecutor::new(empty)).unwrap();
+					(executor, proto_network)
+				},
+				|(mut executor, network)| futures::executor::block_on(executor.update(criterion::black_box(network))),
+				criterion::BatchSize::SmallInput,
+			)
+		});
+	});
+	group.finish();
+}
+
+criterion_group!(benches, update_executor);
+criterion_main!(benches);

node-graph/interpreted-executor/src/dynamic_executor.rs

@@ -14,6 +14,7 @@ use std::panic::UnwindSafe;
 use std::sync::Arc;
 
 /// An executor of a node graph that does not require an online compilation server, and instead uses `Box<dyn ...>`.
+#[derive(Clone)]
 pub struct DynamicExecutor {
 	output: NodeId,
 	/// Stores all of the dynamic node structs.
@@ -170,7 +171,7 @@ impl std::fmt::Display for IntrospectError {
 ///   This maps document paths to node IDs and their associated type information.
 ///
 /// A store of the dynamically typed nodes and also the source map.
-#[derive(Default)]
+#[derive(Default, Clone)]
 pub struct BorrowTree {
 	/// A hashmap of node IDs and dynamically typed nodes.
 	nodes: HashMap<NodeId, (SharedNodeContainer, Path)>,

node-graph/interpreted-executor/src/lib.rs

@@ -1,5 +1,6 @@
 pub mod dynamic_executor;
 pub mod node_registry;
+pub mod util;
 
 #[cfg(test)]
 mod tests {

node-graph/interpreted-executor/src/util.rs

@@ -0,0 +1,83 @@
+use std::sync::Arc;
+
+use graph_craft::{
+	concrete,
+	document::{value::TaggedValue, DocumentNode, DocumentNodeImplementation, NodeInput, NodeNetwork},
+	generic,
+	wasm_application_io::WasmEditorApi,
+	ProtoNodeIdentifier,
+};
+use graphene_std::{transform::Footprint, uuid::NodeId};
+
+// TODO: this is copy pasta from the editor (and does get out of sync)
+pub fn wrap_network_in_scope(mut network: NodeNetwork, editor_api: Arc<WasmEditorApi>) -> NodeNetwork {
+	network.generate_node_paths(&[]);
+
+	let inner_network = DocumentNode {
+		implementation: DocumentNodeImplementation::Network(network),
+		inputs: vec![],
+		..Default::default()
+	};
+
+	// TODO: Replace with "Output" definition?
+	// let render_node = resolve_document_node_type("Output")
+	// 	.expect("Output node type not found")
+	// 	.node_template_input_override(vec![Some(NodeInput::node(NodeId(1), 0)), Some(NodeInput::node(NodeId(0), 1))])
+	// 	.document_node;
+
+	let render_node = graph_craft::document::DocumentNode {
+		inputs: vec![NodeInput::node(NodeId(0), 0), NodeInput::node(NodeId(2), 0)],
+		implementation: graph_craft::document::DocumentNodeImplementation::Network(NodeNetwork {
+			exports: vec![NodeInput::node(NodeId(2), 0)],
+			nodes: [
+				DocumentNode {
+					inputs: vec![NodeInput::scope("editor-api")],
+					manual_composition: Some(concrete!(())),
+					implementation: DocumentNodeImplementation::ProtoNode(ProtoNodeIdentifier::new("wgpu_executor::CreateGpuSurfaceNode")),
+					skip_deduplication: true,
+					..Default::default()
+				},
+				DocumentNode {
+					manual_composition: Some(concrete!(())),
+					inputs: vec![NodeInput::node(NodeId(0), 0)],
+					implementation: DocumentNodeImplementation::ProtoNode(ProtoNodeIdentifier::new("graphene_core::memo::MemoNode")),
+					..Default::default()
+				},
+				// TODO: Add conversion step
+				DocumentNode {
+					manual_composition: Some(concrete!(graphene_std::application_io::RenderConfig)),
+					inputs: vec![
+						NodeInput::scope("editor-api"),
+						NodeInput::network(graphene_core::Type::Fn(Box::new(concrete!(Footprint)), Box::new(generic!(T))), 0),
+						NodeInput::node(NodeId(1), 0),
+					],
+					implementation: DocumentNodeImplementation::ProtoNode(ProtoNodeIdentifier::new("graphene_std::wasm_application_io::RenderNode")),
+					..Default::default()
+				},
+			]
+			.into_iter()
+			.enumerate()
+			.map(|(id, node)| (NodeId(id as u64), node))
+			.collect(),
+			..Default::default()
+		}),
+		..Default::default()
+	};
+
+	// wrap the inner network in a scope
+	let nodes = vec![
+		inner_network,
+		render_node,
+		DocumentNode {
+			implementation: DocumentNodeImplementation::proto("graphene_core::ops::IdentityNode"),
+			inputs: vec![NodeInput::value(TaggedValue::EditorApi(editor_api), false)],
+			..Default::default()
+		},
+	];
+
+	NodeNetwork {
+		exports: vec![NodeInput::node(NodeId(1), 0)],
+		nodes: nodes.into_iter().enumerate().map(|(id, node)| (NodeId(id as u64), node)).collect(),
+		scope_injections: [("editor-api".to_string(), (NodeId(2), concrete!(&WasmEditorApi)))].into_iter().collect(),
+	}
+}