Add manually-runnable benchmarks for runtime profiling (#2005)

* Split benches into two files

* Implement executor update bench

* Restructure benchmarks

* Unify usages of wrap network in scope

* Remove unused imports

* Fix oom bug

* Remove bounding box impl

node-graph/interpreted-executor/Cargo.toml

@@ -28,3 +28,26 @@ once_cell = { workspace = true }
 
 # Optional workspace dependencies
 serde = { workspace = true, optional = true }
+
+[dev-dependencies]
+# Workspace dependencies
+graph-craft = { workspace = true, features = ["loading"] }
+
+# Required dependencies
+criterion = { version = "0.5", features = ["html_reports"]}
+glob = "0.3"
+iai-callgrind = { version = "0.12.3"}
+
+# Benchmarks
+[[bench]]
+name = "update_executor"
+harness = false
+
+[[bench]]
+name = "run_once"
+harness = false
+
+[[bench]]
+name = "run_cached"
+harness = false
+

node-graph/interpreted-executor/benches/benchmark_util.rs

@@ -0,0 +1,23 @@
+use criterion::{measurement::Measurement, BenchmarkGroup};
+use futures::executor::block_on;
+use graph_craft::{
+	proto::ProtoNetwork,
+	util::{compile, load_from_name, DEMO_ART},
+};
+use interpreted_executor::dynamic_executor::DynamicExecutor;
+
+pub fn setup_network(name: &str) -> (DynamicExecutor, ProtoNetwork) {
+	let network = load_from_name(name);
+	let proto_network = compile(network);
+	let executor = block_on(DynamicExecutor::new(proto_network.clone())).unwrap();
+	(executor, proto_network)
+}
+
+pub fn bench_for_each_demo<M: Measurement, F>(group: &mut BenchmarkGroup<M>, f: F)
+where
+	F: Fn(&str, &mut BenchmarkGroup<M>),
+{
+	for name in DEMO_ART {
+		f(name, group);
+	}
+}

node-graph/interpreted-executor/benches/run_cached.rs

@@ -0,0 +1,20 @@
+use criterion::{criterion_group, criterion_main, Criterion};
+use graph_craft::graphene_compiler::Executor;
+use graphene_std::transform::Footprint;
+
+mod benchmark_util;
+use benchmark_util::{bench_for_each_demo, setup_network};
+
+fn subsequent_evaluations(c: &mut Criterion) {
+	let mut group = c.benchmark_group("Subsequent Evaluations");
+	let footprint = Footprint::default();
+	bench_for_each_demo(&mut group, |name, g| {
+		let (executor, _) = setup_network(name);
+		futures::executor::block_on((&executor).execute(criterion::black_box(footprint))).unwrap();
+		g.bench_function(name, |b| b.iter(|| futures::executor::block_on((&executor).execute(criterion::black_box(footprint)))));
+	});
+	group.finish();
+}
+
+criterion_group!(benches, subsequent_evaluations);
+criterion_main!(benches);

node-graph/interpreted-executor/benches/run_demo_art_criterion.rs

@@ -0,0 +1,50 @@
+use criterion::{black_box, criterion_group, criterion_main, measurement::Measurement, BenchmarkGroup, Criterion};
+use graph_craft::{
+	graphene_compiler::Executor,
+	proto::ProtoNetwork,
+	util::{compile, load_from_name, DEMO_ART},
+};
+use graphene_std::transform::Footprint;
+use interpreted_executor::dynamic_executor::DynamicExecutor;
+
+fn update_executor<M: Measurement>(name: &str, c: &mut BenchmarkGroup<M>) {
+	let network = load_from_name(name);
+	let proto_network = compile(network);
+	let empty = ProtoNetwork::default();
+
+	let executor = futures::executor::block_on(DynamicExecutor::new(empty)).unwrap();
+
+	c.bench_function(name, |b| {
+		b.iter_batched(
+			|| (executor.clone(), proto_network.clone()),
+			|(mut executor, network)| futures::executor::block_on(executor.update(black_box(network))),
+			criterion::BatchSize::SmallInput,
+		)
+	});
+}
+
+fn update_executor_demo(c: &mut Criterion) {
+	let mut g = c.benchmark_group("Update Executor");
+	for name in DEMO_ART {
+		update_executor(name, &mut g);
+	}
+}
+
+fn run_once<M: Measurement>(name: &str, c: &mut BenchmarkGroup<M>) {
+	let network = load_from_name(name);
+	let proto_network = compile(network);
+
+	let executor = futures::executor::block_on(DynamicExecutor::new(proto_network)).unwrap();
+	let footprint = Footprint::default();
+
+	c.bench_function(name, |b| b.iter(|| futures::executor::block_on((&executor).execute(footprint))));
+}
+fn run_once_demo(c: &mut Criterion) {
+	let mut g = c.benchmark_group("Run Once no render");
+	for name in DEMO_ART {
+		run_once(name, &mut g);
+	}
+}
+
+criterion_group!(benches, update_executor_demo, run_once_demo);
+criterion_main!(benches);

node-graph/interpreted-executor/benches/run_once.rs

@@ -0,0 +1,24 @@
+use criterion::{criterion_group, criterion_main, Criterion};
+use graph_craft::graphene_compiler::Executor;
+use graphene_std::transform::Footprint;
+
+mod benchmark_util;
+use benchmark_util::{bench_for_each_demo, setup_network};
+
+fn run_once(c: &mut Criterion) {
+	let mut group = c.benchmark_group("Run Once");
+	let footprint = Footprint::default();
+	bench_for_each_demo(&mut group, |name, g| {
+		g.bench_function(name, |b| {
+			b.iter_batched(
+				|| setup_network(name),
+				|(executor, _)| futures::executor::block_on((&executor).execute(criterion::black_box(footprint))),
+				criterion::BatchSize::SmallInput,
+			)
+		});
+	});
+	group.finish();
+}
+
+criterion_group!(benches, run_once);
+criterion_main!(benches);

node-graph/interpreted-executor/benches/update_executor.rs

@@ -0,0 +1,28 @@
+use criterion::{criterion_group, criterion_main, Criterion};
+use graph_craft::proto::ProtoNetwork;
+use interpreted_executor::dynamic_executor::DynamicExecutor;
+
+mod benchmark_util;
+use benchmark_util::{bench_for_each_demo, setup_network};
+
+fn update_executor(c: &mut Criterion) {
+	let mut group = c.benchmark_group("Update Executor");
+	bench_for_each_demo(&mut group, |name, g| {
+		g.bench_function(name, |b| {
+			b.iter_batched(
+				|| {
+					let (_, proto_network) = setup_network(name);
+					let empty = ProtoNetwork::default();
+					let executor = futures::executor::block_on(DynamicExecutor::new(empty)).unwrap();
+					(executor, proto_network)
+				},
+				|(mut executor, network)| futures::executor::block_on(executor.update(criterion::black_box(network))),
+				criterion::BatchSize::SmallInput,
+			)
+		});
+	});
+	group.finish();
+}
+
+criterion_group!(benches, update_executor);
+criterion_main!(benches);

node-graph/interpreted-executor/src/dynamic_executor.rs

@@ -14,6 +14,7 @@ use std::panic::UnwindSafe;
 use std::sync::Arc;
 
 /// An executor of a node graph that does not require an online compilation server, and instead uses `Box<dyn ...>`.
+#[derive(Clone)]
 pub struct DynamicExecutor {
 	output: NodeId,
 	/// Stores all of the dynamic node structs.
@@ -170,7 +171,7 @@ impl std::fmt::Display for IntrospectError {
 ///   This maps document paths to node IDs and their associated type information.
 ///
 /// A store of the dynamically typed nodes and also the source map.
-#[derive(Default)]
+#[derive(Default, Clone)]
 pub struct BorrowTree {
 	/// A hashmap of node IDs and dynamically typed nodes.
 	nodes: HashMap<NodeId, (SharedNodeContainer, Path)>,

node-graph/interpreted-executor/src/lib.rs

@@ -1,5 +1,6 @@
 pub mod dynamic_executor;
 pub mod node_registry;
+pub mod util;
 
 #[cfg(test)]
 mod tests {

node-graph/interpreted-executor/src/util.rs

@@ -0,0 +1,83 @@
+use std::sync::Arc;
+
+use graph_craft::{
+	concrete,
+	document::{value::TaggedValue, DocumentNode, DocumentNodeImplementation, NodeInput, NodeNetwork},
+	generic,
+	wasm_application_io::WasmEditorApi,
+	ProtoNodeIdentifier,
+};
+use graphene_std::{transform::Footprint, uuid::NodeId};
+
+// TODO: this is copy pasta from the editor (and does get out of sync)
+pub fn wrap_network_in_scope(mut network: NodeNetwork, editor_api: Arc<WasmEditorApi>) -> NodeNetwork {
+	network.generate_node_paths(&[]);
+
+	let inner_network = DocumentNode {
+		implementation: DocumentNodeImplementation::Network(network),
+		inputs: vec![],
+		..Default::default()
+	};
+
+	// TODO: Replace with "Output" definition?
+	// let render_node = resolve_document_node_type("Output")
+	// 	.expect("Output node type not found")
+	// 	.node_template_input_override(vec![Some(NodeInput::node(NodeId(1), 0)), Some(NodeInput::node(NodeId(0), 1))])
+	// 	.document_node;
+
+	let render_node = graph_craft::document::DocumentNode {
+		inputs: vec![NodeInput::node(NodeId(0), 0), NodeInput::node(NodeId(2), 0)],
+		implementation: graph_craft::document::DocumentNodeImplementation::Network(NodeNetwork {
+			exports: vec![NodeInput::node(NodeId(2), 0)],
+			nodes: [
+				DocumentNode {
+					inputs: vec![NodeInput::scope("editor-api")],
+					manual_composition: Some(concrete!(())),
+					implementation: DocumentNodeImplementation::ProtoNode(ProtoNodeIdentifier::new("wgpu_executor::CreateGpuSurfaceNode")),
+					skip_deduplication: true,
+					..Default::default()
+				},
+				DocumentNode {
+					manual_composition: Some(concrete!(())),
+					inputs: vec![NodeInput::node(NodeId(0), 0)],
+					implementation: DocumentNodeImplementation::ProtoNode(ProtoNodeIdentifier::new("graphene_core::memo::MemoNode")),
+					..Default::default()
+				},
+				// TODO: Add conversion step
+				DocumentNode {
+					manual_composition: Some(concrete!(graphene_std::application_io::RenderConfig)),
+					inputs: vec![
+						NodeInput::scope("editor-api"),
+						NodeInput::network(graphene_core::Type::Fn(Box::new(concrete!(Footprint)), Box::new(generic!(T))), 0),
+						NodeInput::node(NodeId(1), 0),
+					],
+					implementation: DocumentNodeImplementation::ProtoNode(ProtoNodeIdentifier::new("graphene_std::wasm_application_io::RenderNode")),
+					..Default::default()
+				},
+			]
+			.into_iter()
+			.enumerate()
+			.map(|(id, node)| (NodeId(id as u64), node))
+			.collect(),
+			..Default::default()
+		}),
+		..Default::default()
+	};
+
+	// wrap the inner network in a scope
+	let nodes = vec![
+		inner_network,
+		render_node,
+		DocumentNode {
+			implementation: DocumentNodeImplementation::proto("graphene_core::ops::IdentityNode"),
+			inputs: vec![NodeInput::value(TaggedValue::EditorApi(editor_api), false)],
+			..Default::default()
+		},
+	];
+
+	NodeNetwork {
+		exports: vec![NodeInput::node(NodeId(1), 0)],
+		nodes: nodes.into_iter().enumerate().map(|(id, node)| (NodeId(id as u64), node)).collect(),
+		scope_injections: [("editor-api".to_string(), (NodeId(2), concrete!(&WasmEditorApi)))].into_iter().collect(),
+	}
+}