Include graph runtime benchmarks in CI perf regression runs (#2780)

* Include graph runtime benchmarks in ci regression run

* Update benchmarking workflow

* Add render slowdown

* Fix baseline cache

* Remove benchmark compilation validation run

* Include render node in runtime benchmarks

* Collapse sections without changes

* Readd rulers between the sections

* Add review suggestions

* Rulers rule

* Fix whitespace

node-graph/interpreted-executor/benches/benchmark_util.rs

@@ -3,10 +3,14 @@ use criterion::measurement::Measurement;
 use futures::executor::block_on;
 use graph_craft::proto::ProtoNetwork;
 use graph_craft::util::{DEMO_ART, compile, load_from_name};
+use graphene_std::application_io::EditorApi;
 use interpreted_executor::dynamic_executor::DynamicExecutor;
+use interpreted_executor::util::wrap_network_in_scope;
 
 pub fn setup_network(name: &str) -> (DynamicExecutor, ProtoNetwork) {
 	let network = load_from_name(name);
+	let editor_api = std::sync::Arc::new(EditorApi::default());
+	let network = wrap_network_in_scope(network, editor_api);
 	let proto_network = compile(network);
 	let executor = block_on(DynamicExecutor::new(proto_network.clone())).unwrap();
 	(executor, proto_network)

node-graph/interpreted-executor/benches/run_cached.rs

@@ -2,15 +2,15 @@ mod benchmark_util;
 
 use benchmark_util::{bench_for_each_demo, setup_network};
 use criterion::{Criterion, criterion_group, criterion_main};
-use graphene_std::Context;
+use graphene_std::application_io::RenderConfig;
 
 fn subsequent_evaluations(c: &mut Criterion) {
 	let mut group = c.benchmark_group("Subsequent Evaluations");
-	let context: Context = None;
+	let context = RenderConfig::default();
 	bench_for_each_demo(&mut group, |name, g| {
 		let (executor, _) = setup_network(name);
 		g.bench_function(name, |b| {
-			b.iter(|| futures::executor::block_on(executor.tree().eval_tagged_value(executor.output(), criterion::black_box(context.clone()))).unwrap())
+			b.iter(|| futures::executor::block_on(executor.tree().eval_tagged_value(executor.output(), criterion::black_box(context))).unwrap())
 		});
 	});
 	group.finish();

node-graph/interpreted-executor/benches/run_cached_iai.rs

@@ -1,16 +1,16 @@
-use graph_craft::util::*;
-use graphene_std::Context;
+mod benchmark_util;
+
+use benchmark_util::setup_network;
+use graphene_std::application_io::RenderConfig;
 use iai_callgrind::{black_box, library_benchmark, library_benchmark_group, main};
 use interpreted_executor::dynamic_executor::DynamicExecutor;
 
 fn setup_run_cached(name: &str) -> DynamicExecutor {
-	let network = load_from_name(name);
-	let proto_network = compile(network);
-	let executor = futures::executor::block_on(DynamicExecutor::new(proto_network)).unwrap();
+	let (executor, _) = setup_network(name);
 
 	// Warm up the cache by running once
-	let context: Context = None;
-	let _ = futures::executor::block_on(executor.tree().eval_tagged_value(executor.output(), context.clone()));
+	let context = RenderConfig::default();
+	let _ = futures::executor::block_on(executor.tree().eval_tagged_value(executor.output(), context));
 
 	executor
 }
@@ -18,7 +18,7 @@ fn setup_run_cached(name: &str) -> DynamicExecutor {
 #[library_benchmark]
 #[benches::with_setup(args = ["isometric-fountain", "painted-dreams", "procedural-string-lights", "parametric-dunescape", "red-dress", "valley-of-spires"], setup = setup_run_cached)]
 pub fn run_cached(executor: DynamicExecutor) {
-	let context: Context = None;
+	let context = RenderConfig::default();
 	black_box(futures::executor::block_on(executor.tree().eval_tagged_value(executor.output(), black_box(context))).unwrap());
 }
 

node-graph/interpreted-executor/benches/run_once.rs

@@ -2,16 +2,16 @@ mod benchmark_util;
 
 use benchmark_util::{bench_for_each_demo, setup_network};
 use criterion::{Criterion, criterion_group, criterion_main};
-use graphene_std::Context;
+use graphene_std::application_io::RenderConfig;
 
 fn run_once(c: &mut Criterion) {
 	let mut group = c.benchmark_group("Run Once");
-	let context: Context = None;
+	let context = RenderConfig::default();
 	bench_for_each_demo(&mut group, |name, g| {
 		g.bench_function(name, |b| {
 			b.iter_batched(
 				|| setup_network(name),
-				|(executor, _)| futures::executor::block_on(executor.tree().eval_tagged_value(executor.output(), criterion::black_box(context.clone()))).unwrap(),
+				|(executor, _)| futures::executor::block_on(executor.tree().eval_tagged_value(executor.output(), criterion::black_box(context))).unwrap(),
 				criterion::BatchSize::SmallInput,
 			)
 		});

node-graph/interpreted-executor/benches/run_once_iai.rs

@@ -1,18 +1,19 @@
-use graph_craft::util::*;
-use graphene_std::Context;
+mod benchmark_util;
+
+use benchmark_util::setup_network;
+use graphene_std::application_io;
 use iai_callgrind::{black_box, library_benchmark, library_benchmark_group, main};
 use interpreted_executor::dynamic_executor::DynamicExecutor;
 
 fn setup_run_once(name: &str) -> DynamicExecutor {
-	let network = load_from_name(name);
-	let proto_network = compile(network);
-	futures::executor::block_on(DynamicExecutor::new(proto_network)).unwrap()
+	let (executor, _) = setup_network(name);
+	executor
 }
 
 #[library_benchmark]
 #[benches::with_setup(args = ["isometric-fountain", "painted-dreams", "procedural-string-lights", "parametric-dunescape", "red-dress", "valley-of-spires"], setup = setup_run_once)]
 pub fn run_once(executor: DynamicExecutor) {
-	let context: Context = None;
+	let context = application_io::RenderConfig::default();
 	black_box(futures::executor::block_on(executor.tree().eval_tagged_value(executor.output(), black_box(context))).unwrap());
 }
 

node-graph/interpreted-executor/benches/update_executor_iai.rs

@@ -1,11 +1,12 @@
+mod benchmark_util;
+
+use benchmark_util::setup_network;
 use graph_craft::proto::ProtoNetwork;
-use graph_craft::util::*;
 use iai_callgrind::{black_box, library_benchmark, library_benchmark_group, main};
 use interpreted_executor::dynamic_executor::DynamicExecutor;
 
 fn setup_update_executor(name: &str) -> (DynamicExecutor, ProtoNetwork) {
-	let network = load_from_name(name);
-	let proto_network = compile(network);
+	let (_, proto_network) = setup_network(name);
 	let empty = ProtoNetwork::default();
 	let executor = futures::executor::block_on(DynamicExecutor::new(empty)).unwrap();
 	(executor, proto_network)