Add manually-runnable benchmarks for runtime profiling (#2005)

* Split benches into two files

* Implement executor update bench

* Restructure benchmarks

* Unify usages of wrap network in scope

* Remove unused imports

* Fix oom bug

* Remove bounding box impl

node-graph/interpreted-executor/benches/benchmark_util.rs

@@ -0,0 +1,23 @@
+use criterion::{measurement::Measurement, BenchmarkGroup};
+use futures::executor::block_on;
+use graph_craft::{
+	proto::ProtoNetwork,
+	util::{compile, load_from_name, DEMO_ART},
+};
+use interpreted_executor::dynamic_executor::DynamicExecutor;
+
+pub fn setup_network(name: &str) -> (DynamicExecutor, ProtoNetwork) {
+	let network = load_from_name(name);
+	let proto_network = compile(network);
+	let executor = block_on(DynamicExecutor::new(proto_network.clone())).unwrap();
+	(executor, proto_network)
+}
+
+pub fn bench_for_each_demo<M: Measurement, F>(group: &mut BenchmarkGroup<M>, f: F)
+where
+	F: Fn(&str, &mut BenchmarkGroup<M>),
+{
+	for name in DEMO_ART {
+		f(name, group);
+	}
+}

node-graph/interpreted-executor/benches/run_cached.rs

@@ -0,0 +1,20 @@
+use criterion::{criterion_group, criterion_main, Criterion};
+use graph_craft::graphene_compiler::Executor;
+use graphene_std::transform::Footprint;
+
+mod benchmark_util;
+use benchmark_util::{bench_for_each_demo, setup_network};
+
+fn subsequent_evaluations(c: &mut Criterion) {
+	let mut group = c.benchmark_group("Subsequent Evaluations");
+	let footprint = Footprint::default();
+	bench_for_each_demo(&mut group, |name, g| {
+		let (executor, _) = setup_network(name);
+		futures::executor::block_on((&executor).execute(criterion::black_box(footprint))).unwrap();
+		g.bench_function(name, |b| b.iter(|| futures::executor::block_on((&executor).execute(criterion::black_box(footprint)))));
+	});
+	group.finish();
+}
+
+criterion_group!(benches, subsequent_evaluations);
+criterion_main!(benches);

node-graph/interpreted-executor/benches/run_demo_art_criterion.rs

@@ -0,0 +1,50 @@
+use criterion::{black_box, criterion_group, criterion_main, measurement::Measurement, BenchmarkGroup, Criterion};
+use graph_craft::{
+	graphene_compiler::Executor,
+	proto::ProtoNetwork,
+	util::{compile, load_from_name, DEMO_ART},
+};
+use graphene_std::transform::Footprint;
+use interpreted_executor::dynamic_executor::DynamicExecutor;
+
+fn update_executor<M: Measurement>(name: &str, c: &mut BenchmarkGroup<M>) {
+	let network = load_from_name(name);
+	let proto_network = compile(network);
+	let empty = ProtoNetwork::default();
+
+	let executor = futures::executor::block_on(DynamicExecutor::new(empty)).unwrap();
+
+	c.bench_function(name, |b| {
+		b.iter_batched(
+			|| (executor.clone(), proto_network.clone()),
+			|(mut executor, network)| futures::executor::block_on(executor.update(black_box(network))),
+			criterion::BatchSize::SmallInput,
+		)
+	});
+}
+
+fn update_executor_demo(c: &mut Criterion) {
+	let mut g = c.benchmark_group("Update Executor");
+	for name in DEMO_ART {
+		update_executor(name, &mut g);
+	}
+}
+
+fn run_once<M: Measurement>(name: &str, c: &mut BenchmarkGroup<M>) {
+	let network = load_from_name(name);
+	let proto_network = compile(network);
+
+	let executor = futures::executor::block_on(DynamicExecutor::new(proto_network)).unwrap();
+	let footprint = Footprint::default();
+
+	c.bench_function(name, |b| b.iter(|| futures::executor::block_on((&executor).execute(footprint))));
+}
+fn run_once_demo(c: &mut Criterion) {
+	let mut g = c.benchmark_group("Run Once no render");
+	for name in DEMO_ART {
+		run_once(name, &mut g);
+	}
+}
+
+criterion_group!(benches, update_executor_demo, run_once_demo);
+criterion_main!(benches);

node-graph/interpreted-executor/benches/run_once.rs

@@ -0,0 +1,24 @@
+use criterion::{criterion_group, criterion_main, Criterion};
+use graph_craft::graphene_compiler::Executor;
+use graphene_std::transform::Footprint;
+
+mod benchmark_util;
+use benchmark_util::{bench_for_each_demo, setup_network};
+
+fn run_once(c: &mut Criterion) {
+	let mut group = c.benchmark_group("Run Once");
+	let footprint = Footprint::default();
+	bench_for_each_demo(&mut group, |name, g| {
+		g.bench_function(name, |b| {
+			b.iter_batched(
+				|| setup_network(name),
+				|(executor, _)| futures::executor::block_on((&executor).execute(criterion::black_box(footprint))),
+				criterion::BatchSize::SmallInput,
+			)
+		});
+	});
+	group.finish();
+}
+
+criterion_group!(benches, run_once);
+criterion_main!(benches);

node-graph/interpreted-executor/benches/update_executor.rs

@@ -0,0 +1,28 @@
+use criterion::{criterion_group, criterion_main, Criterion};
+use graph_craft::proto::ProtoNetwork;
+use interpreted_executor::dynamic_executor::DynamicExecutor;
+
+mod benchmark_util;
+use benchmark_util::{bench_for_each_demo, setup_network};
+
+fn update_executor(c: &mut Criterion) {
+	let mut group = c.benchmark_group("Update Executor");
+	bench_for_each_demo(&mut group, |name, g| {
+		g.bench_function(name, |b| {
+			b.iter_batched(
+				|| {
+					let (_, proto_network) = setup_network(name);
+					let empty = ProtoNetwork::default();
+					let executor = futures::executor::block_on(DynamicExecutor::new(empty)).unwrap();
+					(executor, proto_network)
+				},
+				|(mut executor, network)| futures::executor::block_on(executor.update(criterion::black_box(network))),
+				criterion::BatchSize::SmallInput,
+			)
+		});
+	});
+	group.finish();
+}
+
+criterion_group!(benches, update_executor);
+criterion_main!(benches);