Add a profiling action to CI which comments on PRs with notable demo art performance variances (#1925)

* Add profile run to ci

* Remove cargo add

* Rename more main to master

* Don't run on master before the pr has been merged

* Fix pr comment step

* Fix comment v2

* Fix v3

* Fix missing features

* Fix string interpolation

* Only post comment on performance diff

* Fix benchmark runner

* Try adding escaping

* Remove escaped quotes

* Use proper master baseline

* Use proper master baseline

* Fix rebase regression

* Remove unused dependency

node-graph/graph-craft/Cargo.toml

@@ -11,6 +11,8 @@ dealloc_nodes = []
 wgpu = []
 tokio = ["dep:tokio"]
 wayland = []
+criterion = []
+iai = []
 
 [dependencies]
 # Local dependencies
@@ -39,24 +41,27 @@ wgpu-executor = { workspace = true }
 serde = { workspace = true, optional = true }
 tokio = { workspace = true, optional = true }
 
-[target.'cfg(target_arch = "wasm32")'.dependencies]
 # Workspace dependencies
+[target.'cfg(target_arch = "wasm32")'.dependencies]
 web-sys = { workspace = true }
 js-sys = { workspace = true }
 wasm-bindgen = { workspace = true }
 wasm-bindgen-futures = { workspace = true }
 
 [target.'cfg(not(target_arch = "wasm32"))'.dependencies]
-# Workspace dependencies
 winit = { workspace = true }
 
 [dev-dependencies]
-criterion = { version = "0.5", features = ["html_reports"] }
-glob = "0.3"
-pprof = { version = "0.13", features = ["flamegraph"] }
+# Workspace dependencies
 serde_json = { workspace = true }
 graph-craft = { workspace = true, features = ["serde"] }
 
+# Required dependencies
+criterion = { version = "0.5", features = ["html_reports"]}
+glob = "0.3"
+iai-callgrind = { version = "0.12.3"}
+
+# Benchmarks
 [[bench]]
 name = "compile_demo_art"
 harness = false

node-graph/graph-craft/benches/compile_demo_art.rs

@@ -1,28 +1,58 @@
-use criterion::{black_box, criterion_group, criterion_main, Criterion};
 use graph_craft::document::NodeNetwork;
 use graph_craft::graphene_compiler::Compiler;
 use graph_craft::proto::ProtoNetwork;
 
-pub fn compile_to_proto(c: &mut Criterion) {
-	let artworks = glob::glob("../../demo-artwork/*.graphite").expect("failed to read glob pattern");
-	for path in artworks {
-		let Ok(path) = path else { continue };
-		let content = std::fs::read(&path).expect("failed to read file");
-		let network = load_network(std::str::from_utf8(&content).unwrap());
-		let name = path.file_stem().unwrap().to_str().unwrap();
+#[cfg(feature = "criterion")]
+use criterion::{black_box, criterion_group, criterion_main, Criterion};
 
-		c.bench_function(name, |b| b.iter_batched(|| network.clone(), |network| compile(black_box(network)), criterion::BatchSize::SmallInput));
-	}
-}
+#[cfg(all(not(feature = "criterion"), feature = "iai"))]
+use iai_callgrind::{black_box, library_benchmark, library_benchmark_group, main};
 
 fn load_network(document_string: &str) -> NodeNetwork {
 	let document: serde_json::Value = serde_json::from_str(document_string).expect("Failed to parse document");
 	serde_json::from_value::<NodeNetwork>(document["network_interface"]["network"].clone()).expect("Failed to parse document")
 }
+
 fn compile(network: NodeNetwork) -> ProtoNetwork {
 	let compiler = Compiler {};
 	compiler.compile_single(network).unwrap()
 }
 
+#[cfg(all(not(feature = "criterion"), feature = "iai"))]
+fn load_from_name(name: &str) -> NodeNetwork {
+	let content = std::fs::read(&format!("../../demo-artwork/{name}.graphite")).expect("failed to read file");
+	let network = load_network(std::str::from_utf8(&content).unwrap());
+	let content = std::str::from_utf8(&content).unwrap();
+	black_box(compile(black_box(network)));
+	load_network(content)
+}
+
+#[cfg(feature = "criterion")]
+fn compile_to_proto(c: &mut Criterion) {
+	let artworks = glob::glob("../../demo-artwork/*.graphite").expect("failed to read glob pattern");
+	for path in artworks {
+		let Ok(path) = path else { continue };
+		let name = path.file_stem().unwrap().to_str().unwrap();
+		let content = std::fs::read(&path).expect("failed to read file");
+		let network = load_network(std::str::from_utf8(&content).unwrap());
+		c.bench_function(name, |b| b.iter_batched(|| network.clone(), |network| compile(black_box(network)), criterion::BatchSize::SmallInput));
+	}
+}
+
+#[cfg_attr(all(feature = "iai", not(feature = "criterion")), library_benchmark)]
+#[cfg_attr(all(feature = "iai", not(feature="criterion")), benches::with_setup(args = ["isometric-fountain", "painted-dreams", "procedural-string-lights", "red-dress", "valley-of-spires"], setup = load_from_name))]
+pub fn iai_compile_to_proto(input: NodeNetwork) {
+	black_box(compile(input));
+}
+
+#[cfg(feature = "criterion")]
 criterion_group!(benches, compile_to_proto);
+
+#[cfg(feature = "criterion")]
 criterion_main!(benches);
+
+#[cfg(all(not(feature = "criterion"), feature = "iai"))]
+library_benchmark_group!(name = compile_group; benchmarks = iai_compile_to_proto);
+
+#[cfg(all(not(feature = "criterion"), feature = "iai"))]
+main!(library_benchmark_groups = compile_group);