Incremental compilation and stable node IDs (#977)

* Generate stable node ids

* checkpoint

* Implement borrow tree

* Add eval function on borrow tree

* Refactor Node trait to fix lifetime issues

* Compiler infinite loop

* Impl compose pair

* Transition to double lifetime on trait

* Change node trait to use a generic arg for the input

* Start adapting node_macro

* Migrate more nodes to new macro

* Fix raster tests

* Port vector nodes

* Make Node trait object safe

* Fix FlatMapResultNode

* Translate most of gstd

* Fix DowncastBothNode

* Refactor node trait once again to allow for HRTB for type erased nodes

* Start working on type erased nodes

* Try getting DowncastBothNode to work

* Introduce Upcasting node + work on BorrowTree

* Make enough 'static to get the code to compile

* Transition DynamicExecutor to use borrow tree

* Make Compose Node use HRTB's

* Fix MapResultNode

* Disable blur test

* Add workaround for Composing type erased nodes

* Convert more nodes in the node_registry

* Convert more of the node_registry

* Add update tree fn and hook up to frontend

* Fix blur node

* Implement CacheNode

* Make frontend use graph compiler

* Fix document_node_types type declaration for most nodes

* Remove unused imports

* Move comment down

* Reuse nodes via borrow tree

* Deprecate trait based value in favor of TaggedValue

* Remove unsafe code in buffer creation

* Fix blur node

* Fix stable node id generation

* Fix types for Image adjustment document nodes

* Fix Imaginate Node

* Remove unused imports

* Remove log

* Fix off by one error

* Remove macro generated imaginate node entry

* Create parameterized add node

* Fix test case

* Remove link from layer_panel.rs

* Fix formatting

node-graph/graph-craft/src/document.rs

@@ -68,7 +68,7 @@ impl DocumentNode {
 			let (input, mut args) = match first {
 				NodeInput::Value { tagged_value, .. } => {
 					assert_eq!(self.inputs.len(), 0);
-					(ProtoNodeInput::None, ConstructionArgs::Value(tagged_value.to_value()))
+					(ProtoNodeInput::None, ConstructionArgs::Value(tagged_value))
 				}
 				NodeInput::Node(id) => (ProtoNodeInput::Node(id), ConstructionArgs::Nodes(vec![])),
 				NodeInput::Network => (ProtoNodeInput::Network, ConstructionArgs::Nodes(vec![])),
@@ -266,7 +266,7 @@ impl NodeNetwork {
 							) {
 								"Value".to_string()
 							} else {
-								format!("Value: {:?}", tagged_value.clone().to_value())
+								format!("Value: {:?}", tagged_value)
 							};
 							let new_id = map_ids(id, gen_id());
 							let value_node = DocumentNode {
@@ -409,7 +409,6 @@ impl NodeNetwork {
 mod test {
 	use super::*;
 	use crate::proto::{ConstructionArgs, NodeIdentifier, ProtoNetwork, ProtoNode, ProtoNodeInput};
-	use value::IntoValue;
 
 	fn gen_node_id() -> NodeId {
 		static mut NODE_ID: NodeId = 3;
@@ -563,7 +562,7 @@ mod test {
 						construction_args: ConstructionArgs::Nodes(vec![]),
 					},
 				),
-				(14, ProtoNode::value(ConstructionArgs::Value(2_u32.into_any()))),
+				(14, ProtoNode::value(ConstructionArgs::Value(TaggedValue::U32(2)))),
 			]
 			.into_iter()
 			.collect(),
@@ -602,7 +601,7 @@ mod test {
 				(
 					14,
 					DocumentNode {
-						name: "Value: 2".into(),
+						name: "Value: U32(2)".into(),
 						inputs: vec![NodeInput::Value {
 							tagged_value: value::TaggedValue::U32(2),
 							exposed: false,

node-graph/graph-craft/src/document/value.rs

@@ -2,8 +2,11 @@ pub use dyn_any::StaticType;
 use dyn_any::{DynAny, Upcast};
 use dyn_clone::DynClone;
 pub use glam::DVec2;
+use graphene_core::Node;
+use std::hash::Hash;
 pub use std::sync::Arc;
 
+use crate::executor::Any;
 pub use crate::imaginate_input::{ImaginateMaskStartingFill, ImaginateSamplingMethod, ImaginateStatus};
 
 /// A type that is known, allowing serialization (serde::Deserialize is not object safe)
@@ -29,9 +32,83 @@ pub enum TaggedValue {
 	LayerPath(Option<Vec<u64>>),
 }
 
-impl TaggedValue {
+#[allow(clippy::derive_hash_xor_eq)]
+impl Hash for TaggedValue {
+	fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
+		match self {
+			Self::None => 0.hash(state),
+			Self::String(s) => {
+				1.hash(state);
+				s.hash(state)
+			}
+			Self::U32(u) => {
+				2.hash(state);
+				u.hash(state)
+			}
+			Self::F32(f) => {
+				3.hash(state);
+				f.to_bits().hash(state)
+			}
+			Self::F64(f) => {
+				4.hash(state);
+				f.to_bits().hash(state)
+			}
+			Self::Bool(b) => {
+				5.hash(state);
+				b.hash(state)
+			}
+			Self::DVec2(v) => {
+				6.hash(state);
+				v.to_array().iter().for_each(|x| x.to_bits().hash(state))
+			}
+			Self::OptionalDVec2(None) => 7.hash(state),
+			Self::OptionalDVec2(Some(v)) => {
+				8.hash(state);
+				Self::DVec2(*v).hash(state)
+			}
+			Self::Image(i) => {
+				9.hash(state);
+				i.hash(state)
+			}
+			Self::RcImage(i) => {
+				10.hash(state);
+				i.hash(state)
+			}
+			Self::Color(c) => {
+				11.hash(state);
+				c.hash(state)
+			}
+			Self::Subpath(s) => {
+				12.hash(state);
+				s.hash(state)
+			}
+			Self::RcSubpath(s) => {
+				13.hash(state);
+				s.hash(state)
+			}
+			Self::ImaginateSamplingMethod(m) => {
+				14.hash(state);
+				m.hash(state)
+			}
+			Self::ImaginateMaskStartingFill(f) => {
+				15.hash(state);
+				f.hash(state)
+			}
+			Self::ImaginateStatus(s) => {
+				16.hash(state);
+				s.hash(state)
+			}
+			Self::LayerPath(p) => {
+				17.hash(state);
+				p.hash(state)
+			}
+		}
+	}
+}
+
+impl<'a> TaggedValue {
 	/// Converts to a Box<dyn DynAny> - this isn't very neat but I'm not sure of a better approach
-	pub fn to_value(self) -> Value {
+	pub fn to_any(self) -> Any<'a> {
 		match self {
 			TaggedValue::None => Box::new(()),
 			TaggedValue::String(x) => Box::new(x),
@@ -54,19 +131,35 @@ impl TaggedValue {
 	}
 }
 
-pub type Value = Box<dyn ValueTrait>;
+pub struct UpcastNode {
+	value: TaggedValue,
+}
+impl<'input> Node<'input, Box<dyn DynAny<'input> + 'input>> for UpcastNode {
+	type Output = Box<dyn DynAny<'input> + 'input>;
 
-pub trait ValueTrait: DynAny<'static> + Upcast<dyn DynAny<'static>> + std::fmt::Debug + DynClone {}
+	fn eval<'s: 'input>(&'s self, _: Box<dyn DynAny<'input> + 'input>) -> Self::Output {
+		self.value.clone().to_any()
+	}
+}
+impl UpcastNode {
+	pub fn new(value: TaggedValue) -> Self {
+		Self { value }
+	}
+}
 
-pub trait IntoValue: Sized + ValueTrait + 'static {
-	fn into_any(self) -> Value {
+pub type Value<'a> = Box<dyn for<'i> ValueTrait<'i> + 'a>;
+
+pub trait ValueTrait<'a>: DynAny<'a> + Upcast<dyn DynAny<'a> + 'a> + std::fmt::Debug + DynClone + Sync + Send + 'a {}
+
+pub trait IntoValue<'a>: Sized + for<'i> ValueTrait<'i> + 'a {
+	fn into_any(self) -> Value<'a> {
 		Box::new(self)
 	}
 }
 
-impl<T: 'static + StaticType + Upcast<dyn DynAny<'static>> + std::fmt::Debug + PartialEq + Clone> ValueTrait for T {}
+impl<'a, T: 'a + StaticType + Upcast<dyn DynAny<'a> + 'a> + std::fmt::Debug + PartialEq + Clone + Sync + Send + 'a> ValueTrait<'a> for T {}
 
-impl<T: 'static + ValueTrait> IntoValue for T {}
+impl<'a, T: for<'i> ValueTrait<'i> + 'a> IntoValue<'a> for T {}
 
 #[repr(C)]
 pub(crate) struct Vtable {
@@ -81,7 +174,7 @@ pub(crate) struct TraitObject {
 	pub(crate) vtable: &'static Vtable,
 }
 
-impl PartialEq for Box<dyn ValueTrait> {
+impl<'a> PartialEq for Box<dyn for<'i> ValueTrait<'i> + 'a> {
 	#[cfg_attr(miri, ignore)]
 	fn eq(&self, other: &Self) -> bool {
 		if self.type_id() != other.type_id() {
@@ -96,7 +189,16 @@ impl PartialEq for Box<dyn ValueTrait> {
 	}
 }
 
-impl Clone for Value {
+impl<'a> Hash for Value<'a> {
+	fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
+		let self_trait_object = unsafe { std::mem::transmute::<&dyn ValueTrait, TraitObject>(self.as_ref()) };
+		let size = self_trait_object.vtable.size;
+		let self_mem = unsafe { std::slice::from_raw_parts(self_trait_object.self_ptr, size) };
+		self_mem.hash(state);
+	}
+}
+
+impl<'a> Clone for Value<'a> {
 	fn clone(&self) -> Self {
 		let self_trait_object = unsafe { std::mem::transmute::<&dyn ValueTrait, TraitObject>(self.as_ref()) };
 		let size = self_trait_object.vtable.size;
@@ -116,6 +218,7 @@ mod test {
 	use super::*;
 
 	#[test]
+	#[cfg_attr(miri, ignore)]
 	fn test_any_src() {
 		assert!(2_u32.into_any() == 2_u32.into_any());
 		assert!(2_u32.into_any() != 3_u32.into_any());

node-graph/graph-craft/src/executor.rs

@@ -9,10 +9,10 @@ pub struct Compiler {}
 
 impl Compiler {
 	pub fn compile(&self, mut network: NodeNetwork, resolve_inputs: bool) -> ProtoNetwork {
-		let node_count = network.nodes.len();
+		let node_ids = network.nodes.keys().copied().collect::<Vec<_>>();
 		println!("flattening");
-		for id in 0..node_count {
-			network.flatten(id as u64);
+		for id in node_ids {
+			network.flatten(id);
 		}
 		let mut proto_network = network.into_proto_network();
 		if resolve_inputs {
@@ -21,11 +21,12 @@ impl Compiler {
 		}
 		println!("reordering ids");
 		proto_network.reorder_ids();
+		proto_network.generate_stable_node_ids();
 		proto_network
 	}
 }
 pub type Any<'a> = Box<dyn DynAny<'a> + 'a>;
 
 pub trait Executor {
-	fn execute(&self, input: Any<'static>) -> Result<Any<'static>, Box<dyn Error>>;
+	fn execute<'a, 's: 'a>(&'s self, input: Any<'a>) -> Result<Any<'a>, Box<dyn Error>>;
 }

node-graph/graph-craft/src/imaginate_input.rs

@@ -14,6 +14,23 @@ pub enum ImaginateStatus {
 	Terminated,
 }
 
+#[allow(clippy::derive_hash_xor_eq)]
+impl core::hash::Hash for ImaginateStatus {
+	fn hash<H: core::hash::Hasher>(&self, state: &mut H) {
+		match self {
+			Self::Idle => 0.hash(state),
+			Self::Beginning => 1.hash(state),
+			Self::Uploading(f) => {
+				2.hash(state);
+				f.to_bits().hash(state);
+			}
+			Self::Generating => 3.hash(state),
+			Self::Terminating => 4.hash(state),
+			Self::Terminated => 5.hash(state),
+		}
+	}
+}
+
 #[derive(Debug, Clone, PartialEq, specta::Type)]
 #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
 pub struct ImaginateBaseImage {
@@ -31,7 +48,7 @@ pub struct ImaginateMaskImage {
 }
 
 #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
-#[derive(Debug, Default, Clone, Copy, Eq, PartialEq, specta::Type)]
+#[derive(Debug, Default, Clone, Copy, Eq, PartialEq, specta::Type, Hash)]
 pub enum ImaginateMaskPaintMode {
 	#[default]
 	Inpaint,
@@ -39,7 +56,7 @@ pub enum ImaginateMaskPaintMode {
 }
 
 #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
-#[derive(Debug, Default, Clone, Copy, Eq, PartialEq, DynAny, specta::Type)]
+#[derive(Debug, Default, Clone, Copy, Eq, PartialEq, DynAny, specta::Type, Hash)]
 pub enum ImaginateMaskStartingFill {
 	#[default]
 	Fill,
@@ -70,7 +87,7 @@ impl std::fmt::Display for ImaginateMaskStartingFill {
 	}
 }
 
-#[derive(Debug, Default, Copy, Clone, Eq, PartialEq, DynAny, specta::Type)]
+#[derive(Debug, Default, Copy, Clone, Eq, PartialEq, DynAny, specta::Type, Hash)]
 #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
 pub enum ImaginateSamplingMethod {
 	#[default]

node-graph/graph-craft/src/proto.rs

@@ -1,5 +1,6 @@
 use std::borrow::Cow;
 use std::collections::{HashMap, HashSet};
+use std::hash::Hash;
 
 use crate::document::value;
 use crate::document::NodeId;
@@ -87,6 +88,7 @@ impl NodeIdentifier {
 
 #[derive(Debug, Default, PartialEq)]
 pub struct ProtoNetwork {
+	// Should a proto Network even allow inputs? Don't think so
 	pub inputs: Vec<NodeId>,
 	pub output: NodeId,
 	pub nodes: Vec<(NodeId, ProtoNode)>,
@@ -94,7 +96,7 @@ pub struct ProtoNetwork {
 
 #[derive(Debug)]
 pub enum ConstructionArgs {
-	Value(value::Value),
+	Value(value::TaggedValue),
 	Nodes(Vec<NodeId>),
 }
 
@@ -108,6 +110,20 @@ impl PartialEq for ConstructionArgs {
 	}
 }
 
+impl Hash for ConstructionArgs {
+	fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
+		match self {
+			Self::Nodes(nodes) => {
+				"nodes".hash(state);
+				for node in nodes {
+					node.hash(state);
+				}
+			}
+			Self::Value(value) => value.hash(state),
+		}
+	}
+}
+
 impl ConstructionArgs {
 	pub fn new_function_args(&self) -> Vec<String> {
 		match self {
@@ -142,6 +158,19 @@ impl ProtoNodeInput {
 }
 
 impl ProtoNode {
+	pub fn stable_node_id(&self) -> Option<NodeId> {
+		use std::hash::Hasher;
+		let mut hasher = std::collections::hash_map::DefaultHasher::new();
+		self.identifier.fully_qualified_name().hash(&mut hasher);
+		self.construction_args.hash(&mut hasher);
+		match self.input {
+			ProtoNodeInput::None => "none".hash(&mut hasher),
+			ProtoNodeInput::Network => "network".hash(&mut hasher),
+			ProtoNodeInput::Node(id) => id.hash(&mut hasher),
+		};
+		Some(hasher.finish() as NodeId)
+	}
+
 	pub fn value(value: ConstructionArgs) -> Self {
 		Self {
 			identifier: NodeIdentifier::new("graphene_core::value::ValueNode", &[Type::Generic(Cow::Borrowed("T"))]),
@@ -195,6 +224,24 @@ impl ProtoNetwork {
 		edges
 	}
 
+	pub fn generate_stable_node_ids(&mut self) {
+		for i in 0..self.nodes.len() {
+			self.generate_stable_node_id(i);
+		}
+	}
+
+	pub fn generate_stable_node_id(&mut self, index: usize) -> NodeId {
+		let mut lookup = self.nodes.iter().map(|(id, _)| (*id, *id)).collect::<HashMap<_, _>>();
+		if let Some(sni) = self.nodes[index].1.stable_node_id() {
+			lookup.insert(self.nodes[index].0, sni);
+			self.replace_node_references(&lookup);
+			self.nodes[index].0 = sni;
+			sni
+		} else {
+			panic!("failed to generate stable node id for node {:#?}", self.nodes[index].1);
+		}
+	}
+
 	pub fn collect_inwards_edges(&self) -> HashMap<NodeId, Vec<NodeId>> {
 		let mut edges: HashMap<NodeId, Vec<NodeId>> = HashMap::new();
 		for (id, node) in &self.nodes {
@@ -236,7 +283,7 @@ impl ProtoNetwork {
 			self.nodes.push((
 				compose_node_id,
 				ProtoNode {
-					identifier: NodeIdentifier::new("graphene_core::structural::ComposeNode", &[generic!("T"), Type::Generic(Cow::Borrowed("U"))]),
+					identifier: NodeIdentifier::new("graphene_core::structural::ComposeNode<_, _, _>", &[generic!("T"), generic!("U")]),
 					construction_args: ConstructionArgs::Nodes(vec![input_node, id]),
 					input,
 				},
@@ -330,7 +377,6 @@ impl ProtoNetwork {
 mod test {
 	use super::*;
 	use crate::proto::{ConstructionArgs, ProtoNetwork, ProtoNode, ProtoNodeInput};
-	use value::IntoValue;
 
 	#[test]
 	fn topological_sort() {
@@ -378,8 +424,30 @@ mod test {
 		assert_eq!(construction_network.nodes[5].1.construction_args, ConstructionArgs::Nodes(vec![3, 4]));
 	}
 
+	#[test]
+	fn stable_node_id_generation() {
+		let mut construction_network = test_network();
+		construction_network.reorder_ids();
+		construction_network.generate_stable_node_ids();
+		construction_network.resolve_inputs();
+		construction_network.generate_stable_node_ids();
+		assert_eq!(construction_network.nodes[0].1.identifier.name.as_ref(), "value");
+		let ids: Vec<_> = construction_network.nodes.iter().map(|(id, _)| *id).collect();
+		assert_eq!(
+			ids,
+			vec![
+				17495035641492238530,
+				14931179783740213471,
+				2268573767208263092,
+				14616574692620381527,
+				12110007198416821768,
+				11185814750012198757
+			]
+		);
+	}
+
 	fn test_network() -> ProtoNetwork {
-		let construction_network = ProtoNetwork {
+		ProtoNetwork {
 			inputs: vec![10],
 			output: 1,
 			nodes: [
@@ -420,13 +488,12 @@ mod test {
 					ProtoNode {
 						identifier: "value".into(),
 						input: ProtoNodeInput::None,
-						construction_args: ConstructionArgs::Value(2_u32.into_any()),
+						construction_args: ConstructionArgs::Value(value::TaggedValue::U32(2)),
 					},
 				),
 			]
 			.into_iter()
 			.collect(),
-		};
-		construction_network
+		}
 	}
 }