Use ResourceId for referring to Resources (#4176)

* Use ResourceId for referring to Resources

* Use Doc

* use error for preprocessor ResourceId replacement failure

* Keep hashes in DocumentInfo

* Migrate with less mem copy

* Global ResourceMessage -> ResourceStorageMessage

* Add document ResourceMessage

* Move embedding logic

* Fix

* Cleanup

* Review

* Fix

node-graph/graph-craft/src/application_io/resource.rs

@@ -1,7 +1,7 @@
 use std::collections::HashMap;
 use std::sync::{Arc, Mutex};
 
-pub use graphene_application_io::resource::{LoadResource, Resource, ResourceFuture, ResourceHash, ResourceStorage};
+pub use graphene_application_io::resource::*;
 #[cfg(not(target_family = "wasm"))]
 pub mod mmap;
 #[cfg(not(target_family = "wasm"))]

node-graph/graph-craft/src/document/value.rs

@@ -396,7 +396,8 @@ tagged_value! {
 	Footprint(Footprint),
 	VectorModification(Box<VectorModification>),
 	ImageData(Image<Color>),
-	Resource(graphene_application_io::resource::ResourceHash),
+	Resource(graphene_application_io::resource::ResourceId),
+	ResourceHash(graphene_application_io::resource::ResourceHash),
 	// ==========
 	// ENUM TYPES
 	// ==========

node-graph/graphene-cli/src/main.rs

@@ -4,6 +4,7 @@ use clap::{Args, Parser, Subcommand};
 use fern::colors::{Color, ColoredLevelConfig};
 use futures::executor::block_on;
 use graph_craft::application_io::EditorPreferences;
+use graph_craft::application_io::resource::ResourceRegistry;
 use graph_craft::application_io::{PlatformApplicationIo, PlatformEditorApi};
 use graph_craft::document::*;
 use graph_craft::graphene_compiler::Compiler;
@@ -244,7 +245,7 @@ fn compile_graph(document_string: String, editor_api: Arc<PlatformEditorApi>) ->
 	fix_nodes(&mut network);
 
 	let substitutions = preprocessor::generate_node_substitutions();
-	preprocessor::expand_network(&mut network, &substitutions);
+	preprocessor::expand_network(&mut network, &substitutions, &ResourceRegistry::default()).expect("Failed to expand network"); // TODO: actually load the resources from the document
 
 	let wrapped_network = wrap_network_in_scope(network.clone(), editor_api);
 

node-graph/interpreted-executor/benches/benchmark_util.rs

@@ -11,7 +11,7 @@ pub fn setup_network(name: &str) -> (DynamicExecutor, ProtoNetwork) {
 	let mut network = load_from_name(name);
 	let editor_api = std::sync::Arc::new(EditorApi::default());
 	let substitutions = preprocessor::generate_node_substitutions();
-	preprocessor::expand_network(&mut network, &substitutions);
+	preprocessor::expand_network(&mut network, &substitutions, &ResourceRegistry::default()).unwrap();
 	let network = wrap_network_in_scope(network, editor_api);
 	let proto_network = compile(network);
 	let executor = block_on(DynamicExecutor::new(proto_network.clone())).unwrap();

node-graph/libraries/resources/src/lib.rs

@@ -22,7 +22,7 @@ pub trait ResourceStorage: LoadResource {
 #[repr(transparent)]
 #[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Hash, graphene_hash::CacheHash, PartialOrd, Ord, DynAny)]
 #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
-pub struct ResourceId(pub u64);
+pub struct ResourceId(u64);
 
 impl ResourceId {
 	pub fn new() -> Self {
@@ -185,13 +185,6 @@ impl CacheHash for ResourceHash {
 	}
 }
 
-#[derive(Clone, Debug)]
-pub struct ResourceInfo {
-	pub id: ResourceId,
-	pub hash: Option<ResourceHash>,
-	pub inputs: DataSources,
-}
-
 pub type DataSources = Box<[DataSource]>;
 
 #[derive(Clone, Debug, PartialEq, Eq)]
@@ -226,11 +219,11 @@ impl ResourceRegistry {
 		self.hashes.keys().chain(self.sources.keys().filter(|id| !self.hashes.contains_key(id))).copied()
 	}
 
-	pub fn info(&self, id: &ResourceId) -> Option<ResourceInfo> {
+	pub fn info(&self, id: &ResourceId) -> Option<ResourceInfo<'_>> {
 		self.contains(id).then(|| ResourceInfo {
 			id: *id,
-			hash: self.hashes.get(id).copied(),
-			inputs: self.sources.get(id).cloned().unwrap_or_default().into_boxed_slice(),
+			hash: self.hashes.get(id),
+			sources: self.sources.get(id).map(|sources| sources.as_slice()).unwrap_or(&[]),
 		})
 	}
 
@@ -242,17 +235,10 @@ impl ResourceRegistry {
 		self.sources.entry(*id).or_default().insert(0, source);
 	}
 
-	pub fn delete(&mut self, id: &ResourceId) -> Option<ResourceInfo> {
+	pub fn delete(&mut self, id: &ResourceId) -> bool {
 		let hash = self.hashes.remove(id);
 		let sources = self.sources.remove(id);
-		if hash.is_none() && sources.is_none() {
-			return None;
-		}
-		Some(ResourceInfo {
-			id: *id,
-			hash,
-			inputs: sources.unwrap_or_default().into_boxed_slice(),
-		})
+		!(hash.is_none() && sources.is_none())
 	}
 
 	pub fn resolve(&mut self, id: &ResourceId, hash: ResourceHash) -> Option<ResourceHash> {
@@ -263,7 +249,18 @@ impl ResourceRegistry {
 		self.hashes.get(id).copied()
 	}
 
-	pub fn unresolved(&self) -> impl Iterator<Item = (ResourceId, &[DataSource])> + '_ {
-		self.sources.iter().filter(|(id, _)| !self.hashes.contains_key(id)).map(|(id, sources)| (*id, sources.as_slice()))
+	pub fn unresolved(&self) -> impl Iterator<Item = ResourceInfo<'_>> + '_ {
+		self.sources.keys().filter(|id| !self.hashes.contains_key(id)).filter_map(|id| self.info(id))
+	}
+
+	pub fn resolved(&self) -> impl Iterator<Item = ResourceInfo<'_>> + '_ {
+		self.hashes.keys().filter_map(|id| self.info(id))
 	}
 }
+
+#[derive(Clone, Debug)]
+pub struct ResourceInfo<'a> {
+	pub id: ResourceId,
+	pub hash: Option<&'a ResourceHash>,
+	pub sources: &'a [DataSource],
+}

node-graph/preprocessor/src/lib.rs

@@ -1,6 +1,7 @@
 #[macro_use]
 extern crate log;
 
+use graph_craft::application_io::resource::{ResourceId, ResourceRegistry};
 use graph_craft::document::value::*;
 use graph_craft::document::*;
 use graph_craft::proto::RegistryValueSource;
@@ -9,19 +10,20 @@ use graphene_std::registry::*;
 use graphene_std::*;
 use std::collections::{HashMap, HashSet};
 
-pub fn expand_network(network: &mut NodeNetwork, substitutions: &HashMap<ProtoNodeIdentifier, DocumentNode>) {
-	replace_resource_inputs(network);
+pub fn expand_network(network: &mut NodeNetwork, substitutions: &HashMap<ProtoNodeIdentifier, DocumentNode>, resources: &ResourceRegistry) -> Result<(), PreprocessorError> {
+	replace_resource_inputs(network, resources)?;
 	expand_network_inner(network, substitutions);
+	Ok(())
 }
 
 /// Replace every `TaggedValue::Resource(hash)` input with a reference to a freshly inserted `resource` proto node.
-fn replace_resource_inputs(network: &mut NodeNetwork) {
+fn replace_resource_inputs(network: &mut NodeNetwork, resources: &ResourceRegistry) -> Result<(), PreprocessorError> {
 	let mut hash_to_node_id: HashMap<graph_craft::application_io::resource::ResourceHash, NodeId> = HashMap::new();
 	let mut new_resource_nodes: Vec<(NodeId, DocumentNode)> = Vec::new();
 
 	for node in network.nodes.values_mut() {
 		if let DocumentNodeImplementation::Network(nested) = &mut node.implementation {
-			replace_resource_inputs(nested);
+			replace_resource_inputs(nested, resources)?;
 			continue;
 		}
 
@@ -31,12 +33,16 @@ fn replace_resource_inputs(network: &mut NodeNetwork) {
 
 		for input in node.inputs.iter_mut() {
 			let NodeInput::Value { tagged_value, .. } = input else { continue };
-			let TaggedValue::Resource(hash) = **tagged_value else { continue };
+			let TaggedValue::Resource(resource_id) = **tagged_value else { continue };
+
+			let Some(hash) = resources.hash(&resource_id) else {
+				return Err(PreprocessorError::ResourceNotFound(resource_id));
+			};
 
 			let resource_id = *hash_to_node_id.entry(hash).or_insert_with(|| {
 				let id = NodeId::new();
 				let resource_node = DocumentNode {
-					inputs: vec![NodeInput::value(TaggedValue::Resource(hash), false), NodeInput::scope("editor-api")],
+					inputs: vec![NodeInput::value(TaggedValue::ResourceHash(hash), false), NodeInput::scope("editor-api")],
 					implementation: DocumentNodeImplementation::ProtoNode(platform_application_io::resource::IDENTIFIER),
 					..Default::default()
 				};
@@ -51,6 +57,8 @@ fn replace_resource_inputs(network: &mut NodeNetwork) {
 	for (id, node) in new_resource_nodes {
 		network.nodes.insert(id, node);
 	}
+
+	Ok(())
 }
 
 fn expand_network_inner(network: &mut NodeNetwork, substitutions: &HashMap<ProtoNodeIdentifier, DocumentNode>) {
@@ -241,3 +249,16 @@ pub fn node_inputs(fields: &[registry::FieldMetadata], first_node_io: &NodeIOTyp
 		})
 		.collect()
 }
+
+#[derive(Debug)]
+pub enum PreprocessorError {
+	ResourceNotFound(ResourceId),
+}
+
+impl std::fmt::Display for PreprocessorError {
+	fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+		match self {
+			PreprocessorError::ResourceNotFound(id) => write!(f, "Resource not found: {id:?}"),
+		}
+	}
+}