Use ResourceId for referring to Resources (#4176)

* Use ResourceId for referring to Resources

* Use Doc

* use error for preprocessor ResourceId replacement failure

* Keep hashes in DocumentInfo

* Migrate with less mem copy

* Global ResourceMessage -> ResourceStorageMessage

* Add document ResourceMessage

* Move embedding logic

* Fix

* Cleanup

* Review

* Fix

editor/src/application.rs

@@ -30,7 +30,7 @@ impl Editor {
 		};
 
 		let mut application_io = PlatformApplicationIo::default();
-		application_io.inject_resource_proxy(editor.dispatcher.message_handlers.resource_message_handler.resources());
+		application_io.inject_resource_proxy(editor.dispatcher.message_handlers.resource_storage_message_handler.resources());
 		runtime.replace_application_io(application_io);
 
 		(editor, runtime)
@@ -47,7 +47,7 @@ impl Editor {
 	}
 
 	pub fn replace_application_io(&mut self, mut application_io: PlatformApplicationIo) {
-		application_io.inject_resource_proxy(self.dispatcher.message_handlers.resource_message_handler.resources());
+		application_io.inject_resource_proxy(self.dispatcher.message_handlers.resource_storage_message_handler.resources());
 		crate::node_graph_executor::replace_application_io(application_io)
 	}
 }

editor/src/dispatcher.rs

@@ -32,7 +32,7 @@ pub struct DispatcherMessageHandlers {
 	menu_bar_message_handler: MenuBarMessageHandler,
 	pub(crate) portfolio_message_handler: PortfolioMessageHandler,
 	preferences_message_handler: PreferencesMessageHandler,
-	pub(crate) resource_message_handler: ResourceMessageHandler,
+	pub(crate) resource_storage_message_handler: ResourceStorageMessageHandler,
 	tool_message_handler: ToolMessageHandler,
 	viewport_message_handler: ViewportMessageHandler,
 }
@@ -40,7 +40,7 @@ pub struct DispatcherMessageHandlers {
 impl DispatcherMessageHandlers {
 	pub fn with_resource_storage(resource_storage: Box<dyn ResourceStorage>) -> Self {
 		Self {
-			resource_message_handler: ResourceMessageHandler::new(resource_storage),
+			resource_storage_message_handler: ResourceStorageMessageHandler::new(resource_storage),
 			..Self::default()
 		}
 	}
@@ -89,7 +89,7 @@ const DEBUG_MESSAGE_ENDING_BLOCK_LIST: &[&str] = &["PointerMove", "PointerOutsid
 impl Dispatcher {
 	pub fn new(resource_storage: Box<dyn ResourceStorage>) -> Self {
 		let mut s = Self::default();
-		s.message_handlers.resource_message_handler = ResourceMessageHandler::new(resource_storage);
+		s.message_handlers.resource_storage_message_handler = ResourceStorageMessageHandler::new(resource_storage);
 		s
 	}
 
@@ -228,8 +228,10 @@ impl Dispatcher {
 
 					self.message_handlers.layout_message_handler.process_message(message, &mut queue, context);
 				}
-				Message::Resource(message) => {
+				Message::ResourceStorage(message) => {
-					self.message_handlers.resource_message_handler.process_message(message, &mut queue, ResourceMessageContext {});
+					self.message_handlers
+						.resource_storage_message_handler
+						.process_message(message, &mut queue, ResourceStorageMessageContext {});
 				}
 				Message::Portfolio(message) => {
 					self.message_handlers.portfolio_message_handler.process_message(
@@ -243,7 +245,7 @@ impl Dispatcher {
 							timing_information: self.message_handlers.animation_message_handler.timing_information(),
 							animation: &self.message_handlers.animation_message_handler,
 							viewport: &self.message_handlers.viewport_message_handler,
-							resources: &self.message_handlers.resource_message_handler,
+							resource_storage: &self.message_handlers.resource_storage_message_handler,
 						},
 					);
 				}

editor/src/messages/message.rs

@@ -36,7 +36,7 @@ pub enum Message {
 	#[child]
 	Preferences(PreferencesMessage),
 	#[child]
-	Resource(ResourceMessage),
+	ResourceStorage(ResourceStorageMessage),
 	#[child]
 	Tool(ToolMessage),
 	#[child]

editor/src/messages/mod.rs

@@ -17,6 +17,6 @@ pub mod message;
 pub mod portfolio;
 pub mod preferences;
 pub mod prelude;
-pub mod resource;
+pub mod resource_storage;
 pub mod tool;
 pub mod viewport;

editor/src/messages/portfolio/document/document_message.rs

@@ -37,6 +37,8 @@ pub enum DocumentMessage {
 	PropertiesPanel(PropertiesPanelMessage),
 	#[child]
 	DataPanel(DataPanelMessage),
+	#[child]
+	Resource(ResourceMessage),
 
 	// Messages
 	AlignSelectedLayers {

editor/src/messages/portfolio/document/document_message_handler.rs

@@ -19,7 +19,6 @@ use crate::messages::portfolio::document::overlays::grid_overlays::{grid_overlay
 use crate::messages::portfolio::document::overlays::utility_types::{OverlaysType, OverlaysVisibilitySettings, Pivot};
 use crate::messages::portfolio::document::properties_panel::properties_panel_message_handler::PropertiesPanelMessageContext;
 use crate::messages::portfolio::document::utility_types::document_metadata::{DocumentMetadata, LayerNodeIdentifier};
-use crate::messages::portfolio::document::utility_types::embedded_resources::EmbeddedResources;
 use crate::messages::portfolio::document::utility_types::misc::{AlignAggregate, AlignAxis, FlipAxis, PTZ};
 use crate::messages::portfolio::document::utility_types::network_interface::{FlowType, InputConnector, NodeTemplate, OutputConnector};
 use crate::messages::portfolio::utility_types::{CachedData, PanelType};
@@ -30,7 +29,7 @@ use crate::messages::tool::tool_messages::tool_prelude::Key;
 use crate::messages::tool::utility_types::ToolType;
 use crate::node_graph_executor::NodeGraphExecutor;
 use glam::{DAffine2, DVec2};
-use graph_craft::application_io::resource::ResourceHash;
+use graph_craft::application_io::resource::ResourceId;
 use graph_craft::application_io::wgpu_available;
 use graph_craft::descriptor;
 use graph_craft::document::value::TaggedValue;
@@ -61,7 +60,7 @@ pub struct DocumentMessageContext<'a> {
 	pub layers_panel_open: bool,
 	pub properties_panel_open: bool,
 	pub viewport: &'a ViewportMessageHandler,
-	pub resources: &'a ResourceMessageHandler,
+	pub resource_storage: &'a ResourceStorageMessageHandler,
 }
 
 #[derive(Clone, Debug, serde::Serialize, serde::Deserialize, ExtractField)]
@@ -88,6 +87,9 @@ pub struct DocumentMessageHandler {
 	//
 	// Contains the NodeNetwork and acts an an interface to manipulate the NodeNetwork with custom setters in order to keep NetworkMetadata in sync
 	pub network_interface: NodeNetworkInterface,
+	/// Resources embedded in the document.
+	#[serde(default, skip_serializing_if = "ResourceMessageHandler::is_empty")]
+	pub resources: ResourceMessageHandler,
 	/// Tracks which layer occurrences are collapsed in the Layers panel, keyed by tree path.
 	#[serde(deserialize_with = "deserialize_collapsed_layers", default)]
 	pub collapsed: CollapsedLayers,
@@ -110,9 +112,6 @@ pub struct DocumentMessageHandler {
 	pub graph_view_overlay_open: bool,
 	/// The current opacity of the faded node graph background that covers up the artwork.
 	pub graph_fade_artwork_percentage: f64,
-	/// Resources embedded in the document. Only propagated if saving to an external file and never for autosaved documents.
-	#[serde(rename = "resources", default, skip_serializing_if = "EmbeddedResources::is_empty")]
-	pub embedded_resources: EmbeddedResources,
 
 	// =============================================
 	// Fields omitted from the saved document format
@@ -166,6 +165,7 @@ impl Default for DocumentMessageHandler {
 			// Fields that are saved in the document format
 			// ============================================
 			network_interface: default_document_network_interface(),
+			resources: ResourceMessageHandler::default(),
 			collapsed: CollapsedLayers::default(),
 			commit_hash: GRAPHITE_GIT_COMMIT_HASH.to_string(),
 			document_ptz: PTZ::default(),
@@ -175,7 +175,6 @@ impl Default for DocumentMessageHandler {
 			graph_view_overlay_open: false,
 			snapping_state: SnappingState::default(),
 			graph_fade_artwork_percentage: 80.,
-			embedded_resources: EmbeddedResources::default(),
 			// =============================================
 			// Fields omitted from the saved document format
 			// =============================================
@@ -207,7 +206,7 @@ impl MessageHandler<DocumentMessage, DocumentMessageContext<'_>> for DocumentMes
 			data_panel_open,
 			layers_panel_open,
 			properties_panel_open,
-			resources,
+			resource_storage,
 		} = context;
 
 		match message {
@@ -282,6 +281,10 @@ impl MessageHandler<DocumentMessage, DocumentMessageContext<'_>> for DocumentMes
 				let mut graph_operation_message_handler = GraphOperationMessageHandler {};
 				graph_operation_message_handler.process_message(message, responses, context);
 			}
+			DocumentMessage::Resource(message) => {
+				let context = ResourceMessageContext {};
+				self.resources.process_message(message, responses, context);
+			}
 			DocumentMessage::AlignSelectedLayers { axis, aggregate } => {
 				let axis = match axis {
 					AlignAxis::X => DVec2::X,
@@ -920,36 +923,29 @@ impl MessageHandler<DocumentMessage, DocumentMessageContext<'_>> for DocumentMes
 			DocumentMessage::SaveDocument | DocumentMessage::SaveDocumentAs => {
 				responses.add(PortfolioMessage::AutoSaveActiveDocument);
 
+				let name = format!("{}.{}", self.name.clone(), FILE_EXTENSION);
 				let path = if let DocumentMessage::SaveDocumentAs = message { None } else { self.path.clone() };
 				if path.is_some() {
 					responses.add(DocumentMessage::MarkAsSaved);
 				}
 				let folder = self.path.as_ref().and_then(|path| path.parent()).map(|parent| parent.to_path_buf());
 
-				let resource_hashes = Vec::from_iter(self.used_resources(false)).into_boxed_slice();
-				let resources = resources.resources();
 				let mut document = self.clone();
-				let name = format!("{}.{}", self.name.clone(), FILE_EXTENSION);
+				let resources_load_handle = resource_storage.resources();
 
 				responses.add(FrontendMessage::Await {
 					future: FrontendMessageFuture::new(async move {
-						let loads = resource_hashes
+						document.resources.garbage_collect(document.used_resources(false).as_ref());
-							.into_iter()
+						document.resources.embed_resources(resources_load_handle).await;
-							.map(|hash| {
-								let resource = resources.load(hash);
-								async move { resource.await.map(|resource| (hash, resource)) }
-							})
-							.collect::<Vec<_>>();
 
-						document.embedded_resources = EmbeddedResources::from_iter(futures::future::join_all(loads).await.into_iter().flatten());
+						let content = document.serialize_document().into_bytes().into();
-						let content = document.serialize_document();
 
 						FrontendMessage::TriggerSaveDocument {
 							document_id,
 							name,
 							path,
 							folder,
-							content: content.into_bytes().into(),
+							content,
 						}
 					}),
 				});
@@ -3472,14 +3468,19 @@ impl DocumentMessageHandler {
 		self.graph_view_overlay_open
 	}
 
-	pub fn used_resources(&self, include_history: bool) -> HashSet<ResourceHash> {
+	pub fn garbage_collect_resources(&mut self) {
+		let used_resources = self.used_resources(true);
+		self.resources.garbage_collect(&used_resources);
+	}
+
+	pub fn used_resources(&self, include_history: bool) -> Box<[ResourceId]> {
 		let mut resources = HashSet::new();
 		self.network_interface.collect_used_resources(&mut resources);
 		if include_history {
 			self.document_undo_history.iter().for_each(|interface| interface.collect_used_resources(&mut resources));
 			self.document_redo_history.iter().for_each(|interface| interface.collect_used_resources(&mut resources));
 		}
-		resources
+		resources.into_iter().collect::<Vec<_>>().into_boxed_slice()
 	}
 }
 

editor/src/messages/portfolio/document/graph_operation/graph_operation_message_handler.rs

@@ -32,7 +32,7 @@ pub struct GraphOperationMessageHandler {}
 #[message_handler_data]
 impl MessageHandler<GraphOperationMessage, GraphOperationMessageContext<'_>> for GraphOperationMessageHandler {
 	fn process_message(&mut self, message: GraphOperationMessage, responses: &mut VecDeque<Message>, context: GraphOperationMessageContext) {
-		let network_interface = context.network_interface;
+		let GraphOperationMessageContext { network_interface, .. } = context;
 
 		match message {
 			GraphOperationMessage::FillSet { layer, fill } => {

editor/src/messages/portfolio/document/graph_operation/utility_types.rs

@@ -4,6 +4,7 @@ use crate::messages::portfolio::document::utility_types::document_metadata::Laye
 use crate::messages::portfolio::document::utility_types::network_interface::{self, InputConnector, NodeNetworkInterface, OutputConnector};
 use crate::messages::prelude::*;
 use glam::{DAffine2, DVec2};
+use graph_craft::application_io::resource::ResourceId;
 use graph_craft::document::value::TaggedValue;
 use graph_craft::document::{NodeId, NodeInput};
 use graph_craft::{ProtoNodeIdentifier, concrete, descriptor};
@@ -299,13 +300,15 @@ impl<'a> ModifyInputsContext<'a> {
 			.expect("Transform node does not exist")
 			.default_node_template();
 
-		let png_bytes: std::sync::Arc<[u8]> = image.to_png().into();
+		let resource_id = ResourceId::new();
-		let hash = graphene_std::application_io::resource::ResourceHash::from(png_bytes.as_ref());
+		self.responses.add(ResourceMessage::StoreEmbedded {
-		self.responses.add(ResourceMessage::Store { data: png_bytes });
+			resource_id,
+			data: image.to_png().into(),
+		});
 
 		let image_node = resolve_proto_node_type(graphene_std::raster_nodes::std_nodes::image::IDENTIFIER)
 			.expect("Image node does not exist")
-			.node_template_input_override([Some(NodeInput::value(TaggedValue::Resource(hash), false))]);
+			.node_template_input_override([Some(NodeInput::value(TaggedValue::Resource(resource_id), false))]);
 
 		let image_node_id = NodeId::new();
 		self.network_interface.insert_node(image_node_id, image_node, &[]);

editor/src/messages/portfolio/document/mod.rs

@@ -7,6 +7,7 @@ pub mod navigation;
 pub mod node_graph;
 pub mod overlays;
 pub mod properties_panel;
+pub mod resource;
 pub mod utility_types;
 
 #[doc(inline)]

editor/src/messages/portfolio/document/node_graph/node_graph_message_handler.rs

@@ -1557,7 +1557,7 @@ impl<'a> MessageHandler<NodeGraphMessage, NodeGraphMessageContext<'a>> for NodeG
 						// Only disconnect inputs to non selected nodes
 						if network_interface
 							.upstream_output_connector(&input_connector, selection_network_path)
-							.is_some_and(|connector| connector.node_id().map_or(true, |node_id| !all_selected_nodes.contains(&node_id)))
+							.is_some_and(|connector| connector.node_id().is_none_or(|node_id| !all_selected_nodes.contains(&node_id)))
 						{
 							responses.add(NodeGraphMessage::DisconnectInput { input_connector });
 						}

editor/src/messages/portfolio/document/resource/mod.rs

@@ -1,7 +1,8 @@
 mod resource_message;
 mod resource_message_handler;
+pub mod utility_types;
 
 #[doc(inline)]
 pub use resource_message::{ResourceMessage, ResourceMessageDiscriminant};
 #[doc(inline)]
-pub use resource_message_handler::{ResourceMessageContext, ResourceMessageHandler, ResourcesHandle};
+pub use resource_message_handler::{ResourceMessageContext, ResourceMessageHandler};

editor/src/messages/portfolio/document/resource/resource_message.rs

@@ -0,0 +1,9 @@
+use crate::messages::prelude::*;
+use graph_craft::application_io::resource::ResourceId;
+use std::sync::Arc;
+
+#[impl_message(Message, DocumentMessage, Resource)]
+#[derive(PartialEq, Clone, Debug, serde::Serialize, serde::Deserialize)]
+pub enum ResourceMessage {
+	StoreEmbedded { resource_id: ResourceId, data: Arc<[u8]> },
+}

editor/src/messages/portfolio/document/resource/resource_message_handler.rs

@@ -0,0 +1,127 @@
+use crate::messages::portfolio::document::resource::utility_types::EmbeddedResources;
+use crate::messages::prelude::*;
+use base64::Engine;
+use base64::engine::general_purpose::STANDARD as BASE64;
+use graph_craft::application_io::resource::{DataSource, LoadResource, Resource, ResourceHash, ResourceId, ResourceRegistry};
+
+#[derive(ExtractField)]
+pub struct ResourceMessageContext {}
+
+#[derive(Debug, Clone, PartialEq, Default, serde::Serialize, ExtractField)]
+pub struct ResourceMessageHandler {
+	pub registry: ResourceRegistry,
+	pub embedded: EmbeddedResources,
+}
+
+#[message_handler_data]
+impl MessageHandler<ResourceMessage, ResourceMessageContext> for ResourceMessageHandler {
+	fn process_message(&mut self, message: ResourceMessage, responses: &mut VecDeque<Message>, _context: ResourceMessageContext) {
+		match message {
+			ResourceMessage::StoreEmbedded { resource_id, data } => {
+				let hash = ResourceHash::from(data.as_ref());
+				self.registry.push_source_back(&resource_id, DataSource::Embedded);
+				self.registry.resolve(&resource_id, hash);
+				responses.add(ResourceStorageMessage::Store { data });
+			}
+		}
+	}
+
+	fn actions(&self) -> ActionList {
+		actions!(ResourceMessageDiscriminant;)
+	}
+}
+
+impl ResourceMessageHandler {
+	pub fn is_empty(&self) -> bool {
+		self.registry.is_empty() && self.embedded.is_empty()
+	}
+
+	pub async fn embed_resources(&mut self, resources_load_handle: Box<dyn LoadResource>) {
+		let embedded = self
+			.registry
+			.resolved()
+			.filter(|info| info.sources.contains(&DataSource::Embedded))
+			.filter_map(|info| {
+				if let Some(hash) = info.hash {
+					let resource = resources_load_handle.load(*hash);
+					Some(async move { resource.await.map(|resource| (*hash, resource)) })
+				} else {
+					None
+				}
+			})
+			.collect::<Vec<_>>();
+
+		self.embedded = EmbeddedResources::from_iter(futures::future::join_all(embedded).await.into_iter().flatten());
+	}
+
+	pub fn garbage_collect(&mut self, used: &[ResourceId]) {
+		let used = HashSet::<ResourceId>::from_iter(used.iter().cloned());
+		let unused = self.registry.ids().filter(|id| !used.contains(id)).collect::<Vec<_>>();
+		unused.into_iter().for_each(|id| {
+			self.registry.delete(&id);
+		});
+	}
+}
+
+// TODO: Eventually remove this document upgrade code
+impl<'de> serde::Deserialize<'de> for ResourceMessageHandler {
+	fn deserialize<D: serde::Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
+		enum Key {
+			Registry,
+			Embedded,
+			Hash(ResourceHash),
+		}
+
+		impl<'de> serde::Deserialize<'de> for Key {
+			fn deserialize<D: serde::Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
+				let raw = String::deserialize(deserializer)?;
+				Ok(match raw.as_str() {
+					"registry" => Key::Registry,
+					"embedded" => Key::Embedded,
+					_ => Key::Hash(raw.parse().map_err(serde::de::Error::custom)?),
+				})
+			}
+		}
+
+		struct EmbeddedResourcesVisitor {
+			human_readable: bool,
+		}
+
+		impl<'de> serde::de::Visitor<'de> for EmbeddedResourcesVisitor {
+			type Value = ResourceMessageHandler;
+
+			fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result {
+				formatter.write_str("an EmbeddedResources struct or a legacy EmbeddedResourceData map")
+			}
+
+			fn visit_map<A: serde::de::MapAccess<'de>>(self, mut map: A) -> Result<Self::Value, A::Error> {
+				let mut output = ResourceMessageHandler::default();
+
+				while let Some(key) = map.next_key::<Key>()? {
+					match key {
+						Key::Registry => output.registry = map.next_value()?,
+						Key::Embedded => output.embedded = map.next_value()?,
+						Key::Hash(hash) => {
+							let bytes = if self.human_readable {
+								let encoded: String = map.next_value()?;
+								BASE64.decode(&encoded).map_err(serde::de::Error::custom)?
+							} else {
+								let raw: serde_bytes::ByteBuf = map.next_value()?;
+								raw.into_vec()
+							};
+							let data_hash = output.embedded.store(Resource::new(bytes));
+							if data_hash != hash {
+								return Err(serde::de::Error::custom(format!("EmbeddedResource hash mismatch: expected {hash}, got {data_hash}")));
+							}
+						}
+					}
+				}
+
+				Ok(output)
+			}
+		}
+
+		let human_readable = deserializer.is_human_readable();
+		deserializer.deserialize_map(EmbeddedResourcesVisitor { human_readable })
+	}
+}

editor/src/messages/portfolio/document/utility_types/mod.rs

@@ -1,6 +1,5 @@
 pub mod clipboards;
 pub mod document_metadata;
-pub mod embedded_resources;
 pub mod error;
 pub mod misc;
 pub mod network_interface;

editor/src/messages/portfolio/document/utility_types/network_interface.rs

@@ -20,7 +20,7 @@ use crate::messages::tool::tool_messages::tool_prelude::NumberInputMode;
 use deserialization::deserialize_node_persistent_metadata;
 use glam::{DAffine2, DVec2, IVec2};
 use graph_craft::Type;
-use graph_craft::application_io::resource::ResourceHash;
+use graph_craft::application_io::resource::ResourceId;
 use graph_craft::document::value::TaggedValue;
 use graph_craft::document::{DocumentNode, DocumentNodeImplementation, NodeId, NodeInput, NodeNetwork, OldDocumentNodeImplementation, OldNodeNetwork};
 use graphene_std::ContextDependencies;
@@ -511,7 +511,7 @@ impl NodeNetworkInterface {
 		})
 	}
 
-	pub fn collect_used_resources(&self, target: &mut HashSet<ResourceHash>) {
+	pub fn collect_used_resources(&self, target: &mut HashSet<ResourceId>) {
 		collect_network_resources(self.document_network(), target);
 	}
 
@@ -6774,13 +6774,13 @@ pub enum TransactionStatus {
 	Finished,
 }
 
-fn collect_network_resources(network: &NodeNetwork, out: &mut HashSet<ResourceHash>) {
+fn collect_network_resources(network: &NodeNetwork, out: &mut HashSet<ResourceId>) {
 	for node in network.nodes.values() {
 		for input in &node.inputs {
 			if let NodeInput::Value { tagged_value, .. } = input
-				&& let TaggedValue::Resource(hash) = &**tagged_value
+				&& let TaggedValue::Resource(id) = &**tagged_value
 			{
-				out.insert(*hash);
+				out.insert(*id);
 			}
 		}
 		if let DocumentNodeImplementation::Network(nested) = &node.implementation {

editor/src/messages/portfolio/document_migration.rs

@@ -6,6 +6,7 @@ use crate::messages::portfolio::document::utility_types::document_metadata::Laye
 use crate::messages::portfolio::document::utility_types::network_interface::{InputConnector, NodeTemplate, OutputConnector};
 use crate::messages::prelude::DocumentMessageHandler;
 use glam::{DVec2, IVec2};
+use graph_craft::application_io::resource::{DataSource, Resource, ResourceHash, ResourceId};
 use graph_craft::descriptor;
 use graph_craft::document::DocumentNode;
 use graph_craft::document::{DocumentNodeImplementation, NodeInput, value::TaggedValue};
@@ -16,6 +17,7 @@ use graphene_std::uuid::NodeId;
 use graphene_std::vector::style::{PaintOrder, StrokeAlign};
 use std::collections::HashMap;
 use std::f64::consts::PI;
+use std::ops::Range;
 
 const TEXT_REPLACEMENTS: &[(&str, &str)] = &[
 	("graphene_core::vector::vector_nodes::SamplePointsNode", "graphene_core::vector::SamplePolylineNode"),
@@ -1034,6 +1036,77 @@ pub fn document_migration_reset_node_definition(document_serialized_content: &st
 	false
 }
 
+pub fn document_migration_replace_resources_referenced_by_hash(document_serialized_content: String) -> (String, HashMap<ResourceHash, ResourceId>) {
+	fn collect_resources_referenced_by_hash(s: &str) -> HashMap<ResourceHash, Vec<Range<usize>>> {
+		let mut out: HashMap<ResourceHash, Vec<Range<usize>>> = HashMap::new();
+		let mut offset = 0;
+
+		while let Some(i) = s[offset..].find("\"Resource\":") {
+			let abs = offset + i + 11; // pos after `"Resource":`
+			offset = abs;
+
+			if let Some(j) = s[offset..].find('}') {
+				let chunk_start = offset;
+				let chunk = &s[chunk_start..chunk_start + j];
+
+				let quotes: Vec<_> = chunk.match_indices('"').collect();
+				if quotes.len() == 2 {
+					let q0 = chunk_start + quotes[0].0;
+					let q1 = chunk_start + quotes[1].0;
+					let hash = &s[q0 + 1..q1];
+
+					if hash.len() == 64
+						&& hash.bytes().all(|b| matches!(b, b'0'..=b'9' | b'a'..=b'f'))
+						&& let Ok(hash) = hash.parse::<ResourceHash>()
+					{
+						out.entry(hash).or_default().push(q0..q1 + 1);
+					}
+				}
+
+				offset = chunk_start + j + 1;
+			} else {
+				break;
+			}
+		}
+
+		out
+	}
+
+	let resources_by_hash = collect_resources_referenced_by_hash(&document_serialized_content);
+
+	// Require each referenced hash to also appear as an embedded resource map key (more than ranges.len() occurrences).
+	if resources_by_hash.is_empty()
+		|| !resources_by_hash
+			.iter()
+			.all(|(hash, ranges)| document_serialized_content.matches(&hash.to_string()).count() > ranges.len())
+	{
+		return (document_serialized_content, HashMap::new());
+	}
+
+	// Assign a new ResourceId for each hash
+	let mut hash_to_id: HashMap<ResourceHash, ResourceId> = HashMap::new();
+	for hash in resources_by_hash.keys() {
+		#[allow(clippy::unwrap_or_default)]
+		hash_to_id.entry(*hash).or_insert_with(ResourceId::new);
+	}
+
+	// Each range is 66 bytes (64 hex + 2 quotes) and a ResourceId serializes to at most 20 ASCII digits, so the ID always fits.
+	// Overwrite each hash in place with its ID and pad the leftover bytes with spaces, which JSON deserialization discards.
+	let mut bytes = document_serialized_content.into_bytes();
+	for (hash, ranges) in &resources_by_hash {
+		let id_str = format!("{}", hash_to_id[hash]);
+		let id_bytes = id_str.as_bytes();
+		for range in ranges {
+			bytes[range.start..range.start + id_bytes.len()].copy_from_slice(id_bytes);
+			bytes[range.start + id_bytes.len()..range.end].fill(b' ');
+		}
+	}
+
+	let out = String::from_utf8(bytes).expect("in-place hash-to-ID rewrite produced invalid UTF-8");
+
+	(out, hash_to_id)
+}
+
 pub fn document_migration_upgrades(document: &mut DocumentMessageHandler, reset_node_definitions_on_open: bool) {
 	document.network_interface.migrate_path_modify_node();
 
@@ -1597,14 +1670,18 @@ fn migrate_node(node_id: &NodeId, node: &DocumentNode, network_path: &[NodeId],
 		});
 
 		if let Some(image) = image {
-			let hash = document.embedded_resources.store(graphene_std::application_io::resource::Resource::new(image.to_png()));
+			let hash = document.resources.embedded.store(Resource::new(image.to_png()));
+
+			let resource_id = ResourceId::new();
+			document.resources.registry.push_source_back(&resource_id, DataSource::Embedded);
+			document.resources.registry.resolve(&resource_id, hash);
 
 			let mut node_template = resolve_document_node_type(&reference)?.default_node_template();
 			document.network_interface.replace_implementation(node_id, network_path, &mut node_template);
 			let _ = document.network_interface.replace_inputs(node_id, network_path, &mut node_template);
 			document
 				.network_interface
-				.set_input(&InputConnector::node(*node_id, 0), NodeInput::value(TaggedValue::Resource(hash), false), network_path);
+				.set_input(&InputConnector::node(*node_id, 0), NodeInput::value(TaggedValue::Resource(resource_id), false), network_path);
 		}
 	}
 

editor/src/messages/portfolio/portfolio_message_handler.rs

@@ -26,7 +26,7 @@ use crate::messages::tool::utility_types::{HintData, ToolType};
 use crate::messages::viewport::ToPhysical;
 use crate::node_graph_executor::{ExportConfig, NodeGraphExecutor};
 use glam::{DAffine2, DVec2};
-use graph_craft::application_io::resource::ResourceHash;
+use graph_craft::application_io::resource::{DataSource, ResourceHash};
 use graph_craft::document::NodeId;
 use graphene_std::Color;
 use graphene_std::raster_types::Image;
@@ -48,7 +48,7 @@ pub struct PortfolioMessageContext<'a> {
 	pub reset_node_definitions_on_open: bool,
 	pub timing_information: TimingInformation,
 	pub viewport: &'a ViewportMessageHandler,
-	pub resources: &'a ResourceMessageHandler,
+	pub resource_storage: &'a ResourceStorageMessageHandler,
 }
 
 #[derive(Debug, Default, ExtractField)]
@@ -87,7 +87,7 @@ impl MessageHandler<PortfolioMessage, PortfolioMessageContext<'_>> for Portfolio
 			reset_node_definitions_on_open,
 			timing_information,
 			viewport,
-			resources,
+			resource_storage,
 		} = context;
 
 		match message {
@@ -104,7 +104,7 @@ impl MessageHandler<PortfolioMessage, PortfolioMessageContext<'_>> for Portfolio
 						current_tool,
 						preferences,
 						viewport,
-						resources,
+						resource_storage,
 						data_panel_open: self.workspace_panel_layout.is_panel_visible(PanelType::Data) && !self.workspace_panel_layout.focus_document,
 						layers_panel_open: self.workspace_panel_layout.is_panel_visible(PanelType::Layers) && !self.workspace_panel_layout.focus_document,
 						properties_panel_open: self.workspace_panel_layout.is_panel_visible(PanelType::Properties) && !self.workspace_panel_layout.focus_document,
@@ -174,7 +174,7 @@ impl MessageHandler<PortfolioMessage, PortfolioMessageContext<'_>> for Portfolio
 						current_tool,
 						preferences,
 						viewport,
-						resources,
+						resource_storage,
 						data_panel_open: self.workspace_panel_layout.is_panel_visible(PanelType::Data) && !self.workspace_panel_layout.focus_document,
 						layers_panel_open: self.workspace_panel_layout.is_panel_visible(PanelType::Layers) && !self.workspace_panel_layout.focus_document,
 						properties_panel_open: self.workspace_panel_layout.is_panel_visible(PanelType::Properties) && !self.workspace_panel_layout.focus_document,
@@ -465,10 +465,11 @@ impl MessageHandler<PortfolioMessage, PortfolioMessageContext<'_>> for Portfolio
 						return;
 					}
 				}
-				for document in self.documents.values() {
+				for document in self.documents.values_mut() {
-					used_resources.extend(document.used_resources(true));
+					document.garbage_collect_resources();
+					used_resources.extend(document.resources.registry.resolved().filter_map(|info| info.hash.cloned()));
 				}
-				responses.add(ResourceMessage::GarbageCollect {
+				responses.add(ResourceStorageMessage::GarbageCollect {
 					used: Vec::from_iter(used_resources).into_boxed_slice(),
 				});
 			}
@@ -873,6 +874,8 @@ impl MessageHandler<PortfolioMessage, PortfolioMessageContext<'_>> for Portfolio
 				let reset_node_definitions_on_open = reset_node_definitions_on_open || document_migration_reset_node_definition(&document_serialized_content);
 				// Upgrade the document being opened with string replacements on the original JSON
 				let document_serialized_content = document_migration_string_preprocessing(document_serialized_content);
+				// Upgrade resources from being referend by hash to beeing referened by ID
+				let (document_serialized_content, resource_hash_to_id_migration_map) = document_migration_replace_resources_referenced_by_hash(document_serialized_content);
 
 				// Deserialize the document
 				let document = DocumentMessageHandler::deserialize_document(&document_serialized_content);
@@ -922,13 +925,22 @@ impl MessageHandler<PortfolioMessage, PortfolioMessageContext<'_>> for Portfolio
 				document_migration_upgrades(&mut document, reset_node_definitions_on_open);
 
 				// Load the document's embedded resources into the resource storage
-				std::mem::take(&mut document.embedded_resources).into_iter().for_each(|(hash, resource)| {
+				std::mem::take(&mut document.resources.embedded).into_iter().for_each(|(hash, resource)| {
 					let data: Arc<[u8]> = Arc::from(resource.as_ref());
 					if ResourceHash::from(data.as_ref()) != hash {
 						log::error!("Resource hash mismatch for resource with hash {hash}");
 						return;
 					}
-					responses.add(ResourceMessage::Store { data });
+					responses.add(ResourceStorageMessage::Store { data });
+
+					// TODO: Eventually remove this document upgrade code
+					// Register any resources that were previously referenced by hash
+					if let Some(id) = resource_hash_to_id_migration_map.get(&hash)
+						&& !document.resources.registry.contains(id)
+					{
+						document.resources.registry.resolve(id, hash);
+						document.resources.registry.push_source_back(id, DataSource::Embedded);
+					}
 				});
 
 				// Ensure each node has the metadata for its inputs
@@ -2065,7 +2077,7 @@ impl PortfolioMessageHandler {
 				name: document.name.clone(),
 				path: document.path.clone(),
 				is_saved: document.is_saved(),
-				resources: Some(document.used_resources(false).into_iter().collect()),
+				resources: Some(document.resources.registry.resolved().filter_map(|info| info.hash.cloned()).collect::<Vec<_>>().into_boxed_slice()),
 			})
 		} else {
 			self.unloaded_documents.get(&document_id).cloned()

editor/src/messages/prelude.rs

@@ -27,11 +27,12 @@ pub use crate::messages::portfolio::document::navigation::{NavigationMessage, Na
 pub use crate::messages::portfolio::document::node_graph::{NodeGraphMessage, NodeGraphMessageDiscriminant, NodeGraphMessageHandler};
 pub use crate::messages::portfolio::document::overlays::{OverlaysMessage, OverlaysMessageContext, OverlaysMessageDiscriminant, OverlaysMessageHandler};
 pub use crate::messages::portfolio::document::properties_panel::{PropertiesPanelMessage, PropertiesPanelMessageDiscriminant, PropertiesPanelMessageHandler};
+pub use crate::messages::portfolio::document::resource::{ResourceMessage, ResourceMessageContext, ResourceMessageDiscriminant, ResourceMessageHandler};
 pub use crate::messages::portfolio::document::{DocumentMessage, DocumentMessageContext, DocumentMessageDiscriminant, DocumentMessageHandler};
 pub use crate::messages::portfolio::persistent_state::{PersistentStateMessage, PersistentStateMessageContext, PersistentStateMessageDiscriminant, PersistentStateMessageHandler};
 pub use crate::messages::portfolio::{PortfolioMessage, PortfolioMessageContext, PortfolioMessageDiscriminant, PortfolioMessageHandler};
 pub use crate::messages::preferences::{PreferencesMessage, PreferencesMessageDiscriminant, PreferencesMessageHandler};
-pub use crate::messages::resource::{ResourceMessage, ResourceMessageContext, ResourceMessageDiscriminant, ResourceMessageHandler};
+pub use crate::messages::resource_storage::{ResourceStorageMessage, ResourceStorageMessageContext, ResourceStorageMessageDiscriminant, ResourceStorageMessageHandler};
 pub use crate::messages::tool::transform_layer::{TransformLayerMessage, TransformLayerMessageDiscriminant, TransformLayerMessageHandler};
 pub use crate::messages::tool::{ToolMessage, ToolMessageContext, ToolMessageDiscriminant, ToolMessageHandler};
 pub use crate::messages::viewport::{ViewportMessage, ViewportMessageDiscriminant, ViewportMessageHandler};

editor/src/messages/resource_storage/mod.rs

@@ -0,0 +1,7 @@
+mod resource_storage_message;
+mod resource_storage_message_handler;
+
+#[doc(inline)]
+pub use resource_storage_message::{ResourceStorageMessage, ResourceStorageMessageDiscriminant};
+#[doc(inline)]
+pub use resource_storage_message_handler::{ResourceStorageMessageContext, ResourceStorageMessageHandler, ResourcesHandle};

editor/src/messages/resource_storage/resource_storage_message.rs

@@ -2,9 +2,9 @@ use crate::messages::prelude::*;
 use graph_craft::application_io::resource::ResourceHash;
 use std::sync::Arc;
 
-#[impl_message(Message, Resource)]
+#[impl_message(Message, ResourceStorage)]
 #[derive(PartialEq, Clone, Debug, serde::Serialize, serde::Deserialize)]
-pub enum ResourceMessage {
+pub enum ResourceStorageMessage {
 	Store { data: Arc<[u8]> },
 	GarbageCollect { used: Box<[ResourceHash]> },
 }

editor/src/messages/resource_storage/resource_storage_message_handler.rs

@@ -15,11 +15,11 @@ impl LoadResource for ResourcesHandle {
 }
 
 #[derive(ExtractField)]
-pub struct ResourceMessageHandler {
+pub struct ResourceStorageMessageHandler {
 	storage: Option<Arc<RwLock<Box<dyn ResourceStorage>>>>,
 }
 
-impl ResourceMessageHandler {
+impl ResourceStorageMessageHandler {
 	pub fn new(resource_storage: Box<dyn ResourceStorage>) -> Self {
 		Self {
 			storage: Some(Arc::new(RwLock::new(resource_storage))),
@@ -33,13 +33,13 @@ impl ResourceMessageHandler {
 	}
 }
 
-impl std::fmt::Debug for ResourceMessageHandler {
+impl std::fmt::Debug for ResourceStorageMessageHandler {
 	fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-		f.debug_struct("ResourceMessageHandler").finish_non_exhaustive()
+		f.debug_struct("ResourceStorageMessageHandler").finish_non_exhaustive()
 	}
 }
 
-impl Default for ResourceMessageHandler {
+impl Default for ResourceStorageMessageHandler {
 	#[cfg(not(test))]
 	fn default() -> Self {
 		Self { storage: None }
@@ -54,11 +54,11 @@ impl Default for ResourceMessageHandler {
 }
 
 #[derive(ExtractField)]
-pub struct ResourceMessageContext {}
+pub struct ResourceStorageMessageContext {}
 
 #[message_handler_data]
-impl MessageHandler<ResourceMessage, ResourceMessageContext> for ResourceMessageHandler {
+impl MessageHandler<ResourceStorageMessage, ResourceStorageMessageContext> for ResourceStorageMessageHandler {
-	fn process_message(&mut self, message: ResourceMessage, _responses: &mut VecDeque<Message>, _context: ResourceMessageContext) {
+	fn process_message(&mut self, message: ResourceStorageMessage, _responses: &mut VecDeque<Message>, _context: ResourceStorageMessageContext) {
 		let Some(storage) = &self.storage else {
 			log::error!("Received resource message but storage is not initialized");
 			return;
@@ -66,14 +66,14 @@ impl MessageHandler<ResourceMessage, ResourceMessageContext> for ResourceMessage
 		let mut storage = storage.write().unwrap();
 
 		match message {
-			ResourceMessage::Store { data } => {
+			ResourceStorageMessage::Store { data } => {
 				let _hash = storage.store(data.as_ref());
 			}
-			ResourceMessage::GarbageCollect { used } => {
+			ResourceStorageMessage::GarbageCollect { used } => {
 				storage.garbage_collect(&used);
 			}
 		}
 	}
 
-	advertise_actions!(ResourceMessageDiscriminant;);
+	advertise_actions!(ResourceStorageMessageDiscriminant;);
 }

editor/src/node_graph_executor.rs

@@ -113,8 +113,14 @@ impl NodeGraphExecutor {
 		let mut network = document.network_interface.document_network().clone();
 		let instrumented = Instrumented::new(&mut network);
 
+		let resources = document.resources.registry.clone();
+
 		self.runtime_io
-			.send(GraphRuntimeRequest::GraphUpdate(GraphUpdate { network, node_to_inspect: Vec::new() }))
+			.send(GraphRuntimeRequest::GraphUpdate(GraphUpdate {
+				network,
+				resources,
+				node_to_inspect: Vec::new(),
+			}))
 			.map_err(|e| e.to_string())?;
 		Ok(instrumented)
 	}
@@ -128,8 +134,10 @@ impl NodeGraphExecutor {
 			self.previous_node_to_inspect.clone_from(&node_to_inspect);
 			self.node_graph_hash = network_hash;
 
+			let resources = document.resources.registry.clone();
+
 			self.runtime_io
-				.send(GraphRuntimeRequest::GraphUpdate(GraphUpdate { network, node_to_inspect }))
+				.send(GraphRuntimeRequest::GraphUpdate(GraphUpdate { network, resources, node_to_inspect }))
 				.map_err(|e| e.to_string())?;
 		}
 
@@ -237,6 +245,7 @@ impl NodeGraphExecutor {
 	/// Evaluates a node graph for export
 	pub fn submit_document_export(&mut self, document: &mut DocumentMessageHandler, document_id: DocumentId, mut export_config: ExportConfig) -> Result<(), String> {
 		let network = document.network_interface.document_network().clone();
+		let resources = document.resources.registry.clone();
 
 		let export_format = if export_config.file_type == FileType::Svg {
 			graphene_std::application_io::ExportFormat::Svg
@@ -278,7 +287,11 @@ impl NodeGraphExecutor {
 
 		// Execute the node graph
 		self.runtime_io
-			.send(GraphRuntimeRequest::GraphUpdate(GraphUpdate { network, node_to_inspect: Vec::new() }))
+			.send(GraphRuntimeRequest::GraphUpdate(GraphUpdate {
+				network,
+				resources,
+				node_to_inspect: Vec::new(),
+			}))
 			.map_err(|e| e.to_string())?;
 		let execution_id = self.queue_execution(render_config);
 		self.futures.push_back((

editor/src/node_graph_executor/runtime.rs

@@ -1,6 +1,7 @@
 use super::*;
 use crate::messages::frontend::utility_types::{ExportBounds, FileType};
 use glam::{DAffine2, DVec2, UVec2};
+use graph_craft::application_io::resource::ResourceRegistry;
 use graph_craft::application_io::{PlatformApplicationIo, PlatformEditorApi};
 use graph_craft::document::value::{RenderOutput, RenderOutputType, TaggedValue};
 use graph_craft::document::{NodeId, NodeNetwork};
@@ -42,6 +43,7 @@ pub struct NodeRuntime {
 	update_thumbnails: bool,
 
 	editor_api: Arc<PlatformEditorApi>,
+	resources: ResourceRegistry,
 	node_graph_errors: GraphErrors,
 	monitor_nodes: Vec<Vec<NodeId>>,
 
@@ -76,6 +78,7 @@ pub enum GraphRuntimeRequest {
 #[derive(Debug, serde::Serialize, serde::Deserialize)]
 pub struct GraphUpdate {
 	pub(super) network: NodeNetwork,
+	pub(super) resources: ResourceRegistry,
 	/// Full path from the root network to the node that should be temporarily inspected during execution.
 	/// The last element is the inspect target; preceding elements identify the nested subnetwork it lives in,
 	/// so the runtime can splice its monitor node alongside the target instead of only at the top level.
@@ -127,6 +130,7 @@ impl NodeRuntime {
 			sender: InternalNodeGraphUpdateSender(sender.clone()),
 			editor_preferences: EditorPreferences::default(),
 			old_graph: None,
+			resources: ResourceRegistry::default(),
 			update_thumbnails: true,
 
 			editor_api: PlatformEditorApi {
@@ -226,11 +230,16 @@ impl NodeRuntime {
 						let _ = self.update_network(graph).await;
 					}
 				}
-				GraphRuntimeRequest::GraphUpdate(GraphUpdate { mut network, node_to_inspect }) => {
+				GraphRuntimeRequest::GraphUpdate(GraphUpdate {
+					mut network,
+					resources,
+					node_to_inspect,
+				}) => {
 					// Insert the monitor node to manage the inspection
 					self.inspect_state = InspectState::monitor_inspect_node(&mut network, &node_to_inspect);
 
 					self.old_graph = Some(network.clone());
+					self.resources = resources;
 
 					self.node_graph_errors.clear();
 					let result = self.update_network(network).await;
@@ -357,7 +366,9 @@ impl NodeRuntime {
 	}
 
 	async fn update_network(&mut self, mut graph: NodeNetwork) -> Result<ResolvedDocumentNodeTypesDelta, (ResolvedDocumentNodeTypesDelta, String)> {
-		preprocessor::expand_network(&mut graph, &self.substitutions);
+		if let Err(e) = preprocessor::expand_network(&mut graph, &self.substitutions, &self.resources) {
+			return Err((ResolvedDocumentNodeTypesDelta::default(), e.to_string()));
+		}
 
 		let scoped_network = wrap_network_in_scope(graph, self.editor_api.clone());
 

frontend/wrapper/src/editor_wrapper.rs

@@ -9,7 +9,7 @@ use crate::EDITOR;
 #[cfg(not(feature = "native"))]
 use crate::helpers::poll_node_graph_evaluation;
 use crate::helpers::{auto_save_all_documents, calculate_hash, render_image_data_to_canvases, request_animation_frame, set_timeout, translate_key, wrapper};
-use crate::{EDITOR_HAS_CRASHED, EDITOR_WRAPPER, Error, FRONTEND_READY, MESSAGE_BUFFER, PANIC_DIALOG_MESSAGE_CALLBACK};
+use crate::{EDITOR_HAS_CRASHED, Error, FRONTEND_READY, MESSAGE_BUFFER};
 #[cfg(not(feature = "native"))]
 #[cfg(all(not(feature = "native"), target_family = "wasm"))]
 use editor::application::{Editor, Environment, Host, Platform};
@@ -53,7 +53,10 @@ impl EditorWrapper {
 		self.send_frontend_message_to_js(message);
 	}
 
+	#[cfg(any(feature = "native", target_family = "wasm"))]
 	fn initialize_wrapper(frontend_message_handler_callback: js_sys::Function) -> EditorWrapper {
+		use crate::{EDITOR_WRAPPER, PANIC_DIALOG_MESSAGE_CALLBACK};
+
 		let panic_callback = frontend_message_handler_callback.clone();
 		let editor_wrapper = EditorWrapper { frontend_message_handler_callback };
 		if EDITOR_WRAPPER.with(|wrapper| wrapper.lock().ok().map(|mut guard| *guard = Some(editor_wrapper.clone()))).is_none() {

node-graph/graph-craft/src/application_io/resource.rs

@@ -1,7 +1,7 @@
 use std::collections::HashMap;
 use std::sync::{Arc, Mutex};
 
-pub use graphene_application_io::resource::{LoadResource, Resource, ResourceFuture, ResourceHash, ResourceStorage};
+pub use graphene_application_io::resource::*;
 #[cfg(not(target_family = "wasm"))]
 pub mod mmap;
 #[cfg(not(target_family = "wasm"))]

node-graph/graph-craft/src/document/value.rs

@@ -396,7 +396,8 @@ tagged_value! {
 	Footprint(Footprint),
 	VectorModification(Box<VectorModification>),
 	ImageData(Image<Color>),
-	Resource(graphene_application_io::resource::ResourceHash),
+	Resource(graphene_application_io::resource::ResourceId),
+	ResourceHash(graphene_application_io::resource::ResourceHash),
 	// ==========
 	// ENUM TYPES
 	// ==========

node-graph/graphene-cli/src/main.rs

@@ -4,6 +4,7 @@ use clap::{Args, Parser, Subcommand};
 use fern::colors::{Color, ColoredLevelConfig};
 use futures::executor::block_on;
 use graph_craft::application_io::EditorPreferences;
+use graph_craft::application_io::resource::ResourceRegistry;
 use graph_craft::application_io::{PlatformApplicationIo, PlatformEditorApi};
 use graph_craft::document::*;
 use graph_craft::graphene_compiler::Compiler;
@@ -244,7 +245,7 @@ fn compile_graph(document_string: String, editor_api: Arc<PlatformEditorApi>) ->
 	fix_nodes(&mut network);
 
 	let substitutions = preprocessor::generate_node_substitutions();
-	preprocessor::expand_network(&mut network, &substitutions);
+	preprocessor::expand_network(&mut network, &substitutions, &ResourceRegistry::default()).expect("Failed to expand network"); // TODO: actually load the resources from the document
 
 	let wrapped_network = wrap_network_in_scope(network.clone(), editor_api);
 

node-graph/interpreted-executor/benches/benchmark_util.rs

@@ -11,7 +11,7 @@ pub fn setup_network(name: &str) -> (DynamicExecutor, ProtoNetwork) {
 	let mut network = load_from_name(name);
 	let editor_api = std::sync::Arc::new(EditorApi::default());
 	let substitutions = preprocessor::generate_node_substitutions();
-	preprocessor::expand_network(&mut network, &substitutions);
+	preprocessor::expand_network(&mut network, &substitutions, &ResourceRegistry::default()).unwrap();
 	let network = wrap_network_in_scope(network, editor_api);
 	let proto_network = compile(network);
 	let executor = block_on(DynamicExecutor::new(proto_network.clone())).unwrap();

node-graph/libraries/resources/src/lib.rs

@@ -22,7 +22,7 @@ pub trait ResourceStorage: LoadResource {
 #[repr(transparent)]
 #[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Hash, graphene_hash::CacheHash, PartialOrd, Ord, DynAny)]
 #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
-pub struct ResourceId(pub u64);
+pub struct ResourceId(u64);
 
 impl ResourceId {
 	pub fn new() -> Self {
@@ -185,13 +185,6 @@ impl CacheHash for ResourceHash {
 	}
 }
 
-#[derive(Clone, Debug)]
-pub struct ResourceInfo {
-	pub id: ResourceId,
-	pub hash: Option<ResourceHash>,
-	pub inputs: DataSources,
-}
-
 pub type DataSources = Box<[DataSource]>;
 
 #[derive(Clone, Debug, PartialEq, Eq)]
@@ -226,11 +219,11 @@ impl ResourceRegistry {
 		self.hashes.keys().chain(self.sources.keys().filter(|id| !self.hashes.contains_key(id))).copied()
 	}
 
-	pub fn info(&self, id: &ResourceId) -> Option<ResourceInfo> {
+	pub fn info(&self, id: &ResourceId) -> Option<ResourceInfo<'_>> {
 		self.contains(id).then(|| ResourceInfo {
 			id: *id,
-			hash: self.hashes.get(id).copied(),
+			hash: self.hashes.get(id),
-			inputs: self.sources.get(id).cloned().unwrap_or_default().into_boxed_slice(),
+			sources: self.sources.get(id).map(|sources| sources.as_slice()).unwrap_or(&[]),
 		})
 	}
 
@@ -242,17 +235,10 @@ impl ResourceRegistry {
 		self.sources.entry(*id).or_default().insert(0, source);
 	}
 
-	pub fn delete(&mut self, id: &ResourceId) -> Option<ResourceInfo> {
+	pub fn delete(&mut self, id: &ResourceId) -> bool {
 		let hash = self.hashes.remove(id);
 		let sources = self.sources.remove(id);
-		if hash.is_none() && sources.is_none() {
+		!(hash.is_none() && sources.is_none())
-			return None;
-		}
-		Some(ResourceInfo {
-			id: *id,
-			hash,
-			inputs: sources.unwrap_or_default().into_boxed_slice(),
-		})
 	}
 
 	pub fn resolve(&mut self, id: &ResourceId, hash: ResourceHash) -> Option<ResourceHash> {
@@ -263,7 +249,18 @@ impl ResourceRegistry {
 		self.hashes.get(id).copied()
 	}
 
-	pub fn unresolved(&self) -> impl Iterator<Item = (ResourceId, &[DataSource])> + '_ {
+	pub fn unresolved(&self) -> impl Iterator<Item = ResourceInfo<'_>> + '_ {
-		self.sources.iter().filter(|(id, _)| !self.hashes.contains_key(id)).map(|(id, sources)| (*id, sources.as_slice()))
+		self.sources.keys().filter(|id| !self.hashes.contains_key(id)).filter_map(|id| self.info(id))
+	}
+
+	pub fn resolved(&self) -> impl Iterator<Item = ResourceInfo<'_>> + '_ {
+		self.hashes.keys().filter_map(|id| self.info(id))
 	}
 }
+
+#[derive(Clone, Debug)]
+pub struct ResourceInfo<'a> {
+	pub id: ResourceId,
+	pub hash: Option<&'a ResourceHash>,
+	pub sources: &'a [DataSource],
+}

node-graph/preprocessor/src/lib.rs

@@ -1,6 +1,7 @@
 #[macro_use]
 extern crate log;
 
+use graph_craft::application_io::resource::{ResourceId, ResourceRegistry};
 use graph_craft::document::value::*;
 use graph_craft::document::*;
 use graph_craft::proto::RegistryValueSource;
@@ -9,19 +10,20 @@ use graphene_std::registry::*;
 use graphene_std::*;
 use std::collections::{HashMap, HashSet};
 
-pub fn expand_network(network: &mut NodeNetwork, substitutions: &HashMap<ProtoNodeIdentifier, DocumentNode>) {
+pub fn expand_network(network: &mut NodeNetwork, substitutions: &HashMap<ProtoNodeIdentifier, DocumentNode>, resources: &ResourceRegistry) -> Result<(), PreprocessorError> {
-	replace_resource_inputs(network);
+	replace_resource_inputs(network, resources)?;
 	expand_network_inner(network, substitutions);
+	Ok(())
 }
 
 /// Replace every `TaggedValue::Resource(hash)` input with a reference to a freshly inserted `resource` proto node.
-fn replace_resource_inputs(network: &mut NodeNetwork) {
+fn replace_resource_inputs(network: &mut NodeNetwork, resources: &ResourceRegistry) -> Result<(), PreprocessorError> {
 	let mut hash_to_node_id: HashMap<graph_craft::application_io::resource::ResourceHash, NodeId> = HashMap::new();
 	let mut new_resource_nodes: Vec<(NodeId, DocumentNode)> = Vec::new();
 
 	for node in network.nodes.values_mut() {
 		if let DocumentNodeImplementation::Network(nested) = &mut node.implementation {
-			replace_resource_inputs(nested);
+			replace_resource_inputs(nested, resources)?;
 			continue;
 		}
 
@@ -31,12 +33,16 @@ fn replace_resource_inputs(network: &mut NodeNetwork) {
 
 		for input in node.inputs.iter_mut() {
 			let NodeInput::Value { tagged_value, .. } = input else { continue };
-			let TaggedValue::Resource(hash) = **tagged_value else { continue };
+			let TaggedValue::Resource(resource_id) = **tagged_value else { continue };
+
+			let Some(hash) = resources.hash(&resource_id) else {
+				return Err(PreprocessorError::ResourceNotFound(resource_id));
+			};
 
 			let resource_id = *hash_to_node_id.entry(hash).or_insert_with(|| {
 				let id = NodeId::new();
 				let resource_node = DocumentNode {
-					inputs: vec![NodeInput::value(TaggedValue::Resource(hash), false), NodeInput::scope("editor-api")],
+					inputs: vec![NodeInput::value(TaggedValue::ResourceHash(hash), false), NodeInput::scope("editor-api")],
 					implementation: DocumentNodeImplementation::ProtoNode(platform_application_io::resource::IDENTIFIER),
 					..Default::default()
 				};
@@ -51,6 +57,8 @@ fn replace_resource_inputs(network: &mut NodeNetwork) {
 	for (id, node) in new_resource_nodes {
 		network.nodes.insert(id, node);
 	}
+
+	Ok(())
 }
 
 fn expand_network_inner(network: &mut NodeNetwork, substitutions: &HashMap<ProtoNodeIdentifier, DocumentNode>) {
@@ -241,3 +249,16 @@ pub fn node_inputs(fields: &[registry::FieldMetadata], first_node_io: &NodeIOTyp
 		})
 		.collect()
 }
+
+#[derive(Debug)]
+pub enum PreprocessorError {
+	ResourceNotFound(ResourceId),
+}
+
+impl std::fmt::Display for PreprocessorError {
+	fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+		match self {
+			PreprocessorError::ResourceNotFound(id) => write!(f, "Resource not found: {id:?}"),
+		}
+	}
+}