Implement content-addressed resource storage for raster images (#4148)

* Implement content-addressed resource storage

* Implement OPFS resource storage

* Remove ResourceStorage::read

* Review

editor/src/messages/portfolio/document/document_message_handler.rs

@@ -19,6 +19,7 @@ use crate::messages::portfolio::document::overlays::grid_overlays::{grid_overlay
 use crate::messages::portfolio::document::overlays::utility_types::{OverlaysType, OverlaysVisibilitySettings, Pivot};
 use crate::messages::portfolio::document::properties_panel::properties_panel_message_handler::PropertiesPanelMessageContext;
 use crate::messages::portfolio::document::utility_types::document_metadata::{DocumentMetadata, LayerNodeIdentifier};
+use crate::messages::portfolio::document::utility_types::embedded_resources::EmbeddedResources;
 use crate::messages::portfolio::document::utility_types::misc::{AlignAggregate, AlignAxis, FlipAxis, PTZ};
 use crate::messages::portfolio::document::utility_types::network_interface::{FlowType, InputConnector, NodeTemplate, OutputConnector};
 use crate::messages::portfolio::utility_types::{CachedData, PanelType};
@@ -29,13 +30,13 @@ use crate::messages::tool::tool_messages::tool_prelude::Key;
 use crate::messages::tool::utility_types::ToolType;
 use crate::node_graph_executor::NodeGraphExecutor;
 use glam::{DAffine2, DVec2};
+use graph_craft::application_io::{ResourceHash, wgpu_available};
 use graph_craft::descriptor;
 use graph_craft::document::value::TaggedValue;
 use graph_craft::document::{NodeId, NodeInput, NodeNetwork, OldNodeNetwork};
 use graphene_std::math::quad::Quad;
 use graphene_std::path_bool_nodes::boolean_intersect;
 use graphene_std::raster::BlendMode;
-use graphene_std::render_node::wgpu_available;
 use graphene_std::subpath::Subpath;
 use graphene_std::vector::PointId;
 use graphene_std::vector::click_target::{ClickTarget, ClickTargetType};
@@ -59,6 +60,7 @@ pub struct DocumentMessageContext<'a> {
 	pub layers_panel_open: bool,
 	pub properties_panel_open: bool,
 	pub viewport: &'a ViewportMessageHandler,
+	pub resources: &'a ResourceMessageHandler,
 }
 
 #[derive(Clone, Debug, serde::Serialize, serde::Deserialize, ExtractField)]
@@ -107,6 +109,9 @@ pub struct DocumentMessageHandler {
 	pub graph_view_overlay_open: bool,
 	/// The current opacity of the faded node graph background that covers up the artwork.
 	pub graph_fade_artwork_percentage: f64,
+	/// Resources embedded in the document. Only propagated if saving to an external file and never for autosaved documents.
+	#[serde(rename = "resources", default, skip_serializing_if = "EmbeddedResources::is_empty")]
+	pub embedded_resources: EmbeddedResources,
 
 	// =============================================
 	// Fields omitted from the saved document format
@@ -169,6 +174,7 @@ impl Default for DocumentMessageHandler {
 			graph_view_overlay_open: false,
 			snapping_state: SnappingState::default(),
 			graph_fade_artwork_percentage: 80.,
+			embedded_resources: EmbeddedResources::default(),
 			// =============================================
 			// Fields omitted from the saved document format
 			// =============================================
@@ -200,6 +206,7 @@ impl MessageHandler<DocumentMessage, DocumentMessageContext<'_>> for DocumentMes
 			data_panel_open,
 			layers_panel_open,
 			properties_panel_open,
+			resources,
 		} = context;
 
 		match message {
@@ -916,15 +923,34 @@ impl MessageHandler<DocumentMessage, DocumentMessageContext<'_>> for DocumentMes
 				if path.is_some() {
 					responses.add(DocumentMessage::MarkAsSaved);
 				}
-
 				let folder = self.path.as_ref().and_then(|path| path.parent()).map(|parent| parent.to_path_buf());
 
-				responses.add(FrontendMessage::TriggerSaveDocument {
-					document_id,
-					name: format!("{}.{}", self.name.clone(), FILE_EXTENSION),
-					path,
-					folder,
-					content: self.serialize_document().into_bytes().into(),
+				let resource_hashes = Vec::from_iter(self.used_resources(false)).into_boxed_slice();
+				let resources = resources.resources();
+				let mut document = self.clone();
+				let name = format!("{}.{}", self.name.clone(), FILE_EXTENSION);
+
+				responses.add(FrontendMessage::Await {
+					future: FrontendMessageFuture::new(async move {
+						let loads = resource_hashes
+							.into_iter()
+							.map(|hash| {
+								let resource = resources.load(hash);
+								async move { resource.await.map(|resource| (hash, resource)) }
+							})
+							.collect::<Vec<_>>();
+
+						document.embedded_resources = EmbeddedResources::from_iter(futures::future::join_all(loads).await.into_iter().flatten());
+						let content = document.serialize_document();
+
+						FrontendMessage::TriggerSaveDocument {
+							document_id,
+							name,
+							path,
+							folder,
+							content: content.into_bytes().into(),
+						}
+					}),
 				});
 			}
 			DocumentMessage::SavedDocument { path } => {
@@ -1306,11 +1332,10 @@ impl MessageHandler<DocumentMessage, DocumentMessageContext<'_>> for DocumentMes
 				let layer_text_frames = text_frames
 					.into_iter()
 					.filter(|(node_id, _)| self.network_interface.document_network().nodes.contains_key(node_id))
-					.filter_map(|(node_id, frame)| {
-						self.network_interface.is_layer(&node_id, &[]).then(|| {
-							let layer = LayerNodeIdentifier::new(node_id, &self.network_interface);
-							(layer, frame)
-						})
+					.filter(|&(node_id, _)| self.network_interface.is_layer(&node_id, &[]))
+					.map(|(node_id, frame)| {
+						let layer = LayerNodeIdentifier::new(node_id, &self.network_interface);
+						(layer, frame)
 					})
 					.collect();
 				self.network_interface.update_text_frames(layer_text_frames);
@@ -2546,6 +2571,7 @@ impl DocumentMessageHandler {
 
 	/// Helper method for NudgeSelectedLayers message.
 	/// Handles keyboard nudging of selected layers with optional resize mode.
+	#[allow(clippy::too_many_arguments)]
 	fn handle_nudge_selected_layers(
 		&mut self,
 		delta_x: f64,
@@ -3444,6 +3470,16 @@ impl DocumentMessageHandler {
 	pub fn graph_view_overlay_open(&self) -> bool {
 		self.graph_view_overlay_open
 	}
+
+	pub fn used_resources(&self, include_history: bool) -> HashSet<ResourceHash> {
+		let mut resources = HashSet::new();
+		self.network_interface.collect_used_resources(&mut resources);
+		if include_history {
+			self.document_undo_history.iter().for_each(|interface| interface.collect_used_resources(&mut resources));
+			self.document_redo_history.iter().for_each(|interface| interface.collect_used_resources(&mut resources));
+		}
+		resources
+	}
 }
 
 /// Create a network interface with a single export

editor/src/messages/portfolio/document/graph_operation/utility_types.rs

@@ -298,13 +298,18 @@ impl<'a> ModifyInputsContext<'a> {
 		let transform = resolve_proto_node_type(graphene_std::transform_nodes::transform::IDENTIFIER)
 			.expect("Transform node does not exist")
 			.default_node_template();
+
+		let png_bytes: std::sync::Arc<[u8]> = image.to_png().into();
+		let hash = graphene_std::application_io::ResourceHash::from(png_bytes.as_ref());
+		self.responses.add(ResourceMessage::Store { data: png_bytes });
+
 		let image_node = resolve_proto_node_type(graphene_std::raster_nodes::std_nodes::image::IDENTIFIER)
 			.expect("Image node does not exist")
-			.node_template_input_override([Some(NodeInput::value(TaggedValue::None, false)), Some(NodeInput::value(TaggedValue::ImageData(image), false))]);
+			.node_template_input_override([Some(NodeInput::value(TaggedValue::Resource(hash), false))]);
 
-		let image_id = NodeId::new();
-		self.network_interface.insert_node(image_id, image_node, &[]);
-		self.network_interface.move_node_to_chain_start(&image_id, layer, &[], self.import);
+		let image_node_id = NodeId::new();
+		self.network_interface.insert_node(image_node_id, image_node, &[]);
+		self.network_interface.move_node_to_chain_start(&image_node_id, layer, &[], self.import);
 
 		let transform_id = NodeId::new();
 		self.network_interface.insert_node(transform_id, transform, &[]);

editor/src/messages/portfolio/document/utility_types/embedded_resources.rs

@@ -0,0 +1,92 @@
+use base64::Engine;
+use base64::engine::general_purpose::STANDARD as BASE64;
+use graph_craft::application_io::{Resource, ResourceHash};
+use std::collections::HashMap;
+use std::fmt;
+
+#[derive(Clone, Default, Debug, PartialEq)]
+pub struct EmbeddedResources {
+	resources: HashMap<ResourceHash, Resource>,
+}
+
+impl EmbeddedResources {
+	pub fn is_empty(&self) -> bool {
+		self.resources.is_empty()
+	}
+
+	pub fn store(&mut self, resource: Resource) -> ResourceHash {
+		let hash = ResourceHash::from(resource.as_ref());
+		self.resources.insert(hash, resource);
+		hash
+	}
+}
+
+impl FromIterator<(ResourceHash, Resource)> for EmbeddedResources {
+	fn from_iter<T: IntoIterator<Item = (ResourceHash, Resource)>>(iter: T) -> Self {
+		Self {
+			resources: iter.into_iter().collect(),
+		}
+	}
+}
+
+impl IntoIterator for EmbeddedResources {
+	type Item = (ResourceHash, Resource);
+	type IntoIter = std::collections::hash_map::IntoIter<ResourceHash, Resource>;
+
+	fn into_iter(self) -> Self::IntoIter {
+		self.resources.into_iter()
+	}
+}
+
+impl serde::Serialize for EmbeddedResources {
+	fn serialize<S: serde::Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
+		use serde::ser::SerializeMap;
+
+		let human_readable = serializer.is_human_readable();
+		let mut map = serializer.serialize_map(Some(self.resources.len()))?;
+		for (hash, resource) in &self.resources {
+			let bytes: &[u8] = resource.as_ref();
+			if human_readable {
+				map.serialize_entry(hash, &BASE64.encode(bytes))?;
+			} else {
+				map.serialize_entry(hash, serde_bytes::Bytes::new(bytes))?;
+			}
+		}
+		map.end()
+	}
+}
+
+impl<'de> serde::Deserialize<'de> for EmbeddedResources {
+	fn deserialize<D: serde::Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
+		struct EmbeddedResourcesVisitor {
+			human_readable: bool,
+		}
+
+		impl<'de> serde::de::Visitor<'de> for EmbeddedResourcesVisitor {
+			type Value = EmbeddedResources;
+
+			fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
+				formatter.write_str("a map of ResourceHash to resource bytes")
+			}
+
+			fn visit_map<A: serde::de::MapAccess<'de>>(self, mut map: A) -> Result<Self::Value, A::Error> {
+				let mut resources = HashMap::with_capacity(map.size_hint().unwrap_or(0));
+				while let Some(hash) = map.next_key::<ResourceHash>()? {
+					let resource = if self.human_readable {
+						let encoded: String = map.next_value()?;
+						let bytes = BASE64.decode(&encoded).map_err(serde::de::Error::custom)?;
+						Resource::new(bytes)
+					} else {
+						let bytes: serde_bytes::ByteBuf = map.next_value()?;
+						Resource::new(bytes.into_vec())
+					};
+					resources.insert(hash, resource);
+				}
+				Ok(EmbeddedResources { resources })
+			}
+		}
+
+		let human_readable = deserializer.is_human_readable();
+		deserializer.deserialize_map(EmbeddedResourcesVisitor { human_readable })
+	}
+}

editor/src/messages/portfolio/document/utility_types/mod.rs

@@ -1,5 +1,6 @@
 pub mod clipboards;
 pub mod document_metadata;
+pub mod embedded_resources;
 pub mod error;
 pub mod misc;
 pub mod network_interface;

editor/src/messages/portfolio/document/utility_types/network_interface.rs

@@ -20,6 +20,7 @@ use crate::messages::tool::tool_messages::tool_prelude::NumberInputMode;
 use deserialization::deserialize_node_persistent_metadata;
 use glam::{DAffine2, DVec2, IVec2};
 use graph_craft::Type;
+use graph_craft::application_io::ResourceHash;
 use graph_craft::document::value::TaggedValue;
 use graph_craft::document::{DocumentNode, DocumentNodeImplementation, NodeId, NodeInput, NodeNetwork, OldDocumentNodeImplementation, OldNodeNetwork};
 use graphene_std::ContextDependencies;
@@ -510,6 +511,10 @@ impl NodeNetworkInterface {
 		})
 	}
 
+	pub fn collect_used_resources(&self, target: &mut HashSet<ResourceHash>) {
+		collect_network_resources(self.document_network(), target);
+	}
+
 	pub fn frontend_imports(&mut self, network_path: &[NodeId]) -> Vec<Option<FrontendGraphOutput>> {
 		match network_path.split_last() {
 			Some((node_id, encapsulating_network_path)) => {
@@ -6769,6 +6774,21 @@ pub enum TransactionStatus {
 	Finished,
 }
 
+fn collect_network_resources(network: &NodeNetwork, out: &mut HashSet<ResourceHash>) {
+	for node in network.nodes.values() {
+		for input in &node.inputs {
+			if let NodeInput::Value { tagged_value, .. } = input
+				&& let TaggedValue::Resource(hash) = &**tagged_value
+			{
+				out.insert(*hash);
+			}
+		}
+		if let DocumentNodeImplementation::Network(nested) = &node.implementation {
+			collect_network_resources(nested, out);
+		}
+	}
+}
+
 #[cfg(test)]
 mod network_interface_tests {
 	use crate::test_utils::test_prelude::*;

editor/src/messages/portfolio/document_migration.rs

@@ -1588,12 +1588,24 @@ fn migrate_node(node_id: &NodeId, node: &DocumentNode, network_path: &[NodeId],
 		document.network_interface.set_input(&InputConnector::node(*node_id, 4), old_inputs[3].clone(), network_path);
 	}
 
-	if reference == DefinitionIdentifier::ProtoNode(graphene_std::raster_nodes::std_nodes::image::IDENTIFIER) && inputs_count == 1 {
-		let mut node_template = resolve_document_node_type(&reference)?.default_node_template();
-		document.network_interface.replace_implementation(node_id, network_path, &mut node_template);
+	// Upgrade `image` nodes that stored image data as `Image<Color>`` to `image` nodes that store a reference to the image data as a resource.
+	// Encodes the image data as PNG and stores it in the document's embedded resources and rewires the node to reference that resource.
+	if reference == DefinitionIdentifier::ProtoNode(graphene_std::raster_nodes::std_nodes::image::IDENTIFIER) {
+		let image = node.inputs.iter().find_map(|input| match input.as_value()? {
+			TaggedValue::ImageData(image) => Some(image.clone()),
+			_ => None,
+		});
 
-		// Insert a new empty input for the image
-		document.network_interface.add_import(TaggedValue::None, false, 0, "Empty", "", &[*node_id]);
+		if let Some(image) = image {
+			let hash = document.embedded_resources.store(graphene_std::application_io::Resource::new(image.to_png()));
+
+			let mut node_template = resolve_document_node_type(&reference)?.default_node_template();
+			document.network_interface.replace_implementation(node_id, network_path, &mut node_template);
+			let _ = document.network_interface.replace_inputs(node_id, network_path, &mut node_template);
+			document
+				.network_interface
+				.set_input(&InputConnector::node(*node_id, 0), NodeInput::value(TaggedValue::Resource(hash), false), network_path);
+		}
 	}
 
 	if reference == DefinitionIdentifier::ProtoNode(graphene_std::raster_nodes::std_nodes::noise_pattern::IDENTIFIER) && inputs_count == 15 {

editor/src/messages/portfolio/portfolio_message.rs

@@ -50,6 +50,7 @@ pub enum PortfolioMessage {
 	},
 	DestroyAllDocuments,
 	EditorPreferences,
+	GarbageCollectResources,
 	FontCatalogLoaded {
 		catalog: FontCatalog,
 	},

editor/src/messages/portfolio/portfolio_message_handler.rs

@@ -26,6 +26,7 @@ use crate::messages::tool::utility_types::{HintData, ToolType};
 use crate::messages::viewport::ToPhysical;
 use crate::node_graph_executor::{ExportConfig, NodeGraphExecutor};
 use glam::{DAffine2, DVec2};
+use graph_craft::application_io::ResourceHash;
 use graph_craft::document::NodeId;
 use graphene_std::Color;
 use graphene_std::raster_types::Image;
@@ -35,6 +36,7 @@ use graphene_std::text::Font;
 use graphene_std::vector::misc::HandleId;
 use graphene_std::vector::{PointId, SegmentId, Vector, VectorModificationType};
 use std::path::PathBuf;
+use std::sync::Arc;
 use std::vec;
 
 #[derive(ExtractField)]
@@ -46,6 +48,7 @@ pub struct PortfolioMessageContext<'a> {
 	pub reset_node_definitions_on_open: bool,
 	pub timing_information: TimingInformation,
 	pub viewport: &'a ViewportMessageHandler,
+	pub resources: &'a ResourceMessageHandler,
 }
 
 #[derive(Debug, Default, ExtractField)]
@@ -74,6 +77,7 @@ impl MessageHandler<PortfolioMessage, PortfolioMessageContext<'_>> for Portfolio
 			reset_node_definitions_on_open,
 			timing_information,
 			viewport,
+			resources,
 		} = context;
 
 		match message {
@@ -90,6 +94,7 @@ impl MessageHandler<PortfolioMessage, PortfolioMessageContext<'_>> for Portfolio
 						current_tool,
 						preferences,
 						viewport,
+						resources,
 						data_panel_open: self.workspace_panel_layout.is_panel_visible(PanelType::Data) && !self.workspace_panel_layout.focus_document,
 						layers_panel_open: self.workspace_panel_layout.is_panel_visible(PanelType::Layers) && !self.workspace_panel_layout.focus_document,
 						properties_panel_open: self.workspace_panel_layout.is_panel_visible(PanelType::Properties) && !self.workspace_panel_layout.focus_document,
@@ -159,6 +164,7 @@ impl MessageHandler<PortfolioMessage, PortfolioMessageContext<'_>> for Portfolio
 						current_tool,
 						preferences,
 						viewport,
+						resources,
 						data_panel_open: self.workspace_panel_layout.is_panel_visible(PanelType::Data) && !self.workspace_panel_layout.focus_document,
 						layers_panel_open: self.workspace_panel_layout.is_panel_visible(PanelType::Layers) && !self.workspace_panel_layout.focus_document,
 						properties_panel_open: self.workspace_panel_layout.is_panel_visible(PanelType::Properties) && !self.workspace_panel_layout.focus_document,
@@ -183,6 +189,8 @@ impl MessageHandler<PortfolioMessage, PortfolioMessageContext<'_>> for Portfolio
 						responses.add(PortfolioMessage::AutoSaveDocument { document_id: *document_id });
 					}
 				}
+
+				responses.add(PortfolioMessage::GarbageCollectResources);
 			}
 			PortfolioMessage::AutoSaveDocument { document_id } => {
 				let Some(document) = self.document(document_id) else { return };
@@ -437,6 +445,23 @@ impl MessageHandler<PortfolioMessage, PortfolioMessageContext<'_>> for Portfolio
 				}
 			}
 			PortfolioMessage::EditorPreferences => self.executor.update_editor_preferences(preferences.editor_preferences()),
+			PortfolioMessage::GarbageCollectResources => {
+				let mut used_resources = HashSet::new();
+				for (id, info) in self.unloaded_documents.iter() {
+					if let Some(resources) = &info.resources {
+						used_resources.extend(resources.iter());
+					} else {
+						responses.add(PersistentStateMessage::ReadDocument { document_id: *id });
+						return;
+					}
+				}
+				for document in self.documents.values() {
+					used_resources.extend(document.used_resources(true));
+				}
+				responses.add(ResourceMessage::GarbageCollect {
+					used: Vec::from_iter(used_resources).into_boxed_slice(),
+				});
+			}
 			PortfolioMessage::LoadDocumentResources { document_id } => {
 				let catalog = &self.cached_data.font_catalog;
 
@@ -758,6 +783,16 @@ impl MessageHandler<PortfolioMessage, PortfolioMessageContext<'_>> for Portfolio
 				// Upgrade the document's nodes to be compatible with the latest version
 				document_migration_upgrades(&mut document, reset_node_definitions_on_open);
 
+				// Load the document's embedded resources into the resource storage
+				std::mem::take(&mut document.embedded_resources).into_iter().for_each(|(hash, resource)| {
+					let data: Arc<[u8]> = Arc::from(resource.as_ref());
+					if ResourceHash::from(data.as_ref()) != hash {
+						log::error!("Resource hash mismatch for resource with hash {hash}");
+						return;
+					}
+					responses.add(ResourceMessage::Store { data });
+				});
+
 				// Ensure each node has the metadata for its inputs
 				for (node_id, node, path) in document.network_interface.document_network().clone().recursive_nodes() {
 					document.network_interface.validate_input_metadata(node_id, node, &path);
@@ -1869,6 +1904,7 @@ impl PortfolioMessageHandler {
 				name: document.name.clone(),
 				path: document.path.clone(),
 				is_saved: document.is_saved(),
+				resources: Some(document.used_resources(false).into_iter().collect()),
 			})
 		} else {
 			self.unloaded_documents.get(&document_id).cloned()