Implement content-addressed resource storage for raster images (#4148)

* Implement content-addressed resource storage

* Implement OPFS resource storage

* Remove ResourceStorage::read

* Review

node-graph/libraries/application-io/Cargo.toml

@@ -20,6 +20,7 @@ vector-types = { workspace = true }
 text-nodes = { workspace = true }
 
 # Workspace dependencies
+blake3 = { workspace = true }
 glam = { workspace = true }
 log = { workspace = true }
 

node-graph/libraries/application-io/src/lib.rs

@@ -2,15 +2,17 @@ use core_types::transform::Footprint;
 use dyn_any::{DynAny, StaticType, StaticTypeSized};
 use glam::DVec2;
 use std::fmt::Debug;
-use std::future::Future;
 use std::hash::{Hash, Hasher};
-use std::pin::Pin;
 use std::ptr::addr_of;
 use std::sync::Arc;
 use std::time::Duration;
 use text_nodes::FontCache;
 use vector_types::vector::style::RenderMode;
 
+pub mod resource;
+pub use core_types::resource::Resource;
+pub use resource::{LoadResource, ResourceFuture, ResourceHash, ResourceStorage};
+
 #[cfg(feature = "wgpu")]
 #[derive(Debug, Clone, Hash, PartialEq, Eq, DynAny)]
 pub struct ImageTexture(Arc<wgpu::Texture>);
@@ -42,17 +44,12 @@ impl From<ImageTexture> for Arc<wgpu::Texture> {
 #[derive(Debug, Clone, Hash, PartialEq, Eq, DynAny)]
 pub struct ImageTexture;
 
-#[cfg(target_family = "wasm")]
-pub type ResourceFuture = Pin<Box<dyn Future<Output = Result<Arc<[u8]>, ApplicationError>>>>;
-#[cfg(not(target_family = "wasm"))]
-pub type ResourceFuture = Pin<Box<dyn Future<Output = Result<Arc<[u8]>, ApplicationError>> + Send>>;
-
 pub trait ApplicationIo {
 	type Executor;
 	fn gpu_executor(&self) -> Option<&Self::Executor> {
 		None
 	}
-	fn load_resource(&self, url: impl AsRef<str>) -> Result<ResourceFuture, ApplicationError>;
+	fn load_resource(&self, hash: ResourceHash) -> resource::ResourceFuture;
 }
 
 impl<T: ApplicationIo> ApplicationIo for &T {
@@ -62,17 +59,11 @@ impl<T: ApplicationIo> ApplicationIo for &T {
 		(**self).gpu_executor()
 	}
 
-	fn load_resource<'a>(&self, url: impl AsRef<str>) -> Result<ResourceFuture, ApplicationError> {
-		(**self).load_resource(url)
+	fn load_resource(&self, hash: ResourceHash) -> resource::ResourceFuture {
+		(**self).load_resource(hash)
 	}
 }
 
-#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
-pub enum ApplicationError {
-	NotFound,
-	InvalidUrl,
-}
-
 #[derive(Debug, Clone)]
 #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
 pub enum NodeGraphUpdateMessage {}
@@ -139,7 +130,7 @@ impl GetEditorPreferences for DummyPreferences {
 pub struct EditorApi<Io> {
 	/// Font data (for rendering text) made available to the graph through the `PlatformEditorApi`.
 	pub font_cache: FontCache,
-	/// Gives access to APIs like a rendering surface (native window handle or HTML5 canvas) and WGPU (which becomes WebGPU on web).
+	/// Gives access to APIs like resources.
 	pub application_io: Option<Arc<Io>>,
 	pub node_graph_message_sender: Box<dyn NodeGraphUpdateSender + Send + Sync>,
 	/// Editor preferences made available to the graph through the `PlatformEditorApi`.
@@ -162,7 +153,7 @@ impl<Io: Default> Default for EditorApi<Io> {
 impl<Io> Hash for EditorApi<Io> {
 	fn hash<H: Hasher>(&self, state: &mut H) {
 		self.font_cache.hash(state);
-		self.application_io.as_ref().map_or(0, |io| io.as_ref() as *const _ as usize).hash(state);
+		self.application_io.as_ref().map_or(0, |io| io as *const _ as usize).hash(state);
 		(self.node_graph_message_sender.as_ref() as *const dyn NodeGraphUpdateSender).hash(state);
 		(self.editor_preferences.as_ref() as *const dyn GetEditorPreferences).hash(state);
 	}

node-graph/libraries/application-io/src/resource.rs

@@ -0,0 +1,168 @@
+use core_types::CacheHash;
+use core_types::resource::Resource;
+use dyn_any::DynAny;
+use std::fmt;
+use std::future::Future;
+use std::hash::Hash;
+use std::pin::Pin;
+
+pub trait LoadResource: Send + Sync {
+	fn load(&self, hash: ResourceHash) -> ResourceFuture;
+}
+
+pub type ResourceFuture = Pin<Box<dyn Future<Output = Option<Resource>> + Send + 'static>>;
+
+pub trait ResourceStorage: LoadResource {
+	fn store(&mut self, data: &[u8]) -> ResourceHash;
+	fn contains(&mut self, hash: &ResourceHash) -> bool;
+	fn garbage_collect(&mut self, used: &[ResourceHash]);
+}
+
+/// Blake3 content hash of a resource, represented as 32 bytes
+#[derive(Clone, Copy, Debug, Default, Hash, PartialEq, Eq, PartialOrd, Ord, DynAny)]
+pub struct ResourceHash([u8; 32]);
+
+impl From<&[u8]> for ResourceHash {
+	fn from(data: &[u8]) -> Self {
+		Self(blake3::hash(data).into())
+	}
+}
+
+impl From<[u8; 32]> for ResourceHash {
+	fn from(bytes: [u8; 32]) -> Self {
+		Self(bytes)
+	}
+}
+
+impl From<&ResourceHash> for [u8; 32] {
+	fn from(hash: &ResourceHash) -> Self {
+		hash.0
+	}
+}
+
+impl From<&ResourceHash> for String {
+	fn from(hash: &ResourceHash) -> Self {
+		const HEX: &[u8; 16] = b"0123456789abcdef";
+		let mut out = String::with_capacity(hash.0.len() * 2);
+		for byte in &hash.0 {
+			out.push(HEX[(byte >> 4) as usize] as char);
+			out.push(HEX[(byte & 0x0f) as usize] as char);
+		}
+		out
+	}
+}
+
+impl fmt::Display for ResourceHash {
+	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+		f.write_str(&String::from(self))
+	}
+}
+
+impl std::str::FromStr for ResourceHash {
+	type Err = ResourceHashParseError;
+
+	fn from_str(s: &str) -> Result<Self, Self::Err> {
+		fn decode_hex_nibble(byte: u8, position: usize) -> Result<u8, ResourceHashParseError> {
+			match byte {
+				b'0'..=b'9' => Ok(byte - b'0'),
+				b'a'..=b'f' => Ok(byte - b'a' + 10),
+				b'A'..=b'F' => Ok(byte - b'A' + 10),
+				_ => Err(ResourceHashParseError::InvalidCharacter { byte, position }),
+			}
+		}
+
+		let bytes = s.as_bytes();
+		if bytes.len() != 64 {
+			return Err(ResourceHashParseError::InvalidLength { found: bytes.len() });
+		}
+
+		let mut out = [0u8; 32];
+		for (index, chunk) in bytes.chunks_exact(2).enumerate() {
+			let high = decode_hex_nibble(chunk[0], index * 2)?;
+			let low = decode_hex_nibble(chunk[1], index * 2 + 1)?;
+			out[index] = (high << 4) | low;
+		}
+
+		Ok(Self(out))
+	}
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum ResourceHashParseError {
+	InvalidLength { found: usize },
+	InvalidCharacter { byte: u8, position: usize },
+}
+
+impl fmt::Display for ResourceHashParseError {
+	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+		match self {
+			Self::InvalidLength { found } => write!(f, "resource hash must be 64 hex characters, got {found}"),
+			Self::InvalidCharacter { byte, position } => write!(f, "resource hash contains non-hex byte {byte:#04x} at position {position}"),
+		}
+	}
+}
+
+impl std::error::Error for ResourceHashParseError {}
+
+impl TryFrom<&str> for ResourceHash {
+	type Error = ResourceHashParseError;
+
+	fn try_from(value: &str) -> Result<Self, Self::Error> {
+		value.parse()
+	}
+}
+
+#[cfg(feature = "serde")]
+impl serde::Serialize for ResourceHash {
+	fn serialize<S: serde::Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
+		if serializer.is_human_readable() {
+			serializer.serialize_str(&String::from(self))
+		} else {
+			serializer.serialize_bytes(&self.0)
+		}
+	}
+}
+
+#[cfg(feature = "serde")]
+impl<'de> serde::Deserialize<'de> for ResourceHash {
+	fn deserialize<D: serde::Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
+		struct ResourceHashVisitor;
+
+		impl<'de> serde::de::Visitor<'de> for ResourceHashVisitor {
+			type Value = ResourceHash;
+
+			fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
+				formatter.write_str("a 64-character hex string or 32 raw bytes")
+			}
+
+			fn visit_str<E: serde::de::Error>(self, value: &str) -> Result<Self::Value, E> {
+				ResourceHash::try_from(value).map_err(E::custom)
+			}
+
+			fn visit_bytes<E: serde::de::Error>(self, value: &[u8]) -> Result<Self::Value, E> {
+				let bytes: [u8; 32] = value.try_into().map_err(|_| E::invalid_length(value.len(), &"32 bytes"))?;
+				Ok(ResourceHash(bytes))
+			}
+
+			fn visit_seq<A: serde::de::SeqAccess<'de>>(self, mut seq: A) -> Result<Self::Value, A::Error> {
+				let mut bytes = [0u8; 32];
+				for (i, slot) in bytes.iter_mut().enumerate() {
+					*slot = seq.next_element()?.ok_or_else(|| serde::de::Error::invalid_length(i, &"32 bytes"))?;
+				}
+				Ok(ResourceHash(bytes))
+			}
+		}
+
+		if deserializer.is_human_readable() {
+			deserializer.deserialize_str(ResourceHashVisitor)
+		} else {
+			deserializer.deserialize_bytes(ResourceHashVisitor)
+		}
+	}
+}
+
+impl CacheHash for ResourceHash {
+	fn cache_hash<H: core::hash::Hasher>(&self, state: &mut H) {
+		core::hash::Hash::hash(self, state);
+	}
+}

node-graph/libraries/core-types/src/lib.rs

@@ -11,6 +11,7 @@ pub mod misc;
 pub mod ops;
 pub mod registry;
 pub mod render_complexity;
+pub mod resource;
 pub mod transform;
 pub mod uuid;
 pub mod value;

node-graph/libraries/core-types/src/resource.rs

@@ -0,0 +1,54 @@
+use dyn_any::DynAny;
+use graphene_hash::CacheHash;
+use std::hash::Hash;
+use std::ops::Deref;
+use std::sync::Arc;
+
+#[derive(Clone, DynAny)]
+pub struct Resource {
+	inner: Arc<dyn AsRef<[u8]> + Send + Sync>,
+}
+
+impl Resource {
+	pub fn new<T: AsRef<[u8]> + Send + Sync + 'static>(data: T) -> Self {
+		Self { inner: Arc::new(data) }
+	}
+}
+
+impl From<&Resource> for Arc<dyn AsRef<[u8]> + Send + Sync> {
+	fn from(val: &Resource) -> Self {
+		val.inner.clone()
+	}
+}
+
+impl Deref for Resource {
+	type Target = [u8];
+
+	fn deref(&self) -> &[u8] {
+		(*self.inner).as_ref()
+	}
+}
+
+impl AsRef<[u8]> for Resource {
+	fn as_ref(&self) -> &[u8] {
+		(*self.inner).as_ref()
+	}
+}
+
+impl std::fmt::Debug for Resource {
+	fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+		f.debug_struct("Resource").field("len", &self.len()).finish()
+	}
+}
+
+impl PartialEq for Resource {
+	fn eq(&self, other: &Self) -> bool {
+		self.as_ref() == other.as_ref()
+	}
+}
+
+impl CacheHash for Resource {
+	fn cache_hash<H: core::hash::Hasher>(&self, state: &mut H) {
+		self.as_ref().hash(state);
+	}
+}