Unify file opening, importing, and pasting into a file ingest handler (#4548)

* Unify file and data ingest

* Code review

---------

Co-authored-by: Keavon Chambers <keavon@keavon.com>

frontend/src/stores/portfolio.ts

@@ -84,7 +84,7 @@ export function createPortfolioStore(subscriptions: SubscriptionsRouter, editor:
 		try {
 			const url = new URL(`demo-artwork/${data.filename}`, document.location.href);
 			const response = await fetch(url);
-			editor.openFile(data.filename, await response.bytes());
+			editor.ingestPicked(data.filename, "", await response.bytes(), "Open");
 		} catch {
 			// Needs to be delayed until the end of the current call stack so the existing demo artwork dialog can be closed first, otherwise this dialog won't show
 			setTimeout(() => {
@@ -93,19 +93,10 @@ export function createPortfolioStore(subscriptions: SubscriptionsRouter, editor:
 		}
 	});
 
-	subscriptions.subscribeFrontendMessage("TriggerOpen", async ({ filters }) => {
-		const files = await upload(acceptStringFromFilters(filters), "data", true);
-		files.forEach((file) => editor.openFile(file.filename, file.content));
-	});
-
-	subscriptions.subscribeFrontendMessage("TriggerImport", async ({ filters }) => {
-		const data = await upload(acceptStringFromFilters(filters), "data");
-		editor.importFile(data.filename, data.content);
-	});
-
-	subscriptions.subscribeFrontendMessage("TriggerUploadResource", async ({ filters }) => {
-		const data = await upload(acceptStringFromFilters(filters), "data");
-		editor.uploadResource(data.filename, data.content);
+	subscriptions.subscribeFrontendMessage("TriggerBrowse", async ({ options, action }) => {
+		const accept = acceptStringFromFilters(options.filters);
+		const files = options.multiple ? await upload(accept, "data", true) : [await upload(accept, "data")];
+		files.forEach((file) => editor.ingestPicked(file.filename, file.type, file.content, action));
 	});
 
 	subscriptions.subscribeFrontendMessage("TriggerSaveDocument", (data) => {
@@ -195,9 +186,7 @@ export function destroyPortfolioStore() {
 	subscriptions.unsubscribeFrontendMessage("UpdateOpenDocumentsList");
 	subscriptions.unsubscribeFrontendMessage("UpdateActiveDocument");
 	subscriptions.unsubscribeFrontendMessage("TriggerFetchAndOpenDocument");
-	subscriptions.unsubscribeFrontendMessage("TriggerOpen");
-	subscriptions.unsubscribeFrontendMessage("TriggerImport");
-	subscriptions.unsubscribeFrontendMessage("TriggerUploadResource");
+	subscriptions.unsubscribeFrontendMessage("TriggerBrowse");
 	subscriptions.unsubscribeFrontendMessage("TriggerSaveDocument");
 	subscriptions.unsubscribeFrontendMessage("TriggerSaveFile");
 	subscriptions.unsubscribeFrontendMessage("TriggerExportImage");

frontend/src/utility-functions/clipboard.ts

@@ -1,4 +1,3 @@
-import { extractPixelData } from "/src/utility-functions/rasterization";
 import { stripIndents } from "/src/utility-functions/strip-indents";
 import type { EditorWrapper } from "/wrapper/pkg/graphite_wasm_wrapper";
 
@@ -114,28 +113,9 @@ export async function triggerClipboardRead(editor: EditorWrapper) {
 				// Read an image from the clipboard and pass it to the editor to be loaded
 				const imageType = item.types.find((type) => type.startsWith("image/"));
 
-				// Import the actual SVG content if it's an SVG
-				if (imageType?.includes("svg")) {
-					const blob = await item.getType("text/plain");
-					const reader = new FileReader();
-					reader.onload = () => {
-						if (typeof reader.result === "string") editor.pasteSvg(undefined, reader.result);
-					};
-					reader.readAsText(blob);
-					return true;
-				}
-
-				// Import the bitmap image if it's an image
 				if (imageType) {
 					const blob = await item.getType(imageType);
-					const reader = new FileReader();
-					reader.onload = async () => {
-						if (reader.result instanceof ArrayBuffer) {
-							const imageData = await extractPixelData(new Blob([reader.result], { type: imageType }));
-							editor.pasteImage(undefined, new Uint8Array(imageData.data), imageData.width, imageData.height);
-						}
-					};
-					reader.readAsArrayBuffer(blob);
+					editor.ingestFile(undefined, imageType, new Uint8Array(await blob.arrayBuffer()));
 					return true;
 				}
 

frontend/src/utility-functions/files.ts

@@ -1,4 +1,3 @@
-import { extractPixelData } from "/src/utility-functions/rasterization";
 import type { EditorWrapper, FileFilter } from "/wrapper/pkg/graphite_wasm_wrapper";
 
 export function downloadFileURL(filename: string, url: string) {
@@ -85,24 +84,9 @@ export async function pasteFile(item: DataTransferItem, editor: EditorWrapper, m
 	const file = item.getAsFile();
 	if (!file) return;
 
-	const extension = file.name.split(".").pop()?.toLowerCase() ?? "";
-	if (file.type.startsWith("image/svg")) {
-		const svg = await file.text();
-		editor.pasteSvg(file.name, svg, mouse?.[0], mouse?.[1], insertParentId, insertIndex);
-	} else if (editor.rasterImageExtensions().includes(extension)) {
-		// Formats the editor decodes itself keep their original bytes instead of being rasterized by the browser
-		editor.pasteImageFile(file.name, await file.bytes(), mouse?.[0], mouse?.[1], insertParentId, insertIndex);
-	} else if (file.type.startsWith("image/")) {
-		const imageData = await extractPixelData(file);
-		editor.pasteImage(file.name, new Uint8Array(imageData.data), imageData.width, imageData.height, mouse?.[0], mouse?.[1], insertParentId, insertIndex);
-	} else {
-		// TODO: When we eventually have sub-documents, this should be changed to import the document as a node instead of opening it in a separate tab
-		editor.openFile(file.name, await file.bytes());
-	}
+	editor.ingestFile(file.name, file.type, await file.bytes(), mouse?.[0], mouse?.[1], insertParentId, insertIndex);
 }
 
 export function acceptStringFromFilters(filters: FileFilter[]): string {
-	const extensions = filters.flatMap((filter) => filter.extensions);
-	const imageMime = extensions.some((extension) => ["svg", "png", "jpg", "jpeg", "bmp", "gif", "webp", "avif", "tif", "tiff"].includes(extension)) ? ["image/*"] : [];
-	return [...imageMime, ...extensions.map((extension) => `.${extension}`)].join(",");
+	return filters.flatMap((filter) => [...filter.mimeTypes, ...filter.extensions.map((extension) => `.${extension}`)]).join(",");
 }

frontend/src/utility-functions/network.ts

@@ -46,7 +46,7 @@ export async function loadDemoArtwork(editor: EditorWrapper) {
 
 		const filename = url.pathname.split("/").pop() || "Untitled.graphite";
 		const content = await response.bytes();
-		editor.openFile(filename, content);
+		editor.ingestPicked(filename, "", content, "Open");
 
 		history.replaceState("", "", `${window.location.pathname}${window.location.search}`);
 	} catch {

frontend/src/utility-functions/rasterization.ts

@@ -51,15 +51,6 @@ export async function rasterizeSVG(svg: string, width: number, height: number, m
 	return blob;
 }
 
-/// Convert an image source (e.g. PNG document) into pixel data, a width, and a height
-export async function extractPixelData(imageData: ImageBitmapSource): Promise<ImageData> {
-	const canvasContext = await imageToCanvasContext(imageData);
-	const width = canvasContext.canvas.width;
-	const height = canvasContext.canvas.height;
-
-	return canvasContext.getImageData(0, 0, width, height);
-}
-
 export async function imageToCanvasContext(imageData: ImageBitmapSource): Promise<CanvasRenderingContext2D> {
 	// Special handling to rasterize an SVG file
 	let svgImageData;

frontend/wrapper/src/editor_commands.rs

@@ -20,12 +20,10 @@ mod editor_commands {
 	use editor::messages::portfolio::document::node_graph::document_node_definitions::DefinitionIdentifier;
 	use editor::messages::portfolio::document::utility_types::document_metadata::LayerNodeIdentifier;
 	use editor::messages::portfolio::document::utility_types::network_interface::ImportOrExport;
-	use editor::messages::portfolio::resource_upload::utility_types::UploadTarget;
 	use editor::messages::portfolio::utility_types::PanelGroupId;
 	use editor::messages::prelude::*;
 	use editor::messages::tool::tool_messages::tool_prelude::WidgetId;
 	use graph_craft::document::NodeId;
-	use graphene_std::raster::Image;
 	use graphene_std::raster::color::Color;
 	use graphene_std::vector::style::FillChoice;
 	use std::path::PathBuf;
@@ -213,14 +211,6 @@ mod editor_commands {
 		DialogMessage::RequestNewDocumentDialog.into()
 	}
 
-	fn open_file(path: String, content: Vec<u8>) -> Message {
-		PortfolioMessage::OpenFile { path: PathBuf::from(path), content }.into()
-	}
-
-	fn import_file(path: String, content: Vec<u8>) -> Message {
-		PortfolioMessage::ImportFile { path: PathBuf::from(path), content }.into()
-	}
-
 	fn trigger_auto_save(document_id: u64) -> Message {
 		PortfolioMessage::AutoSaveDocument { document_id: DocumentId(document_id) }.into()
 	}
@@ -607,6 +597,36 @@ mod editor_commands {
 		ClipboardMessage::ReadSelection { content, cut }.into()
 	}
 
+	/// A file picked in the dialog that `TriggerBrowse` opened
+	fn ingest_picked(name: String, mime_type: String, data: Vec<u8>, action: IngestAction) -> Message {
+		IngestMessage::Ingest {
+			data,
+			action,
+			mime_type,
+			path: Some(PathBuf::from(name)),
+		}
+		.into()
+	}
+
+	/// A file dropped on a panel or pasted, placed by the drop position or the layer slot it landed in
+	fn ingest_file(name: Option<String>, mime_type: String, data: Vec<u8>, mouse_x: Option<f64>, mouse_y: Option<f64>, insert_parent_id: Option<u64>, insert_index: Option<u32>) -> Message {
+		let action = match (insert_parent_id.zip(insert_index), mouse_x.zip(mouse_y)) {
+			(Some((parent, insert_index)), _) => IngestAction::DropOnLayers {
+				parent: LayerNodeIdentifier::new_unchecked(NodeId(parent)),
+				insert_index,
+			},
+			(None, Some(mouse)) => IngestAction::DropOnCanvas { mouse },
+			(None, None) => IngestAction::Paste,
+		};
+		IngestMessage::Ingest {
+			data,
+			action,
+			mime_type,
+			path: name.map(PathBuf::from),
+		}
+		.into()
+	}
+
 	/// Paste from a serialized JSON representation
 	fn paste_text(data: String) -> Message {
 		ClipboardMessage::ReadClipboard {
@@ -615,82 +635,6 @@ mod editor_commands {
 		.into()
 	}
 
-	/// Pastes decoded RGBA8 pixels as an image layer, encoded as PNG for storage
-	fn paste_image(
-		name: Option<String>,
-		image_data: Vec<u8>,
-		width: u32,
-		height: u32,
-		mouse_x: Option<f64>,
-		mouse_y: Option<f64>,
-		insert_parent_id: Option<u64>,
-		insert_index: Option<usize>,
-	) -> Message {
-		let mouse = mouse_x.and_then(|x| mouse_y.map(|y| (x, y)));
-		let data = Image::from_image_data(&image_data, width, height).to_png();
-
-		let parent_and_insert_index = if let (Some(insert_parent_id), Some(insert_index)) = (insert_parent_id, insert_index) {
-			let insert_parent_id = NodeId(insert_parent_id);
-			let parent = LayerNodeIdentifier::new_unchecked(insert_parent_id);
-			Some((parent, insert_index))
-		} else {
-			None
-		};
-
-		ResourceUploadMessage::Upload {
-			name,
-			data: data.into(),
-			target: UploadTarget::Layer { mouse, parent_and_insert_index },
-		}
-		.into()
-	}
-
-	/// Pastes an image file as an image layer, keeping its original encoding
-	fn paste_image_file(name: Option<String>, data: Vec<u8>, mouse_x: Option<f64>, mouse_y: Option<f64>, insert_parent_id: Option<u64>, insert_index: Option<usize>) -> Message {
-		let mouse = mouse_x.and_then(|x| mouse_y.map(|y| (x, y)));
-
-		let parent_and_insert_index = if let (Some(insert_parent_id), Some(insert_index)) = (insert_parent_id, insert_index) {
-			let insert_parent_id = NodeId(insert_parent_id);
-			let parent = LayerNodeIdentifier::new_unchecked(insert_parent_id);
-			Some((parent, insert_index))
-		} else {
-			None
-		};
-
-		ResourceUploadMessage::Upload {
-			name,
-			data: data.into(),
-			target: UploadTarget::Layer { mouse, parent_and_insert_index },
-		}
-		.into()
-	}
-
-	/// Hands the file picked for a requested resource upload to the editor
-	fn upload_resource(name: String, data: Vec<u8>) -> Message {
-		ResourceUploadMessage::ReceiveUpload { name: Some(name), data: data.into() }.into()
-	}
-
-	/// Pastes an SVG given its string representation
-	fn paste_svg(name: Option<String>, svg: String, mouse_x: Option<f64>, mouse_y: Option<f64>, insert_parent_id: Option<u64>, insert_index: Option<usize>) -> Message {
-		let mouse = mouse_x.and_then(|x| mouse_y.map(|y| (x, y)));
-
-		let parent_and_insert_index = if let (Some(insert_parent_id), Some(insert_index)) = (insert_parent_id, insert_index) {
-			let insert_parent_id = NodeId(insert_parent_id);
-			let parent = LayerNodeIdentifier::new_unchecked(insert_parent_id);
-			Some((parent, insert_index))
-		} else {
-			None
-		};
-
-		PortfolioMessage::InsertSvg {
-			name,
-			svg,
-			mouse,
-			parent_and_insert_index,
-		}
-		.into()
-	}
-
 	/// Toggle visibility of a layer or node given its node ID
 	fn toggle_node_visibility_layer_panel(id: u64) -> Message {
 		NodeGraphMessage::ToggleVisibility {
@@ -792,6 +736,7 @@ macro_rules! editor_proxy_types {
 }
 
 editor_proxy_types! {
+	IngestAction = editor::messages::portfolio::ingest::utility_types::IngestAction;
 	LayoutTarget = editor::messages::layout::utility_types::layout_widget::LayoutTarget;
 	DockingSplitDirection = editor::messages::portfolio::utility_types::DockingSplitDirection;
 	PanelTypes = Vec<editor::messages::portfolio::utility_types::PanelType>;

frontend/wrapper/src/editor_wrapper.rs

@@ -240,22 +240,6 @@ impl EditorWrapper {
 		cfg!(debug_assertions)
 	}
 
-	/// The file extensions of raster images the editor decodes itself (web only; on desktop, dropped files are imported natively and this is never called)
-	#[cfg(all(feature = "web", not(feature = "native")))]
-	#[wasm_bindgen(js_name = rasterImageExtensions)]
-	pub fn raster_image_extensions(&self) -> Vec<String> {
-		editor::messages::portfolio::resource_upload::utility_types::RASTER_IMAGE_EXTENSIONS
-			.iter()
-			.map(|extension| extension.to_string())
-			.collect()
-	}
-	#[cfg(feature = "native")]
-	#[wasm_bindgen(js_name = rasterImageExtensions)]
-	pub fn raster_image_extensions(&self) -> Vec<String> {
-		log::error!("rasterImageExtensions is unavailable on desktop, where dropped files are imported natively");
-		Vec::new()
-	}
-
 	/// Load persisted browser storage state (web only; on desktop, persistence is handled natively and this is never triggered)
 	#[cfg(all(feature = "web", not(feature = "native")))]
 	#[wasm_bindgen(js_name = loadPersistedState)]