Add embedable images (#564)

* Add embedable bitmaps

* Initial work on blob urls

* Finish implementing data url

* Fix some bugs

* Rename bitmap to image

* Fix loading image on document load

* Add transform properties for image

* Remove some logging

* Add image dimensions

* Implement system copy and paste

* Fix pasting images

* Fix test

* Address code review

Co-authored-by: Keavon Chambers <keavon@keavon.com>

editor/src/communication/dispatcher.rs

@@ -64,6 +64,12 @@ impl Dispatcher {
 				#[remain::unsorted]
 				NoOp => {}
 				Frontend(message) => {
+					// Image data should be immediatly handled
+					if let FrontendMessage::UpdateImageData { .. } = message {
+						self.responses.push(message);
+						return;
+					}
+
 					// `FrontendMessage`s are saved and will be sent to the frontend after the message queue is done being processed
 					self.responses.push(message);
 				}
@@ -169,9 +175,9 @@ mod test {
 		let mut editor = create_editor_with_three_layers();
 
 		let document_before_copy = editor.dispatcher.message_handlers.portfolio_message_handler.active_document().graphene_document.clone();
-		editor.handle_message(PortfolioMessage::Copy { clipboard: Clipboard::User });
+		editor.handle_message(PortfolioMessage::Copy { clipboard: Clipboard::Internal });
 		editor.handle_message(PortfolioMessage::PasteIntoFolder {
-			clipboard: Clipboard::User,
+			clipboard: Clipboard::Internal,
 			folder_path: vec![],
 			insert_index: -1,
 		});
@@ -208,9 +214,9 @@ mod test {
 		editor.handle_message(DocumentMessage::SetSelectedLayers {
 			replacement_selected_layers: vec![vec![shape_id]],
 		});
-		editor.handle_message(PortfolioMessage::Copy { clipboard: Clipboard::User });
+		editor.handle_message(PortfolioMessage::Copy { clipboard: Clipboard::Internal });
 		editor.handle_message(PortfolioMessage::PasteIntoFolder {
-			clipboard: Clipboard::User,
+			clipboard: Clipboard::Internal,
 			folder_path: vec![],
 			insert_index: -1,
 		});
@@ -273,15 +279,15 @@ mod test {
 
 		let document_before_copy = editor.dispatcher.message_handlers.portfolio_message_handler.active_document().graphene_document.clone();
 
-		editor.handle_message(PortfolioMessage::Copy { clipboard: Clipboard::User });
+		editor.handle_message(PortfolioMessage::Copy { clipboard: Clipboard::Internal });
 		editor.handle_message(DocumentMessage::DeleteSelectedLayers);
 		editor.handle_message(PortfolioMessage::PasteIntoFolder {
-			clipboard: Clipboard::User,
+			clipboard: Clipboard::Internal,
 			folder_path: vec![],
 			insert_index: -1,
 		});
 		editor.handle_message(PortfolioMessage::PasteIntoFolder {
-			clipboard: Clipboard::User,
+			clipboard: Clipboard::Internal,
 			folder_path: vec![],
 			insert_index: -1,
 		});
@@ -344,16 +350,16 @@ mod test {
 		editor.handle_message(DocumentMessage::SetSelectedLayers {
 			replacement_selected_layers: vec![vec![rect_id], vec![ellipse_id]],
 		});
-		editor.handle_message(PortfolioMessage::Copy { clipboard: Clipboard::User });
+		editor.handle_message(PortfolioMessage::Copy { clipboard: Clipboard::Internal });
 		editor.handle_message(DocumentMessage::DeleteSelectedLayers);
 		editor.draw_rect(0., 800., 12., 200.);
 		editor.handle_message(PortfolioMessage::PasteIntoFolder {
-			clipboard: Clipboard::User,
+			clipboard: Clipboard::Internal,
 			folder_path: vec![],
 			insert_index: -1,
 		});
 		editor.handle_message(PortfolioMessage::PasteIntoFolder {
-			clipboard: Clipboard::User,
+			clipboard: Clipboard::Internal,
 			folder_path: vec![],
 			insert_index: -1,
 		});

editor/src/document/clipboards.rs

@@ -7,12 +7,14 @@ use serde::{Deserialize, Serialize};
 #[repr(u8)]
 #[derive(Serialize, Deserialize, Clone, Copy, PartialEq, Eq, Debug)]
 pub enum Clipboard {
-	System,
-	User,
-	_ClipboardCount, // Keep this as the last entry since it is used for counting the number of enum variants
+	Internal,
+
+	_InternalClipboardCount, // Keep this as the last entry in internal clipboards since it is used for counting the number of enum variants
+
+	Device,
 }
 
-pub const CLIPBOARD_COUNT: u8 = Clipboard::_ClipboardCount as u8;
+pub const INTERNAL_CLIPBOARD_COUNT: u8 = Clipboard::_InternalClipboardCount as u8;
 
 #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
 pub struct CopyBufferEntry {

editor/src/document/document_message.rs

@@ -79,6 +79,11 @@ pub enum DocumentMessage {
 		delta_x: f64,
 		delta_y: f64,
 	},
+	PasteImage {
+		mime: String,
+		image_data: Vec<u8>,
+		mouse: Option<(f64, f64)>,
+	},
 	Redo,
 	RenameLayer {
 		layer_path: Vec<LayerId>,

editor/src/document/document_message_handler.rs

@@ -6,6 +6,7 @@ use super::{ArtboardMessageHandler, MovementMessageHandler, OverlaysMessageHandl
 use crate::consts::{
 	ASYMPTOTIC_EFFECT, DEFAULT_DOCUMENT_NAME, FILE_EXPORT_SUFFIX, FILE_SAVE_SUFFIX, GRAPHITE_DOCUMENT_VERSION, SCALE_EFFECT, SCROLLBAR_SPACING, VIEWPORT_ZOOM_TO_FIT_PADDING_SCALE_FACTOR,
 };
+use crate::frontend::utility_types::FrontendImageData;
 use crate::input::InputPreprocessorMessageHandler;
 use crate::layout::widgets::{
 	IconButton, LayoutRow, NumberInput, NumberInputIncrementBehavior, OptionalInput, PopoverButton, PropertyHolder, RadioEntryData, RadioInput, Separator, SeparatorDirection, SeparatorType, Widget,
@@ -470,6 +471,33 @@ impl DocumentMessageHandler {
 		path.push(generate_uuid());
 		path
 	}
+
+	/// Creates the blob URLs for the image data in the document
+	pub fn load_image_data(&self, responses: &mut VecDeque<Message>, root: &LayerDataType, mut path: Vec<LayerId>) {
+		let mut image_data = Vec::new();
+		fn walk_layers(data: &LayerDataType, path: &mut Vec<LayerId>, responses: &mut VecDeque<Message>, image_data: &mut Vec<FrontendImageData>) {
+			match data {
+				LayerDataType::Folder(f) => {
+					for (id, layer) in f.layer_ids.iter().zip(f.layers().iter()) {
+						path.push(*id);
+						walk_layers(&layer.data, path, responses, image_data);
+						path.pop();
+					}
+				}
+				LayerDataType::Image(img) => image_data.push(FrontendImageData {
+					path: path.clone(),
+					image_data: img.image_data.clone(),
+					mime: img.mime.clone(),
+				}),
+				_ => {}
+			}
+		}
+
+		walk_layers(root, &mut path, responses, &mut image_data);
+		if !image_data.is_empty() {
+			responses.push_front(FrontendMessage::UpdateImageData { image_data }.into());
+		}
+	}
 }
 
 impl PropertyHolder for DocumentMessageHandler {
@@ -783,7 +811,7 @@ impl MessageHandler<DocumentMessage, &InputPreprocessorMessageHandler> for Docum
 			}
 			DirtyRenderDocument => {
 				// Mark all non-overlay caches as dirty
-				GrapheneDocument::visit_all_shapes(&mut self.graphene_document.root, &mut |_| {});
+				GrapheneDocument::mark_children_as_dirty(&mut self.graphene_document.root);
 
 				responses.push_back(DocumentMessage::RenderDocument.into());
 			}
@@ -865,13 +893,13 @@ impl MessageHandler<DocumentMessage, &InputPreprocessorMessageHandler> for Docum
 
 				new_folder_path.push(generate_uuid());
 
-				responses.push_back(PortfolioMessage::Copy { clipboard: Clipboard::System }.into());
+				responses.push_back(PortfolioMessage::Copy { clipboard: Clipboard::Internal }.into());
 				responses.push_back(DocumentMessage::DeleteSelectedLayers.into());
 				responses.push_back(DocumentOperation::CreateFolder { path: new_folder_path.clone() }.into());
 				responses.push_back(DocumentMessage::ToggleLayerExpansion { layer_path: new_folder_path.clone() }.into());
 				responses.push_back(
 					PortfolioMessage::PasteIntoFolder {
-						clipboard: Clipboard::System,
+						clipboard: Clipboard::Internal,
 						folder_path: new_folder_path.clone(),
 						insert_index: -1,
 					}
@@ -904,11 +932,11 @@ impl MessageHandler<DocumentMessage, &InputPreprocessorMessageHandler> for Docum
 
 				let insert_index = self.update_insert_index(&selected_layers, &folder_path, insert_index, reverse_index).unwrap();
 
-				responses.push_back(PortfolioMessage::Copy { clipboard: Clipboard::System }.into());
+				responses.push_back(PortfolioMessage::Copy { clipboard: Clipboard::Internal }.into());
 				responses.push_back(DocumentMessage::DeleteSelectedLayers.into());
 				responses.push_back(
 					PortfolioMessage::PasteIntoFolder {
-						clipboard: Clipboard::System,
+						clipboard: Clipboard::Internal,
 						folder_path,
 						insert_index,
 					}
@@ -926,6 +954,39 @@ impl MessageHandler<DocumentMessage, &InputPreprocessorMessageHandler> for Docum
 				}
 				responses.push_back(ToolMessage::DocumentIsDirty.into());
 			}
+			PasteImage { mime, image_data, mouse } => {
+				let path = vec![generate_uuid()];
+				responses.push_front(
+					FrontendMessage::UpdateImageData {
+						image_data: vec![FrontendImageData {
+							path: path.clone(),
+							image_data: image_data.clone(),
+							mime: mime.clone(),
+						}],
+					}
+					.into(),
+				);
+				responses.push_back(
+					DocumentOperation::AddImage {
+						path: path.clone(),
+						transform: DAffine2::ZERO.to_cols_array(),
+						insert_index: -1,
+						mime,
+						image_data,
+					}
+					.into(),
+				);
+				responses.push_back(
+					DocumentMessage::SetSelectedLayers {
+						replacement_selected_layers: vec![path.clone()],
+					}
+					.into(),
+				);
+
+				let mouse = mouse.map_or(ipp.mouse.position, |pos| pos.into());
+				let transform = DAffine2::from_translation(mouse - ipp.viewport_bounds.top_left).to_cols_array();
+				responses.push_back(DocumentOperation::SetLayerTransformInViewport { path, transform }.into());
+			}
 			Redo => {
 				responses.push_back(SelectToolMessage::Abort.into());
 				responses.push_back(DocumentHistoryForward.into());
@@ -1200,10 +1261,10 @@ impl MessageHandler<DocumentMessage, &InputPreprocessorMessageHandler> for Docum
 					// Select them
 					DocumentMessage::SetSelectedLayers { replacement_selected_layers: select }.into(),
 					// Copy them
-					PortfolioMessage::Copy { clipboard: Clipboard::System }.into(),
+					PortfolioMessage::Copy { clipboard: Clipboard::Internal }.into(),
 					// Paste them into the folder above
 					PortfolioMessage::PasteIntoFolder {
-						clipboard: Clipboard::System,
+						clipboard: Clipboard::Internal,
 						folder_path: folder_path[..folder_path.len() - 1].to_vec(),
 						insert_index: -1,
 					}

editor/src/document/layer_panel.rs

@@ -100,6 +100,7 @@ pub enum LayerDataTypeDiscriminant {
 	Folder,
 	Shape,
 	Text,
+	Image,
 }
 
 impl fmt::Display for LayerDataTypeDiscriminant {
@@ -108,6 +109,7 @@ impl fmt::Display for LayerDataTypeDiscriminant {
 			LayerDataTypeDiscriminant::Folder => "Folder",
 			LayerDataTypeDiscriminant::Shape => "Shape",
 			LayerDataTypeDiscriminant::Text => "Text",
+			LayerDataTypeDiscriminant::Image => "Image",
 		};
 
 		formatter.write_str(name)
@@ -122,6 +124,7 @@ impl From<&LayerDataType> for LayerDataTypeDiscriminant {
 			Folder(_) => LayerDataTypeDiscriminant::Folder,
 			Shape(_) => LayerDataTypeDiscriminant::Shape,
 			Text(_) => LayerDataTypeDiscriminant::Text,
+			Image(_) => LayerDataTypeDiscriminant::Image,
 		}
 	}
 }

editor/src/document/portfolio_message.rs

@@ -55,6 +55,9 @@ pub enum PortfolioMessage {
 		folder_path: Vec<LayerId>,
 		insert_index: isize,
 	},
+	PasteSerializedData {
+		data: String,
+	},
 	PrevDocument,
 	RequestAboutGraphiteDialog,
 	SelectDocument {

editor/src/document/portfolio_message_handler.rs

@@ -1,4 +1,4 @@
-use super::clipboards::{CopyBufferEntry, CLIPBOARD_COUNT};
+use super::clipboards::{CopyBufferEntry, INTERNAL_CLIPBOARD_COUNT};
 use super::DocumentMessageHandler;
 use crate::consts::{DEFAULT_DOCUMENT_NAME, GRAPHITE_DOCUMENT_VERSION};
 use crate::frontend::utility_types::FrontendDocumentDetails;
@@ -17,7 +17,7 @@ pub struct PortfolioMessageHandler {
 	documents: HashMap<u64, DocumentMessageHandler>,
 	document_ids: Vec<u64>,
 	active_document_id: u64,
-	copy_buffer: [Vec<CopyBufferEntry>; CLIPBOARD_COUNT as usize],
+	copy_buffer: [Vec<CopyBufferEntry>; INTERNAL_CLIPBOARD_COUNT as usize],
 }
 
 impl PortfolioMessageHandler {
@@ -78,6 +78,8 @@ impl PortfolioMessageHandler {
 				.collect::<Vec<_>>(),
 		);
 
+		new_document.load_image_data(responses, &new_document.graphene_document.root.data, Vec::new());
+
 		self.documents.insert(document_id, new_document);
 
 		// Send the new list of document tab names
@@ -119,7 +121,7 @@ impl Default for PortfolioMessageHandler {
 		Self {
 			documents: documents_map,
 			document_ids: vec![starting_key],
-			copy_buffer: [EMPTY_VEC; CLIPBOARD_COUNT as usize],
+			copy_buffer: [EMPTY_VEC; INTERNAL_CLIPBOARD_COUNT as usize],
 			active_document_id: starting_key,
 		}
 	}
@@ -228,16 +230,28 @@ impl MessageHandler<PortfolioMessage, &InputPreprocessorMessageHandler> for Port
 				// We can't use `self.active_document()` because it counts as an immutable borrow of the entirety of `self`
 				let active_document = self.documents.get(&self.active_document_id).unwrap();
 
-				let copy_buffer = &mut self.copy_buffer;
-				copy_buffer[clipboard as usize].clear();
-
-				for layer_path in active_document.selected_layers_without_children() {
-					match (active_document.graphene_document.layer(layer_path).map(|t| t.clone()), *active_document.layer_metadata(layer_path)) {
-						(Ok(layer), layer_metadata) => {
-							copy_buffer[clipboard as usize].push(CopyBufferEntry { layer, layer_metadata });
+				let copy_val = |buffer: &mut Vec<CopyBufferEntry>| {
+					for layer_path in active_document.selected_layers_without_children() {
+						match (active_document.graphene_document.layer(layer_path).map(|t| t.clone()), *active_document.layer_metadata(layer_path)) {
+							(Ok(layer), layer_metadata) => {
+								buffer.push(CopyBufferEntry { layer, layer_metadata });
+							}
+							(Err(e), _) => warn!("Could not access selected layer {:?}: {:?}", layer_path, e),
 						}
-						(Err(e), _) => warn!("Could not access selected layer {:?}: {:?}", layer_path, e),
 					}
+				};
+
+				if clipboard == Clipboard::Device {
+					let mut buffer = Vec::new();
+					copy_val(&mut buffer);
+					let mut copy_text = String::from("graphite/layer: ");
+					copy_text += &serde_json::to_string(&buffer).expect("Could not serialize paste");
+
+					responses.push_back(FrontendMessage::TriggerTextCopy { copy_text }.into());
+				} else {
+					let copy_buffer = &mut self.copy_buffer;
+					copy_buffer[clipboard as usize].clear();
+					copy_val(&mut copy_buffer[clipboard as usize]);
 				}
 			}
 			Cut { clipboard } => {
@@ -331,6 +345,7 @@ impl MessageHandler<PortfolioMessage, &InputPreprocessorMessageHandler> for Port
 						}
 						.into(),
 					);
+					self.active_document().load_image_data(responses, &entry.layer.data, destination_path.clone());
 					responses.push_front(
 						DocumentOperation::InsertLayer {
 							layer: entry.layer.clone(),
@@ -351,6 +366,40 @@ impl MessageHandler<PortfolioMessage, &InputPreprocessorMessageHandler> for Port
 					}
 				}
 			}
+			PasteSerializedData { data } => {
+				if let Ok(data) = serde_json::from_str::<Vec<CopyBufferEntry>>(&data) {
+					let document = self.active_document();
+					let shallowest_common_folder = document
+						.graphene_document
+						.shallowest_common_folder(document.selected_layers())
+						.expect("While pasting from serialized, the selected layers did not exist while attempting to find the appropriate folder path for insertion");
+					responses.push_back(DeselectAllLayers.into());
+					responses.push_back(StartTransaction.into());
+
+					for entry in data {
+						let destination_path = [shallowest_common_folder.to_vec(), vec![generate_uuid()]].concat();
+
+						responses.push_front(
+							DocumentMessage::UpdateLayerMetadata {
+								layer_path: destination_path.clone(),
+								layer_metadata: entry.layer_metadata,
+							}
+							.into(),
+						);
+						self.active_document().load_image_data(responses, &entry.layer.data, destination_path.clone());
+						responses.push_front(
+							DocumentOperation::InsertLayer {
+								layer: entry.layer.clone(),
+								destination_path,
+								insert_index: -1,
+							}
+							.into(),
+						);
+					}
+
+					responses.push_back(CommitTransaction.into());
+				}
+			}
 			PrevDocument => {
 				let len = self.document_ids.len();
 				let current_index = self.document_index(self.active_document_id);

editor/src/document/properties_panel_message_handler.rs

@@ -231,6 +231,10 @@ fn register_layer_properties(layer: &Layer, responses: &mut VecDeque<Message>) {
 					icon: "NodeText".into(),
 					gap_after: true,
 				})),
+				LayerDataType::Image(_) => WidgetHolder::new(Widget::IconLabel(IconLabel {
+					icon: "NodeImage".into(),
+					gap_after: true,
+				})),
 			},
 			WidgetHolder::new(Widget::Separator(Separator {
 				separator_type: SeparatorType::Related,
@@ -260,9 +264,6 @@ fn register_layer_properties(layer: &Layer, responses: &mut VecDeque<Message>) {
 	}];
 
 	let properties_body = match &layer.data {
-		LayerDataType::Folder(_) => {
-			vec![]
-		}
 		LayerDataType::Shape(shape) => {
 			if let Some(fill_layout) = node_section_fill(shape.style.fill()) {
 				vec![node_section_transform(layer), fill_layout, node_section_stroke(&shape.style.stroke().unwrap_or_default())]
@@ -277,6 +278,12 @@ fn register_layer_properties(layer: &Layer, responses: &mut VecDeque<Message>) {
 				node_section_stroke(&text.style.stroke().unwrap_or_default()),
 			]
 		}
+		LayerDataType::Image(_) => {
+			vec![node_section_transform(layer)]
+		}
+		_ => {
+			vec![]
+		}
 	};
 
 	responses.push_back(

editor/src/frontend/frontend_message.rs

@@ -1,4 +1,4 @@
-use super::utility_types::{FrontendDocumentDetails, MouseCursorIcon};
+use super::utility_types::{FrontendDocumentDetails, FrontendImageData, MouseCursorIcon};
 use crate::document::layer_panel::{LayerPanelEntry, RawBuffer};
 use crate::layout::layout_message::LayoutTarget;
 use crate::layout::widgets::SubLayout;
@@ -29,6 +29,7 @@ pub enum FrontendMessage {
 	TriggerIndexedDbRemoveDocument { document_id: u64 },
 	TriggerIndexedDbWriteDocument { document: String, details: FrontendDocumentDetails, version: String },
 	TriggerTextCommit,
+	TriggerTextCopy { copy_text: String },
 	TriggerViewportResize,
 
 	// Update prefix: give the frontend a new value or state for it to use
@@ -43,6 +44,7 @@ pub enum FrontendMessage {
 	UpdateDocumentOverlays { svg: String },
 	UpdateDocumentRulers { origin: (f64, f64), spacing: f64, interval: f64 },
 	UpdateDocumentScrollbars { position: (f64, f64), size: (f64, f64), multiplier: (f64, f64) },
+	UpdateImageData { image_data: Vec<FrontendImageData> },
 	UpdateInputHints { hint_data: HintData },
 	UpdateMouseCursor { cursor: MouseCursorIcon },
 	UpdateOpenDocumentsList { open_documents: Vec<FrontendDocumentDetails> },

editor/src/frontend/utility_types.rs

@@ -1,3 +1,4 @@
+use graphene::LayerId;
 use serde::{Deserialize, Serialize};
 
 #[derive(PartialEq, Clone, Deserialize, Serialize, Debug)]
@@ -7,6 +8,13 @@ pub struct FrontendDocumentDetails {
 	pub id: u64,
 }
 
+#[derive(PartialEq, Clone, Deserialize, Serialize, Debug)]
+pub struct FrontendImageData {
+	pub path: Vec<LayerId>,
+	pub mime: String,
+	pub image_data: Vec<u8>,
+}
+
 #[derive(Clone, Copy, Debug, Eq, Deserialize, PartialEq, Serialize)]
 pub enum MouseCursorIcon {
 	Default,

editor/src/input/input_mapper.rs

@@ -188,9 +188,8 @@ impl Default for Mapping {
 			entry! {action=PortfolioMessage::PrevDocument, key_down=KeyTab, modifiers=[KeyControl, KeyShift]},
 			entry! {action=PortfolioMessage::CloseAllDocumentsWithConfirmation, key_down=KeyW, modifiers=[KeyControl, KeyAlt]},
 			entry! {action=PortfolioMessage::CloseActiveDocumentWithConfirmation, key_down=KeyW, modifiers=[KeyControl]},
-			entry! {action=PortfolioMessage::Paste { clipboard: Clipboard::User }, key_down=KeyV, modifiers=[KeyControl]},
-			entry! {action=PortfolioMessage::Copy { clipboard: Clipboard::User }, key_down=KeyC, modifiers=[KeyControl]},
-			entry! {action=PortfolioMessage::Cut { clipboard: Clipboard::User }, key_down=KeyX, modifiers=[KeyControl]},
+			entry! {action=PortfolioMessage::Copy { clipboard: Clipboard::Device }, key_down=KeyC, modifiers=[KeyControl]},
+			entry! {action=PortfolioMessage::Cut { clipboard: Clipboard::Device }, key_down=KeyX, modifiers=[KeyControl]},
 			// Nudging
 			entry! {action=DocumentMessage::NudgeSelectedLayers { delta_x: -SHIFT_NUDGE_AMOUNT, delta_y: -SHIFT_NUDGE_AMOUNT }, key_down=KeyArrowUp, modifiers=[KeyShift, KeyArrowLeft]},
 			entry! {action=DocumentMessage::NudgeSelectedLayers { delta_x: SHIFT_NUDGE_AMOUNT, delta_y: -SHIFT_NUDGE_AMOUNT }, key_down=KeyArrowUp, modifiers=[KeyShift, KeyArrowRight]},