Add inpainting and outpainting to Imaginate (#864)

* Do not select layer immediatly on drag

* Add LayerReferenceInput MVP widget

* Properties Panel

* Fix dragging marker flicker

* Change mask shape to outline

* Add mask rendering

* Simplify select code

* Remove colours

* Fix inpaint/outpaint and rearrage widget UX

* Add mask blur and mask starting fill parameters

* Guard for the case when the layer is missing

* Add icon to LayerReferenceInput to finalize its UI

Co-authored-by: Keavon Chambers <keavon@keavon.com>

graphene/src/document.rs

@@ -82,6 +82,22 @@ impl Document {
 		}
 	}
 
+	/// Renders a layer and its children
+	pub fn render_layer(&mut self, layer_path: &[LayerId], render_data: RenderData) -> Option<String> {
+		// Note: it is bad practice to directly clone and modify the Graphene document structure, this is a temporary hack until this whole system is replaced by the node graph
+		let mut temp_clone = self.layer_mut(layer_path).ok()?.clone();
+
+		// Render and append to the defs section
+		let mut svg_defs = String::from("<defs>");
+		temp_clone.render(&mut vec![], &mut svg_defs, render_data);
+		svg_defs.push_str("</defs>");
+
+		// Append the cached rendered SVG
+		svg_defs.push_str(&temp_clone.cache);
+
+		Some(svg_defs)
+	}
+
 	pub fn current_state_identifier(&self) -> u64 {
 		self.state_identifier.finish()
 	}
@@ -895,6 +911,36 @@ impl Document {
 				self.mark_as_dirty(&path)?;
 				Some(vec![LayerChanged { path }])
 			}
+			Operation::ImaginateSetMaskBlurPx { path, mask_blur_px } => {
+				let layer = self.layer_mut(&path).expect("Setting Imaginate mask blur for invalid layer");
+				if let LayerDataType::Imaginate(imaginate) = &mut layer.data {
+					imaginate.mask_blur_px = mask_blur_px;
+				} else {
+					panic!("Incorrectly trying to set the mask blur for a layer that is not an Imaginate layer type");
+				}
+				self.mark_as_dirty(&path)?;
+				Some(vec![LayerChanged { path }])
+			}
+			Operation::ImaginateSetMaskFillContent { path, mode } => {
+				let layer = self.layer_mut(&path).expect("Setting Imaginate mask fill content for invalid layer");
+				if let LayerDataType::Imaginate(imaginate) = &mut layer.data {
+					imaginate.mask_fill_content = mode;
+				} else {
+					panic!("Incorrectly trying to set the mask fill content for a layer that is not an Imaginate layer type");
+				}
+				self.mark_as_dirty(&path)?;
+				Some(vec![LayerChanged { path }])
+			}
+			Operation::ImaginateSetMaskPaintMode { path, paint } => {
+				let layer = self.layer_mut(&path).expect("Setting Imaginate mask paint mode for invalid layer");
+				if let LayerDataType::Imaginate(imaginate) = &mut layer.data {
+					imaginate.mask_paint_mode = paint;
+				} else {
+					panic!("Incorrectly trying to set the mask paint mode for a layer that is not an Imaginate layer type");
+				}
+				self.mark_as_dirty(&path)?;
+				Some(vec![LayerChanged { path }])
+			}
 			Operation::ImaginateSetCfgScale { path, cfg_scale } => {
 				let layer = self.layer_mut(&path).expect("Setting Imaginate CFG scale for invalid layer");
 				if let LayerDataType::Imaginate(imaginate) = &mut layer.data {
@@ -915,6 +961,16 @@ impl Document {
 				self.mark_as_dirty(&path)?;
 				Some(vec![LayerChanged { path }])
 			}
+			Operation::ImaginateSetLayerPath { path, layer_path } => {
+				let layer = self.layer_mut(&path).expect("Setting Imaginate layer path strength for invalid layer");
+				if let LayerDataType::Imaginate(imaginate) = &mut layer.data {
+					imaginate.mask_layer_ref = layer_path;
+				} else {
+					panic!("Incorrectly trying to set the layer path for a layer that is not an Imaginate layer type");
+				}
+				self.mark_as_dirty(&path)?;
+				Some(vec![LayerChanged { path }])
+			}
 			Operation::ImaginateSetSamples { path, samples } => {
 				let layer = self.layer_mut(&path).expect("Setting Imaginate samples for invalid layer");
 				if let LayerDataType::Imaginate(imaginate) = &mut layer.data {

graphene/src/layers/imaginate_layer.rs

@@ -18,6 +18,10 @@ pub struct ImaginateLayer {
 	pub sampling_method: ImaginateSamplingMethod,
 	pub use_img2img: bool,
 	pub denoising_strength: f64,
+	pub mask_layer_ref: Option<Vec<LayerId>>,
+	pub mask_paint_mode: ImaginateMaskPaintMode,
+	pub mask_blur_px: u32,
+	pub mask_fill_content: ImaginateMaskFillContent,
 	pub cfg_scale: f64,
 	pub prompt: String,
 	pub negative_prompt: String,
@@ -62,6 +66,44 @@ pub struct ImaginateBaseImage {
 	pub size: DVec2,
 }
 
+#[derive(Debug, Default, Clone, Copy, Eq, PartialEq, Deserialize, Serialize)]
+pub enum ImaginateMaskPaintMode {
+	#[default]
+	Inpaint,
+	Outpaint,
+}
+
+#[derive(Debug, Default, Clone, Copy, Eq, PartialEq, Deserialize, Serialize)]
+pub enum ImaginateMaskFillContent {
+	#[default]
+	Fill,
+	Original,
+	LatentNoise,
+	LatentNothing,
+}
+
+impl ImaginateMaskFillContent {
+	pub fn list() -> [ImaginateMaskFillContent; 4] {
+		[
+			ImaginateMaskFillContent::Fill,
+			ImaginateMaskFillContent::Original,
+			ImaginateMaskFillContent::LatentNoise,
+			ImaginateMaskFillContent::LatentNothing,
+		]
+	}
+}
+
+impl std::fmt::Display for ImaginateMaskFillContent {
+	fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+		match self {
+			ImaginateMaskFillContent::Fill => write!(f, "Smeared Surroundings"),
+			ImaginateMaskFillContent::Original => write!(f, "Original Base Image"),
+			ImaginateMaskFillContent::LatentNoise => write!(f, "Randomness (Latent Noise)"),
+			ImaginateMaskFillContent::LatentNothing => write!(f, "Neutral (Latent Nothing)"),
+		}
+	}
+}
+
 #[derive(Debug, Default, Copy, Clone, Eq, PartialEq, Deserialize, Serialize)]
 pub enum ImaginateSamplingMethod {
 	#[default]
@@ -182,6 +224,10 @@ impl Default for ImaginateLayer {
 			sampling_method: Default::default(),
 			use_img2img: false,
 			denoising_strength: 0.66,
+			mask_paint_mode: ImaginateMaskPaintMode::default(),
+			mask_layer_ref: None,
+			mask_blur_px: 4,
+			mask_fill_content: ImaginateMaskFillContent::default(),
 			cfg_scale: 10.,
 			prompt: "".into(),
 			negative_prompt: "".into(),

graphene/src/layers/layer_info.rs

@@ -76,7 +76,7 @@ impl fmt::Display for LayerDataTypeDiscriminant {
 			LayerDataTypeDiscriminant::Text => write!(f, "Text"),
 			LayerDataTypeDiscriminant::Image => write!(f, "Image"),
 			LayerDataTypeDiscriminant::Imaginate => write!(f, "Imaginate"),
-			LayerDataTypeDiscriminant::NodeGraphFrame => write!(f, "NodeGraphFrame"),
+			LayerDataTypeDiscriminant::NodeGraphFrame => write!(f, "Node Graph Frame"),
 		}
 	}
 }

graphene/src/operation.rs

@@ -1,6 +1,6 @@
 use crate::boolean_ops::BooleanOperation as BooleanOperationType;
 use crate::layers::blend_mode::BlendMode;
-use crate::layers::imaginate_layer::{ImaginateSamplingMethod, ImaginateStatus};
+use crate::layers::imaginate_layer::{ImaginateMaskFillContent, ImaginateMaskPaintMode, ImaginateSamplingMethod, ImaginateStatus};
 use crate::layers::layer_info::Layer;
 use crate::layers::style::{self, Stroke};
 use crate::layers::vector::consts::ManipulatorType;
@@ -95,6 +95,18 @@ pub enum Operation {
 		path: Vec<LayerId>,
 		prompt: String,
 	},
+	ImaginateSetMaskBlurPx {
+		path: Vec<LayerId>,
+		mask_blur_px: u32,
+	},
+	ImaginateSetMaskFillContent {
+		path: Vec<LayerId>,
+		mode: ImaginateMaskFillContent,
+	},
+	ImaginateSetMaskPaintMode {
+		path: Vec<LayerId>,
+		paint: ImaginateMaskPaintMode,
+	},
 	ImaginateSetCfgScale {
 		path: Vec<LayerId>,
 		cfg_scale: f64,
@@ -118,6 +130,10 @@ pub enum Operation {
 		path: Vec<LayerId>,
 		denoising_strength: f64,
 	},
+	ImaginateSetLayerPath {
+		path: Vec<LayerId>,
+		layer_path: Option<Vec<LayerId>>,
+	},
 	ImaginateSetUseImg2Img {
 		path: Vec<LayerId>,
 		use_img2img: bool,