Integrate Stable Diffusion with the Imaginate layer (#784)

* Add AI Artist layer

* WIP add a button to download the rendered folder under an AI Artist layer

* Successfully download the correct image

* Break out image downloading JS into helper function

* Change file download from using data URLs to blob URLs

* WIP rasterize to blob

* Remove dimensions from AI Artist layer

* Successfully draw rasterized image on layer after calculation

* Working txt2img generation based on user prompt

* Add img2img and the main parameters

* Fix ability to rasterize multi-depth documents with blob URL images by switching them to base64

* Fix test

* Rasterize with artboard background color

* Allow aspect ratio stretch of AI Artist images

* Add automatic resolution choosing

* Add a terminate button, and make the lifecycle more robust

* Add negative prompt

* Add range bounds for parameter inputs

* Add seed

* Add tiling and restore faces

* Add server status check, server hostname customization, and resizing layer to fit AI Artist resolution

* Fix background color of infinite canvas rasterization

* Escape prompt text sent in the JSON

* Revoke blob URLs when cleared/replaced to reduce memory leak

* Fix welcome screen logo color

* Add PreferencesMessageHandler

* Add persistent storage of preferences

* Fix crash introduced in previous commit when moving mouse on page load

* Add tooltips to the AI Artist layer properties

* Integrate AI Artist tool into the raster section of the tool shelf

* Add a refresh button to the connection status

* Fix crash when generating and switching to a different document tab

* Add persistent image storage to AI Artist layers and fix duplication bugs

* Add a generate with random seed button

* Simplify and standardize message names

* Majorly improve robustness of networking code

* Fix race condition causing default server hostname to show disconnected when app loads with AI Artist layer selected (probably, not confirmed fixed)

* Clean up messages and function calls by changing arguments into structs

* Update API to more recent server commit

* Add support for picking the sampling method

* Add machinery for filtering selected layers with type

* Replace placeholder button icons

* Improve the random icon by tilting the dice

* Use selected_layers() instead of repeating that code

* Fix borrow error

* Change message flow in progress towards fixing #797

* Allow loading image on non-active document (fixes #797)

* Reduce code duplication with rasterization

* Add AI Artist tool and layer icons, and remove ugly node layer icon style

* Rename "AI Artist" codename to "Imaginate" feature name

Co-authored-by: otdavies <oliver@psyfer.io>
Co-authored-by: 0hypercube <0hypercube@gmail.com>
This commit is contained in:
Keavon Chambers
2022-10-18 22:33:27 -07:00
committed by GitHub
co-authored by otdavies 0hypercube
parent 06acd45a81
commit 30719bdc72
118 changed files with 3767 additions and 678 deletions
+236 -7
View File
@@ -2,6 +2,7 @@ use crate::boolean_ops::composite_boolean_operation;
use crate::intersection::Quad;
use crate::layers::folder_layer::FolderLayer;
use crate::layers::image_layer::ImageLayer;
use crate::layers::imaginate_layer::{ImaginateImageData, ImaginateLayer, ImaginateStatus};
use crate::layers::layer_info::{Layer, LayerData, LayerDataType, LayerDataTypeDiscriminant};
use crate::layers::shape_layer::ShapeLayer;
use crate::layers::style::RenderData;
@@ -42,16 +43,44 @@ impl Default for Document {
impl Document {
/// Wrapper around render, that returns the whole document as a Response.
pub fn render_root(&mut self, render_data: RenderData) -> String {
// Render and append to the defs section
let mut svg_defs = String::from("<defs>");
self.root.render(&mut vec![], &mut svg_defs, render_data);
svg_defs.push_str("</defs>");
// Append the cached rendered SVG
svg_defs.push_str(&self.root.cache);
svg_defs
}
/// Renders everything below the given layer contained within its parent folder.
pub fn render_layers_below(&mut self, below_layer_path: &[LayerId], render_data: RenderData) -> Option<String> {
// Split the path into the layer ID and its parent folder
let (layer_id_to_render_below, parent_folder_path) = below_layer_path.split_last()?;
// Note: it is bad practice to directly clone and modify the Graphene document structure, this is a temporary hack until this whole system is replaced by the node graph
let mut temp_subset_folder = self.layer_mut(parent_folder_path).ok()?.clone();
if let LayerDataType::Folder(ref mut folder) = temp_subset_folder.data {
// Remove the upper layers to leave behind the lower subset for rendering
let count_of_layers_below = folder.layer_ids.iter().position(|id| id == layer_id_to_render_below).unwrap();
folder.layer_ids.truncate(count_of_layers_below);
folder.layers.truncate(count_of_layers_below);
// Render and append to the defs section
let mut svg_defs = String::from("<defs>");
temp_subset_folder.render(&mut vec![], &mut svg_defs, render_data);
svg_defs.push_str("</defs>");
// Append the cached rendered SVG
svg_defs.push_str(&temp_subset_folder.cache);
Some(svg_defs)
} else {
None
}
}
pub fn current_state_identifier(&self) -> u64 {
self.state_identifier.finish()
}
@@ -412,6 +441,7 @@ impl Document {
Ok(())
}
/// For the purposes of rendering, this invalidates the render cache for the layer so it must be re-rendered next time.
pub fn mark_as_dirty(&mut self, path: &[LayerId]) -> Result<(), DocumentError> {
self.mark_upstream_as_dirty(path)?;
Ok(())
@@ -558,6 +588,13 @@ impl Document {
Some([vec![DocumentChanged, CreatedLayer { path: path.clone() }], update_thumbnails_upstream(&path)].concat())
}
Operation::AddImaginateFrame { path, insert_index, transform } => {
let layer = Layer::new(LayerDataType::Imaginate(ImaginateLayer::default()), transform);
self.set_layer(&path, layer, insert_index)?;
Some([vec![DocumentChanged, CreatedLayer { path: path.clone() }], update_thumbnails_upstream(&path)].concat())
}
Operation::SetTextEditability { path, editable } => {
self.layer_mut(&path)?.as_text_mut()?.editable = editable;
self.mark_as_dirty(&path)?;
@@ -673,7 +710,7 @@ impl Document {
} => {
let (folder_path, layer_id) = split_path(&destination_path)?;
let folder = self.folder_mut(folder_path)?;
folder.add_layer(layer, Some(layer_id), insert_index).ok_or(DocumentError::IndexOutOfBounds)?;
folder.add_layer(*layer, Some(layer_id), insert_index).ok_or(DocumentError::IndexOutOfBounds)?;
self.mark_as_dirty(&destination_path)?;
fn aggregate_insertions(folder: &FolderLayer, path: &mut Vec<LayerId>, responses: &mut Vec<DocumentResponse>) {
@@ -753,13 +790,180 @@ impl Document {
self.mark_as_dirty(&path)?;
Some([vec![DocumentChanged], update_thumbnails_upstream(&path)].concat())
}
Operation::SetImageBlobUrl { path, blob_url, dimensions } => {
let image = self.layer_mut(&path).expect("Blob url for invalid layer").as_image_mut().unwrap();
image.blob_url = Some(blob_url);
image.dimensions = dimensions.into();
Operation::SetLayerBlobUrl { layer_path, blob_url, resolution } => {
let layer = self.layer_mut(&layer_path).unwrap_or_else(|_| panic!("Blob url for invalid layer with path '{:?}'", layer_path));
match &mut layer.data {
LayerDataType::Image(image) => {
image.blob_url = Some(blob_url);
image.dimensions = resolution.into();
}
LayerDataType::Imaginate(imaginate) => {
imaginate.blob_url = Some(blob_url);
imaginate.dimensions = resolution.into();
}
_ => panic!("Incorrectly trying to set the image blob URL for a layer that is not an Image or Imaginate layer type"),
}
self.mark_as_dirty(&layer_path)?;
Some([vec![DocumentChanged, LayerChanged { path: layer_path.clone() }], update_thumbnails_upstream(&layer_path)].concat())
}
Operation::ImaginateSetImageData { layer_path, image_data } => {
let layer = self.layer_mut(&layer_path).expect("Setting Imaginate image data for invalid layer");
if let LayerDataType::Imaginate(imaginate) = &mut layer.data {
imaginate.image_data = Some(ImaginateImageData { image_data });
} else {
panic!("Incorrectly trying to set image data for a layer that is not an Imaginate layer type");
}
Some(vec![LayerChanged { path: layer_path.clone() }])
}
Operation::ImaginateSetGeneratingStatus { path, percent, status } => {
let layer = self.layer_mut(&path).expect("Generating Imaginate for invalid layer");
if let LayerDataType::Imaginate(imaginate) = &mut layer.data {
if let Some(percentage) = percent {
imaginate.percent_complete = percentage;
}
if status == ImaginateStatus::Generating {
imaginate.image_data = None;
}
imaginate.status = status;
} else {
panic!("Incorrectly trying to set the generating status for a layer that is not an Imaginate layer type");
}
Some(vec![LayerChanged { path: path.clone() }])
}
Operation::ImaginateClear { path } => {
let layer = self.layer_mut(&path).expect("Clearing Imaginate image for invalid layer");
if let LayerDataType::Imaginate(imaginate) = &mut layer.data {
imaginate.image_data = None;
imaginate.blob_url = None;
imaginate.status = ImaginateStatus::Idle;
imaginate.percent_complete = 0.;
} else {
panic!("Incorrectly trying to clear the blob URL for a layer that is not an Imaginate layer type");
}
self.mark_as_dirty(&path)?;
Some([vec![DocumentChanged, LayerChanged { path: path.clone() }], update_thumbnails_upstream(&path)].concat())
}
Operation::ImaginateSetNegativePrompt { path, negative_prompt } => {
let layer = self.layer_mut(&path).expect("Setting Imaginate negative prompt for invalid layer");
if let LayerDataType::Imaginate(imaginate) = &mut layer.data {
imaginate.negative_prompt = negative_prompt;
} else {
panic!("Incorrectly trying to set the negative prompt for a layer that is not an Imaginate layer type");
}
self.mark_as_dirty(&path)?;
Some(vec![LayerChanged { path }])
}
Operation::ImaginateSetPrompt { path, prompt } => {
let layer = self.layer_mut(&path).expect("Setting Imaginate prompt for invalid layer");
if let LayerDataType::Imaginate(imaginate) = &mut layer.data {
imaginate.prompt = prompt;
} else {
panic!("Incorrectly trying to set the prompt for a layer that is not an Imaginate layer type");
}
self.mark_as_dirty(&path)?;
Some(vec![LayerChanged { path }])
}
Operation::ImaginateSetCfgScale { path, cfg_scale } => {
let layer = self.layer_mut(&path).expect("Setting Imaginate CFG scale for invalid layer");
if let LayerDataType::Imaginate(imaginate) = &mut layer.data {
imaginate.cfg_scale = cfg_scale;
} else {
panic!("Incorrectly trying to set the CFG scale for a layer that is not an Imaginate layer type");
}
self.mark_as_dirty(&path)?;
Some(vec![LayerChanged { path }])
}
Operation::ImaginateSetDenoisingStrength { path, denoising_strength } => {
let layer = self.layer_mut(&path).expect("Setting Imaginate denoising strength for invalid layer");
if let LayerDataType::Imaginate(imaginate) = &mut layer.data {
imaginate.denoising_strength = denoising_strength;
} else {
panic!("Incorrectly trying to set the denoising strength for a layer that is not an Imaginate layer type");
}
self.mark_as_dirty(&path)?;
Some(vec![LayerChanged { path }])
}
Operation::ImaginateSetSamples { path, samples } => {
let layer = self.layer_mut(&path).expect("Setting Imaginate samples for invalid layer");
if let LayerDataType::Imaginate(imaginate) = &mut layer.data {
imaginate.samples = samples;
} else {
panic!("Incorrectly trying to set the samples for a layer that is not an Imaginate layer type");
}
self.mark_as_dirty(&path)?;
Some(vec![LayerChanged { path }])
}
Operation::SetImaginateSamplingMethod { path, method } => {
let layer = self.layer_mut(&path).expect("Setting Imaginate sampling method for invalid layer");
if let LayerDataType::Imaginate(imaginate) = &mut layer.data {
imaginate.sampling_method = method;
} else {
panic!("Incorrectly trying to set the sampling method for a layer that is not an Imaginate layer type");
}
self.mark_as_dirty(&path)?;
Some(vec![LayerChanged { path }])
}
Operation::ImaginateSetScaleFromResolution { path } => {
let layer = self.layer_mut(&path).expect("Setting Imaginate scale from resolution for invalid layer");
let (width, height) = pick_layer_safe_imaginate_resolution(layer, font_cache);
let current_width = layer.transform.transform_vector2((1., 0.).into()).length();
let current_height = layer.transform.transform_vector2((0., 1.).into()).length();
let scale_x_by = width as f64 / current_width;
let scale_y_by = height as f64 / current_height;
let scale_by_vector = DVec2::new(scale_x_by, scale_y_by);
let scale_by_matrix = DAffine2::from_scale_angle_translation(scale_by_vector, 0., (0., 0.).into());
layer.transform = layer.transform * scale_by_matrix;
self.mark_as_dirty(&path)?;
Some([update_thumbnails_upstream(&path), vec![DocumentChanged, LayerChanged { path }]].concat())
}
Operation::ImaginateSetSeed { path, seed } => {
let layer = self.layer_mut(&path).expect("Setting Imaginate seed for invalid layer");
if let LayerDataType::Imaginate(imaginate) = &mut layer.data {
imaginate.seed = seed;
} else {
panic!("Incorrectly trying to set the seed for a layer that is not an Imaginate layer type");
}
self.mark_as_dirty(&path)?;
Some(vec![LayerChanged { path }])
}
Operation::ImaginateSetUseImg2Img { path, use_img2img } => {
let layer = self.layer_mut(&path).expect("Calling Imaginate img2img for invalid layer");
if let LayerDataType::Imaginate(imaginate) = &mut layer.data {
imaginate.use_img2img = use_img2img;
} else {
panic!("Incorrectly trying to set the img2img status for a layer that is not an Imaginate layer type");
}
self.mark_as_dirty(&path)?;
Some(vec![LayerChanged { path }])
}
Operation::ImaginateSetRestoreFaces { path, restore_faces } => {
let layer = self.layer_mut(&path).expect("Setting Imaginate restore faces for invalid layer");
if let LayerDataType::Imaginate(imaginate) = &mut layer.data {
imaginate.restore_faces = restore_faces;
} else {
panic!("Incorrectly trying to set the restore faces status for a layer that is not an Imaginate layer type");
}
self.mark_as_dirty(&path)?;
Some(vec![LayerChanged { path }])
}
Operation::ImaginateSetTiling { path, tiling } => {
let layer = self.layer_mut(&path).expect("Setting Imaginate tiling for invalid layer");
if let LayerDataType::Imaginate(imaginate) = &mut layer.data {
imaginate.tiling = tiling;
} else {
panic!("Incorrectly trying to set the tiling status for a layer that is not an Imaginate layer type");
}
self.mark_as_dirty(&path)?;
Some(vec![LayerChanged { path }])
}
Operation::SetPivot { layer_path, pivot } => {
let layer = self.layer_mut(&layer_path).expect("Setting pivot for invalid layer");
layer.pivot = pivot.into();
@@ -1041,3 +1245,28 @@ fn update_thumbnails_upstream(path: &[LayerId]) -> Vec<DocumentResponse> {
}
responses
}
pub fn pick_layer_safe_imaginate_resolution(layer: &Layer, font_cache: &FontCache) -> (u64, u64) {
let layer_bounds = layer.bounding_transform(font_cache);
let layer_bounds_size = (layer_bounds.transform_vector2((1., 0.).into()).length(), layer_bounds.transform_vector2((0., 1.).into()).length());
pick_safe_imaginate_resolution(layer_bounds_size)
}
pub fn pick_safe_imaginate_resolution((width, height): (f64, f64)) -> (u64, u64) {
const MAX_RESOLUTION: u64 = 1000 * 1000;
let mut scale_factor = 1.;
let round_to_increment = |size: f64| (size / 64.).round() as u64 * 64;
loop {
let possible_solution = (round_to_increment(width * scale_factor), round_to_increment(height * scale_factor));
if possible_solution.0 * possible_solution.1 <= MAX_RESOLUTION {
return possible_solution;
}
scale_factor -= 0.1;
}
}
+1
View File
@@ -12,6 +12,7 @@ pub enum DocumentError {
NotAShape,
NotText,
NotAnImage,
NotAnImaginate,
InvalidFile(String),
}
+18 -19
View File
@@ -41,30 +41,29 @@ pub enum BlendMode {
impl fmt::Display for BlendMode {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
let text = match self {
BlendMode::Normal => "Normal".to_string(),
match self {
BlendMode::Normal => write!(f, "Normal"),
BlendMode::Multiply => "Multiply".to_string(),
BlendMode::Darken => "Darken".to_string(),
BlendMode::ColorBurn => "Color Burn".to_string(),
BlendMode::Multiply => write!(f, "Multiply"),
BlendMode::Darken => write!(f, "Darken"),
BlendMode::ColorBurn => write!(f, "Color Burn"),
BlendMode::Screen => "Screen".to_string(),
BlendMode::Lighten => "Lighten".to_string(),
BlendMode::ColorDodge => "Color Dodge".to_string(),
BlendMode::Screen => write!(f, "Screen"),
BlendMode::Lighten => write!(f, "Lighten"),
BlendMode::ColorDodge => write!(f, "Color Dodge"),
BlendMode::Overlay => "Overlay".to_string(),
BlendMode::SoftLight => "Soft Light".to_string(),
BlendMode::HardLight => "Hard Light".to_string(),
BlendMode::Overlay => write!(f, "Overlay"),
BlendMode::SoftLight => write!(f, "Soft Light"),
BlendMode::HardLight => write!(f, "Hard Light"),
BlendMode::Difference => "Difference".to_string(),
BlendMode::Exclusion => "Exclusion".to_string(),
BlendMode::Difference => write!(f, "Difference"),
BlendMode::Exclusion => write!(f, "Exclusion"),
BlendMode::Hue => "Hue".to_string(),
BlendMode::Saturation => "Saturation".to_string(),
BlendMode::Color => "Color".to_string(),
BlendMode::Luminosity => "Luminosity".to_string(),
};
write!(f, "{}", text)
BlendMode::Hue => write!(f, "Hue"),
BlendMode::Saturation => write!(f, "Saturation"),
BlendMode::Color => write!(f, "Color"),
BlendMode::Luminosity => write!(f, "Luminosity"),
}
}
}
+1 -1
View File
@@ -18,7 +18,7 @@ pub struct FolderLayer {
/// The IDs of the [Layer]s contained within the Folder
pub layer_ids: Vec<LayerId>,
/// The [Layer]s contained in the folder
layers: Vec<Layer>,
pub layers: Vec<Layer>,
}
impl LayerData for FolderLayer {
+25 -19
View File
@@ -1,3 +1,4 @@
use super::base64_serde;
use super::layer_info::LayerData;
use super::style::{RenderData, ViewMode};
use crate::intersection::{intersect_quad_bez_path, Quad};
@@ -9,17 +10,12 @@ use kurbo::{Affine, BezPath, Shape as KurboShape};
use serde::{Deserialize, Serialize};
use std::fmt::Write;
mod base64_serde;
fn glam_to_kurbo(transform: DAffine2) -> Affine {
Affine::new(transform.to_cols_array())
}
#[derive(Debug, Clone, PartialEq, Deserialize, Serialize)]
#[derive(Clone, PartialEq, Deserialize, Serialize)]
pub struct ImageLayer {
pub mime: String,
#[serde(serialize_with = "base64_serde::as_base64", deserialize_with = "base64_serde::from_base64")]
pub image_data: Vec<u8>,
// TODO: Have the browser dispose of this blob URL when this is dropped (like when the layer is deleted)
#[serde(skip)]
pub blob_url: Option<String>,
#[serde(skip)]
@@ -50,15 +46,12 @@ impl LayerData for ImageLayer {
.collect::<String>();
let _ = write!(
svg,
r#"<image width="{}" height="{}" transform="matrix({})" href=""#,
self.dimensions.x, self.dimensions.y, svg_transform,
r#"<image width="{}" height="{}" transform="matrix({})" href="{}"/>"#,
self.dimensions.x,
self.dimensions.y,
svg_transform,
self.blob_url.as_ref().unwrap_or(&String::new())
);
if render_data.embed_images {
let _ = write!(svg, "data:{};base64,{}", self.mime, base64::encode(&self.image_data));
} else {
let _ = write!(svg, "{}", self.blob_url.as_ref().unwrap_or(&String::new()));
}
let _ = svg.write_str(r#""/>"#);
let _ = svg.write_str("</g>");
}
@@ -83,13 +76,11 @@ impl LayerData for ImageLayer {
impl ImageLayer {
pub fn new(mime: String, image_data: Vec<u8>) -> Self {
let blob_url = None;
let dimensions = DVec2::ONE;
Self {
mime,
image_data,
blob_url,
dimensions,
blob_url: None,
dimensions: DVec2::ONE,
}
}
@@ -105,3 +96,18 @@ impl ImageLayer {
kurbo::Rect::from_origin_size(kurbo::Point::ZERO, kurbo::Size::new(self.dimensions.x, self.dimensions.y)).to_path(0.)
}
}
impl std::fmt::Debug for ImageLayer {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("ImageLayer")
.field("mime", &self.mime)
.field("image_data", &"...")
.field("blob_url", &self.blob_url)
.field("dimensions", &self.dimensions)
.finish()
}
}
fn glam_to_kurbo(transform: DAffine2) -> Affine {
Affine::new(transform.to_cols_array())
}
+286
View File
@@ -0,0 +1,286 @@
use super::base64_serde;
use super::layer_info::LayerData;
use super::style::{RenderData, ViewMode};
use crate::intersection::{intersect_quad_bez_path, Quad};
use crate::layers::text_layer::FontCache;
use crate::LayerId;
use glam::{DAffine2, DMat2, DVec2};
use kurbo::{Affine, BezPath, Shape as KurboShape};
use serde::{Deserialize, Serialize};
use std::fmt::Write;
#[derive(Clone, PartialEq, Deserialize, Serialize)]
pub struct ImaginateLayer {
// User-configurable layer parameters
pub seed: u64,
pub samples: u32,
pub sampling_method: ImaginateSamplingMethod,
pub use_img2img: bool,
pub denoising_strength: f64,
pub cfg_scale: f64,
pub prompt: String,
pub negative_prompt: String,
pub restore_faces: bool,
pub tiling: bool,
// Image stored in layer after generation completes
pub image_data: Option<ImaginateImageData>,
pub mime: String,
/// 0 is not started, 100 is complete.
pub percent_complete: f64,
// TODO: Have the browser dispose of this blob URL when this is dropped (like when the layer is deleted)
#[serde(skip)]
pub blob_url: Option<String>,
#[serde(skip)]
pub status: ImaginateStatus,
#[serde(skip)]
pub dimensions: DVec2,
}
#[derive(Default, Debug, Clone, PartialEq, Deserialize, Serialize)]
pub enum ImaginateStatus {
#[default]
Idle,
Beginning,
Uploading(f64),
Generating,
Terminating,
Terminated,
}
#[derive(Debug, Clone, Eq, PartialEq, Deserialize, Serialize)]
pub struct ImaginateImageData {
#[serde(serialize_with = "base64_serde::as_base64", deserialize_with = "base64_serde::from_base64")]
pub image_data: Vec<u8>,
}
#[derive(Debug, Clone, PartialEq, Deserialize, Serialize)]
pub struct ImaginateBaseImage {
pub svg: String,
pub size: DVec2,
}
#[derive(Debug, Default, Copy, Clone, Eq, PartialEq, Deserialize, Serialize)]
pub enum ImaginateSamplingMethod {
#[default]
EulerA,
Euler,
LMS,
Heun,
DPM2,
DPM2A,
DPMFast,
DPMAdaptive,
LMSKarras,
DPM2Karras,
DPM2AKarras,
DDIM,
PLMS,
}
impl ImaginateSamplingMethod {
pub fn api_value(&self) -> &str {
match self {
ImaginateSamplingMethod::EulerA => "Euler a",
ImaginateSamplingMethod::Euler => "Euler",
ImaginateSamplingMethod::LMS => "LMS",
ImaginateSamplingMethod::Heun => "Heun",
ImaginateSamplingMethod::DPM2 => "DPM2",
ImaginateSamplingMethod::DPM2A => "DPM2 a",
ImaginateSamplingMethod::DPMFast => "DPM fast",
ImaginateSamplingMethod::DPMAdaptive => "DPM adaptive",
ImaginateSamplingMethod::LMSKarras => "LMS Karras",
ImaginateSamplingMethod::DPM2Karras => "DPM2 Karras",
ImaginateSamplingMethod::DPM2AKarras => "DPM2 a Karras",
ImaginateSamplingMethod::DDIM => "DDIM",
ImaginateSamplingMethod::PLMS => "PLMS",
}
}
pub fn list() -> [ImaginateSamplingMethod; 13] {
[
ImaginateSamplingMethod::EulerA,
ImaginateSamplingMethod::Euler,
ImaginateSamplingMethod::LMS,
ImaginateSamplingMethod::Heun,
ImaginateSamplingMethod::DPM2,
ImaginateSamplingMethod::DPM2A,
ImaginateSamplingMethod::DPMFast,
ImaginateSamplingMethod::DPMAdaptive,
ImaginateSamplingMethod::LMSKarras,
ImaginateSamplingMethod::DPM2Karras,
ImaginateSamplingMethod::DPM2AKarras,
ImaginateSamplingMethod::DDIM,
ImaginateSamplingMethod::PLMS,
]
}
}
impl std::fmt::Display for ImaginateSamplingMethod {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
ImaginateSamplingMethod::EulerA => write!(f, "Euler A (Recommended)"),
ImaginateSamplingMethod::Euler => write!(f, "Euler"),
ImaginateSamplingMethod::LMS => write!(f, "LMS"),
ImaginateSamplingMethod::Heun => write!(f, "Heun"),
ImaginateSamplingMethod::DPM2 => write!(f, "DPM2"),
ImaginateSamplingMethod::DPM2A => write!(f, "DPM2 A"),
ImaginateSamplingMethod::DPMFast => write!(f, "DPM Fast"),
ImaginateSamplingMethod::DPMAdaptive => write!(f, "DPM Adaptive"),
ImaginateSamplingMethod::LMSKarras => write!(f, "LMS Karras"),
ImaginateSamplingMethod::DPM2Karras => write!(f, "DPM2 Karras"),
ImaginateSamplingMethod::DPM2AKarras => write!(f, "DPM2 A Karras"),
ImaginateSamplingMethod::DDIM => write!(f, "DDIM"),
ImaginateSamplingMethod::PLMS => write!(f, "PLMS"),
}
}
}
#[derive(Debug, Clone, PartialEq, Deserialize, Serialize)]
pub struct ImaginateGenerationParameters {
pub seed: u64,
pub samples: u32,
/// Use `ImaginateSamplingMethod::api_value()` to generate this string
#[serde(rename = "samplingMethod")]
pub sampling_method: String,
#[serde(rename = "denoisingStrength")]
pub denoising_strength: Option<f64>,
#[serde(rename = "cfgScale")]
pub cfg_scale: f64,
pub prompt: String,
#[serde(rename = "negativePrompt")]
pub negative_prompt: String,
pub resolution: (u64, u64),
#[serde(rename = "restoreFaces")]
pub restore_faces: bool,
pub tiling: bool,
}
impl Default for ImaginateLayer {
fn default() -> Self {
Self {
seed: 0,
samples: 30,
sampling_method: Default::default(),
use_img2img: false,
denoising_strength: 0.66,
cfg_scale: 10.,
prompt: "".into(),
negative_prompt: "".into(),
restore_faces: false,
tiling: false,
image_data: None,
mime: "image/png".into(),
blob_url: None,
percent_complete: 0.,
status: Default::default(),
dimensions: Default::default(),
}
}
}
impl LayerData for ImaginateLayer {
fn render(&mut self, svg: &mut String, _svg_defs: &mut String, transforms: &mut Vec<DAffine2>, render_data: RenderData) {
let transform = self.transform(transforms, render_data.view_mode);
let inverse = transform.inverse();
let matrix_values = transform.matrix2.to_cols_array();
let (width, height) = (matrix_values[0], matrix_values[3]);
if !inverse.is_finite() {
let _ = write!(svg, "<!-- SVG shape has an invalid transform -->");
return;
}
let _ = writeln!(svg, r#"<g transform="matrix("#);
inverse.to_cols_array().iter().enumerate().for_each(|(i, entry)| {
let _ = svg.write_str(&(entry.to_string() + if i == 5 { "" } else { "," }));
});
let _ = svg.write_str(r#")">"#);
if let Some(blob_url) = &self.blob_url {
let _ = write!(
svg,
r#"<image width="{}" height="{}" transform="matrix(1,0,0,1,{},{})" preserveAspectRatio="none" href="{}"/>"#,
width.abs(),
height.abs(),
if width >= 0. { transform.translation.x } else { transform.translation.x + width },
if height >= 0. { transform.translation.y } else { transform.translation.y + height },
blob_url
);
} else {
let _ = write!(
svg,
r#"<rect width="{}" height="{}" transform="matrix(1,0,0,1,{},{})" fill="none" stroke="var(--color-data-raster)" stroke-width="3" stroke-dasharray="8"/>"#,
width.abs(),
height.abs(),
if width >= 0. { transform.translation.x } else { transform.translation.x + width },
if height >= 0. { transform.translation.y } else { transform.translation.y + height },
);
}
let _ = svg.write_str("</g>");
}
fn bounding_box(&self, transform: glam::DAffine2, _font_cache: &FontCache) -> Option<[DVec2; 2]> {
let mut path = self.bounds();
if transform.matrix2 == DMat2::ZERO {
return None;
}
path.apply_affine(glam_to_kurbo(transform));
let kurbo::Rect { x0, y0, x1, y1 } = path.bounding_box();
Some([(x0, y0).into(), (x1, y1).into()])
}
fn intersects_quad(&self, quad: Quad, path: &mut Vec<LayerId>, intersections: &mut Vec<Vec<LayerId>>, _font_cache: &FontCache) {
if intersect_quad_bez_path(quad, &self.bounds(), true) {
intersections.push(path.clone());
}
}
}
impl ImaginateLayer {
pub fn transform(&self, transforms: &[DAffine2], mode: ViewMode) -> DAffine2 {
let start = match mode {
ViewMode::Outline => 0,
_ => (transforms.len() as i32 - 1).max(0) as usize,
};
transforms.iter().skip(start).cloned().reduce(|a, b| a * b).unwrap_or(DAffine2::IDENTITY)
}
fn bounds(&self) -> BezPath {
kurbo::Rect::from_origin_size(kurbo::Point::ZERO, kurbo::Size::new(1., 1.)).to_path(0.)
}
}
fn glam_to_kurbo(transform: DAffine2) -> Affine {
Affine::new(transform.to_cols_array())
}
impl std::fmt::Debug for ImaginateLayer {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("ImaginateLayer")
.field("seed", &self.seed)
.field("samples", &self.samples)
.field("use_img2img", &self.use_img2img)
.field("denoising_strength", &self.denoising_strength)
.field("cfg_scale", &self.cfg_scale)
.field("prompt", &self.prompt)
.field("negative_prompt", &self.negative_prompt)
.field("restore_faces", &self.restore_faces)
.field("tiling", &self.tiling)
.field("image_data", &self.image_data.as_ref().map(|_| "..."))
.field("mime", &self.mime)
.field("percent_complete", &self.percent_complete)
.field("blob_url", &self.blob_url)
.field("status", &self.status)
.field("dimensions", &self.dimensions)
.finish()
}
}
+35 -11
View File
@@ -1,6 +1,7 @@
use super::blend_mode::BlendMode;
use super::folder_layer::FolderLayer;
use super::image_layer::ImageLayer;
use super::imaginate_layer::ImaginateLayer;
use super::shape_layer::ShapeLayer;
use super::style::{PathStyle, RenderData};
use super::text_layer::TextLayer;
@@ -26,6 +27,8 @@ pub enum LayerDataType {
Text(TextLayer),
/// A layer that wraps an [ImageLayer] struct.
Image(ImageLayer),
/// A layer that wraps an [ImageLayer] struct.
Imaginate(ImaginateLayer),
}
impl LayerDataType {
@@ -35,6 +38,7 @@ impl LayerDataType {
LayerDataType::Folder(f) => f,
LayerDataType::Text(t) => t,
LayerDataType::Image(i) => i,
LayerDataType::Imaginate(a) => a,
}
}
@@ -44,6 +48,7 @@ impl LayerDataType {
LayerDataType::Folder(f) => f,
LayerDataType::Text(t) => t,
LayerDataType::Image(i) => i,
LayerDataType::Imaginate(a) => a,
}
}
}
@@ -54,18 +59,18 @@ pub enum LayerDataTypeDiscriminant {
Shape,
Text,
Image,
Imaginate,
}
impl fmt::Display for LayerDataTypeDiscriminant {
fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
let name = match self {
LayerDataTypeDiscriminant::Folder => "Folder",
LayerDataTypeDiscriminant::Shape => "Shape",
LayerDataTypeDiscriminant::Text => "Text",
LayerDataTypeDiscriminant::Image => "Image",
};
formatter.write_str(name)
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
match self {
LayerDataTypeDiscriminant::Folder => write!(f, "Folder"),
LayerDataTypeDiscriminant::Shape => write!(f, "Shape"),
LayerDataTypeDiscriminant::Text => write!(f, "Text"),
LayerDataTypeDiscriminant::Image => write!(f, "Image"),
LayerDataTypeDiscriminant::Imaginate => write!(f, "Imaginate"),
}
}
}
@@ -78,6 +83,7 @@ impl From<&LayerDataType> for LayerDataTypeDiscriminant {
Shape(_) => LayerDataTypeDiscriminant::Shape,
Text(_) => LayerDataTypeDiscriminant::Text,
Image(_) => LayerDataTypeDiscriminant::Image,
Imaginate(_) => LayerDataTypeDiscriminant::Imaginate,
}
}
}
@@ -98,7 +104,7 @@ pub trait LayerData {
///
/// // Render the shape without any transforms, in normal view mode
/// # let font_cache = Default::default();
/// let render_data = RenderData::new(ViewMode::Normal, &font_cache, None, false);
/// let render_data = RenderData::new(ViewMode::Normal, &font_cache, None);
/// shape.render(&mut svg, &mut String::new(), &mut vec![], render_data);
///
/// assert_eq!(
@@ -361,7 +367,7 @@ impl Layer {
let dimensions = b - a;
DAffine2::from_scale(dimensions)
}
_ => DAffine2::IDENTITY,
None => DAffine2::IDENTITY,
};
self.transform * scale
@@ -447,6 +453,24 @@ impl Layer {
}
}
/// Get a mutable reference to the Imaginate element wrapped by the layer.
/// This operation will fail if the [Layer type](Layer::data) is not `LayerDataType::Imaginate`.
pub fn as_imaginate_mut(&mut self) -> Result<&mut ImaginateLayer, DocumentError> {
match &mut self.data {
LayerDataType::Imaginate(imaginate) => Ok(imaginate),
_ => Err(DocumentError::NotAnImaginate),
}
}
/// Get a reference to the Imaginate element wrapped by the layer.
/// This operation will fail if the [Layer type](Layer::data) is not `LayerDataType::Imaginate`.
pub fn as_imaginate(&self) -> Result<&ImaginateLayer, DocumentError> {
match &self.data {
LayerDataType::Imaginate(imaginate) => Ok(imaginate),
_ => Err(DocumentError::NotAnImaginate),
}
}
pub fn style(&self) -> Result<&PathStyle, DocumentError> {
match &self.data {
LayerDataType::Shape(s) => Ok(&s.style),
+4
View File
@@ -6,6 +6,7 @@
//! * [Shape layers](shape_layer::ShapeLayer), which contain generic SVG [`<path>`](https://developer.mozilla.org/en-US/docs/Web/SVG/Element/path)s
//! * [Text layers](text_layer::TextLayer), which contain a description of laid out text
//! * [Image layers](image_layer::ImageLayer), which contain a bitmap image
//! * [Imaginate layers](imaginate_layer::ImaginateLayer), which contain a bitmap image generated based on a prompt and optionally the layers beneath it
//!
//! Refer to the module-level documentation for detailed information on each layer.
//!
@@ -14,6 +15,7 @@
//! When different layers overlap, they are blended together according to the [BlendMode](blend_mode::BlendMode)
//! using the CSS [`mix-blend-mode`](https://developer.mozilla.org/en-US/docs/Web/CSS/mix-blend-mode) property and the layer opacity.
pub mod base64_serde;
/// Different ways of combining overlapping SVG elements.
pub mod blend_mode;
/// Contains the [FolderLayer](folder_layer::FolderLayer) type that encapsulates other layers, including more folders.
@@ -21,6 +23,8 @@ pub mod folder_layer;
pub mod id_vec;
/// Contains the [ImageLayer](image_layer::ImageLayer) type that contains a bitmap image.
pub mod image_layer;
/// Contains the [ImaginateLayer](imaginate_layer::ImaginateLayer) type that contains a bitmap image generated based on a prompt and optionally the layers beneath it.
pub mod imaginate_layer;
/// Contains the base [Layer](layer_info::Layer) type, an abstraction over the different types of layers.
pub mod layer_info;
/// Contains the [ShapeLayer](shape_layer::ShapeLayer) type, a generic SVG element defined using Bezier paths.
+11 -13
View File
@@ -43,16 +43,14 @@ pub struct RenderData<'a> {
pub view_mode: ViewMode,
pub font_cache: &'a FontCache,
pub culling_bounds: Option<[DVec2; 2]>,
pub embed_images: bool,
}
impl<'a> RenderData<'a> {
pub fn new(view_mode: ViewMode, font_cache: &'a FontCache, culling_bounds: Option<[DVec2; 2]>, embed_images: bool) -> Self {
pub fn new(view_mode: ViewMode, font_cache: &'a FontCache, culling_bounds: Option<[DVec2; 2]>) -> Self {
Self {
view_mode,
font_cache,
culling_bounds,
embed_images,
}
}
}
@@ -204,11 +202,11 @@ pub enum LineCap {
impl Display for LineCap {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.write_str(match &self {
LineCap::Butt => "butt",
LineCap::Round => "round",
LineCap::Square => "square",
})
match self {
LineCap::Butt => write!(f, "butt"),
LineCap::Round => write!(f, "round"),
LineCap::Square => write!(f, "square"),
}
}
}
@@ -222,11 +220,11 @@ pub enum LineJoin {
impl Display for LineJoin {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.write_str(match &self {
LineJoin::Bevel => "bevel",
LineJoin::Miter => "miter",
LineJoin::Round => "round",
})
match self {
LineJoin::Bevel => write!(f, "bevel"),
LineJoin::Miter => write!(f, "miter"),
LineJoin::Round => write!(f, "round"),
}
}
}
+79 -14
View File
@@ -1,5 +1,6 @@
use crate::boolean_ops::BooleanOperation as BooleanOperationType;
use crate::layers::blend_mode::BlendMode;
use crate::layers::imaginate_layer::{ImaginateSamplingMethod, ImaginateStatus};
use crate::layers::layer_info::Layer;
use crate::layers::style::{self, Stroke};
use crate::layers::vector::consts::ManipulatorType;
@@ -36,25 +37,89 @@ pub enum Operation {
},
AddText {
path: Vec<LayerId>,
transform: [f64; 6],
insert_index: isize,
text: String,
transform: [f64; 6],
style: style::PathStyle,
text: String,
size: f64,
font_name: String,
font_style: String,
},
AddImage {
path: Vec<LayerId>,
transform: [f64; 6],
insert_index: isize,
transform: [f64; 6],
mime: String,
image_data: Vec<u8>,
},
SetImageBlobUrl {
AddImaginateFrame {
path: Vec<LayerId>,
insert_index: isize,
transform: [f64; 6],
},
/// Sets a blob URL as the image source for an Image or Imaginate layer type.
/// **Be sure to call `FrontendMessage::TriggerRevokeBlobUrl` together with this.**
SetLayerBlobUrl {
layer_path: Vec<LayerId>,
blob_url: String,
dimensions: (f64, f64),
resolution: (f64, f64),
},
/// Clears the image to leave the Imaginate layer un-rendered.
/// **Be sure to call `FrontendMessage::TriggerRevokeBlobUrl` together with this.**
ImaginateClear {
path: Vec<LayerId>,
},
ImaginateSetGeneratingStatus {
path: Vec<LayerId>,
percent: Option<f64>,
status: ImaginateStatus,
},
ImaginateSetImageData {
layer_path: Vec<LayerId>,
image_data: Vec<u8>,
},
ImaginateSetNegativePrompt {
path: Vec<LayerId>,
negative_prompt: String,
},
ImaginateSetPrompt {
path: Vec<LayerId>,
prompt: String,
},
ImaginateSetCfgScale {
path: Vec<LayerId>,
cfg_scale: f64,
},
ImaginateSetSamples {
path: Vec<LayerId>,
samples: u32,
},
SetImaginateSamplingMethod {
path: Vec<LayerId>,
method: ImaginateSamplingMethod,
},
ImaginateSetScaleFromResolution {
path: Vec<LayerId>,
},
ImaginateSetSeed {
path: Vec<LayerId>,
seed: u64,
},
ImaginateSetDenoisingStrength {
path: Vec<LayerId>,
denoising_strength: f64,
},
ImaginateSetUseImg2Img {
path: Vec<LayerId>,
use_img2img: bool,
},
ImaginateSetRestoreFaces {
path: Vec<LayerId>,
restore_faces: bool,
},
ImaginateSetTiling {
path: Vec<LayerId>,
tiling: bool,
},
SetPivot {
layer_path: Vec<LayerId>,
@@ -70,32 +135,32 @@ pub enum Operation {
},
AddPolyline {
path: Vec<LayerId>,
transform: [f64; 6],
insert_index: isize,
points: Vec<(f64, f64)>,
transform: [f64; 6],
style: style::PathStyle,
points: Vec<(f64, f64)>,
},
AddSpline {
path: Vec<LayerId>,
transform: [f64; 6],
insert_index: isize,
points: Vec<(f64, f64)>,
transform: [f64; 6],
style: style::PathStyle,
points: Vec<(f64, f64)>,
},
AddNgon {
path: Vec<LayerId>,
insert_index: isize,
transform: [f64; 6],
sides: u32,
style: style::PathStyle,
sides: u32,
},
AddShape {
path: Vec<LayerId>,
transform: [f64; 6],
insert_index: isize,
transform: [f64; 6],
style: style::PathStyle,
// TODO This will become a compound path once we support them.
subpath: Subpath,
style: style::PathStyle,
},
BooleanOperation {
operation: BooleanOperationType,
@@ -120,8 +185,8 @@ pub enum Operation {
ModifyFont {
path: Vec<LayerId>,
font_family: String,
font_style: String,
size: f64,
font_style: String,
},
MoveSelectedManipulatorPoints {
layer_path: Vec<LayerId>,
@@ -144,7 +209,7 @@ pub enum Operation {
new_name: String,
},
InsertLayer {
layer: Layer,
layer: Box<Layer>,
destination_path: Vec<LayerId>,
insert_index: isize,
},
+22 -14
View File
@@ -6,23 +6,31 @@ use std::fmt;
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Hash)]
#[repr(C)]
pub enum DocumentResponse {
/// For the purposes of rendering, this triggers a re-render of the entire document.
DocumentChanged,
FolderChanged { path: Vec<LayerId> },
CreatedLayer { path: Vec<LayerId> },
DeletedLayer { path: Vec<LayerId> },
LayerChanged { path: Vec<LayerId> },
FolderChanged {
path: Vec<LayerId>,
},
CreatedLayer {
path: Vec<LayerId>,
},
DeletedLayer {
path: Vec<LayerId>,
},
/// Triggers an update of the layer in the layer panel.
LayerChanged {
path: Vec<LayerId>,
},
}
impl fmt::Display for DocumentResponse {
fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
let name = match self {
DocumentResponse::DocumentChanged { .. } => "DocumentChanged",
DocumentResponse::FolderChanged { .. } => "FolderChanged",
DocumentResponse::CreatedLayer { .. } => "CreatedLayer",
DocumentResponse::LayerChanged { .. } => "LayerChanged",
DocumentResponse::DeletedLayer { .. } => "DeleteLayer",
};
formatter.write_str(name)
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
match self {
DocumentResponse::DocumentChanged { .. } => write!(f, "DocumentChanged"),
DocumentResponse::FolderChanged { .. } => write!(f, "FolderChanged"),
DocumentResponse::CreatedLayer { .. } => write!(f, "CreatedLayer"),
DocumentResponse::LayerChanged { .. } => write!(f, "LayerChanged"),
DocumentResponse::DeletedLayer { .. } => write!(f, "DeleteLayer"),
}
}
}