Thumbnails for the layer node (#1210)

* Thumbnails for the layer node

* Raster node graph frames

* Downscale to a random resolution

* Cleanup and bug fixes

* Generate paths before duplicating outputs

* Fix stable id test

* Code review changes

* Code review pass with minor changes

---------

Co-authored-by: Keavon Chambers <keavon@keavon.com>
This commit is contained in:
0HyperCube
2023-05-18 05:12:50 +01:00
committed by Keavon Chambers
co-authored by Keavon Chambers
parent 727e5bdeb1
commit 6400953af5
24 changed files with 609 additions and 131 deletions
@@ -18,6 +18,8 @@ pub struct FrontendImageData {
#[serde(skip)]
pub image_data: std::sync::Arc<Vec<u8>>,
pub transform: Option<[f64; 6]>,
#[serde(rename = "nodeId")]
pub node_id: Option<graph_craft::document::NodeId>,
}
#[derive(Clone, Copy, Debug, Default, Eq, PartialEq, Serialize, Deserialize, specta::Type)]
@@ -187,8 +187,7 @@ impl MessageHandler<DocumentMessage, (u64, &InputPreprocessorMessageHandler, &Pe
}
#[remain::unsorted]
NodeGraph(message) => {
let selected_layers = &mut self.layer_metadata.iter().filter_map(|(path, data)| data.selected.then_some(path.as_slice()));
self.node_graph_handler.process_message(message, responses, (&mut self.document_legacy, selected_layers));
self.node_graph_handler.process_message(message, responses, (&mut self.document_legacy, executor));
}
#[remain::unsorted]
GraphOperation(message) => GraphOperationMessageHandler.process_message(message, responses, (&mut self.document_legacy, &mut self.node_graph_handler)),
@@ -3,6 +3,7 @@ use crate::messages::input_mapper::utility_types::macros::action_keys;
use crate::messages::layout::utility_types::layout_widget::{Layout, LayoutGroup, Widget, WidgetCallback, WidgetHolder, WidgetLayout};
use crate::messages::layout::utility_types::widgets::button_widgets::TextButton;
use crate::messages::prelude::*;
use crate::node_graph_executor::NodeGraphExecutor;
use document_legacy::document::Document;
use document_legacy::layers::layer_layer::LayerLayer;
@@ -84,6 +85,8 @@ pub struct FrontendNode {
pub position: (i32, i32),
pub disabled: bool,
pub previewed: bool,
#[serde(rename = "thumbnailSvg")]
pub thumbnail_svg: Option<String>,
}
// (link_start, link_end, link_end_input_index)
@@ -254,9 +257,11 @@ impl NodeGraphMessageHandler {
}
}
fn send_graph(network: &NodeNetwork, responses: &mut VecDeque<Message>) {
fn send_graph(network: &NodeNetwork, executor: &NodeGraphExecutor, layer_path: &Option<Vec<LayerId>>, responses: &mut VecDeque<Message>) {
responses.add(PropertiesPanelMessage::ResendActiveProperties);
let layer_id = layer_path.as_ref().and_then(|path| path.last().copied());
// List of links in format (link_start, link_end, link_end_input_index)
let links = network
.nodes
@@ -288,37 +293,49 @@ impl NodeGraphMessageHandler {
warn!("Node '{}' does not exist in library", node.name);
continue;
};
let primary_input = node
.inputs
.first()
.filter(|input| input.is_exposed())
.and_then(|_| node_type.inputs.get(0))
.map(|input_type| input_type.data_type);
let exposed_inputs = node
.inputs
.iter()
.zip(node_type.inputs.iter())
.skip(1)
.filter(|(input, _)| input.is_exposed())
.map(|(_, input_type)| NodeGraphInput {
data_type: input_type.data_type,
name: input_type.name.to_string(),
})
.collect();
let outputs = node_type
.outputs
.iter()
.map(|output_type| NodeGraphOutput {
data_type: output_type.data_type,
name: output_type.name.to_string(),
})
.collect();
let thumbnail_svg = layer_id
.and_then(|layer_id| executor.thumbnails.get(&layer_id))
.and_then(|layer| layer.get(id))
.map(|svg| svg.to_string());
nodes.push(FrontendNode {
id: *id,
display_name: node.name.clone(),
primary_input: node
.inputs
.first()
.filter(|input| input.is_exposed())
.and_then(|_| node_type.inputs.get(0))
.map(|input_type| input_type.data_type),
exposed_inputs: node
.inputs
.iter()
.zip(node_type.inputs.iter())
.skip(1)
.filter(|(input, _)| input.is_exposed())
.map(|(_, input_type)| NodeGraphInput {
data_type: input_type.data_type,
name: input_type.name.to_string(),
})
.collect(),
outputs: node_type
.outputs
.iter()
.map(|output_type| NodeGraphOutput {
data_type: output_type.data_type,
name: output_type.name.to_string(),
})
.collect(),
primary_input,
exposed_inputs,
outputs,
position: node.metadata.position.into(),
previewed: network.outputs_contain(*id),
disabled: network.disabled.contains(id),
thumbnail_svg,
})
}
responses.add(FrontendMessage::UpdateNodeGraph { nodes, links });
@@ -405,9 +422,9 @@ impl NodeGraphMessageHandler {
}
}
impl MessageHandler<NodeGraphMessage, (&mut Document, &mut dyn Iterator<Item = &[LayerId]>)> for NodeGraphMessageHandler {
impl MessageHandler<NodeGraphMessage, (&mut Document, &NodeGraphExecutor)> for NodeGraphMessageHandler {
#[remain::check]
fn process_message(&mut self, message: NodeGraphMessage, responses: &mut VecDeque<Message>, (document, _selected): (&mut Document, &mut dyn Iterator<Item = &[LayerId]>)) {
fn process_message(&mut self, message: NodeGraphMessage, responses: &mut VecDeque<Message>, (document, executor): (&mut Document, &NodeGraphExecutor)) {
#[remain::sorted]
match message {
NodeGraphMessage::CloseNodeGraph => {
@@ -536,7 +553,7 @@ impl MessageHandler<NodeGraphMessage, (&mut Document, &mut dyn Iterator<Item = &
}
}
if let Some(network) = self.get_active_network(document) {
Self::send_graph(network, responses);
Self::send_graph(network, executor, &self.layer_path, responses);
}
self.collect_nested_addresses(document, responses);
self.update_selected(document, responses);
@@ -561,7 +578,7 @@ impl MessageHandler<NodeGraphMessage, (&mut Document, &mut dyn Iterator<Item = &
responses.add(NodeGraphMessage::InsertNode { node_id, document_node });
}
Self::send_graph(network, responses);
Self::send_graph(network, executor, &self.layer_path, responses);
self.update_selected(document, responses);
}
}
@@ -571,7 +588,7 @@ impl MessageHandler<NodeGraphMessage, (&mut Document, &mut dyn Iterator<Item = &
self.nested_path.pop();
}
if let Some(network) = self.get_active_network(document) {
Self::send_graph(network, responses);
Self::send_graph(network, executor, &self.layer_path, responses);
}
self.collect_nested_addresses(document, responses);
self.update_selected(document, responses);
@@ -622,7 +639,7 @@ impl MessageHandler<NodeGraphMessage, (&mut Document, &mut dyn Iterator<Item = &
node.metadata.position += IVec2::new(displacement_x, displacement_y)
}
}
Self::send_graph(network, responses);
Self::send_graph(network, executor, &self.layer_path, responses);
}
NodeGraphMessage::OpenNodeGraph { layer_path } => {
self.layer_path = Some(layer_path);
@@ -630,7 +647,7 @@ impl MessageHandler<NodeGraphMessage, (&mut Document, &mut dyn Iterator<Item = &
if let Some(network) = self.get_active_network(document) {
self.selected_nodes.clear();
Self::send_graph(network, responses);
Self::send_graph(network, executor, &self.layer_path, responses);
let node_types = document_node_types::collect_node_types();
responses.add(FrontendMessage::UpdateNodeTypes { node_types });
@@ -689,7 +706,7 @@ impl MessageHandler<NodeGraphMessage, (&mut Document, &mut dyn Iterator<Item = &
}
NodeGraphMessage::SendGraph { should_rerender } => {
if let Some(network) = self.get_active_network(document) {
Self::send_graph(network, responses);
Self::send_graph(network, executor, &self.layer_path, responses);
if should_rerender {
if let Some(layer_path) = self.layer_path.clone() {
responses.add(DocumentMessage::InputFrameRasterizeRegionBelowLayer { layer_path });
@@ -816,7 +833,7 @@ impl MessageHandler<NodeGraphMessage, (&mut Document, &mut dyn Iterator<Item = &
.disabled
.extend(self.selected_nodes.iter().filter(|&id| !network.inputs.contains(id) && !original_outputs.contains(id)));
}
Self::send_graph(network, responses);
Self::send_graph(network, executor, &self.layer_path, responses);
// Only generate node graph if one of the selected nodes is connected to the output
if self.selected_nodes.iter().any(|&node_id| network.connected_to_output(node_id, true)) {
@@ -842,7 +859,7 @@ impl MessageHandler<NodeGraphMessage, (&mut Document, &mut dyn Iterator<Item = &
} else {
return;
}
Self::send_graph(network, responses);
Self::send_graph(network, executor, &self.layer_path, responses);
}
self.update_selection_action_buttons(document, responses);
if let Some(layer_path) = self.layer_path.clone() {
@@ -121,7 +121,40 @@ fn static_nodes() -> Vec<DocumentNodeType> {
DocumentNodeType {
name: "Layer",
category: "General",
identifier: NodeImplementation::proto("graphene_core::ConstructLayerNode<_, _, _, _, _, _, _>"),
identifier: NodeImplementation::DocumentNode(NodeNetwork {
inputs: vec![0; 8],
outputs: vec![NodeOutput::new(1, 0)],
nodes: [
(
0,
DocumentNode {
inputs: vec![
NodeInput::Network(concrete!(graphene_core::vector::VectorData)),
NodeInput::Network(concrete!(String)),
NodeInput::Network(concrete!(BlendMode)),
NodeInput::Network(concrete!(f32)),
NodeInput::Network(concrete!(bool)),
NodeInput::Network(concrete!(bool)),
NodeInput::Network(concrete!(bool)),
NodeInput::Network(concrete!(graphene_core::GraphicGroup)),
],
implementation: DocumentNodeImplementation::proto("graphene_core::ConstructLayerNode<_, _, _, _, _, _, _>"),
..Default::default()
},
),
// The monitor node is used to display a thumbnail in the UI.
(
1,
DocumentNode {
inputs: vec![NodeInput::node(0, 0)],
implementation: DocumentNodeImplementation::proto("graphene_std::memo::MonitorNode<_>"),
..Default::default()
},
),
]
.into(),
..Default::default()
}),
inputs: vec![
DocumentInputType::value("Vector Data", TaggedValue::VectorData(graphene_core::vector::VectorData::empty()), true),
DocumentInputType::value("Name", TaggedValue::String(String::new()), false),
@@ -528,7 +561,7 @@ fn static_nodes() -> Vec<DocumentNodeType> {
identifier: NodeImplementation::DocumentNode(NodeNetwork {
inputs: vec![0],
outputs: vec![NodeOutput::new(1, 0)],
nodes: vec![
nodes: [
(
0,
DocumentNode {
@@ -548,8 +581,7 @@ fn static_nodes() -> Vec<DocumentNodeType> {
},
),
]
.into_iter()
.collect(),
.into(),
..Default::default()
}),
inputs: vec![DocumentInputType::value("Image", TaggedValue::ImageFrame(ImageFrame::empty()), true)],
@@ -727,12 +727,12 @@ pub fn adjust_selective_color_properties(document_node: &DocumentNode, node_id:
} = &document_node.inputs[colors_index]
{
use SelectiveColorChoice::*;
let entries = [vec![Reds, Yellows, Greens, Cyans, Blues, Magentas], vec![Whites, Neutrals, Blacks]]
let entries = [[Reds, Yellows, Greens, Cyans, Blues, Magentas].as_slice(), [Whites, Neutrals, Blacks].as_slice()]
.into_iter()
.map(|section| {
section
.into_iter()
.map(|choice| DropdownEntryData::new(choice.to_string()).on_update(update_value(move |_| TaggedValue::SelectiveColorChoice(choice), node_id, colors_index)))
.map(|choice| DropdownEntryData::new(choice.to_string()).on_update(update_value(move |_| TaggedValue::SelectiveColorChoice(*choice), node_id, colors_index)))
.collect()
})
.collect();
@@ -128,6 +128,7 @@ pub enum PortfolioMessage {
SetImageBlobUrl {
document_id: u64,
layer_path: Vec<LayerId>,
node_id: Option<NodeId>,
blob_url: String,
resolution: (f64, f64),
},
@@ -503,9 +503,15 @@ impl MessageHandler<PortfolioMessage, (&InputPreprocessorMessageHandler, &Prefer
PortfolioMessage::SetImageBlobUrl {
document_id,
layer_path,
node_id,
blob_url,
resolution,
} => {
if let (Some(layer_id), Some(node_id)) = (layer_path.last().copied(), node_id) {
self.executor.insert_thumbnail_blob_url(blob_url, layer_id, node_id, responses);
return;
}
let message = DocumentMessage::SetImageBlobUrl {
layer_path,
blob_url,
+167 -65
View File
@@ -7,10 +7,13 @@ use crate::messages::prelude::*;
use document_legacy::{document::pick_safe_imaginate_resolution, layers::layer_info::LayerDataType};
use document_legacy::{LayerId, Operation};
use dyn_any::DynAny;
use graph_craft::document::{generate_uuid, NodeId, NodeInput, NodeNetwork, NodeOutput};
use graph_craft::document::{generate_uuid, DocumentNodeImplementation, NodeId, NodeInput, NodeNetwork, NodeOutput};
use graph_craft::executor::Compiler;
use graph_craft::imaginate_input::*;
use graph_craft::{concrete, Type, TypeDescriptor};
use graphene_core::raster::{Image, ImageFrame};
use graphene_core::renderer::{SvgSegment, SvgSegmentList};
use graphene_core::vector::style::ViewMode;
use graphene_core::vector::VectorData;
use graphene_core::{Color, EditorApi};
use interpreted_executor::executor::DynamicExecutor;
@@ -24,16 +27,33 @@ pub struct NodeGraphExecutor {
pub(crate) executor: DynamicExecutor,
// TODO: This is a memory leak since layers are never removed
pub(crate) last_output_type: HashMap<Vec<LayerId>, Option<Type>>,
pub(crate) thumbnails: HashMap<LayerId, HashMap<NodeId, SvgSegmentList>>,
}
fn get_imaginate_index(name: &str) -> usize {
use crate::messages::portfolio::document::node_graph::IMAGINATE_NODE;
IMAGINATE_NODE.inputs.iter().position(|input| input.name == name).unwrap_or_else(|| panic!("Input {name} not found"))
}
impl NodeGraphExecutor {
/// Execute the network by flattening it and creating a borrow stack.
fn execute_network<'a>(&'a mut self, network: NodeNetwork, editor_api: EditorApi<'a>) -> Result<Box<dyn dyn_any::DynAny + 'a>, String> {
/// Wraps a network in a scope and returns the new network and the paths to the monitor nodes.
fn wrap_network(network: NodeNetwork) -> (NodeNetwork, Vec<Vec<NodeId>>) {
let mut scoped_network = wrap_network_in_scope(network);
scoped_network.generate_node_paths(&[]);
let monitor_nodes = scoped_network
.recursive_nodes()
.filter(|(node, _, _)| node.implementation == DocumentNodeImplementation::proto("graphene_std::memo::MonitorNode<_>"))
.map(|(_, _, path)| path)
.collect();
scoped_network.duplicate_outputs(&mut generate_uuid);
scoped_network.remove_dead_nodes();
(scoped_network, monitor_nodes)
}
/// Executes the network by flattening it and creating a borrow stack.
fn execute_network<'a>(&'a mut self, scoped_network: NodeNetwork, editor_api: EditorApi<'a>) -> Result<Box<dyn dyn_any::DynAny + 'a>, String> {
// We assume only one output
assert_eq!(scoped_network.outputs.len(), 1, "Graph with multiple outputs not yet handled");
let c = Compiler {};
@@ -100,6 +120,7 @@ impl NodeGraphExecutor {
}
}
let (network, _) = Self::wrap_network(network);
let boxed = self.execute_network(network, editor_api.into_owned())?;
dyn_any::downcast::<T>(boxed).map(|v| *v)
@@ -125,73 +146,61 @@ impl NodeGraphExecutor {
Ok::<_, String>((image_data, size))
}
fn generate_imaginate(
&mut self,
network: NodeNetwork,
imaginate_node_path: Vec<NodeId>,
(document, document_id): (&mut DocumentMessageHandler, u64),
layer_path: Vec<LayerId>,
mut editor_api: EditorApi<'_>,
(preferences, persistent_data): (&PreferencesMessageHandler, &PersistentData),
) -> Result<Message, String> {
use crate::messages::portfolio::document::node_graph::IMAGINATE_NODE;
use graph_craft::imaginate_input::*;
let image = editor_api.image_frame.take();
let get = |name: &str| IMAGINATE_NODE.inputs.iter().position(|input| input.name == name).unwrap_or_else(|| panic!("Input {name} not found"));
// Get the node graph layer
let layer = document.document_legacy.layer(&layer_path).map_err(|e| format!("No layer: {e:?}"))?;
let transform = layer.transform;
let resolution: Option<glam::DVec2> = self.compute_input(&network, &imaginate_node_path, get("Resolution"), Cow::Borrowed(&editor_api))?;
let resolution = resolution.unwrap_or_else(|| {
let (x, y) = pick_safe_imaginate_resolution((transform.transform_vector2(DVec2::new(1., 0.)).length(), transform.transform_vector2(DVec2::new(0., 1.)).length()));
DVec2::new(x as f64, y as f64)
});
let parameters = ImaginateGenerationParameters {
seed: self.compute_input::<f64>(&network, &imaginate_node_path, get("Seed"), Cow::Borrowed(&editor_api))? as u64,
fn imaginate_parameters(&mut self, network: &NodeNetwork, node_path: &[LayerId], resolution: DVec2, editor_api: &EditorApi) -> Result<ImaginateGenerationParameters, String> {
let get = get_imaginate_index;
Ok(ImaginateGenerationParameters {
seed: self.compute_input::<f64>(network, node_path, get("Seed"), Cow::Borrowed(editor_api))? as u64,
resolution: resolution.as_uvec2().into(),
samples: self.compute_input::<f64>(&network, &imaginate_node_path, get("Samples"), Cow::Borrowed(&editor_api))? as u32,
samples: self.compute_input::<f64>(network, node_path, get("Samples"), Cow::Borrowed(editor_api))? as u32,
sampling_method: self
.compute_input::<ImaginateSamplingMethod>(&network, &imaginate_node_path, get("Sampling Method"), Cow::Borrowed(&editor_api))?
.compute_input::<ImaginateSamplingMethod>(network, node_path, get("Sampling Method"), Cow::Borrowed(editor_api))?
.api_value()
.to_string(),
text_guidance: self.compute_input(&network, &imaginate_node_path, get("Prompt Guidance"), Cow::Borrowed(&editor_api))?,
text_prompt: self.compute_input(&network, &imaginate_node_path, get("Prompt"), Cow::Borrowed(&editor_api))?,
negative_prompt: self.compute_input(&network, &imaginate_node_path, get("Negative Prompt"), Cow::Borrowed(&editor_api))?,
image_creativity: Some(self.compute_input::<f64>(&network, &imaginate_node_path, get("Image Creativity"), Cow::Borrowed(&editor_api))? / 100.),
restore_faces: self.compute_input(&network, &imaginate_node_path, get("Improve Faces"), Cow::Borrowed(&editor_api))?,
tiling: self.compute_input(&network, &imaginate_node_path, get("Tiling"), Cow::Borrowed(&editor_api))?,
};
let use_base_image = self.compute_input::<bool>(&network, &imaginate_node_path, get("Adapt Input Image"), Cow::Borrowed(&editor_api))?;
text_guidance: self.compute_input(network, node_path, get("Prompt Guidance"), Cow::Borrowed(editor_api))?,
text_prompt: self.compute_input(network, node_path, get("Prompt"), Cow::Borrowed(editor_api))?,
negative_prompt: self.compute_input(network, node_path, get("Negative Prompt"), Cow::Borrowed(editor_api))?,
image_creativity: Some(self.compute_input::<f64>(network, node_path, get("Image Creativity"), Cow::Borrowed(editor_api))? / 100.),
restore_faces: self.compute_input(network, node_path, get("Improve Faces"), Cow::Borrowed(editor_api))?,
tiling: self.compute_input(network, node_path, get("Tiling"), Cow::Borrowed(editor_api))?,
})
}
editor_api.image_frame = image;
fn imaginate_base_image(&mut self, network: &NodeNetwork, imaginate_node_path: &[LayerId], resolution: DVec2, editor_api: &EditorApi) -> Result<Option<(ImaginateBaseImage, DAffine2)>, String> {
let use_base_image = self.compute_input::<bool>(&network, &imaginate_node_path, get_imaginate_index("Adapt Input Image"), Cow::Borrowed(editor_api))?;
let input_image_frame: Option<ImageFrame<Color>> = if use_base_image {
Some(self.compute_input::<ImageFrame<Color>>(&network, &imaginate_node_path, get("Input Image"), Cow::Borrowed(&editor_api))?)
Some(self.compute_input::<ImageFrame<Color>>(&network, &imaginate_node_path, get_imaginate_index("Input Image"), Cow::Borrowed(editor_api))?)
} else {
None
};
let image_transform = input_image_frame.as_ref().map(|image_frame| image_frame.transform);
let base_image = if let Some(ImageFrame { image, .. }) = input_image_frame {
let base_image = if let Some(ImageFrame { image, transform }) = input_image_frame {
// Only use if has size
if image.width > 0 && image.height > 0 {
let (image_data, size) = Self::encode_img(image, Some(resolution), image::ImageOutputFormat::Png)?;
let size = DVec2::new(size.0 as f64, size.1 as f64);
let mime = "image/png".to_string();
Some(ImaginateBaseImage { image_data, size, mime })
Some((ImaginateBaseImage { image_data, size, mime }, transform))
} else {
info!("Base image is input but has no size.");
None
}
} else {
None
};
Ok(base_image)
}
let mask_image = if let Some(transform) = image_transform {
let mask_path: Option<Vec<LayerId>> = self.compute_input(&network, &imaginate_node_path, get("Masking Layer"), Cow::Borrowed(&editor_api))?;
fn imaginate_mask_image(
&mut self,
network: &NodeNetwork,
node_path: &[LayerId],
editor_api: &EditorApi<'_>,
image_transform: Option<DAffine2>,
document: &mut DocumentMessageHandler,
persistent_data: &PersistentData,
) -> Result<Option<ImaginateMaskImage>, String> {
if let Some(transform) = image_transform {
let mask_path: Option<Vec<LayerId>> = self.compute_input(&network, &node_path, get_imaginate_index("Masking Layer"), Cow::Borrowed(&editor_api))?;
// Calculate the size of the frame
let size = DVec2::new(transform.transform_vector2(DVec2::new(1., 0.)).length(), transform.transform_vector2(DVec2::new(0., 1.)).length());
@@ -214,22 +223,53 @@ impl NodeGraphExecutor {
}
document.restore_document_transform(old_transforms);
mask_image
Ok(mask_image)
} else {
None
};
Ok(None)
}
}
fn generate_imaginate(
&mut self,
network: NodeNetwork,
imaginate_node_path: Vec<NodeId>,
(document, document_id): (&mut DocumentMessageHandler, u64),
layer_path: Vec<LayerId>,
mut editor_api: EditorApi<'_>,
(preferences, persistent_data): (&PreferencesMessageHandler, &PersistentData),
) -> Result<Message, String> {
let image = editor_api.image_frame.take();
// Get the node graph layer
let layer = document.document_legacy.layer(&layer_path).map_err(|e| format!("No layer: {e:?}"))?;
let transform = layer.transform;
let resolution: Option<glam::DVec2> = self.compute_input(&network, &imaginate_node_path, get_imaginate_index("Resolution"), Cow::Borrowed(&editor_api))?;
let resolution = resolution.unwrap_or_else(|| {
let (x, y) = pick_safe_imaginate_resolution((transform.transform_vector2(DVec2::new(1., 0.)).length(), transform.transform_vector2(DVec2::new(0., 1.)).length()));
DVec2::new(x as f64, y as f64)
});
let parameters = self.imaginate_parameters(&network, &imaginate_node_path, resolution, &editor_api)?;
editor_api.image_frame = image;
let base = self.imaginate_base_image(&network, &imaginate_node_path, resolution, &editor_api)?;
let image_transform = base.as_ref().map(|base| base.1);
let base_image = base.map(|base| base.0);
let mask_image = self.imaginate_mask_image(&network, &imaginate_node_path, &editor_api, image_transform, document, persistent_data)?;
Ok(FrontendMessage::TriggerImaginateGenerate {
parameters: Box::new(parameters),
base_image: base_image.map(Box::new),
mask_image: mask_image.map(Box::new),
mask_paint_mode: if self.compute_input::<bool>(&network, &imaginate_node_path, get("Inpaint"), Cow::Borrowed(&editor_api))? {
mask_paint_mode: if self.compute_input::<bool>(&network, &imaginate_node_path, get_imaginate_index("Inpaint"), Cow::Borrowed(&editor_api))? {
ImaginateMaskPaintMode::Inpaint
} else {
ImaginateMaskPaintMode::Outpaint
},
mask_blur_px: self.compute_input::<f64>(&network, &imaginate_node_path, get("Mask Blur"), Cow::Borrowed(&editor_api))? as u32,
imaginate_mask_starting_fill: self.compute_input(&network, &imaginate_node_path, get("Mask Starting Fill"), Cow::Borrowed(&editor_api))?,
mask_blur_px: self.compute_input::<f64>(&network, &imaginate_node_path, get_imaginate_index("Mask Blur"), Cow::Borrowed(&editor_api))? as u32,
imaginate_mask_starting_fill: self.compute_input(&network, &imaginate_node_path, get_imaginate_index("Mask Starting Fill"), Cow::Borrowed(&editor_api))?,
hostname: preferences.imaginate_server_hostname.clone(),
refresh_frequency: preferences.imaginate_refresh_frequency,
document_id,
@@ -239,6 +279,22 @@ impl NodeGraphExecutor {
.into())
}
/// Generate a new [`FrontendImageData`] from the [`Image`].
fn to_frontend_image_data(image: Image<Color>, transform: Option<[f64; 6]>, layer_path: &[LayerId], node_id: Option<u64>, resize: Option<DVec2>) -> Result<FrontendImageData, String> {
let (image_data, _size) = Self::encode_img(image, resize, image::ImageOutputFormat::Bmp)?;
let mime = "image/bmp".to_string();
let image_data = std::sync::Arc::new(image_data);
Ok(FrontendImageData {
path: layer_path.to_vec(),
node_id,
image_data,
mime,
transform,
})
}
/// Evaluates a node graph, computing either the Imaginate node or the entire graph
pub fn evaluate_node_graph(
&mut self,
@@ -276,6 +332,7 @@ impl NodeGraphExecutor {
return Ok(());
}
// Execute the node graph
let (network, monitor_nodes) = Self::wrap_network(network);
let boxed_node_graph_output = self.execute_network(network, editor_api)?;
// Check if the output is vector data
@@ -303,29 +360,74 @@ impl NodeGraphExecutor {
responses.add(Operation::SetLayerTransform { path: layer_path.clone(), transform });
}
} else {
// Update the image data
let (image_data, _size) = Self::encode_img(image, None, image::ImageOutputFormat::Bmp)?;
let mime = "image/bmp".to_string();
let image_data = std::sync::Arc::new(image_data);
let image_data = vec![FrontendImageData {
path: layer_path.clone(),
image_data,
mime,
transform,
}];
let image_data = vec![Self::to_frontend_image_data(image, transform, &layer_path, None, None)?];
responses.add(FrontendMessage::UpdateImageData { document_id, image_data });
}
} else if core::any::TypeId::of::<graphene_core::Artboard>() == DynAny::type_id(boxed_node_graph_output.as_ref()) {
let artboard: graphene_core::Artboard = dyn_any::downcast(boxed_node_graph_output).map(|artboard| *artboard)?;
info!("{artboard:#?}");
self.update_thumbnails(&layer_path, monitor_nodes, responses);
return Err(format!("Artboard (see console)"));
} else if core::any::TypeId::of::<graphene_core::GraphicGroup>() == DynAny::type_id(boxed_node_graph_output.as_ref()) {
let graphic_group: graphene_core::GraphicGroup = dyn_any::downcast(boxed_node_graph_output).map(|graphic| *graphic)?;
info!("{graphic_group:#?}");
self.update_thumbnails(&layer_path, monitor_nodes, responses);
return Err(format!("Graphic group (see console)"));
}
Ok(())
}
/// Recomputes the thumbnails for the layers in the graph, modifying the state and updating the UI.
pub fn update_thumbnails(&mut self, layer_path: &[LayerId], monitor_nodes: Vec<Vec<u64>>, responses: &mut VecDeque<Message>) {
let mut thumbnails_changed: bool = false;
let mut image_data: Vec<_> = Vec::new();
for node_path in monitor_nodes {
let Some(value) = self.executor.introspect(&node_path).flatten() else {
warn!("No introspect");
continue;
};
let Some(graphic_group) = value.downcast_ref::<graphene_core::GraphicGroup>() else {
warn!("Not graphic");
continue;
};
use graphene_core::renderer::*;
let bounds = graphic_group.bounding_box(DAffine2::IDENTITY);
let render_params = RenderParams::new(ViewMode::Normal, bounds, true);
let mut render = SvgRender::new();
graphic_group.render_svg(&mut render, &render_params);
let [min, max] = bounds.unwrap_or_default();
render.format_svg(min, max);
info!("SVG {}", render.svg);
if let (Some(layer_id), Some(node_id)) = (layer_path.last().copied(), node_path.get(node_path.len() - 2).copied()) {
let old_thumbnail = self.thumbnails.entry(layer_id).or_default().entry(node_id).or_default();
if *old_thumbnail != render.svg {
*old_thumbnail = render.svg;
thumbnails_changed = true;
}
}
let resize = Some(DVec2::splat(100.));
let create_image_data = |(node_id, image)| Self::to_frontend_image_data(image, None, layer_path, Some(node_id), resize).ok();
image_data.extend(render.image_data.into_iter().filter_map(create_image_data))
}
if !image_data.is_empty() {
responses.add(FrontendMessage::UpdateImageData { document_id: 0, image_data });
} else if thumbnails_changed {
responses.add(NodeGraphMessage::SendGraph { should_rerender: false });
}
}
/// When a blob url for a thumbnail is loaded, update the state and the UI.
pub fn insert_thumbnail_blob_url(&mut self, blob_url: String, layer_id: LayerId, node_id: NodeId, responses: &mut VecDeque<Message>) {
if let Some(layer) = self.thumbnails.get_mut(&layer_id) {
if let Some(segment) = layer.values_mut().flat_map(|segments| segments.iter_mut()).find(|segment| **segment == SvgSegment::BlobUrl(node_id)) {
*segment = SvgSegment::String(blob_url);
responses.add(NodeGraphMessage::SendGraph { should_rerender: false });
return;
}
}
}
}