Replace the image layer type with an Image node (#948)

* Use builder pattern for widgets

* Arguments to new function

* Add node graph when dragging in image

* Fix duplicate import

* Skip processing under node graph frame if unused

* Reduce node graph rerenders

* DUPLICATE ALL frontend changes into other frontend

* DUPLICATE more changes to another frontend

* Code review

* Allow importing SVG files as bitmaps

Co-authored-by: Keavon Chambers <keavon@keavon.com>
This commit is contained in:
0HyperCube
2023-01-27 10:01:09 +00:00
committed by Keavon Chambers
co-authored by Keavon Chambers
parent 66e8325362
commit 64e62699fc
32 changed files with 444 additions and 261 deletions
@@ -10,6 +10,7 @@ use document_legacy::layers::style::ViewMode;
use document_legacy::LayerId;
use document_legacy::Operation as DocumentOperation;
use graph_craft::document::NodeId;
use graphene_core::raster::Image;
use serde::{Deserialize, Serialize};
#[remain::sorted]
@@ -115,8 +116,7 @@ pub enum DocumentMessage {
delta_y: f64,
},
PasteImage {
mime: String,
image_data: Vec<u8>,
image: Image,
mouse: Option<(f64, f64)>,
},
Redo,
@@ -546,25 +546,36 @@ impl MessageHandler<DocumentMessage, (u64, &InputPreprocessorMessageHandler, &Pe
}
responses.push_back(BroadcastEvent::DocumentIsDirty.into());
}
PasteImage { mime, image_data, mouse } => {
PasteImage { image, mouse } => {
let image_size = DVec2::new(image.width as f64, image.height as f64);
responses.push_back(DocumentMessage::StartTransaction.into());
let path = vec![generate_uuid()];
responses.push_back(
DocumentOperation::AddImage {
path: path.clone(),
transform: DAffine2::ZERO.to_cols_array(),
insert_index: -1,
image_data: image_data.clone(),
mime: mime.clone(),
}
.into(),
let image_node_id = 2;
let mut network = graph_craft::document::NodeNetwork::new_network(32, image_node_id);
let Some(image_node_type) = crate::messages::portfolio::document::node_graph::resolve_document_node_type("Image") else {
warn!("Image node should be in registry");
return;
};
network.nodes.insert(
image_node_id,
graph_craft::document::DocumentNode {
name: image_node_type.name.to_string(),
inputs: vec![graph_craft::document::NodeInput::value(graph_craft::document::value::TaggedValue::Image(image), false)],
implementation: image_node_type.generate_implementation(),
metadata: graph_craft::document::DocumentNodeMetadata { position: (20, 4).into() },
},
);
let image_data = std::sync::Arc::new(image_data);
responses.push_back(
FrontendMessage::UpdateImageData {
document_id,
image_data: vec![FrontendImageData { path: path.clone(), image_data, mime }],
DocumentOperation::AddNodeGraphFrame {
path: path.clone(),
insert_index: -1,
transform: DAffine2::ZERO.to_cols_array(),
network,
}
.into(),
);
@@ -575,9 +586,21 @@ impl MessageHandler<DocumentMessage, (u64, &InputPreprocessorMessageHandler, &Pe
.into(),
);
let mouse = mouse.map_or(ipp.viewport_bounds.center(), |pos| pos.into());
let transform = DAffine2::from_translation(mouse - ipp.viewport_bounds.top_left).to_cols_array();
responses.push_back(DocumentOperation::SetLayerTransformInViewport { path, transform }.into());
// Transform of parent folder
let to_parent_folder = self.document_legacy.generate_transform_across_scope(&path[..path.len() - 1], None).unwrap_or_default();
// Align the layer with the mouse or center of viewport
let viewport_location = mouse.map_or(ipp.viewport_bounds.center(), |pos| pos.into());
let center_in_viewport = DAffine2::from_translation(viewport_location - ipp.viewport_bounds.top_left);
let center_in_viewport_layerspace = to_parent_folder.inverse() * center_in_viewport;
// Make layer the size of the image
let fit_image_size = DAffine2::from_scale_angle_translation(image_size, 0., image_size / -2.);
let transform = (center_in_viewport_layerspace * fit_image_size).to_cols_array();
responses.push_back(DocumentOperation::SetLayerTransform { path, transform }.into());
responses.push_back(DocumentMessage::NodeGraphFrameGenerate.into());
}
Redo => {
responses.push_back(SelectToolMessage::Abort.into());
@@ -949,6 +972,24 @@ impl DocumentMessageHandler {
// Prepare the node graph input image
let Some(node_network) = self.document_legacy.layer(&layer_path).ok().and_then(|layer|layer.as_node_graph().ok()) else {
return None;
};
// Skip processing under node graph frame input if not connected
if !node_network.connected_to_output(node_network.inputs[0]) {
return Some(
PortfolioMessage::ProcessNodeGraphFrame {
document_id,
layer_path,
image_data: Default::default(),
size: (0, 0),
imaginate_node,
}
.into(),
);
}
// Calculate the size of the region to be exported
let old_transforms = self.remove_document_transform();
@@ -1366,7 +1407,7 @@ impl DocumentMessageHandler {
responses.push_back(DocumentMessage::LayerChanged { affected_layer_path: layer.clone() }.into())
}
responses.push_back(NodeGraphMessage::SendGraph.into());
responses.push_back(NodeGraphMessage::SendGraph { should_rerender: true }.into());
Ok(())
}
@@ -1400,7 +1441,7 @@ impl DocumentMessageHandler {
responses.push_back(DocumentMessage::LayerChanged { affected_layer_path: layer.clone() }.into())
}
responses.push_back(NodeGraphMessage::SendGraph.into());
responses.push_back(NodeGraphMessage::SendGraph { should_rerender: true }.into());
Ok(())
}
@@ -61,7 +61,9 @@ pub enum NodeGraphMessage {
SelectNodes {
nodes: Vec<NodeId>,
},
SendGraph,
SendGraph {
should_rerender: bool,
},
SetDrawing {
new_drawing: bool,
},
@@ -1,18 +1,17 @@
pub use self::document_node_types::*;
use crate::messages::input_mapper::utility_types::macros::action_keys;
use crate::messages::layout::utility_types::layout_widget::{Layout, LayoutGroup, Widget, WidgetCallback, WidgetHolder, WidgetLayout};
use crate::messages::layout::utility_types::widgets::button_widgets::{BreadcrumbTrailButtons, TextButton};
use crate::messages::prelude::*;
use document_legacy::document::Document;
use document_legacy::layers::layer_info::{LayerDataType, LayerDataTypeDiscriminant};
use document_legacy::layers::layer_info::LayerDataTypeDiscriminant;
use document_legacy::layers::nodegraph_layer::NodeGraphFrameLayer;
use document_legacy::LayerId;
use graph_craft::document::value::TaggedValue;
use graph_craft::document::{DocumentNode, DocumentNodeImplementation, DocumentNodeMetadata, NodeId, NodeInput, NodeNetwork};
use graph_craft::document::{DocumentNode, DocumentNodeImplementation, NodeId, NodeInput, NodeNetwork};
mod document_node_types;
mod node_properties;
pub use self::document_node_types::*;
use glam::IVec2;
@@ -112,37 +111,21 @@ pub struct NodeGraphMessageHandler {
impl NodeGraphMessageHandler {
fn get_root_network<'a>(&self, document: &'a Document) -> Option<&'a graph_craft::document::NodeNetwork> {
self.layer_path.as_ref().and_then(|path| document.layer(path).ok()).and_then(|layer| match &layer.data {
LayerDataType::NodeGraphFrame(n) => Some(&n.network),
_ => None,
})
self.layer_path.as_ref().and_then(|path| document.layer(path).ok()).and_then(|layer| layer.as_node_graph().ok())
}
fn get_root_network_mut<'a>(&self, document: &'a mut Document) -> Option<&'a mut graph_craft::document::NodeNetwork> {
self.layer_path.as_ref().and_then(|path| document.layer_mut(path).ok()).and_then(|layer| match &mut layer.data {
LayerDataType::NodeGraphFrame(n) => Some(&mut n.network),
_ => None,
})
self.layer_path.as_ref().and_then(|path| document.layer_mut(path).ok()).and_then(|layer| layer.as_node_graph_mut().ok())
}
/// Get the active graph_craft NodeNetwork struct
fn get_active_network<'a>(&self, document: &'a Document) -> Option<&'a graph_craft::document::NodeNetwork> {
let mut network = self.get_root_network(document);
for segement in &self.nested_path {
network = network.and_then(|network| network.nodes.get(segement)).and_then(|node| node.implementation.get_network());
}
network
self.get_root_network(document).and_then(|network| network.nested_network(&self.nested_path))
}
/// Get the active graph_craft NodeNetwork struct
fn get_active_network_mut<'a>(&self, document: &'a mut Document) -> Option<&'a mut graph_craft::document::NodeNetwork> {
let mut network = self.get_root_network_mut(document);
for segement in &self.nested_path {
network = network.and_then(|network| network.nodes.get_mut(segement)).and_then(|node| node.implementation.get_network_mut());
}
network
self.get_root_network_mut(document).and_then(|network| network.nested_network_mut(&self.nested_path))
}
/// Send the cached layout for the bar at the top of the node panel to the frontend
@@ -239,8 +222,8 @@ impl NodeGraphMessageHandler {
pub fn collate_properties(&self, node_graph_frame: &NodeGraphFrameLayer, context: &mut NodePropertiesContext, sections: &mut Vec<LayoutGroup>) {
let mut network = &node_graph_frame.network;
for segement in &self.nested_path {
network = network.nodes.get(segement).and_then(|node| node.implementation.get_network()).unwrap();
for segment in &self.nested_path {
network = network.nodes.get(segment).and_then(|node| node.implementation.get_network()).unwrap();
}
// If empty, show all nodes in the network starting with the output
@@ -293,7 +276,7 @@ impl NodeGraphMessageHandler {
for (id, node) in &network.nodes {
let Some(node_type) = document_node_types::resolve_document_node_type(&node.name) else {
warn!("Node '{}' does not exist in library", node.name);
continue
continue;
};
nodes.push(FrontendNode {
id: *id,
@@ -442,7 +425,8 @@ impl MessageHandler<NodeGraphMessage, (&mut Document, &mut dyn Iterator<Item = &
let input = NodeInput::Node(output_node);
responses.push_back(NodeGraphMessage::SetNodeInput { node_id, input_index, input }.into());
responses.push_back(NodeGraphMessage::SendGraph.into());
let should_rerender = network.connected_to_output(node_id);
responses.push_back(NodeGraphMessage::SendGraph { should_rerender }.into());
}
NodeGraphMessage::Copy => {
let Some(network) = self.get_active_network(document) else {
@@ -468,38 +452,17 @@ impl MessageHandler<NodeGraphMessage, (&mut Document, &mut dyn Iterator<Item = &
return;
};
let num_inputs = document_node_type.inputs.len();
let inner_network = NodeNetwork {
inputs: (0..num_inputs).map(|_| 0).collect(),
output: 0,
nodes: [(
0,
DocumentNode {
name: format!("{}_impl", document_node_type.name),
// TODO: Allow inserting nodes that contain other nodes.
implementation: DocumentNodeImplementation::Unresolved(document_node_type.identifier.clone()),
inputs: (0..num_inputs).map(|_| NodeInput::Network).collect(),
metadata: DocumentNodeMetadata::default(),
},
)]
.into_iter()
.collect(),
..Default::default()
};
responses.push_back(DocumentMessage::StartTransaction.into());
let document_node = DocumentNode {
name: node_type.clone(),
inputs: document_node_type.inputs.iter().map(|input| input.default.clone()).collect(),
// TODO: Allow inserting nodes that contain other nodes.
implementation: DocumentNodeImplementation::Network(inner_network),
implementation: document_node_type.generate_implementation(),
metadata: graph_craft::document::DocumentNodeMetadata { position: (x, y).into() },
};
responses.push_back(NodeGraphMessage::InsertNode { node_id, document_node }.into());
responses.push_back(NodeGraphMessage::SendGraph.into());
responses.push_back(NodeGraphMessage::SendGraph { should_rerender: false }.into());
}
NodeGraphMessage::Cut => {
responses.push_back(NodeGraphMessage::Copy.into());
@@ -518,8 +481,14 @@ impl MessageHandler<NodeGraphMessage, (&mut Document, &mut dyn Iterator<Item = &
responses.push_back(NodeGraphMessage::DeleteNode { node_id }.into());
}
responses.push_back(NodeGraphMessage::SendGraph.into());
responses.push_back(DocumentMessage::NodeGraphFrameGenerate.into());
responses.push_back(NodeGraphMessage::SendGraph { should_rerender: false }.into());
if let Some(network) = self.get_active_network(document) {
// Only generate node graph if one of the selected nodes is connected to the output
if self.selected_nodes.iter().any(|&node_id| network.connected_to_output(node_id)) {
responses.push_back(DocumentMessage::NodeGraphFrameGenerate.into());
}
}
}
NodeGraphMessage::DisconnectNodes { node_id, input_index } => {
let Some(network) = self.get_active_network(document) else {
@@ -540,7 +509,8 @@ impl MessageHandler<NodeGraphMessage, (&mut Document, &mut dyn Iterator<Item = &
let input = node_type.inputs[input_index].default.clone();
responses.push_back(NodeGraphMessage::SetNodeInput { node_id, input_index, input }.into());
responses.push_back(NodeGraphMessage::SendGraph.into());
let should_rerender = network.connected_to_output(node_id);
responses.push_back(NodeGraphMessage::SendGraph { should_rerender }.into());
}
NodeGraphMessage::DoubleClickNode { node } => {
if let Some(network) = self.get_active_network(document) {
@@ -615,7 +585,8 @@ impl MessageHandler<NodeGraphMessage, (&mut Document, &mut dyn Iterator<Item = &
}
responses.push_back(NodeGraphMessage::SetNodeInput { node_id, input_index, input }.into());
responses.push_back(NodeGraphMessage::SendGraph.into());
let should_rerender = network.connected_to_output(node_id);
responses.push_back(NodeGraphMessage::SendGraph { should_rerender }.into());
responses.push_back(PropertiesPanelMessage::ResendActiveProperties.into());
}
NodeGraphMessage::InsertNode { node_id, document_node } => {
@@ -697,7 +668,7 @@ impl MessageHandler<NodeGraphMessage, (&mut Document, &mut dyn Iterator<Item = &
let nodes = new_ids.values().copied().collect();
responses.push_back(NodeGraphMessage::SelectNodes { nodes }.into());
responses.push_back(NodeGraphMessage::SendGraph.into());
responses.push_back(NodeGraphMessage::SendGraph { should_rerender: false }.into());
}
NodeGraphMessage::SelectNodes { nodes } => {
self.selected_nodes = nodes;
@@ -705,10 +676,12 @@ impl MessageHandler<NodeGraphMessage, (&mut Document, &mut dyn Iterator<Item = &
self.update_selected(document, responses);
responses.push_back(PropertiesPanelMessage::ResendActiveProperties.into());
}
NodeGraphMessage::SendGraph => {
NodeGraphMessage::SendGraph { should_rerender } => {
if let Some(network) = self.get_active_network(document) {
Self::send_graph(network, responses);
responses.push_back(DocumentMessage::NodeGraphFrameGenerate.into());
if should_rerender {
responses.push_back(DocumentMessage::NodeGraphFrameGenerate.into());
}
}
}
NodeGraphMessage::SetDrawing { new_drawing } => {
@@ -738,7 +711,7 @@ impl MessageHandler<NodeGraphMessage, (&mut Document, &mut dyn Iterator<Item = &
let input = NodeInput::Value { tagged_value: value, exposed: false };
responses.push_back(NodeGraphMessage::SetNodeInput { node_id, input_index, input }.into());
responses.push_back(PropertiesPanelMessage::ResendActiveProperties.into());
if node.name != "Imaginate" || input_index == 0 {
if (node.name != "Imaginate" || input_index == 0) && network.connected_to_output(node_id) {
responses.push_back(DocumentMessage::NodeGraphFrameGenerate.into());
}
}
@@ -757,18 +730,16 @@ impl MessageHandler<NodeGraphMessage, (&mut Document, &mut dyn Iterator<Item = &
input_index,
value,
} => {
let mut network = document.layer_mut(&layer_path).ok().and_then(|layer| match &mut layer.data {
LayerDataType::NodeGraphFrame(n) => Some(&mut n.network),
_ => None,
});
let Some((node_id, node_path)) = node_path.split_last() else {
error!("Node path is empty");
return
return;
};
for segement in node_path {
network = network.and_then(|network| network.nodes.get_mut(segement)).and_then(|node| node.implementation.get_network_mut());
}
let network = document
.layer_mut(&layer_path)
.ok()
.and_then(|layer| layer.as_node_graph_mut().ok())
.and_then(|network| network.nested_network_mut(node_path));
if let Some(network) = network {
if let Some(node) = network.nodes.get_mut(node_id) {
@@ -777,7 +748,9 @@ impl MessageHandler<NodeGraphMessage, (&mut Document, &mut dyn Iterator<Item = &
node.inputs.extend(((node.inputs.len() - 1)..input_index).map(|_| NodeInput::Network));
}
node.inputs[input_index] = NodeInput::Value { tagged_value: value, exposed: false };
responses.push_back(DocumentMessage::NodeGraphFrameGenerate.into());
if network.connected_to_output(*node_id) {
responses.push_back(DocumentMessage::NodeGraphFrameGenerate.into());
}
}
}
}
@@ -826,7 +799,7 @@ impl MessageHandler<NodeGraphMessage, (&mut Document, &mut dyn Iterator<Item = &
stack.extend(outwards_links.get(&id).unwrap_or(&Vec::new()).iter().copied())
}
}
responses.push_back(NodeGraphMessage::SendGraph.into());
responses.push_back(NodeGraphMessage::SendGraph { should_rerender: false }.into());
}
NodeGraphMessage::ToggleHidden => {
responses.push_back(DocumentMessage::StartTransaction.into());
@@ -844,9 +817,13 @@ impl MessageHandler<NodeGraphMessage, (&mut Document, &mut dyn Iterator<Item = &
network.disabled.extend(self.selected_nodes.iter().filter(|&id| !network.inputs.contains(id) && original_output != *id));
}
Self::send_graph(network, responses);
// Only generate node graph if one of the selected nodes is connected to the output
if self.selected_nodes.iter().any(|&node_id| network.connected_to_output(node_id)) {
responses.push_back(DocumentMessage::NodeGraphFrameGenerate.into());
}
}
self.update_selection_action_buttons(document, responses);
responses.push_back(DocumentMessage::NodeGraphFrameGenerate.into());
}
NodeGraphMessage::TogglePreview { node_id } => {
responses.push_back(DocumentMessage::StartTransaction.into());
@@ -860,6 +837,8 @@ impl MessageHandler<NodeGraphMessage, (&mut Document, &mut dyn Iterator<Item = &
network.output = node_id;
} else if let Some(output) = network.previous_output.take() {
network.output = output
} else {
return;
}
Self::send_graph(network, responses);
}
@@ -3,7 +3,7 @@ use crate::messages::layout::utility_types::layout_widget::LayoutGroup;
use graph_craft::concrete;
use graph_craft::document::value::*;
use graph_craft::document::{DocumentNode, NodeId, NodeInput};
use graph_craft::document::*;
use graph_craft::imaginate_input::ImaginateSamplingMethod;
use graph_craft::proto::{NodeIdentifier, Type};
use graphene_core::raster::Image;
@@ -61,11 +61,19 @@ static DOCUMENT_NODE_TYPES: &[DocumentNodeType] = &[
default: NodeInput::Node(0),
}],
outputs: &[FrontendGraphDataType::General],
properties: |_document_node, _node_id, _context| node_properties::string_properties("The identity node simply returns the input".to_string()),
properties: |_document_node, _node_id, _context| node_properties::string_properties("The identity node simply returns the input"),
},
DocumentNodeType {
name: "Image",
category: "Ignore",
identifier: NodeIdentifier::new("graphene_core::ops::IdNode", &[concrete!("Any<'_>")]),
inputs: &[DocumentInputType::new("Image", TaggedValue::Image(Image::empty()), false)],
outputs: &[FrontendGraphDataType::Raster],
properties: |_document_node, _node_id, _context| node_properties::string_properties("A bitmap image embedded in this node"),
},
DocumentNodeType {
name: "Input",
category: "Meta",
category: "Ignore",
identifier: NodeIdentifier::new("graphene_core::ops::IdNode", &[concrete!("Any<'_>")]),
inputs: &[DocumentInputType {
name: "In",
@@ -77,7 +85,7 @@ static DOCUMENT_NODE_TYPES: &[DocumentNodeType] = &[
},
DocumentNodeType {
name: "Output",
category: "Meta",
category: "Ignore",
identifier: NodeIdentifier::new("graphene_core::ops::IdNode", &[concrete!("Any<'_>")]),
inputs: &[DocumentInputType {
name: "In",
@@ -337,7 +345,32 @@ pub fn resolve_document_node_type(name: &str) -> Option<&DocumentNodeType> {
pub fn collect_node_types() -> Vec<FrontendNodeType> {
DOCUMENT_NODE_TYPES
.iter()
.filter(|node_type| !matches!(node_type.name, "Input" | "Output"))
.filter(|node_type| !node_type.category.eq_ignore_ascii_case("ignore"))
.map(|node_type| FrontendNodeType::new(node_type.name, node_type.category))
.collect()
}
impl DocumentNodeType {
/// Generate a [`DocumentNodeImplementation`] from this node type, using a nested network.
pub fn generate_implementation(&self) -> DocumentNodeImplementation {
let number_of_inputs = self.inputs.len();
let network = NodeNetwork {
inputs: (0..number_of_inputs).map(|_| 0).collect(),
output: 0,
nodes: [(
0,
DocumentNode {
name: format!("{}_impl", self.name),
// TODO: Allow inserting nodes that contain other nodes.
implementation: DocumentNodeImplementation::Unresolved(self.identifier.clone()),
inputs: (0..number_of_inputs).map(|_| NodeInput::Network).collect(),
metadata: DocumentNodeMetadata::default(),
},
)]
.into_iter()
.collect(),
..Default::default()
};
DocumentNodeImplementation::Network(network)
}
}
@@ -633,7 +633,7 @@ pub fn imaginate_properties(document_node: &DocumentNode, node_id: NodeId, conte
LayoutGroup::Row { widgets }.with_tooltip(
"Amplification of the text prompt's influence over the outcome. At 0, the prompt is entirely ignored.\n\
\n\
Lower values are more creative and exploratory. Higher values are more literal and uninspired, but may be lower quality.\n\
Lower values are more creative and exploratory. Higher values are more literal and uninspired.\n\
\n\
This parameter is otherwise known as CFG (classifier-free guidance).",
)