Implement Infrastructure to reuse previous frames for brush drawing

Implement Infrastructuro to reuse the previous evaluation of the
node graph to blend the new stroke with instead of drawing the
entire trace from scratch.
This does not transition to a blending based approach because that still
caused regressions but allows the brush node to work with input data
natively.

Test Plan:
- Use the brush tool in the editor and check for regressions
- Evaluate the performance

Reviewers: Keavon

Pull Request: https://github.com/GraphiteEditor/Graphite/pull/1190
This commit is contained in:
Dennis Kobert
2023-05-03 13:14:41 +02:00
committed by Keavon Chambers
parent ebf67eaa82
commit 3adcc3031a
19 changed files with 282 additions and 172 deletions

View File

@@ -39,8 +39,9 @@ pub trait Node<'i, Input: 'i>: 'i {
type Output: 'i;
fn eval(&'i self, input: Input) -> Self::Output;
fn reset(self: Pin<&mut Self>) {}
#[cfg(feature = "alloc")]
fn serialize(&self) -> Option<String> {
#[cfg(feature = "std")]
fn serialize(&self) -> Option<std::sync::Arc<dyn core::any::Any>> {
log::warn!("Node::serialize not implemented for {}", core::any::type_name::<Self>());
None
}
}

View File

@@ -139,6 +139,7 @@ pub trait UnassociatedAlpha: RGB + Alpha {
pub trait Alpha {
type AlphaChannel: Channel;
const TRANSPARENT: Self;
fn alpha(&self) -> Self::AlphaChannel;
fn a(&self) -> Self::AlphaChannel {
self.alpha()

View File

@@ -96,6 +96,8 @@ impl Pixel for Color {
impl Alpha for Color {
type AlphaChannel = f32;
const TRANSPARENT: Self = Self::TRANSPARENT;
fn alpha(&self) -> f32 {
self.alpha
}

View File

@@ -9,15 +9,45 @@ pub struct DynAnyNode<I, O, Node> {
_i: PhantomData<I>,
_o: PhantomData<O>,
}
#[node_macro::node_fn(DynAnyNode<_I, _O>)]
fn any_node<_I: StaticType, _O: StaticType, N>(input: Any<'input>, node: &'any_input N) -> Any<'input>
impl<'input, _I: 'input + StaticType, _O: 'input + StaticType, N: 'input, S0: 'input> Node<'input, Any<'input>> for DynAnyNode<_I, _O, S0>
where
N: for<'any_input> Node<'any_input, _I, Output = _O>,
S0: for<'any_input> Node<'any_input, (), Output = &'any_input N>,
{
let node_name = core::any::type_name::<N>();
let input: Box<_I> = dyn_any::downcast(input).unwrap_or_else(|e| panic!("DynAnyNode Input, {e} in:\n{node_name}"));
Box::new(node.eval(*input))
type Output = Any<'input>;
#[inline]
fn eval(&'input self, input: Any<'input>) -> Self::Output {
let node = self.node.eval(());
{
let node_name = core::any::type_name::<N>();
let input: Box<_I> = dyn_any::downcast(input).unwrap_or_else(|e| panic!("DynAnyNode Input, {0} in:\n{1}", e, node_name));
Box::new(node.eval(*input))
}
}
fn reset(self: std::pin::Pin<&mut Self>) {
let wrapped_node = unsafe { self.map_unchecked_mut(|e| &mut e.node) };
Node::reset(wrapped_node);
}
fn serialize(&self) -> Option<std::sync::Arc<dyn core::any::Any>> {
self.node.eval(()).serialize()
}
}
impl<'input, _I: StaticType, _O: StaticType, N, S0: 'input> DynAnyNode<_I, _O, S0>
where
S0: for<'any_input> Node<'any_input, (), Output = &'any_input N>,
{
pub const fn new(node: S0) -> Self {
Self {
node,
_i: core::marker::PhantomData,
_o: core::marker::PhantomData,
}
}
}
pub struct DynAnyRefNode<I, O, Node> {
node: Node,
_i: PhantomData<(I, O)>,
@@ -28,11 +58,9 @@ where
{
type Output = Any<'input>;
fn eval(&'input self, input: Any<'input>) -> Self::Output {
{
let node_name = core::any::type_name::<N>();
let input: Box<_I> = dyn_any::downcast(input).unwrap_or_else(|e| panic!("DynAnyRefNode Input, {e} in:\n{node_name}"));
Box::new(self.node.eval(*input))
}
let node_name = core::any::type_name::<N>();
let input: Box<_I> = dyn_any::downcast(input).unwrap_or_else(|e| panic!("DynAnyRefNode Input, {e} in:\n{node_name}"));
Box::new(self.node.eval(*input))
}
fn reset(self: std::pin::Pin<&mut Self>) {
let wrapped_node = unsafe { self.map_unchecked_mut(|e| &mut e.node) };

View File

@@ -7,40 +7,6 @@ use graphene_core::vector::VectorData;
use graphene_core::Node;
use node_macro::node_fn;
// Spacing is a consistent 0.2 apart, even when tiled across pixels (from 0.9 to the neighboring 0.1), to avoid bias
const MULTISAMPLE_GRID: [(f64, f64); 25] = [
// Row 1
(0.1, 0.1),
(0.1, 0.3),
(0.1, 0.5),
(0.1, 0.7),
(0.1, 0.9),
// Row 2
(0.3, 0.1),
(0.3, 0.3),
(0.3, 0.5),
(0.3, 0.7),
(0.3, 0.9),
// Row 3
(0.5, 0.1),
(0.5, 0.3),
(0.5, 0.5),
(0.5, 0.7),
(0.5, 0.9),
// Row 4
(0.7, 0.1),
(0.7, 0.3),
(0.7, 0.5),
(0.7, 0.7),
(0.7, 0.9),
// Row 5
(0.9, 0.1),
(0.9, 0.3),
(0.9, 0.5),
(0.9, 0.7),
(0.9, 0.9),
];
#[derive(Clone, Debug, PartialEq)]
pub struct ReduceNode<Initial, Lambda> {
pub initial: Initial,

View File

@@ -1,11 +1,12 @@
use graphene_core::Node;
#[cfg(feature = "serde")]
use serde::Serialize;
use std::hash::{Hash, Hasher};
use std::marker::PhantomData;
use std::pin::Pin;
use std::sync::atomic::AtomicBool;
use std::sync::Mutex;
use std::sync::{Arc, Mutex};
use xxhash_rust::xxh3::Xxh3;
/// Caches the output of a given Node and acts as a proxy
@@ -53,32 +54,25 @@ impl<T, CachedNode> CacheNode<T, CachedNode> {
/// Caches the output of the last graph evaluation for introspection
#[derive(Default)]
pub struct MonitorNode<T, CachedNode> {
output: Mutex<Option<T>>,
node: CachedNode,
pub struct MonitorNode<T> {
output: Mutex<Option<Arc<T>>>,
}
impl<'i, T: 'i + Serialize + Clone, I: 'i + Hash, CachedNode: 'i> Node<'i, I> for MonitorNode<T, CachedNode>
where
CachedNode: for<'any_input> Node<'any_input, I, Output = T>,
{
impl<'i, T: 'static + Clone> Node<'i, T> for MonitorNode<T> {
type Output = T;
fn eval(&'i self, input: I) -> Self::Output {
let output = self.node.eval(input);
*self.output.lock().unwrap() = Some(output.clone());
output
fn eval(&'i self, input: T) -> Self::Output {
*self.output.lock().unwrap() = Some(Arc::new(input.clone()));
input
}
fn serialize(&self) -> Option<String> {
fn serialize(&self) -> Option<Arc<dyn core::any::Any>> {
let output = self.output.lock().unwrap();
(*output).as_ref().and_then(|output| serde_json::to_string(output).ok())
(*output).as_ref().and_then(|output| Some(output.clone() as Arc<dyn core::any::Any>))
}
}
impl<T, CachedNode> std::marker::Unpin for MonitorNode<T, CachedNode> {}
impl<T, CachedNode> MonitorNode<T, CachedNode> {
pub const fn new(node: CachedNode) -> MonitorNode<T, CachedNode> {
MonitorNode { output: Mutex::new(None), node }
impl<T> MonitorNode<T> {
pub const fn new() -> MonitorNode<T> {
MonitorNode { output: Mutex::new(None) }
}
}

View File

@@ -1,9 +1,10 @@
use dyn_any::{DynAny, StaticType};
use glam::{DAffine2, DVec2};
use graphene_core::raster::{Alpha, Channel, Image, ImageFrame, Luminance, Pixel, RasterMut, Sample};
use graphene_core::raster::{Alpha, BlendMode, BlendNode, Channel, Image, ImageFrame, Luminance, Pixel, RasterMut, Sample};
use graphene_core::transform::Transform;
use graphene_core::Node;
use graphene_core::value::CopiedNode;
use graphene_core::{Color, Node};
use std::fmt::Debug;
use std::marker::PhantomData;
@@ -48,51 +49,6 @@ fn buffer_node<R: std::io::Read>(reader: R) -> Result<Vec<u8>, Error> {
Ok(std::io::Read::bytes(reader).collect::<Result<Vec<_>, _>>()?)
}
/*
pub fn file_node<'i, 's: 'i, P: AsRef<Path> + 'i>() -> impl Node<'i, 's, P, Output = Result<Vec<u8>, Error>> {
let fs = ValueNode(StdFs).then(CloneNode::new());
let file = FileNode::new(fs);
file.then(FlatMapResultNode::new(ValueNode::new(BufferNode)))
}
pub fn image_node<'i, 's: 'i, P: AsRef<Path> + 'i>() -> impl Node<'i, 's, P, Output = Result<Image, Error>> {
let file = file_node();
let image_loader = FnNode::new(|data: Vec<u8>| image::load_from_memory(&data).map_err(Error::Image).map(|image| image.into_rgba32f()));
let image = file.then(FlatMapResultNode::new(ValueNode::new(image_loader)));
let convert_image = FnNode::new(|image: image::ImageBuffer<_, _>| {
let data = image
.enumerate_pixels()
.map(|(_, _, pixel): (_, _, &image::Rgba<f32>)| {
let c = pixel.channels();
Color::from_rgbaf32(c[0], c[1], c[2], c[3]).unwrap()
})
.collect();
Image {
width: image.width(),
height: image.height(),
data,
}
});
image.then(MapResultNode::new(convert_image))
}
pub fn export_image_node<'i, 's: 'i>() -> impl Node<'i, 's, (Image, &'i str), Output = Result<(), Error>> {
FnNode::new(|input: (Image, &str)| {
let (image, path) = input;
let mut new_image = image::ImageBuffer::new(image.width, image.height);
for ((x, y, pixel), color) in new_image.enumerate_pixels_mut().zip(image.data.iter()) {
let color: Color = *color;
assert!(x < image.width);
assert!(y < image.height);
*pixel = image::Rgba(color.to_rgba8())
}
new_image.save(path).map_err(Error::Image)
})
}
*/
pub struct DownresNode<P> {
_p: PhantomData<P>,
}
@@ -168,6 +124,17 @@ impl AxisAlignedBbox {
end: DVec2::new(self.end.x.max(other.end.x), self.end.y.max(other.end.y)),
}
}
pub fn union_non_empty(&self, other: &AxisAlignedBbox) -> Option<AxisAlignedBbox> {
match (self.size() == DVec2::ZERO, other.size() == DVec2::ZERO) {
(true, true) => None,
(true, _) => Some(other.clone()),
(_, true) => Some(self.clone()),
_ => Some(AxisAlignedBbox {
start: DVec2::new(self.start.x.min(other.start.x), self.start.y.min(other.start.y)),
end: DVec2::new(self.end.x.max(other.end.x), self.end.y.max(other.end.y)),
}),
}
}
}
#[derive(Debug, Clone)]
@@ -266,7 +233,7 @@ pub struct BlendImageTupleNode<P, Fg, MapFn> {
}
#[node_macro::node_fn(BlendImageTupleNode<_P, _Fg>)]
fn blend_image_tuple<_P: Pixel + Debug, MapFn, _Fg: Sample<Pixel = _P> + Transform>(images: (ImageFrame<_P>, _Fg), map_fn: &'any_input MapFn) -> ImageFrame<_P>
fn blend_image_tuple<_P: Alpha + Pixel + Debug, MapFn, _Fg: Sample<Pixel = _P> + Transform>(images: (ImageFrame<_P>, _Fg), map_fn: &'any_input MapFn) -> ImageFrame<_P>
where
MapFn: for<'any_input> Node<'any_input, (_P, _P), Output = _P> + 'input + Clone,
{
@@ -282,25 +249,67 @@ pub struct BlendImageNode<P, Background, MapFn> {
_p: PhantomData<P>,
}
// TODO: Implement proper blending
#[node_macro::node_fn(BlendImageNode<_P>)]
fn blend_image_node<_P: Clone, MapFn, Frame: Sample<Pixel = _P> + Transform, Background: RasterMut<Pixel = _P> + Transform>(
foreground: Frame,
background: Background,
map_fn: &'any_input MapFn,
) -> Background
fn blend_image_node<_P: Alpha + Pixel + Debug, MapFn, Forground: Sample<Pixel = _P> + Transform>(foreground: Forground, background: ImageFrame<_P>, map_fn: &'any_input MapFn) -> ImageFrame<_P>
where
MapFn: for<'any_input> Node<'any_input, (_P, _P), Output = _P> + 'input,
{
blend_image(foreground, background, map_fn)
blend_new_image(foreground, background, map_fn)
}
fn blend_image<_P: Clone, MapFn, Frame: Sample<Pixel = _P> + Transform, Background: RasterMut<Pixel = _P> + Transform>(foreground: Frame, mut background: Background, map_fn: &MapFn) -> Background
#[derive(Debug, Clone, Copy)]
pub struct BlendReverseImageNode<P, Background, MapFn> {
background: Background,
map_fn: MapFn,
_p: PhantomData<P>,
}
#[node_macro::node_fn(BlendReverseImageNode<_P>)]
fn blend_image_node<_P: Alpha + Pixel + Debug, MapFn, Background: Transform + Sample<Pixel = _P>>(foreground: ImageFrame<_P>, background: Background, map_fn: &'any_input MapFn) -> ImageFrame<_P>
where
MapFn: for<'any_input> Node<'any_input, (_P, _P), Output = _P> + 'input,
{
blend_new_image(background, foreground, map_fn)
}
fn blend_new_image<_P: Alpha + Pixel + Debug, MapFn, Frame: Sample<Pixel = _P> + Transform>(foreground: Frame, background: ImageFrame<_P>, map_fn: &MapFn) -> ImageFrame<_P>
where
MapFn: for<'any_input> Node<'any_input, (_P, _P), Output = _P>,
{
let foreground_aabb = compute_transformed_bounding_box(foreground.transform()).axis_aligned_bbox();
let background_aabb = compute_transformed_bounding_box(background.transform()).axis_aligned_bbox();
let Some(aabb) = foreground_aabb.union_non_empty(&background_aabb) else {return ImageFrame::empty()};
if background_aabb.contains(foreground_aabb.start) && background_aabb.contains(foreground_aabb.end) {
return blend_image(foreground, background, map_fn);
}
// Clamp the foreground image to the background image
let start = aabb.start.as_uvec2();
let end = aabb.end.as_uvec2();
let new_background = Image::new(end.x - start.x, end.y - start.y, _P::TRANSPARENT);
let size = DVec2::new(new_background.width as f64, new_background.height as f64);
let transfrom = DAffine2::from_scale_angle_translation(size, 0., start.as_dvec2());
let mut new_background = ImageFrame {
image: new_background,
transform: transfrom,
};
new_background = blend_image(background, new_background, map_fn);
blend_image(foreground, new_background, map_fn)
}
fn blend_image<_P: Alpha + Pixel + Debug, MapFn, Frame: Sample<Pixel = _P> + Transform, Background: RasterMut<Pixel = _P> + Transform + Sample<Pixel = _P>>(
foreground: Frame,
mut background: Background,
map_fn: &MapFn,
) -> Background
where
MapFn: for<'any_input> Node<'any_input, (_P, _P), Output = _P>,
{
let background_size = DVec2::new(background.width() as f64, background.height() as f64);
// Transforms a point from the background image to the forground image
let bg_to_fg = background.transform() * DAffine2::from_scale(1. / background_size);
@@ -319,7 +328,7 @@ where
if let Some(src_pixel) = foreground.sample(fg_point, area) {
if let Some(dst_pixel) = background.get_pixel_mut(x, y) {
*dst_pixel = map_fn.eval((src_pixel, dst_pixel.clone()));
*dst_pixel = map_fn.eval((src_pixel, *dst_pixel));
}
}
}
@@ -328,6 +337,23 @@ where
background
}
#[derive(Debug, Clone, Copy)]
pub struct ExtendImageNode<Background> {
background: Background,
}
#[node_macro::node_fn(ExtendImageNode)]
fn extend_image_node(foreground: ImageFrame<Color>, background: ImageFrame<Color>) -> ImageFrame<Color> {
let foreground_aabb = compute_transformed_bounding_box(foreground.transform()).axis_aligned_bbox();
let background_aabb = compute_transformed_bounding_box(background.transform()).axis_aligned_bbox();
if foreground_aabb.contains(background_aabb.start) && foreground_aabb.contains(background_aabb.end) {
return foreground;
}
blend_image(foreground, background, &BlendNode::new(CopiedNode::new(BlendMode::Normal), CopiedNode::new(100.)))
}
#[derive(Clone, Debug, PartialEq)]
pub struct MergeBoundingBoxNode<Data> {
_data: PhantomData<Data>,
@@ -340,7 +366,7 @@ fn merge_bounding_box_node<_Data: Transform>(input: (Option<AxisAlignedBbox>, _D
let snd_aabb = compute_transformed_bounding_box(data.transform()).axis_aligned_bbox();
if let Some(fst_aabb) = initial_aabb {
Some(fst_aabb.union(&snd_aabb))
fst_aabb.union_non_empty(&snd_aabb)
} else {
Some(snd_aabb)
}

View File

@@ -17,6 +17,8 @@ pub struct DynamicExecutor {
output: NodeId,
tree: BorrowTree,
typing_context: TypingContext,
// This allows us to keep the nodes around for one more frame which is used for introspection
orphaned_nodes: Vec<NodeId>,
}
impl Default for DynamicExecutor {
@@ -25,6 +27,7 @@ impl Default for DynamicExecutor {
output: Default::default(),
tree: Default::default(),
typing_context: TypingContext::new(&node_registry::NODE_REGISTRY),
orphaned_nodes: Vec::new(),
}
}
}
@@ -36,21 +39,27 @@ impl DynamicExecutor {
let output = proto_network.output;
let tree = BorrowTree::new(proto_network, &typing_context)?;
Ok(Self { tree, output, typing_context })
Ok(Self {
tree,
output,
typing_context,
orphaned_nodes: Vec::new(),
})
}
pub fn update(&mut self, proto_network: ProtoNetwork) -> Result<(), String> {
self.output = proto_network.output;
self.typing_context.update(&proto_network)?;
trace!("setting output to {}", self.output);
let orphans = self.tree.update(proto_network, &self.typing_context)?;
let mut orphans = self.tree.update(proto_network, &self.typing_context)?;
core::mem::swap(&mut self.orphaned_nodes, &mut orphans);
for node_id in orphans {
self.tree.free_node(node_id)
}
Ok(())
}
pub fn introspect(&self, node_path: &[NodeId]) -> Option<Option<String>> {
pub fn introspect(&self, node_path: &[NodeId]) -> Option<Option<Arc<dyn std::any::Any>>> {
self.tree.introspect(node_path)
}
@@ -128,7 +137,7 @@ impl BorrowTree {
node.reset();
}
old_nodes.remove(&id);
self.source_map.retain(|_, nid| *nid != id);
self.source_map.retain(|_, nid| !old_nodes.contains(nid));
}
Ok(old_nodes.into_iter().collect())
}
@@ -145,7 +154,7 @@ impl BorrowTree {
node
}
pub fn introspect(&self, node_path: &[NodeId]) -> Option<Option<String>> {
pub fn introspect(&self, node_path: &[NodeId]) -> Option<Option<Arc<dyn std::any::Any>>> {
let id = self.source_map.get(node_path)?;
let node = self.nodes.get(id)?;
let reader = node.read().unwrap();

View File

@@ -154,7 +154,7 @@ fn node_registry() -> HashMap<NodeIdentifier, HashMap<NodeIOTypes, NodeConstruct
register_node!(graphene_std::raster::MaskImageNode<_, _, _>, input: ImageFrame<Color>, params: [ImageFrame<Color>]),
register_node!(graphene_std::raster::MaskImageNode<_, _, _>, input: ImageFrame<Color>, params: [ImageFrame<Luma>]),
register_node!(graphene_std::raster::EmptyImageNode<_, _>, input: DAffine2, params: [Color]),
register_node!(graphene_std::memo::MonitorNode<_, _>, input: (), params: [ImageFrame<Color>]),
register_node!(graphene_std::memo::MonitorNode<_>, input: ImageFrame<Color>, params: []),
#[cfg(feature = "gpu")]
register_node!(graphene_std::executor::MapGpuSingleImageNode<_>, input: Image<Color>, params: [String]),
vec![(
@@ -175,14 +175,16 @@ fn node_registry() -> HashMap<NodeIdentifier, HashMap<NodeIOTypes, NodeConstruct
vec![(
NodeIdentifier::new("graphene_std::brush::BrushNode"),
|args| {
use graphene_core::structural::*;
use graphene_core::value::*;
use graphene_std::brush::*;
let trace: DowncastBothNode<(), Vec<DVec2>> = DowncastBothNode::new(args[0]);
let diameter: DowncastBothNode<(), f64> = DowncastBothNode::new(args[1]);
let hardness: DowncastBothNode<(), f64> = DowncastBothNode::new(args[2]);
let flow: DowncastBothNode<(), f64> = DowncastBothNode::new(args[3]);
let color: DowncastBothNode<(), Color> = DowncastBothNode::new(args[4]);
let image: DowncastBothNode<(), ImageFrame<Color>> = DowncastBothNode::new(args[0]);
let trace: DowncastBothNode<(), Vec<DVec2>> = DowncastBothNode::new(args[1]);
let diameter: DowncastBothNode<(), f64> = DowncastBothNode::new(args[2]);
let hardness: DowncastBothNode<(), f64> = DowncastBothNode::new(args[3]);
let flow: DowncastBothNode<(), f64> = DowncastBothNode::new(args[4]);
let color: DowncastBothNode<(), Color> = DowncastBothNode::new(args[5]);
let stamp = BrushStampGeneratorNode::new(color, CopiedNode::new(hardness.eval(())), CopiedNode::new(flow.eval(())));
let stamp = stamp.eval(diameter.eval(()));
@@ -191,16 +193,19 @@ fn node_registry() -> HashMap<NodeIdentifier, HashMap<NodeIOTypes, NodeConstruct
let frames = MapNode::new(ValueNode::new(frames));
let frames = frames.eval(trace.eval(()).into_iter()).collect::<Vec<_>>();
let background_bounds = ReduceNode::new(DebugClonedNode::new(None), ValueNode::new(MergeBoundingBoxNode::new()));
let background_bounds = ReduceNode::new(ClonedNode::new(None), ValueNode::new(MergeBoundingBoxNode::new()));
let background_bounds = background_bounds.eval(frames.clone().into_iter());
let background_bounds = DebugClonedNode::new(background_bounds.unwrap().to_transform());
let background_bounds = MergeBoundingBoxNode::new().eval((background_bounds, image.eval(())));
let background_bounds = ClonedNode::new(background_bounds.unwrap().to_transform());
let background_image = background_bounds.then(EmptyImageNode::new(CopiedNode::new(Color::TRANSPARENT)));
let blend_node = graphene_core::raster::BlendNode::new(CopiedNode::new(BlendMode::Normal), CopiedNode::new(100.));
let background = ExtendImageNode::new(background_image);
let background_image = image.then(background);
let final_image = ReduceNode::new(background_image, ValueNode::new(BlendImageTupleNode::new(ValueNode::new(blend_node))));
let final_image = DebugClonedNode::new(frames.into_iter()).then(final_image);
let final_image = ClonedNode::new(frames.into_iter()).then(final_image);
let any: DynAnyNode<(), _, _> = graphene_std::any::DynAnyNode::new(ValueNode::new(final_image));
Box::pin(any)
@@ -208,7 +213,7 @@ fn node_registry() -> HashMap<NodeIdentifier, HashMap<NodeIOTypes, NodeConstruct
NodeIOTypes::new(
concrete!(()),
concrete!(ImageFrame<Color>),
vec![value_fn!(Vec<DVec2>), value_fn!(f64), value_fn!(f64), value_fn!(f64), value_fn!(Color)],
vec![value_fn!(ImageFrame<Color>), value_fn!(Vec<DVec2>), value_fn!(f64), value_fn!(f64), value_fn!(f64), value_fn!(Color)],
),
)],
vec![(