Flip the Node trait from eval onto the frame claim's serve

This commit is contained in:
Dennis Kobert
2026-08-29 12:26:46 +00:00
parent 277641d27e
commit 3af6834d3c
22 changed files with 1226 additions and 1156 deletions

View File

@@ -17,7 +17,7 @@ pub enum BatchStatus<'a> {
/// hint is as for `Lent`.
Filled(RecordBatchMut<'a>, Finality, Extent),
/// No batch implementation behind this edge; a driver answers with the
/// per-lane eval and copy-out loop ([`crate::record::fill_frames`]).
/// per-lane serve and copy-out loop ([`crate::record::fill_frames`]).
Unbatched,
Pending,
Error(GraphError),
@@ -314,43 +314,34 @@ impl<T: Copy> Iterator for ListIter<'_, T> {
}
}
/// The output marker of a node that serves records through the caller's
/// frame claim instead of returning a plain value. It satisfies the lift
/// bounds so [`Node::serve`] stays callable and overridable on the erased
/// surface.
#[derive(Clone, Copy, Debug, Default, PartialEq, dyn_any::DynAny)]
pub struct Records;
pub trait Node<Input> {
type Output;
fn eval(&self, input: &Input) -> GPoll<Self::Output>;
/// Serves the node's record through the caller's claim: the writes land
/// in the claim and the returned proof is mintable only by its closing
/// methods, so the served record is of the claimed layout by
/// construction. The default lifts a plain output's element; record
/// servers override it.
/// construction. The caller claims the frame at [`Node::layout`] before
/// the call, so the node advances the record stack by exactly that frame.
fn serve<'e, 'l>(&self, input: &Input, slot: crate::record::FrameClaim<'l>) -> GPoll<crate::record::Served<'e>>
where
Self::Output: Send + Sync + dyn_any::StaticTypeSized,
Input: crate::context::ExtractArena<ArenaRef = &'e crate::arena::Arena>,
{
slot.lift_served(self.eval(input), input.arena())
}
Input: crate::context::ExtractArena<ArenaRef = &'e crate::arena::Arena>;
/// The count of items at one absolute nesting level (innermost `0`). The
/// leveled primitive a structure node overrides to report a pushed level's
/// size; the scalar base is one item at every level. Uncertainty rides the
/// `GPoll` status axis.
fn extent_at(&self, _input: &Input, _level: u8) -> GPoll<Extent> {
fn extent_at<'e>(&self, _input: &Input, _level: u8) -> GPoll<Extent>
where
Input: crate::context::ExtractArena<ArenaRef = &'e crate::arena::Arena>,
{
GPoll::Final(Extent::Exactly(1))
}
/// The composite domain query derived from [`extent_at`](Node::extent_at):
/// one level, the product of the levels below or above it, or the whole
/// domain's flat count. Consumers query this; nodes only write `extent_at`.
fn extent(&self, input: &Input, at: Level) -> GPoll<Extent> {
fn extent<'e>(&self, input: &Input, at: Level) -> GPoll<Extent>
where
Input: crate::context::ExtractArena<ArenaRef = &'e crate::arena::Arena>,
{
let product = |range: core::ops::Range<u8>| range.fold(GPoll::Final(Extent::Exactly(1)), |acc, level| Extent::mul(acc, self.extent_at(input, level)));
match at {
Level::At(level) => self.extent_at(input, level),
@@ -384,13 +375,13 @@ pub trait Node<Input> {
/// Batched evaluation of `range` into caller-provided frame storage of
/// `range.len() * layout.lane_stride()` bytes; see [`BatchStatus`]. The
/// default advertises no support and drivers fall back to per-lane eval
/// default advertises no support and drivers fall back to per-lane serves
/// with copy-out ([`crate::record::fill_frames`]); overrides exist to beat
/// that loop (resident lanes, direct fills, fewer erased calls), never for
/// correctness.
fn eval_batch<'a>(&'a self, input: &'a Input, range: Range<u64>, scratch: Option<&'a mut [MaybeUninit<u64>]>) -> BatchStatus<'a>
fn eval_batch<'a, 'e>(&'a self, input: &'a Input, range: Range<u64>, scratch: Option<&'a mut [MaybeUninit<u64>]>) -> BatchStatus<'a>
where
Input: InjectIndex + Copy,
Input: InjectIndex + Copy + crate::context::ExtractArena<ArenaRef = &'e crate::arena::Arena>,
{
let _ = (input, range, scratch);
BatchStatus::Unbatched
@@ -401,13 +392,17 @@ impl<Input, N> Node<Input> for &N
where
N: Node<Input> + ?Sized,
{
type Output = N::Output;
fn eval(&self, input: &Input) -> GPoll<Self::Output> {
(**self).eval(input)
fn serve<'e, 'l>(&self, input: &Input, slot: crate::record::FrameClaim<'l>) -> GPoll<crate::record::Served<'e>>
where
Input: crate::context::ExtractArena<ArenaRef = &'e crate::arena::Arena>,
{
(**self).serve(input, slot)
}
fn extent_at(&self, input: &Input, level: u8) -> GPoll<Extent> {
fn extent_at<'e>(&self, input: &Input, level: u8) -> GPoll<Extent>
where
Input: crate::context::ExtractArena<ArenaRef = &'e crate::arena::Arena>,
{
(**self).extent_at(input, level)
}
@@ -419,9 +414,9 @@ where
(**self).layout()
}
fn eval_batch<'a>(&'a self, input: &'a Input, range: Range<u64>, scratch: Option<&'a mut [MaybeUninit<u64>]>) -> BatchStatus<'a>
fn eval_batch<'a, 'e>(&'a self, input: &'a Input, range: Range<u64>, scratch: Option<&'a mut [MaybeUninit<u64>]>) -> BatchStatus<'a>
where
Input: InjectIndex + Copy,
Input: InjectIndex + Copy + crate::context::ExtractArena<ArenaRef = &'e crate::arena::Arena>,
{
(**self).eval_batch(input, range, scratch)
}
@@ -431,13 +426,17 @@ impl<Input, N> Node<Input> for Box<N>
where
N: Node<Input> + ?Sized,
{
type Output = N::Output;
fn eval(&self, input: &Input) -> GPoll<Self::Output> {
(**self).eval(input)
fn serve<'e, 'l>(&self, input: &Input, slot: crate::record::FrameClaim<'l>) -> GPoll<crate::record::Served<'e>>
where
Input: crate::context::ExtractArena<ArenaRef = &'e crate::arena::Arena>,
{
(**self).serve(input, slot)
}
fn extent_at(&self, input: &Input, level: u8) -> GPoll<Extent> {
fn extent_at<'e>(&self, input: &Input, level: u8) -> GPoll<Extent>
where
Input: crate::context::ExtractArena<ArenaRef = &'e crate::arena::Arena>,
{
(**self).extent_at(input, level)
}
@@ -449,9 +448,9 @@ where
(**self).layout()
}
fn eval_batch<'a>(&'a self, input: &'a Input, range: Range<u64>, scratch: Option<&'a mut [MaybeUninit<u64>]>) -> BatchStatus<'a>
fn eval_batch<'a, 'e>(&'a self, input: &'a Input, range: Range<u64>, scratch: Option<&'a mut [MaybeUninit<u64>]>) -> BatchStatus<'a>
where
Input: InjectIndex + Copy,
Input: InjectIndex + Copy + crate::context::ExtractArena<ArenaRef = &'e crate::arena::Arena>,
{
(**self).eval_batch(input, range, scratch)
}
@@ -461,13 +460,17 @@ impl<Input, N> Node<Input> for std::sync::Arc<N>
where
N: Node<Input> + ?Sized,
{
type Output = N::Output;
fn eval(&self, input: &Input) -> GPoll<Self::Output> {
(**self).eval(input)
fn serve<'e, 'l>(&self, input: &Input, slot: crate::record::FrameClaim<'l>) -> GPoll<crate::record::Served<'e>>
where
Input: crate::context::ExtractArena<ArenaRef = &'e crate::arena::Arena>,
{
(**self).serve(input, slot)
}
fn extent_at(&self, input: &Input, level: u8) -> GPoll<Extent> {
fn extent_at<'e>(&self, input: &Input, level: u8) -> GPoll<Extent>
where
Input: crate::context::ExtractArena<ArenaRef = &'e crate::arena::Arena>,
{
(**self).extent_at(input, level)
}
@@ -479,9 +482,9 @@ where
(**self).layout()
}
fn eval_batch<'a>(&'a self, input: &'a Input, range: Range<u64>, scratch: Option<&'a mut [MaybeUninit<u64>]>) -> BatchStatus<'a>
fn eval_batch<'a, 'e>(&'a self, input: &'a Input, range: Range<u64>, scratch: Option<&'a mut [MaybeUninit<u64>]>) -> BatchStatus<'a>
where
Input: InjectIndex + Copy,
Input: InjectIndex + Copy + crate::context::ExtractArena<ArenaRef = &'e crate::arena::Arena>,
{
(**self).eval_batch(input, range, scratch)
}
@@ -513,20 +516,27 @@ impl StatusCell {
Self { no_partial: true, ..Self::new() }
}
/// Claims the edge's own frame, serves through it, and folds the poll's
/// status into the cell. The claim is the caller's, so the edge's frame is
/// entered exactly once per evaluation.
#[inline(always)]
pub fn eval_input<Input, N: Node<Input>>(&self, input_index: usize, node: &N, input: &Input) -> Result<N::Output, Interrupt> {
match node.eval(input) {
GPoll::Final(value) => Ok(value),
pub fn eval_input<'e, Input, N: Node<Input> + ?Sized>(&self, input_index: usize, node: &N, input: &Input) -> Result<crate::record::RecordValue<'e>, Interrupt>
where
Input: crate::context::ExtractArena<ArenaRef = &'e crate::arena::Arena>,
{
let slot = crate::record::FrameClaim::enter(node.layout());
match node.serve(input, slot) {
GPoll::Final(served) => Ok(served.value()),
GPoll::Partial(_) if self.no_partial => Err(Interrupt::Pending),
GPoll::Partial(value) => {
GPoll::Partial(served) => {
self.finality.set(Finality::Partial);
Ok(value)
Ok(served.value())
}
GPoll::Fallback(boxed) => {
let (value, error) = *boxed;
let (served, error) = *boxed;
let first = self.error.take();
self.error.set(first.or(Some(error.traced(input_index))));
Ok(value)
Ok(served.value())
}
GPoll::Pending => Err(Interrupt::Pending),
GPoll::Error(mut error) => {
@@ -590,19 +600,20 @@ impl<'a, N> LazyInput<'a, N> {
}
#[inline(always)]
pub fn eval<Input>(&self, ctx: &Input) -> Result<N::Output, Interrupt>
pub fn eval<'e, Input>(&self, ctx: &Input) -> Result<crate::record::RecordValue<'e>, Interrupt>
where
N: Node<Input>,
N: crate::record::DerivedRecordEdge<'e, Input>,
{
self.cell.eval_input(self.input_index, self.node, ctx)
self.node.eval_derived(self.cell, self.input_index, ctx)
}
/// The edge's composite extent, for kernels that split or shift indices
/// over their sources.
#[inline(always)]
pub fn extent<Input>(&self, ctx: &Input, at: Level) -> GPoll<Extent>
pub fn extent<'e, Input>(&self, ctx: &Input, at: Level) -> GPoll<Extent>
where
N: Node<Input>,
Input: crate::context::ExtractArena<ArenaRef = &'e crate::arena::Arena>,
{
self.node.extent(ctx, at)
}
@@ -612,40 +623,60 @@ impl<'a, N> LazyInput<'a, N> {
mod tests {
use super::*;
use crate::arena::Arena;
use crate::context::ExtractArena;
use crate::record::{LiftedSource, serve_edge};
#[derive(Clone, Copy)]
struct TestInput {
struct TestInput<'a> {
index: u64,
arena: &'a Arena,
}
impl InjectIndex for TestInput {
impl InjectIndex for TestInput<'_> {
fn set_index(&mut self, index: u64) {
self.index = index;
}
}
struct Double;
impl<'a> ExtractArena for TestInput<'a> {
type ArenaRef = &'a Arena;
impl Node<TestInput> for Double {
type Output = u64;
fn eval(&self, input: &TestInput) -> GPoll<u64> {
GPoll::Final(input.index * 2)
fn arena(&self) -> &'a Arena {
self.arena
}
}
fn double<'a>() -> LiftedSource<u64, impl Fn(&TestInput<'a>) -> GPoll<u64>> {
LiftedSource::new(|input: &TestInput<'a>| GPoll::Final(input.index * 2))
}
#[test]
fn the_default_advertises_no_batch_support() {
let input = TestInput { index: 0 };
let arena = Arena::new(1024).unwrap();
let input = TestInput { index: 0, arena: &arena };
let mut scratch = [const { MaybeUninit::uninit() }; 4];
assert!(matches!(Double.eval_batch(&input, 2..6, Some(&mut scratch)), BatchStatus::Unbatched));
assert!(matches!(Double.eval_batch(&input, 2..6, None), BatchStatus::Unbatched));
let node = double();
assert!(matches!(node.eval_batch(&input, 2..6, Some(&mut scratch)), BatchStatus::Unbatched));
assert!(matches!(node.eval_batch(&input, 2..6, None), BatchStatus::Unbatched));
}
#[test]
fn trait_is_object_safe_across_erased_edges() {
let erased: Box<dyn Node<TestInput, Output = u64>> = Box::new(Double);
let input = TestInput { index: 21 };
assert_eq!(erased.eval(&input), GPoll::Final(42));
let arena = Arena::new(1024).unwrap();
let input = TestInput { index: 21, arena: &arena };
let node = double();
let layout = Node::<TestInput>::layout(&node).clone();
let erased: Box<dyn Node<TestInput>> = Box::new(node);
// SAFETY: between evaluations, nothing served on the stack is live.
unsafe {
crate::record::stack::reserve(1 << 12);
}
let GPoll::Final(value) = serve_edge(&*erased, &input) else {
panic!("the erased edge must serve a final record");
};
// SAFETY: the record was served at `layout`, whose element is the output.
assert_eq!(unsafe { crate::record::read_element::<u64>(layout.rec(&value)) }, 42);
assert!(matches!(erased.eval_batch(&input, 0..2, None), BatchStatus::Unbatched));
}
}

View File

@@ -485,23 +485,30 @@ impl<'e> RecordValue<'e> {
/// A record edge evaluable at a derived context, yielding the record at that
/// context's lifetime. The lifetime is a trait parameter because a bound like
/// `for<'d> Node<Derived<'d, C>, Output = RecordValue<'d>>` is rejected: in a
/// higher-ranked bound the lifetime must appear in a constrained input
/// position, and both the `Derived` projection and the `Output` binding are
/// unconstrained ones.
/// `for<'d> Node<Derived<'d, C>>` cannot also say the derived context's arena
/// is at `'d`: the equality binding `ExtractArena<ArenaRef = &'d Arena>` is an
/// unconstrained position under a higher rank.
pub trait DerivedRecordEdge<'derived, C> {
fn eval_derived(&self, cell: &crate::node::StatusCell, input_index: usize, ctx: &C) -> Result<RecordValue<'derived>, crate::gpoll::Interrupt>;
/// [`serve_edge`] at the derived context, for poll kernels that carry the
/// status themselves.
fn serve_derived(&self, ctx: &C) -> GPoll<RecordValue<'derived>>;
fn extent_at_derived(&self, ctx: &C, level: u8) -> GPoll<crate::gpoll::Extent>;
}
impl<'derived, C, N> DerivedRecordEdge<'derived, C> for N
where
N: Node<C, Output = RecordValue<'derived>>,
N: Node<C>,
C: crate::context::ExtractArena<ArenaRef = &'derived crate::arena::Arena>,
{
fn eval_derived(&self, cell: &crate::node::StatusCell, input_index: usize, ctx: &C) -> Result<RecordValue<'derived>, crate::gpoll::Interrupt> {
cell.eval_input(input_index, self, ctx)
}
fn serve_derived(&self, ctx: &C) -> GPoll<RecordValue<'derived>> {
serve_edge(self, ctx)
}
fn extent_at_derived(&self, ctx: &C, level: u8) -> GPoll<crate::gpoll::Extent> {
self.extent_at(ctx, level)
}
@@ -513,8 +520,8 @@ where
/// are distinct. Frame bytes carry no drop glue, so the copy is a move.
pub fn fill_frames<'a, 'e, C, N>(node: &'a N, input: &C, range: std::ops::Range<u64>, scratch: Option<&'a mut [std::mem::MaybeUninit<u64>]>) -> crate::node::BatchStatus<'a>
where
C: crate::context::InjectIndex + Copy,
N: Node<C, Output = RecordValue<'e>>,
C: crate::context::InjectIndex + Copy + crate::context::ExtractArena<ArenaRef = &'e crate::arena::Arena>,
N: Node<C>,
{
use crate::node::BatchStatus;
let Some(scratch) = scratch else {
@@ -538,7 +545,7 @@ where
// SAFETY: the lane's record is copied out before the scope releases it,
// and valueless exits serve nothing above the entry.
let _lane_scope = unsafe { stack::ScopeGuard::enter() };
let value = match node.eval(&local) {
let value = match serve_edge(node, &local) {
GPoll::Final(value) => value,
GPoll::Partial(value) => {
finality = crate::gpoll::Finality::Partial;
@@ -568,8 +575,8 @@ where
/// scratch, and an unbatched edge falls back to the [`fill_frames`] loop.
pub fn materialize_batch<'a, 'e, C, N>(node: &'a N, input: &'a C, range: std::ops::Range<u64>, arena: &'a crate::arena::Arena) -> crate::node::BatchStatus<'a>
where
C: crate::context::InjectIndex + Copy,
N: Node<C, Output = RecordValue<'e>>,
C: crate::context::InjectIndex + Copy + crate::context::ExtractArena<ArenaRef = &'e crate::arena::Arena>,
N: Node<C>,
{
use crate::node::BatchStatus;
let Some(len) = range.end.checked_sub(range.start).and_then(|len| usize::try_from(len).ok()) else {
@@ -608,8 +615,8 @@ pub enum LevelStatus<'a> {
/// reducers inline the same protocol with their span offsets.
pub fn materialize_level<'a, 'e, C, N>(node: &'a N, input: &'a C, arena: &'a crate::arena::Arena) -> LevelStatus<'a>
where
C: crate::context::InjectIndex + Copy,
N: Node<C, Output = RecordValue<'e>>,
C: crate::context::InjectIndex + Copy + crate::context::ExtractArena<ArenaRef = &'e crate::arena::Arena>,
N: Node<C>,
{
use crate::gpoll::{Extent, GraphError, Level};
use crate::node::BatchStatus;
@@ -652,47 +659,6 @@ where
}
}
/// A record edge at a caller-chosen lifetime; the lifetime is a trait
/// parameter for the same constrained-position reason as
/// [`DerivedRecordEdge`].
pub trait RecordEdge<'e, C>: Node<C, Output = RecordValue<'e>> {}
impl<'e, C, N: Node<C, Output = RecordValue<'e>>> RecordEdge<'e, C> for N {}
/// Builds an element-only record from a kernel's poll: inline layouts land
/// in the value, larger ones spill to the record stack, arena exhaustion of
/// a parked element reports as an error poll.
pub(crate) fn lift_poll<'e, T: Send + Sync>(poll: GPoll<T>, layout: &Layout, arena: &'e crate::arena::Arena) -> GPoll<RecordValue<'e>> {
let build = |element: T| {
if layout.frame_bytes() == 0 {
let mut value = RecordValue::zeroed();
unsafe { write_element(value.as_mut_ptr(), element, arena)? };
Some(value)
} else {
let dst = stack::push(layout.frame_bytes());
let written = unsafe { write_element(dst, element, arena) };
stack::truncate_above(dst, layout.frame_bytes());
written.map(|()| RecordValue::spilled(unsafe { Rec::new(dst.cast_const()) }))
}
};
let exhausted = || {
GPoll::Error(Box::new(crate::gpoll::GraphError {
kind: crate::gpoll::ErrorKind::ArenaExhausted,
trace: Vec::new(),
}))
};
match poll {
GPoll::Final(element) => build(element).map_or_else(exhausted, GPoll::Final),
GPoll::Partial(element) => build(element).map_or_else(exhausted, GPoll::Partial),
GPoll::Fallback(boxed) => {
let (element, error) = *boxed;
build(element).map_or_else(exhausted, |value| GPoll::Fallback(Box::new((value, error))))
}
GPoll::Pending => GPoll::Pending,
GPoll::Error(error) => GPoll::Error(error),
}
}
/// The raw lazy record edge handed to a record-opaque kernel: the wire plus
/// its wiring-proven layout, the pairing the kernel's unsafe record
/// operations rely on. The kernel must only pair the layout with values this
@@ -711,19 +677,22 @@ impl<'a, N> RecordEdgeInput<'a, N> {
self.layout
}
pub fn eval<'e, C>(&self, ctx: &C) -> GPoll<RecordValue<'e>>
/// Serves the edge through the kernel's own claim: the kernel's output
/// layout is the edge's, so the claim it was handed is the edge's frame.
pub fn serve<'e, 'l, C>(&self, ctx: &C, slot: FrameClaim<'l>) -> GPoll<Served<'e>>
where
N: Node<C, Output = RecordValue<'e>>,
N: Node<C>,
C: crate::context::ExtractArena<ArenaRef = &'e crate::arena::Arena>,
{
self.node.eval(ctx)
self.node.serve(ctx, slot)
}
/// [`materialize_level`] over the edge: the wire's whole flat span as one
/// batch.
pub fn materialize_level<'e, 'b, C>(&'b self, ctx: &'b C, arena: &'b crate::arena::Arena) -> LevelStatus<'b>
where
N: Node<C, Output = RecordValue<'e>>,
C: crate::context::InjectIndex + Copy,
N: Node<C>,
C: crate::context::InjectIndex + Copy + crate::context::ExtractArena<ArenaRef = &'e crate::arena::Arena>,
{
materialize_level(self.node, ctx, arena)
}
@@ -765,12 +734,12 @@ impl<'a, Out, N> ElementEdge<'a, Out, N> {
pub fn eval<'d, C>(&self, ctx: &C) -> GPoll<Out>
where
N: Node<C, Output = RecordValue<'d>>,
N: DerivedRecordEdge<'d, C>,
{
// SAFETY: the read copies out by value, so no record above the entry
// (the edge's own frame) is live past the scope.
let _scope = unsafe { stack::ScopeGuard::enter() };
self.node.eval(ctx).map(|value| unsafe { (self.read)(self.layout.rec(&value), self.reads) })
self.node.serve_derived(ctx).map(|value| unsafe { (self.read)(self.layout.rec(&value), self.reads) })
}
}
@@ -816,13 +785,13 @@ impl<'a, Out, N> ElementLazyInput<'a, Out, N> {
pub fn eval<'d, C>(&self, ctx: &C) -> Result<Out, crate::gpoll::Interrupt>
where
N: Node<C, Output = RecordValue<'d>>,
N: DerivedRecordEdge<'d, C>,
{
// SAFETY: the read copies the element and declared attributes out by
// value, so no record above the entry (the edge's own frame) is live
// past the scope.
let _scope = unsafe { stack::ScopeGuard::enter() };
let value = self.cell.eval_input(self.input_index, self.node, ctx)?;
let value = self.node.eval_derived(self.cell, self.input_index, ctx)?;
Ok(unsafe { (self.read)(self.layout.rec(&value), self.reads) })
}
}
@@ -1583,7 +1552,6 @@ pub struct SourcePlan {
moves: Vec<(usize, usize, usize)>,
fills: Vec<(usize, Box<[u8]>)>,
source: Layout,
union: Layout,
}
impl SourcePlan {
@@ -1602,7 +1570,6 @@ impl SourcePlan {
moves,
fills,
source: source.clone(),
union: union.clone(),
})
}
@@ -1650,47 +1617,6 @@ pub unsafe fn copy_record_bytes(layout: &Layout, rec: Rec) -> Box<[u8]> {
unsafe { std::slice::from_raw_parts(rec.ptr(), layout.size) }.into()
}
/// Serves a record whose frame lives in storage the current evaluation does
/// not own (a cached batch, published bytes): claims the node's frame over a
/// copy of the source frame, so the contract that every node advances the
/// record stack by exactly its own frame holds for cache hits too.
///
/// # Safety
/// `src` must point at a live record of `layout` whose parked references
/// outlive the serving evaluation.
pub unsafe fn serve_frame<'e>(layout: &Layout, src: *const u8) -> RecordValue<'e> {
if layout.frame_bytes() == 0 {
let mut value = RecordValue::zeroed();
unsafe { std::ptr::copy_nonoverlapping(src, value.as_mut_ptr(), layout.size) };
value
} else {
let dst = stack::push(layout.frame_bytes());
unsafe { std::ptr::copy_nonoverlapping(src, dst, layout.size) };
RecordValue::spilled(unsafe { Rec::new(dst.cast_const()) })
}
}
/// Claims a node's frame with nothing to serve, for exits that yield no
/// record (past-end, pending): the frame contract holds on every exit,
/// error included.
pub fn claim_frame(layout: &Layout) {
if layout.frame_bytes() != 0 {
stack::push(layout.frame_bytes());
}
}
/// Closes an interrupted eval: rewinds to the eval's entry pointer (interrupt
/// exits carry no value, so frames claimed above it are dead) and claims the
/// node's own frame, keeping the frame contract on interrupt exits.
///
/// # Safety
/// `entry` must be the stack pointer at the eval's entry, and no record
/// above it may be referenced after the close.
pub unsafe fn interrupt_frame(entry: usize, layout: &Layout) {
unsafe { stack::rewind(entry) };
claim_frame(layout);
}
/// A node's own output frame, claimed at eval entry: the one closing surface
/// for every exit. Writes land through it, [`Self::lift`] and [`Self::finish`]
/// serve the record, and its drop releases everything claimed above the frame
@@ -1806,6 +1732,37 @@ impl<'l> FrameClaim<'l> {
pub fn lift_served<'e, T: Send + Sync + dyn_any::StaticTypeSized>(self, poll: GPoll<T>, arena: &'e crate::arena::Arena) -> GPoll<Served<'e>> {
self.lift(poll, arena).map(|value| Served { value })
}
/// [`Self::finish`] with the proof-bearing return for [`Node::serve`].
///
/// # Safety
/// As [`Self::finish`].
pub unsafe fn finish_served<'e>(self) -> Served<'e> {
Served { value: unsafe { self.finish() } }
}
/// Fills the frame from a record a forwarded wire already served, and
/// closes it: the source's frame sits above this claim and dies with its
/// drop, so the served record is this claim's own.
///
/// # Safety
/// `value` must be a live record of this frame's layout.
pub unsafe fn forward<'e>(mut self, value: &RecordValue<'_>) -> Served<'e> {
let src = self.layout.rec(value).ptr();
unsafe {
self.fill_copy(src);
self.finish_served()
}
}
/// Translates a source record into the frame through a wiring-resolved plan.
///
/// # Safety
/// `src` must be a live record of `plan`'s source layout, and `plan` must
/// translate into this frame's layout.
pub unsafe fn translate(&mut self, src: Rec, plan: &SourcePlan) {
unsafe { plan.translate(src, self.dst()) };
}
}
/// The proof a record was served through a frame claim: mintable only by the
@@ -1819,6 +1776,23 @@ impl<'e> Served<'e> {
pub fn value(self) -> RecordValue<'e> {
self.value
}
/// The served record in place, for producers that read it before passing
/// the proof on.
pub fn record(&self) -> &RecordValue<'e> {
&self.value
}
}
/// Claims `node`'s own frame and serves through it: the caller-side half of
/// [`Node::serve`], for drivers that want the record rather than the proof.
pub fn serve_edge<'e, C, N>(node: &N, input: &C) -> GPoll<RecordValue<'e>>
where
N: Node<C> + ?Sized,
C: crate::context::ExtractArena<ArenaRef = &'e crate::arena::Arena>,
{
let slot = FrameClaim::enter(node.layout());
node.serve(input, slot).map(Served::value)
}
impl Drop for FrameClaim<'_> {
@@ -1876,6 +1850,13 @@ impl OwnedRecord {
written.map(|()| value)
}
/// [`Self::replay`] into a caller's claim rather than a fresh frame; the
/// claim's layout is the one the copy was taken at.
pub fn replay_into(&self, slot: &mut FrameClaim<'_>, arena: &crate::arena::Arena) -> Option<()> {
let layout = slot.layout;
self.write_into(layout, slot.dst(), arena)
}
fn write_into(&self, layout: &Layout, dst: *mut u8, arena: &crate::arena::Arena) -> Option<()> {
unsafe { std::ptr::copy_nonoverlapping(self.bytes.as_ptr(), dst, self.bytes.len()) };
if let Some(element) = &self.element {
@@ -1948,12 +1929,16 @@ impl ServedRecord {
/// rewinds to its entry state, so the result is owned and no stack slot
/// stays claimed. Assertion scaffolding for law tests; production consumers
/// read served records in place.
pub fn capture<'e, C, N: Node<C, Output = RecordValue<'e>>>(node: &N, ctx: &C) -> GPoll<ServedRecord> {
pub fn capture<'e, C, N>(node: &N, ctx: &C) -> GPoll<ServedRecord>
where
N: Node<C> + ?Sized,
C: crate::context::ExtractArena<ArenaRef = &'e crate::arena::Arena>,
{
// SAFETY: any served record is deep-copied out inside the scope, so
// nothing served above the entry escapes it.
let _scope = unsafe { stack::ScopeGuard::enter() };
let layout = node.layout().clone();
node.eval(ctx).map(|value| ServedRecord {
serve_edge(node, ctx).map(|value| ServedRecord {
// SAFETY: the poll served `value` at the node's declared layout and
// nothing has claimed frames since.
record: unsafe { OwnedRecord::copy_out(&layout, layout.rec(&value)) },
@@ -1961,39 +1946,39 @@ pub fn capture<'e, C, N: Node<C, Output = RecordValue<'e>>>(node: &N, ctx: &C) -
})
}
/// Law-test scaffolding: wraps an arbitrary plain node onto a record wire
/// (the element lands at offset 0 of a fresh element-only record, parked when
/// it carries drop glue). No production path constructs one; value edges are
/// Law-test scaffolding: a kernel closure served onto an element-only record
/// wire (the element lands at offset 0, parked when it carries drop glue). No
/// production path constructs one; value edges are
/// [`crate::value::ValueSource`].
pub struct RecordLift<El, N> {
edge: N,
pub struct LiftedSource<El, F> {
kernel: F,
layout: Layout,
_marker: std::marker::PhantomData<fn() -> El>,
}
impl<El: Clone + Send + Sync + dyn_any::StaticTypeSized, N> RecordLift<El, N>
impl<El: Clone + Send + Sync + dyn_any::StaticTypeSized, F> LiftedSource<El, F>
where
El::Static: Clone + Send + Sync,
{
pub fn new(edge: N) -> Self {
pub fn new(kernel: F) -> Self {
Self {
edge,
kernel,
layout: Layout::default().with_writes(0, element_write::<El>(), &[]),
_marker: std::marker::PhantomData,
}
}
}
impl<'e, C, El, N> Node<C> for RecordLift<El, N>
impl<C, El, F> Node<C> for LiftedSource<El, F>
where
C: crate::context::ExtractArena<ArenaRef = &'e crate::arena::Arena>,
El: Send + Sync + 'static,
N: Node<C, Output = El>,
El: Send + Sync + dyn_any::StaticTypeSized,
F: Fn(&C) -> GPoll<El>,
{
type Output = RecordValue<'e>;
fn eval(&self, input: &C) -> GPoll<RecordValue<'e>> {
lift_poll(self.edge.eval(input), &self.layout, input.arena())
fn serve<'e, 'l>(&self, input: &C, slot: FrameClaim<'l>) -> GPoll<Served<'e>>
where
C: crate::context::ExtractArena<ArenaRef = &'e crate::arena::Arena>,
{
slot.lift_served((self.kernel)(input), input.arena())
}
fn layout(&self) -> &Layout {
@@ -2021,54 +2006,63 @@ impl<El, N> RecordExtract<El, N> {
}
}
impl<'e, C, El, N> Node<C> for RecordExtract<El, N>
where
El: Clone + 'static,
N: Node<C, Output = RecordValue<'e>>,
{
type Output = El;
fn eval(&self, input: &C) -> GPoll<El> {
impl<El: Clone + 'static, N> RecordExtract<El, N> {
/// The edge's element, copied out of its record.
pub fn eval<'e, C>(&self, input: &C) -> GPoll<El>
where
N: Node<C>,
C: crate::context::ExtractArena<ArenaRef = &'e crate::arena::Arena>,
{
// SAFETY: the element copies out by value, so no record above the
// entry (the edge's frame) is live past the scope; a plain output
// claims no frame itself.
// entry (the edge's frame) is live past the scope.
let _scope = unsafe { stack::ScopeGuard::enter() };
self.edge.eval(input).map(|value| unsafe { read_element::<El>(self.layout.rec(&value)) })
serve_edge(&self.edge, input).map(|value| unsafe { read_element::<El>(self.layout.rec(&value)) })
}
}
impl<'e, C, N> Node<C> for RecordSource<N>
impl<C, N> Node<C> for RecordSource<N>
where
N: Node<C, Output = RecordValue<'e>>,
N: Node<C>,
{
type Output = RecordValue<'e>;
fn eval(&self, input: &C) -> GPoll<RecordValue<'e>> {
match &self.plan {
None => self.edge.eval(input),
Some(plan) if plan.union.frame_bytes() == 0 => {
// SAFETY: the translation copies the record into the inline
// value, so no record above the entry is live past the scope.
let _scope = unsafe { stack::ScopeGuard::enter() };
self.edge.eval(input).map(|value| {
let mut out = RecordValue::zeroed();
unsafe { plan.translate(plan.source.rec(&value), out.as_mut_ptr()) };
out
})
fn serve<'e, 'l>(&self, input: &C, mut slot: FrameClaim<'l>) -> GPoll<Served<'e>>
where
C: crate::context::ExtractArena<ArenaRef = &'e crate::arena::Arena>,
{
// The source's frame is claimed above this one and dies with the
// claim's drop; the translated union record stays.
let Some(plan) = &self.plan else {
return self.edge.serve(input, slot);
};
match serve_edge(&self.edge, input) {
GPoll::Final(value) => {
// SAFETY: the value came from this edge, so it carries the
// plan's source layout.
unsafe { slot.translate(plan.source.rec(&value), plan) };
// SAFETY: the translation completes the union record.
GPoll::Final(unsafe { slot.finish_served() })
}
Some(plan) => {
let dst = stack::push(plan.union.frame_bytes());
let value = self.edge.eval(input);
let result = value.map(|value| RecordValue::spilled(unsafe { plan.translate(plan.source.rec(&value), dst) }));
// The source's frame dies with the translation; the claimed
// frame above stays as this node's output.
stack::truncate_above(dst, plan.union.frame_bytes());
result
GPoll::Partial(value) => {
// SAFETY: as for the final arm.
unsafe { slot.translate(plan.source.rec(&value), plan) };
// SAFETY: as for the final arm.
GPoll::Partial(unsafe { slot.finish_served() })
}
GPoll::Fallback(boxed) => {
let (value, error) = *boxed;
// SAFETY: as for the final arm.
unsafe { slot.translate(plan.source.rec(&value), plan) };
// SAFETY: as for the final arm.
GPoll::Fallback(Box::new((unsafe { slot.finish_served() }, error)))
}
GPoll::Pending => GPoll::Pending,
GPoll::Error(error) => GPoll::Error(error),
}
}
fn extent_at(&self, input: &C, level: u8) -> GPoll<crate::gpoll::Extent> {
fn extent_at<'x>(&self, input: &C, level: u8) -> GPoll<crate::gpoll::Extent>
where
C: crate::context::ExtractArena<ArenaRef = &'x crate::arena::Arena>,
{
self.edge.extent_at(input, level)
}
@@ -3034,17 +3028,17 @@ mod tests {
layout: Layout,
}
impl<'e> Node<&'e crate::arena::Arena> for Fixture {
type Output = RecordValue<'e>;
fn eval(&self, arena: &&'e crate::arena::Arena) -> GPoll<RecordValue<'e>> {
let mut frame = FrameBuilder::new(&self.layout, arena);
impl<C> Node<C> for Fixture {
fn serve<'e, 'l>(&self, input: &C, slot: FrameClaim<'l>) -> GPoll<Served<'e>>
where
C: crate::context::ExtractArena<ArenaRef = &'e crate::arena::Arena>,
{
let mut frame = FrameBuilder::new(&self.layout, crate::context::ExtractArena::arena(input));
frame.element(String::from("parked"));
frame.attr::<Transform>(DAffine2::from_translation(DVec2::new(3., 4.)));
match frame.finish() {
Some(value) => GPoll::Final(value),
None => GPoll::error("arena exhausted"),
}
let Some(value) = frame.finish() else { return GPoll::error("arena exhausted") };
// SAFETY: the builder served a record of this node's layout.
GPoll::Final(unsafe { slot.forward(&value) })
}
fn layout(&self) -> &Layout {
@@ -3054,9 +3048,15 @@ mod tests {
let layout = Layout::default().with_writes(0, element_write::<String>(), &[FieldWrite::of::<Transform>(0), FieldWrite::of::<Opacity>(0)]);
// SAFETY: between evaluations, nothing served on the stack is live.
unsafe { stack::reserve(1 << 10); } let arena = crate::arena::Arena::new(1024).unwrap();
unsafe {
stack::reserve(1 << 10);
}
let arena = crate::arena::Arena::new(1024).unwrap();
let generations = [];
let scope = crate::context::EvalScope::new(None, None, None, &generations, &arena);
let ctx = crate::context::ContextImpl::root(&scope);
let mark = stack::sp();
let GPoll::Final(served) = capture(&Fixture { layout }, &&arena) else {
let GPoll::Final(served) = capture(&Fixture { layout }, &ctx) else {
panic!("the fixture serves finally");
};
assert_eq!(stack::sp(), mark, "capture returns every claimed slot");

View File

@@ -74,9 +74,9 @@ pub use crate::NodeIOTypes;
/// Element-independent by erasure; the wire's `Type::Record(El)` keeps element reads proven at wiring.
#[cfg(not(target_family = "wasm"))]
pub type ErasedRecordNode = dyn for<'c> Node<ContextImpl<'c>, Output = crate::record::RecordValue<'c>> + Send + Sync;
pub type ErasedRecordNode = dyn for<'c> Node<ContextImpl<'c>> + Send + Sync;
#[cfg(target_family = "wasm")]
pub type ErasedRecordNode = dyn for<'c> Node<ContextImpl<'c>, Output = crate::record::RecordValue<'c>>;
pub type ErasedRecordNode = dyn for<'c> Node<ContextImpl<'c>>;
#[cfg(not(target_family = "wasm"))]
type DynEdge = dyn std::any::Any + Send + Sync;
@@ -136,12 +136,13 @@ impl<Input, N> Node<Input> for SharedEdge<N>
where
N: Node<Input> + ?Sized,
{
type Output = N::Output;
fn eval(&self, input: &Input) -> crate::gpoll::GPoll<Self::Output> {
// Every node advances the record stack by exactly its own frame: it keeps
// its output and reclaims its inputs. A mismatch is a leaked or
// over-released frame.
fn serve<'e, 'l>(&self, input: &Input, slot: crate::record::FrameClaim<'l>) -> crate::gpoll::GPoll<crate::record::Served<'e>>
where
Input: crate::context::ExtractArena<ArenaRef = &'e crate::arena::Arena>,
{
// Every node advances the record stack by exactly its own frame: the
// caller claimed it before the call, so serving must leave the stack
// where it found it. A mismatch is a leaked or over-released frame.
#[cfg(debug_assertions)]
let sp_before = crate::record::stack::sp();
#[cfg(debug_assertions)]
@@ -160,7 +161,7 @@ where
}
// SAFETY: `own` keeps the payload alive for `self`'s lifetime and Arc
// payloads are address stable.
let result = unsafe { self.ptr.as_ref() }.eval(input);
let result = unsafe { self.ptr.as_ref() }.serve(input, slot);
#[cfg(debug_assertions)]
if trace {
eprintln!("sp> exit frame_bytes {} sp {} -> {}", self.layout().frame_bytes(), sp_before, crate::record::stack::sp());
@@ -168,7 +169,7 @@ where
#[cfg(debug_assertions)]
debug_assert_eq!(
crate::record::stack::sp(),
sp_before + self.layout().frame_bytes(),
sp_before,
"{} left the record stack misaligned (frame_bytes {}, depth {}, fields [{}], poll {})",
std::any::type_name::<N>(),
self.layout().frame_bytes(),
@@ -185,26 +186,29 @@ where
result
}
fn extent_at(&self, input: &Input, level: u8) -> crate::gpoll::GPoll<crate::gpoll::Extent> {
// SAFETY: as in eval.
fn extent_at<'x>(&self, input: &Input, level: u8) -> crate::gpoll::GPoll<crate::gpoll::Extent>
where
Input: crate::context::ExtractArena<ArenaRef = &'x crate::arena::Arena>,
{
// SAFETY: as in serve.
unsafe { self.ptr.as_ref() }.extent_at(input, level)
}
fn serialize(&self) -> Option<std::sync::Arc<dyn std::any::Any + Send + Sync>> {
// SAFETY: as in eval.
// SAFETY: as in serve.
unsafe { self.ptr.as_ref() }.serialize()
}
fn layout(&self) -> &crate::record::Layout {
// SAFETY: as in eval.
// SAFETY: as in serve.
unsafe { self.ptr.as_ref() }.layout()
}
fn eval_batch<'a>(&'a self, input: &'a Input, range: std::ops::Range<u64>, scratch: Option<&'a mut [std::mem::MaybeUninit<u64>]>) -> crate::node::BatchStatus<'a>
fn eval_batch<'a, 'x>(&'a self, input: &'a Input, range: std::ops::Range<u64>, scratch: Option<&'a mut [std::mem::MaybeUninit<u64>]>) -> crate::node::BatchStatus<'a>
where
Input: crate::context::InjectIndex + Copy,
Input: crate::context::InjectIndex + Copy + crate::context::ExtractArena<ArenaRef = &'x crate::arena::Arena>,
{
// SAFETY: as in eval.
// SAFETY: as in serve.
unsafe { self.ptr.as_ref() }.eval_batch(input, range, scratch)
}
}
@@ -341,186 +345,244 @@ mod tests {
use super::*;
use crate::SourceId;
use crate::arena::Arena;
use crate::context::{Ctx, EvalScope, ExtractArena};
use crate::context::{Ctx, EvalScope, ExtractArena, ExtractIndices};
use crate::gpoll::GPoll;
use std::sync::Arc;
use std::sync::atomic::{AtomicU32, Ordering};
struct CountingNode(AtomicU32);
use crate::record::{FrameClaim, Layout, LiftedSource, Served, element_write, read_element, serve_edge, stack};
impl<Input> Node<Input> for CountingNode {
type Output = u32;
fn eval(&self, _input: &Input) -> GPoll<u32> {
GPoll::Final(self.0.fetch_add(1, Ordering::Relaxed) + 1)
}
}
struct LendNode(String);
impl<'e, Input: Ctx + ExtractArena<ArenaRef = &'e Arena>> Node<Input> for LendNode {
type Output = &'e String;
fn eval(&self, input: &Input) -> GPoll<&'e String> {
match input.arena().alloc(self.0.clone()) {
Some((parked, _)) => GPoll::Final(parked),
None => GPoll::arena_exhausted(),
}
}
fn counting() -> LiftedSource<u32, impl for<'c> Fn(&ContextImpl<'c>) -> GPoll<u32>> {
let count = AtomicU32::new(0);
LiftedSource::new(move |_: &ContextImpl<'_>| GPoll::Final(count.fetch_add(1, Ordering::Relaxed) + 1))
}
fn scope_fixture<'a>(generations: &'a [(SourceId, u64)], arena: &'a Arena) -> EvalScope<'a> {
EvalScope::new(Some(0.5), None, None, generations, arena)
}
#[test]
fn borrow_carrying_value_types_wire_through_the_general_constructor() {
struct SplitBorrow<'c>(&'c str, usize);
/// Serves a parked borrow of its own value: the element is a reference into
/// the evaluation's arena, so its lifetime is the serving one.
struct LendNode {
value: String,
layout: Layout,
}
struct SplitNode<Node0> {
content: Node0,
impl LendNode {
fn new(value: &str) -> Self {
Self {
value: value.to_string(),
layout: Layout::default().with_writes(0, element_write::<&'static String>(), &[]),
}
}
}
impl<'e, Input, Node0> Node<Input> for SplitNode<Node0>
impl<Input: Ctx> Node<Input> for LendNode {
fn serve<'e, 'l>(&self, input: &Input, slot: FrameClaim<'l>) -> GPoll<Served<'e>>
where
Input: Ctx,
Node0: Node<Input, Output = &'e String>,
Input: ExtractArena<ArenaRef = &'e Arena>,
{
type Output = SplitBorrow<'e>;
fn eval(&self, input: &Input) -> GPoll<SplitBorrow<'e>> {
self.content.eval(input).map(|value| SplitBorrow(value, value.len()))
match input.arena().alloc(self.value.clone()) {
Some((parked, _)) => slot.lift_served(GPoll::Final(parked), input.arena()),
None => GPoll::arena_exhausted(),
}
}
type ErasedSplitEdge = dyn for<'c> Node<ContextImpl<'c>, Output = SplitBorrow<'c>> + Send + Sync;
fn layout(&self) -> &Layout {
&self.layout
}
}
#[test]
fn borrow_carrying_value_types_wire_through_the_general_constructor() {
let arena = Arena::new(4096).unwrap();
let generations = [];
let scope = scope_fixture(&generations, &arena);
let ctx = ContextImpl::root(&scope);
// SAFETY: between evaluations, nothing served on the stack is live.
unsafe {
stack::reserve(1 << 12);
}
let node: Arc<ErasedSplitEdge> = Arc::new(SplitNode {
content: LendNode("held".to_string()),
});
let handle = EdgeHandle::new_erased(node, concrete!(SplitBorrow<'static>));
assert_eq!(*handle.ty(), concrete!(SplitBorrow<'static>));
let node = LendNode::new("held");
let layout = Node::<ContextImpl>::layout(&node).clone();
let handle = EdgeHandle::new_erased(Arc::new(node) as Arc<ErasedRecordNode>, concrete!(String));
assert_eq!(*handle.ty(), concrete!(String));
let wired = handle.downcast_erased::<ErasedSplitEdge>(concrete!(SplitBorrow<'static>)).unwrap();
let GPoll::Final(split) = wired.eval(&ctx) else {
panic!("borrow-carrying output must eval through the erased edge");
let wired = handle.downcast_erased::<ErasedRecordNode>(concrete!(String)).unwrap();
let GPoll::Final(value) = serve_edge(&wired, &ctx) else {
panic!("borrow-carrying output must serve through the erased edge");
};
assert_eq!(split.0, "held");
assert_eq!(split.1, 4);
// SAFETY: the record was served at `layout`, whose element is the borrow.
let held = unsafe { read_element::<&String>(layout.rec(&value)) };
assert_eq!(held, "held");
assert_eq!(held.len(), 4);
}
/// Evaluates its content at three promoted index levels and serves the
/// collected elements.
struct RepeatNode<Node0, T> {
content: Node0,
inner: Layout,
layout: Layout,
_marker: std::marker::PhantomData<fn() -> T>,
}
impl<Node0, T: Clone + Send + Sync + dyn_any::StaticTypeSized> RepeatNode<Node0, T>
where
Vec<T>: Clone + Send + Sync + dyn_any::StaticTypeSized,
<Vec<T> as dyn_any::StaticTypeSized>::Static: Clone + Send + Sync,
{
fn new(content: Node0, inner: Layout) -> Self {
Self {
content,
inner,
layout: Layout::default().with_writes(0, element_write::<Vec<T>>(), &[]),
_marker: std::marker::PhantomData,
}
}
}
impl<C, T, Node0> Node<C> for RepeatNode<Node0, T>
where
C: Ctx + crate::context::DeriveCtx,
T: Clone + 'static,
Vec<T>: Send + Sync + dyn_any::StaticTypeSized,
Node0: for<'x> crate::record::DerivedRecordEdge<'x, crate::context::Derived<'x, C>>,
{
fn serve<'e, 'l>(&self, input: &C, slot: FrameClaim<'l>) -> GPoll<Served<'e>>
where
C: ExtractArena<ArenaRef = &'e Arena>,
{
let cell = crate::node::StatusCell::new();
let spilled = input.index_head();
let mut result = Vec::new();
for index in 0..3 {
// SAFETY: the element copies out by value, so the content's
// frame is dead by the time the scope releases it.
let _scope = unsafe { stack::ScopeGuard::enter() };
let derived = input.promoted(&spilled, index);
match self.content.eval_derived(&cell, 0, &derived) {
// SAFETY: the content served at its own layout, whose
// element is `T`.
Ok(value) => result.push(unsafe { read_element::<T>(self.inner.rec(&value)) }),
Err(interrupt) => return interrupt.into(),
}
}
slot.lift_served(cell.finish(result), input.arena())
}
fn layout(&self) -> &Layout {
&self.layout
}
}
#[test]
fn derive_ctx_repeat_pushes_index_levels_through_the_erased_edge() {
use crate::context::{DeriveCtx, Derived, ExtractIndex};
struct RepeatNode<Node0> {
content: Node0,
}
impl<C, T, Node0> Node<C> for RepeatNode<Node0>
where
C: Ctx + DeriveCtx,
Node0: for<'x> Node<Derived<'x, C>, Output = T>,
{
type Output = Vec<T>;
fn eval(&self, input: &C) -> GPoll<Vec<T>> {
let spilled = input.index_head();
let mut result = Vec::new();
for index in 0..3 {
let derived = input.promoted(&spilled, index);
match self.content.eval(&derived) {
GPoll::Final(value) => result.push(value),
other => return other.map(|_| Vec::new()),
}
}
GPoll::Final(result)
}
}
struct LevelsNode;
impl<Input: ExtractIndex> Node<Input> for LevelsNode {
type Output = Vec<usize>;
fn eval(&self, input: &Input) -> GPoll<Vec<usize>> {
GPoll::Final(input.try_index().map(|levels| levels.collect()).unwrap_or_default())
}
}
let arena = Arena::new(1024).unwrap();
let generations = [];
let scope = scope_fixture(&generations, &arena);
let ctx = ContextImpl::root(&scope);
let nested = crate::record::RecordLift::<Vec<Vec<Vec<usize>>>, _>::new(RepeatNode {
content: RepeatNode { content: LevelsNode },
});
let levels = LiftedSource::<Vec<usize>, _>::new(|input: &ContextImpl| GPoll::Final(input.try_index().map(|levels| levels.collect()).unwrap_or_default()));
let levels_layout = Node::<ContextImpl>::layout(&levels).clone();
let inner = RepeatNode::<_, Vec<usize>>::new(levels, levels_layout);
let inner_layout = Node::<ContextImpl>::layout(&inner).clone();
let nested = RepeatNode::<_, Vec<Vec<usize>>>::new(inner, inner_layout);
let layout = Node::<ContextImpl>::layout(&nested).clone();
let erased: Box<ErasedRecordNode> = Box::new(nested);
// SAFETY: between evaluations, nothing served on the stack is live.
unsafe {
crate::record::stack::reserve(1 << 12);
stack::reserve(1 << 12);
}
let GPoll::Final(value) = erased.eval(&ctx) else {
let GPoll::Final(value) = serve_edge(&*erased, &ctx) else {
panic!("nested repeat must evaluate");
};
// SAFETY: the record was served at `layout`, whose element is the output.
let outer = unsafe { crate::record::read_element::<Vec<Vec<Vec<usize>>>>(layout.rec(&value)) };
let outer = unsafe { read_element::<Vec<Vec<Vec<usize>>>>(layout.rec(&value)) };
assert_eq!(outer.len(), 3);
assert_eq!(outer[2][1], vec![1, 2, 0]);
assert_eq!(outer[0][0], vec![0, 0, 0]);
}
/// Shifts the footprint's resolution and serves its content's element under
/// the derived context.
struct ShiftFootprintNode<Node0> {
content: Node0,
inner: Layout,
layout: Layout,
}
impl<Node0> ShiftFootprintNode<Node0> {
fn new(content: Node0, inner: Layout) -> Self {
Self {
content,
inner,
layout: Layout::default().with_writes(0, element_write::<u32>(), &[]),
}
}
}
impl<C, Node0> Node<C> for ShiftFootprintNode<Node0>
where
C: Ctx + crate::context::DeriveCtx + crate::context::ExtractFootprint,
Node0: for<'x> crate::record::DerivedRecordEdge<'x, crate::context::Derived<'x, C>>,
{
fn serve<'e, 'l>(&self, input: &C, slot: FrameClaim<'l>) -> GPoll<Served<'e>>
where
C: ExtractArena<ArenaRef = &'e Arena>,
{
use crate::transform::Footprint;
let cell = crate::node::StatusCell::new();
let mut footprint = input.try_footprint().copied().unwrap_or(Footprint::DEFAULT);
footprint.resolution.x += 7;
// SAFETY: the element copies out by value, so the content's frame is
// dead by the time the scope releases it.
let value = {
let _scope = unsafe { stack::ScopeGuard::enter() };
let derived = input.with_footprint(&footprint);
match self.content.eval_derived(&cell, 0, &derived) {
// SAFETY: the content served at its own layout, whose
// element is the resolution.
Ok(value) => unsafe { read_element::<u32>(self.inner.rec(&value)) },
Err(interrupt) => return interrupt.into(),
}
};
slot.lift_served(cell.finish(value), input.arena())
}
fn layout(&self) -> &Layout {
&self.layout
}
}
#[test]
fn derive_ctx_footprint_replace_reaches_the_content() {
use crate::context::{DeriveCtx, Derived, ExtractFootprint};
use crate::context::ExtractFootprint;
use crate::transform::Footprint;
struct ShiftFootprintNode<Node0> {
content: Node0,
}
impl<C, T, Node0> Node<C> for ShiftFootprintNode<Node0>
where
C: Ctx + DeriveCtx + ExtractFootprint,
Node0: for<'x> Node<Derived<'x, C>, Output = T>,
{
type Output = T;
fn eval(&self, input: &C) -> GPoll<T> {
let mut footprint = input.try_footprint().copied().unwrap_or(Footprint::DEFAULT);
footprint.resolution.x += 7;
let derived = input.with_footprint(&footprint);
self.content.eval(&derived)
}
}
struct ResolutionNode;
impl<Input: ExtractFootprint> Node<Input> for ResolutionNode {
type Output = u32;
fn eval(&self, input: &Input) -> GPoll<u32> {
GPoll::Final(input.try_footprint().map(|footprint| footprint.resolution.x).unwrap_or(0))
}
}
let arena = Arena::new(1024).unwrap();
let generations = [];
let scope = scope_fixture(&generations, &arena);
let ctx = ContextImpl::root(&scope);
// SAFETY: between evaluations, nothing served on the stack is live.
unsafe {
stack::reserve(1 << 12);
}
let graph = ShiftFootprintNode {
content: ShiftFootprintNode { content: ResolutionNode },
let resolution = LiftedSource::<u32, _>::new(|input: &ContextImpl| GPoll::Final(input.try_footprint().map(|footprint| footprint.resolution.x).unwrap_or(0)));
let resolution_layout = Node::<ContextImpl>::layout(&resolution).clone();
let shifted = ShiftFootprintNode::new(resolution, resolution_layout);
let shifted_layout = Node::<ContextImpl>::layout(&shifted).clone();
let graph = ShiftFootprintNode::new(shifted, shifted_layout);
let layout = Node::<ContextImpl>::layout(&graph).clone();
let GPoll::Final(value) = serve_edge(&graph, &ctx) else {
panic!("the footprint shift must reach the content");
};
assert_eq!(graph.eval(&ctx), GPoll::Final(Footprint::DEFAULT.resolution.x + 14));
// SAFETY: the record was served at `layout`, whose element is the resolution.
assert_eq!(unsafe { read_element::<u32>(layout.rec(&value)) }, Footprint::DEFAULT.resolution.x + 14);
}
#[test]
@@ -559,24 +621,24 @@ mod tests {
let scope = scope_fixture(&generations, &arena);
let ctx = ContextImpl::root(&scope);
let counting = crate::record::RecordLift::<u32, _>::new(CountingNode(AtomicU32::new(0)));
let counting = counting();
let layout = Node::<ContextImpl>::layout(&counting).clone();
let handle = EdgeHandle::new_record::<u32>(Arc::new(counting) as Arc<ErasedRecordNode>);
let duplicate = handle.duplicate();
assert_eq!(*duplicate.ty(), record_edge_type::<u32>());
// SAFETY: between evaluations, nothing served on the stack is live.
unsafe {
crate::record::stack::reserve(1 << 12);
stack::reserve(1 << 12);
}
let first = handle.downcast_record::<u32>().unwrap();
let second = duplicate.downcast_record::<u32>().unwrap();
// SAFETY: each record was served at `layout`, whose element is the count.
let count = |value| unsafe { layout.rec(&value).element::<u32>() };
assert_eq!(first.eval(&ctx).map(count), GPoll::Final(1));
assert_eq!(second.eval(&ctx).map(count), GPoll::Final(2));
assert_eq!(serve_edge(&first, &ctx).map(count), GPoll::Final(1));
assert_eq!(serve_edge(&second, &ctx).map(count), GPoll::Final(2));
drop(first);
assert_eq!(second.eval(&ctx).map(count), GPoll::Final(3));
assert_eq!(serve_edge(&second, &ctx).map(count), GPoll::Final(3));
}
}

View File

@@ -171,7 +171,7 @@ mod tests {
use crate::context::{ContextImpl, Ctx, EvalScope, ExtractFootprint, ExtractVarArgs, VarArgLink, VarArgSlots};
use crate::gpoll::GPoll;
use crate::node::Node;
use crate::record::{Layout, RecordExtract, RecordLift, element_write, stack};
use crate::record::{Layout, LiftedSource, RecordExtract, element_write, stack};
use crate::transform::Footprint;
use std::sync::Mutex;
use std::sync::atomic::{AtomicU32, Ordering};
@@ -248,16 +248,6 @@ mod tests {
}
}
struct SourceNode<T>(T);
impl<T: Clone, Input> Node<Input> for SourceNode<T> {
type Output = T;
fn eval(&self, _input: &Input) -> GPoll<T> {
GPoll::Final(self.0.clone())
}
}
fn element_layout<T: Clone + Send + Sync + dyn_any::StaticTypeSized>() -> Layout
where
T::Static: Clone + Send + Sync,
@@ -265,11 +255,11 @@ mod tests {
Layout::default().with_writes(0, element_write::<T>(), &[])
}
fn lifted<T: Clone + Send + Sync + dyn_any::StaticTypeSized>(value: T) -> RecordLift<T, SourceNode<T>>
fn lifted<T: Clone + Send + Sync + dyn_any::StaticTypeSized>(value: T) -> LiftedSource<T, impl for<'c> Fn(&ContextImpl<'c>) -> GPoll<T>>
where
T::Static: Clone + Send + Sync,
{
RecordLift::new(SourceNode(value))
LiftedSource::new(move |_: &ContextImpl<'_>| GPoll::Final(value.clone()))
}
fn extract<El: Clone + Send + Sync + dyn_any::StaticTypeSized, N: Node<ContextImpl<'static>>>(mut graph: N) -> RecordExtract<El, N>
@@ -277,7 +267,10 @@ mod tests {
El::Static: Clone + Send + Sync,
{
// SAFETY: between evaluations, nothing served on the stack is live.
unsafe { stack::reserve(1 << 12); } let layout = element_layout::<El>();
unsafe {
stack::reserve(1 << 12);
}
let layout = element_layout::<El>();
graph.set_layout(crate::record::RecordLayout {
frame_bytes: layout.frame_bytes(),
plan: Vec::new(),
@@ -333,17 +326,11 @@ mod tests {
Ok(Box::pin(async move { value + addend }))
}
struct GatedSource(Arc<std::sync::atomic::AtomicBool>, f64);
impl<Input> Node<Input> for GatedSource {
type Output = f64;
fn eval(&self, _input: &Input) -> GPoll<f64> {
match self.0.load(Ordering::Relaxed) {
true => GPoll::Final(self.1),
false => GPoll::Pending,
}
}
fn gated(gate: Arc<std::sync::atomic::AtomicBool>, value: f64) -> LiftedSource<f64, impl for<'c> Fn(&ContextImpl<'c>) -> GPoll<f64>> {
LiftedSource::new(move |_: &ContextImpl<'_>| match gate.load(Ordering::Relaxed) {
true => GPoll::Final(value),
false => GPoll::Pending,
})
}
fn scope_fixture<'a>(generations: &'a [(SourceId, u64)], arena: &'a Arena) -> EvalScope<'a> {
@@ -367,13 +354,13 @@ mod tests {
&element_layout::<u64>(),
));
assert_eq!(Node::eval(&graph, &ctx), GPoll::Pending);
assert_eq!(Node::eval(&graph, &ctx), GPoll::Pending);
assert_eq!(graph.eval(&ctx), GPoll::Pending);
assert_eq!(graph.eval(&ctx), GPoll::Pending);
assert_eq!(SLOW_DOUBLE_RUNS.load(Ordering::Relaxed), 0);
assert_eq!(runtime.drain(), vec![7]);
assert_eq!(SLOW_DOUBLE_RUNS.load(Ordering::Relaxed), 1);
assert_eq!(Node::eval(&graph, &ctx), GPoll::Final(42.0));
assert_eq!(Node::eval(&graph, &ctx), GPoll::Final(42.0));
assert_eq!(graph.eval(&ctx), GPoll::Final(42.0));
assert_eq!(graph.eval(&ctx), GPoll::Final(42.0));
assert_eq!(SLOW_DOUBLE_RUNS.load(Ordering::Relaxed), 1);
}
@@ -394,9 +381,9 @@ mod tests {
&element_layout::<u64>(),
));
assert_eq!(Node::eval(&graph, &ctx), GPoll::Partial(-1.0));
assert_eq!(graph.eval(&ctx), GPoll::Partial(-1.0));
runtime.drain();
assert_eq!(Node::eval(&graph, &ctx), GPoll::Final(42.0));
assert_eq!(graph.eval(&ctx), GPoll::Final(42.0));
}
#[test]
@@ -416,9 +403,9 @@ mod tests {
&element_layout::<u64>(),
));
assert_eq!(Node::eval(&graph, &ctx), GPoll::Pending);
assert_eq!(graph.eval(&ctx), GPoll::Pending);
runtime.drain();
assert_eq!(Node::eval(&graph, &ctx), GPoll::Final(42.0));
assert_eq!(graph.eval(&ctx), GPoll::Final(42.0));
}
#[test]
@@ -438,12 +425,12 @@ mod tests {
&element_layout::<u64>(),
));
assert_eq!(Node::eval(&graph, &ctx), GPoll::Pending);
assert_eq!(graph.eval(&ctx), GPoll::Pending);
assert_eq!(STAGED_RUNS.load(Ordering::Relaxed), 1, "the prologue runs synchronously on the miss");
assert_eq!(Node::eval(&graph, &ctx), GPoll::Pending);
assert_eq!(graph.eval(&ctx), GPoll::Pending);
assert_eq!(STAGED_RUNS.load(Ordering::Relaxed), 1, "in flight must not rerun the prologue");
assert_eq!(runtime.drain(), vec![8]);
assert_eq!(Node::eval(&graph, &ctx), GPoll::Final(42.0));
assert_eq!(graph.eval(&ctx), GPoll::Final(42.0));
assert_eq!(STAGED_RUNS.load(Ordering::Relaxed), 1);
}
@@ -458,7 +445,7 @@ mod tests {
let runtime = Arc::new(MockRuntime::default());
let graph = extract::<f64, _>(StagedSumNode::new(
lifted(40.0f64),
RecordLift::<f64, _>::new(GatedSource(gate.clone(), 2.0)),
gated(gate.clone(), 2.0),
lifted(RuntimeHandle(runtime.clone())),
lifted(9u64),
&element_layout::<f64>(),
@@ -467,12 +454,12 @@ mod tests {
&element_layout::<u64>(),
));
assert_eq!(Node::eval(&graph, &ctx), GPoll::Pending);
assert_eq!(graph.eval(&ctx), GPoll::Pending);
assert_eq!(runtime.drain(), Vec::<SourceId>::new(), "an interrupted prologue must not spawn or claim the slot");
gate.store(true, Ordering::Relaxed);
assert_eq!(Node::eval(&graph, &ctx), GPoll::Pending);
assert_eq!(graph.eval(&ctx), GPoll::Pending);
assert_eq!(runtime.drain(), vec![9]);
assert_eq!(Node::eval(&graph, &ctx), GPoll::Final(42.0));
assert_eq!(graph.eval(&ctx), GPoll::Final(42.0));
}
#[test]
@@ -498,9 +485,9 @@ mod tests {
&element_layout::<u64>(),
));
assert_eq!(Node::eval(&graph, &ctx), GPoll::Pending);
assert_eq!(graph.eval(&ctx), GPoll::Pending);
runtime.drain();
assert_eq!(Node::eval(&graph, &ctx), GPoll::Final(21.5));
assert_eq!(graph.eval(&ctx), GPoll::Final(21.5));
}
#[test]
@@ -522,9 +509,9 @@ mod tests {
&element_layout::<u64>(),
));
assert_eq!(Node::eval(&graph, &ctx), GPoll::Pending);
assert_eq!(graph.eval(&ctx), GPoll::Pending);
runtime.drain();
assert_eq!(Node::eval(&graph, &ctx), GPoll::Final(Footprint::DEFAULT.resolution.x));
assert_eq!(graph.eval(&ctx), GPoll::Final(Footprint::DEFAULT.resolution.x));
}
#[test]
@@ -638,7 +625,7 @@ mod tests {
let snapshot = runtime.snapshot();
let scope = EvalScope::new(None, None, None, &snapshot, &arena);
let ctx = ContextImpl::root(&scope);
assert_eq!(Node::eval(&graph, &ctx), GPoll::Final(42.0));
assert_eq!(graph.eval(&ctx), GPoll::Final(42.0));
assert!(!runtime.take_dirty());
assert_eq!(runtime.snapshot(), vec![(13, 0)]);
assert_eq!(runtime.spawner().drain(), 0);
@@ -661,7 +648,7 @@ mod tests {
let snapshot = runtime.snapshot();
let scope = EvalScope::new(None, None, None, &snapshot, &arena);
let ctx = ContextImpl::root(&scope);
assert_eq!(Node::eval(&graph, &ctx), GPoll::Pending);
assert_eq!(graph.eval(&ctx), GPoll::Pending);
assert!(!runtime.take_dirty());
assert_eq!(runtime.spawner().drain(), 1);
@@ -671,7 +658,7 @@ mod tests {
let bumped_scope = EvalScope::new(None, None, None, &bumped, &arena);
let bumped_ctx = ContextImpl::root(&bumped_scope);
assert_eq!(Node::eval(&graph, &bumped_ctx), GPoll::Final(42.0), "the own-generation-excluded key replays the landed slot");
assert_eq!(graph.eval(&bumped_ctx), GPoll::Final(42.0), "the own-generation-excluded key replays the landed slot");
assert_eq!(runtime.spawner().drain(), 0, "a slot hit must not respawn");
let downstream_key = crate::registry::cache_key(&ContextImpl::root(&scope));

View File

@@ -17,15 +17,15 @@ where
}
}
impl<'e, C, T> crate::node::Node<C> for ValueSource<T>
impl<C, T> crate::node::Node<C> for ValueSource<T>
where
C: crate::context::ExtractArena<ArenaRef = &'e crate::arena::Arena>,
T: Clone + Send + Sync + 'static,
T: Clone + Send + Sync + dyn_any::StaticTypeSized,
{
type Output = crate::record::RecordValue<'e>;
fn eval(&self, input: &C) -> crate::gpoll::GPoll<crate::record::RecordValue<'e>> {
crate::record::lift_poll(crate::gpoll::GPoll::Final(self.value.clone()), &self.layout, input.arena())
fn serve<'e, 'l>(&self, input: &C, slot: crate::record::FrameClaim<'l>) -> crate::gpoll::GPoll<crate::record::Served<'e>>
where
C: crate::context::ExtractArena<ArenaRef = &'e crate::arena::Arena>,
{
slot.lift_served(crate::gpoll::GPoll::Final(self.value.clone()), input.arena())
}
fn layout(&self) -> &crate::record::Layout {
@@ -60,21 +60,25 @@ where
}
}
impl<'e, C, T> crate::node::Node<C> for LeveledValueSource<T>
impl<C, T> crate::node::Node<C> for LeveledValueSource<T>
where
C: crate::context::ExtractArena<ArenaRef = &'e crate::arena::Arena> + crate::context::ExtractIndex,
T: Clone + Send + Sync + 'static,
C: crate::context::ExtractIndex,
T: Clone + Send + Sync + dyn_any::StaticTypeSized,
{
type Output = crate::record::RecordValue<'e>;
fn eval(&self, input: &C) -> crate::gpoll::GPoll<crate::record::RecordValue<'e>> {
fn serve<'e, 'l>(&self, input: &C, slot: crate::record::FrameClaim<'l>) -> crate::gpoll::GPoll<crate::record::Served<'e>>
where
C: crate::context::ExtractArena<ArenaRef = &'e crate::arena::Arena>,
{
let Some(value) = self.values.get(input.innermost_index() as usize) else {
return crate::gpoll::GPoll::error("value level addressed past its items");
};
crate::record::lift_poll(crate::gpoll::GPoll::Final(value.clone()), &self.layout, input.arena())
slot.lift_served(crate::gpoll::GPoll::Final(value.clone()), input.arena())
}
fn extent_at(&self, _input: &C, level: u8) -> crate::gpoll::GPoll<crate::gpoll::Extent> {
fn extent_at<'x>(&self, _input: &C, level: u8) -> crate::gpoll::GPoll<crate::gpoll::Extent>
where
C: crate::context::ExtractArena<ArenaRef = &'x crate::arena::Arena>,
{
match level {
0 => crate::gpoll::GPoll::Final(crate::gpoll::Extent::Exactly(self.values.len())),
_ => crate::gpoll::GPoll::Final(crate::gpoll::Extent::Exactly(1)),

View File

@@ -10,7 +10,7 @@ use core_types::arena::Arena;
use core_types::context::InjectIndex;
use core_types::gpoll::{Finality, GraphError};
use core_types::node::Node;
use core_types::record::{Group, GroupItem, LevelStatus, RecordValue, materialize_level};
use core_types::record::{Group, GroupItem, LevelStatus, materialize_level};
use core_types::uuid::NodeId;
use glam::{DAffine2, DVec2};
use vector_types::GradientStops;
@@ -26,8 +26,8 @@ pub enum LevelGroup<'e> {
/// group over the level's records, ready for the group render bridge.
pub fn materialize_group<'a, 'e, C, N>(node: &'a N, input: &'a C, arena: &'a Arena) -> LevelGroup<'a>
where
C: InjectIndex + Copy,
N: Node<C, Output = RecordValue<'e>>,
C: InjectIndex + Copy + core_types::context::ExtractArena<ArenaRef = &'e Arena>,
N: Node<C>,
{
match materialize_level(node, input, arena) {
LevelStatus::Batch(batch, finality) => {