Convert the node catalog to rank-polymorphic kernels, materialize stored values as ranked wires, and display wire rank in the graph

Co-authored-by:    Dennis Kobert <dennis@kobert.dev>
This commit is contained in:
Keavon Chambers
2026-07-20 21:27:13 -07:00
committed by Dennis Kobert
parent 83cfd0225a
commit 04d6c0d5cf
55 changed files with 1781 additions and 995 deletions

View File

@@ -1,9 +1,11 @@
use core_types::list::{Item, List};
use core_types::{ATTR_TYPE, Ctx};
use crate::{expanded_count, locate_expanded, unescape_string};
use core_types::attribute::{Attr, Type};
use core_types::extent::{LevelIn, ListIn, ValueIn};
use core_types::gpoll::{Extent, GPoll, GraphError, Interrupt};
use core_types::node::Lane;
use core_types::{Ctx, ExtractIndex, InjectIndex};
use serde_json::Value;
use crate::unescape_string;
// ===========
// Format JSON
// ===========
@@ -221,12 +223,12 @@ fn query_json(
/// • **Index Elements**: access the `N`th query result.
/// • **String to Number**: convert numeric query results to numbers.
/// • **String Value** → **Equals**: convert "true", "false", or "null" query results to bools.
#[node_macro::node(name("Query JSON All"), category("Text: JSON"))]
fn query_json_all(
_: impl Ctx,
/// The JSON string to extract values from.
#[node_macro::node(name("Query JSON All"), category("Text: JSON"), extent(query_json_all_extent))]
fn query_json_all<'e>(
ctx: impl Ctx + ExtractArena<'e> + ExtractIndex + InjectIndex + Copy,
/// The JSON strings to extract values from.
#[name("JSON")]
json: String,
json: IList<String>,
/// Determines which contained values to extract from within the JSON.
///
/// The path syntax is like JavaScript's accessor syntax that follows an array/object value. It also supports negative indexing to count backwards from the end. Additionally, `[]` accesses all array and object values instead of just one.
@@ -240,15 +242,33 @@ fn query_json_all(
/// Strips the surrounding double quotes from string values, returning the raw text. Other types are never wrapped in quotes.
#[default(true)]
unquote_strings: bool,
) -> List<String> {
let cleaned = strip_trailing_commas(&json);
let Ok(value): Result<Value, _> = serde_json::from_str(&cleaned) else { return List::new() };
let Some(segments) = parse_json_path(path.trim()) else { return List::new() };
) -> Result<IList<(Lane<String>, Attr<'e, Type>)>, Interrupt> {
let (row, (text, ty)) = locate_expanded(json, ctx.index() as usize, |json| query_all(json, &path, unquote_strings)).ok_or_else(|| Interrupt::from(GraphError::past_end()))?;
Ok((json.lane(row).map_element(text), Attr(ty)))
}
/// Every value `path` matches in `json` with its JSON type, none for invalid
/// JSON or an invalid path.
fn query_all(json: &str, path: &str, unquote_strings: bool) -> Vec<(String, &'static str)> {
let cleaned = strip_trailing_commas(json);
let Ok(value): Result<Value, _> = serde_json::from_str(&cleaned) else { return Vec::new() };
let Some(segments) = parse_json_path(path.trim()) else { return Vec::new() };
let mut results = Vec::new();
resolve_all(&value, &segments, !unquote_strings, &mut results);
results
}
results.into_iter().map(|(text, ty)| Item::new_from_element(text).with_attribute(ATTR_TYPE, ty.to_string())).collect()
/// The level holds every string's matched values in order.
fn query_json_all_extent(json: ListIn<'_, String>, path: ValueIn<'_, String>, unquote_strings: ValueIn<'_, bool>, level: LevelIn) -> GPoll<Extent> {
match level.top() {
true => json
.get()
.zip(path.get())
.zip(unquote_strings.get())
.map(|((json, path), unquote_strings)| expanded_count(json, |json| query_all(json, &path, unquote_strings).len())),
false => GPoll::Final(Extent::Exactly(1)),
}
}
/// A parsed segment of a JSON access path.

View File

@@ -8,11 +8,12 @@ mod text_context;
mod to_path;
use convert_case::{Boundary, Converter, pattern};
use core_types::gpoll::Interrupt;
use core_types::extent::{LevelIn, ListIn, ValueIn};
use core_types::gpoll::{Extent, GPoll, GraphError, Interrupt};
use core_types::graphene_hash::CacheHash;
use core_types::list::{Item, List};
use core_types::node::Lane;
use core_types::registry::types::{SignedInteger, TextArea};
use core_types::{Context, Ctx, DeriveCtx, ExtractVarArgs};
use core_types::{Ctx, ExtractIndex, InjectIndex};
use dyn_any::DynAny;
use glam::{DAffine2, DVec2};
use unicode_segmentation::UnicodeSegmentation;
@@ -361,7 +362,7 @@ fn format_number(
}
/// Parses a string into a number. Falls back to the chosen value if the string is not a valid number.
#[node_macro::node(category("Text"))]
#[node_macro::node(category("Text"), name("String to Number"))]
fn string_to_number(
_: impl Ctx,
/// The string containing a number. Surrounding whitespace is ignored, a decimal point (.) may be included, sign prefixes (+/-) are respected, and scientific notation (e.g. "1e-3") is supported.
@@ -727,14 +728,43 @@ fn string_length(_: impl Ctx, string: String) -> f64 {
string.graphemes(true).count() as f64
}
/// Splits a string into a list of substrings based on the specified delimiter. This is the inverse of the **String Join** node.
/// The `lane`-th row of the level made by expanding every string in order,
/// with the row of the string it came from.
pub(crate) fn locate_expanded<R>(strings: core_types::node::List<'_, String>, lane: usize, expand: impl Fn(&str) -> Vec<R>) -> Option<(usize, R)> {
let mut remaining = lane;
for row in 0..strings.len() {
let mut expanded = expand(strings.element_ref(row));
if remaining >= expanded.len() {
remaining -= expanded.len();
continue;
}
return Some((row, expanded.swap_remove(remaining)));
}
None
}
/// The rows every string expands to, summed.
pub(crate) fn expanded_count(strings: core_types::node::List<'_, String>, expand: impl Fn(&str) -> usize) -> Extent {
Extent::Exactly((0..strings.len()).map(|row| expand(strings.element_ref(row))).sum())
}
/// The parts of `string` around `delimiter`, unescaped when asked.
fn split_parts(string: &str, delimiter: &str, delimiter_escaping: bool) -> Vec<String> {
let delimiter = match delimiter_escaping {
true => unescape_string(delimiter.to_string()),
false => delimiter.to_string(),
};
string.split(&delimiter).map(str::to_string).collect()
}
/// Splits each string into substrings based on the specified delimiter, producing one flat list of all the substrings. This is the inverse of the **String Join** node.
///
/// For example, splitting "a, b, c" with delimiter ", " produces `["a", "b", "c"]`.
#[node_macro::node(category("Text"))]
#[node_macro::node(category("Text"), extent(string_split_extent))]
fn string_split(
_: impl Ctx,
/// The string to split into substrings.
string: String,
ctx: impl Ctx + ExtractIndex + InjectIndex + Copy,
/// The strings to split into substrings.
strings: IList<String>,
/// The character(s) that separate the substrings. These are not included in the outputs.
#[default("\\n")]
delimiter: String,
@@ -742,10 +772,21 @@ fn string_split(
/// "\n" (newline), "\r" (carriage return), "\t" (tab), "\0" (null), and "\\" (backslash).
#[default(true)]
delimiter_escaping: bool,
) -> List<String> {
let delimiter = if delimiter_escaping { unescape_string(delimiter) } else { delimiter };
) -> Result<IList<Lane<String>>, Interrupt> {
let (row, part) = locate_expanded(strings, ctx.index() as usize, |string| split_parts(string, &delimiter, delimiter_escaping)).ok_or_else(|| Interrupt::from(GraphError::past_end()))?;
Ok(strings.lane(row).map_element(part))
}
string.split(&delimiter).map(str::to_string).map(Item::new_from_element).collect()
/// The level holds every string's parts in order.
fn string_split_extent(strings: ListIn<'_, String>, delimiter: ValueIn<'_, String>, delimiter_escaping: ValueIn<'_, bool>, level: LevelIn) -> GPoll<Extent> {
match level.top() {
true => strings
.get()
.zip(delimiter.get())
.zip(delimiter_escaping.get())
.map(|((strings, delimiter), escaping)| expanded_count(strings, |string| split_parts(string, &delimiter, escaping).len())),
false => GPoll::Final(Extent::Exactly(1)),
}
}
/// Joins a list of strings together with a separator between each pair. This is the inverse of the **String Split** node.
@@ -755,7 +796,7 @@ fn string_split(
fn string_join(
_: impl Ctx,
/// The list of strings to join together.
strings: List<String>,
strings: IList<String>,
/// The text placed between each pair of strings.
#[default(", ")]
separator: String,
@@ -766,39 +807,7 @@ fn string_join(
) -> String {
let separator = if separator_escaping { unescape_string(separator) } else { separator };
strings.iter_element_values().map(|s| s.as_str()).collect::<Vec<_>>().join(&separator)
}
/// Iterates over a list of strings, evaluating the mapped operation for each one. Use the **Read String** node to access the current string inside the loop.
#[node_macro::node(category("Text"))]
fn map_string(
ctx: impl Ctx + DeriveCtx,
strings: List<String>,
#[expose]
#[implementations(Context -> String)]
mapped: impl Node<Context<'_>, Output = String>,
) -> Result<List<String>, Interrupt> {
let spilled = ctx.index_head();
let mut result = List::new();
for (i, row) in strings.into_iter().enumerate() {
let string = row.into_element();
let scoped = ctx.push_vararg(&string);
let mapped_string = mapped.eval(&scoped.ctx().promoted(&spilled, i as u64))?;
result.push(Item::new_from_element(mapped_string));
}
Ok(result)
}
/// Reads the current string from within a **Map String** node's loop.
#[node_macro::node(category("Context"))]
fn read_string(ctx: impl Ctx + ExtractVarArgs) -> String {
let Ok(var_arg) = ctx.vararg(0) else { return String::new() };
let var_arg = var_arg as &dyn std::any::Any;
var_arg.downcast_ref::<String>().cloned().unwrap_or_default()
(0..strings.len()).map(|row| strings.element_ref(row).as_str()).collect::<Vec<_>>().join(&separator)
}
/// Converts a value to a JSON string representation.

View File

@@ -202,7 +202,7 @@ impl PathBuilder {
}
}
// "Separate Glyphs" off: widen the accumulated AABBs and bundle as one override `Vector`
// Glyph separation off: widen the accumulated AABBs and bundle as one override `Vector`
if !self.merged_click_target_bboxes.is_empty() {
let mut bboxes = self.merged_click_target_bboxes;
widen_horizontal_gaps(&mut bboxes, &self.merged_click_target_baselines);

View File

@@ -1,6 +1,10 @@
use core_types::list::{Item, List};
use crate::{expanded_count, locate_expanded};
use core_types::attribute::{Attr, End, Name, Start};
use core_types::extent::{LevelIn, ListIn, ValueIn};
use core_types::gpoll::{Extent, GPoll, GraphError, Interrupt};
use core_types::node::Lane;
use core_types::registry::types::SignedInteger;
use core_types::{ATTR_END, ATTR_NAME, ATTR_START, Ctx};
use core_types::{Ctx, ExtractIndex, InjectIndex};
/// Checks whether the string contains a match for the given regular expression pattern. Optionally restricts the match to only the start and/or end of the string.
#[node_macro::node(category("Text: Regex"))]
@@ -77,17 +81,101 @@ fn regex_replace(
}
}
/// Finds a regex match in the string and returns its components. The result is a list where the first item is the whole match (`$0`) and subsequent items are the capture groups (`$1`, `$2`, etc., if any).
/// The pattern with its flag prefix compiled, or nothing for an empty or
/// invalid pattern (the latter logged).
fn compile_regex(pattern: &str, case_insensitive: bool, multiline: bool) -> Option<fancy_regex::Regex> {
if pattern.is_empty() {
return None;
}
let flags = match (case_insensitive, multiline) {
(false, false) => "",
(true, false) => "(?i)",
(false, true) => "(?m)",
(true, true) => "(?im)",
};
match fancy_regex::Regex::new(&format!("{flags}{pattern}")) {
Ok(regex) => Some(regex),
Err(_) => {
log::error!("Invalid regex pattern: {pattern}");
None
}
}
}
/// One matched substring with its byte range in the searched string and, for
/// a capture, the group's name.
struct Span {
text: String,
start: u64,
end: u64,
name: String,
}
/// The whole match then each capture group of the `match_index`-th match,
/// empty where the index resolves to no match.
fn capture_spans(regex: &fancy_regex::Regex, string: &str, match_index: f64) -> Vec<Span> {
// Capture group names indexed positionally; index 0 (the whole match) is always None.
let capture_names: Vec<Option<String>> = regex.capture_names().map(|name| name.map(str::to_string)).collect();
// Collect all matches since we need to support negative indexing
let matches: Vec<_> = regex.captures_iter(string).filter_map(|c| c.ok()).collect();
let match_index = match_index as i32;
let resolved_index = match match_index < 0 {
true => match matches.len().checked_sub((-match_index) as usize) {
Some(index) => index,
None => return Vec::new(),
},
false => match_index as usize,
};
let Some(captures) = matches.get(resolved_index) else {
return Vec::new();
};
(0..captures.len())
.map(|i| {
let captured = captures.get(i);
Span {
text: captured.map_or(String::new(), |m| m.as_str().to_string()),
start: captured.map_or(0, |m| m.start() as u64),
end: captured.map_or(0, |m| m.end() as u64),
name: capture_names.get(i).cloned().flatten().unwrap_or_default(),
}
})
.collect()
}
fn match_spans(regex: &fancy_regex::Regex, string: &str) -> Vec<Span> {
regex
.find_iter(string)
.filter_map(|m| m.ok())
.map(|m| Span {
text: m.as_str().to_string(),
start: m.start() as u64,
end: m.end() as u64,
name: String::new(),
})
.collect()
}
/// The parts of `string` between matches, the whole string without a usable pattern.
fn split_parts(regex: Option<&fancy_regex::Regex>, string: &str) -> Vec<String> {
match regex {
Some(regex) => regex.split(string).filter_map(|s| s.ok()).map(str::to_string).collect(),
None => vec![string.to_string()],
}
}
/// Finds a regex match in each string and returns its components, as one flat list where a match contributes the whole match (`$0`) followed by its capture groups (`$1`, `$2`, etc., if any).
///
/// The match index selects which non-overlapping occurrence to return (0 for the first match). Returns an empty list if no match is found at the given index.
/// The match index selects which non-overlapping occurrence to return (0 for the first match). A string contributes nothing if no match is found at the given index.
///
/// Each item carries `start` and `end` byte-offset attributes pointing into the original string, plus a `name` attribute holding
/// Each item carries `start` and `end` byte-offset attributes pointing into its original string, plus a `name` attribute holding
/// the capture group's name (empty for unnamed groups, and for index 0 which is the whole match).
#[node_macro::node(category(""))]
fn regex_find(
_: impl Ctx,
/// The string to search within.
string: String,
#[node_macro::node(category(""), extent(regex_find_extent))]
fn regex_find<'e>(
ctx: impl Ctx + ExtractArena<'e> + ExtractIndex + InjectIndex + Copy,
/// The strings to search within.
strings: IList<String>,
/// The regular expression pattern to search for.
pattern: String,
/// Which non-overlapping occurrence of the pattern to return, starting from 0 for the first match. Negative indices count backwards from the last match.
@@ -96,135 +184,109 @@ fn regex_find(
case_insensitive: bool,
/// Make `^` and `$` match the start and end of each line, not just the whole string.
multiline: bool,
) -> List<String> {
if pattern.is_empty() {
return List::new();
}
let flags = match (case_insensitive, multiline) {
(false, false) => "",
(true, false) => "(?i)",
(false, true) => "(?m)",
(true, true) => "(?im)",
};
let full_pattern = format!("{flags}{pattern}");
let Ok(regex) = fancy_regex::Regex::new(&full_pattern) else {
log::error!("Invalid regex pattern: {pattern}");
return List::new();
};
// Capture group names indexed positionally; index 0 (the whole match) is always None.
let capture_names: Vec<Option<String>> = regex.capture_names().map(|name| name.map(str::to_string)).collect();
// Collect all matches since we need to support negative indexing
let matches: Vec<_> = regex.captures_iter(&string).filter_map(|c| c.ok()).collect();
let match_index = match_index as i32;
let resolved_index = if match_index < 0 {
let from_end = (-match_index) as usize;
if from_end > matches.len() {
return List::new();
}
matches.len() - from_end
} else {
match_index as usize
};
let Some(captures) = matches.get(resolved_index) else {
return List::new();
};
// Index 0 is the whole match, 1+ are capture groups
(0..captures.len())
.map(|i| {
let captured = captures.get(i);
let text = captured.map_or(String::new(), |m| m.as_str().to_string());
let start = captured.map_or(0_u64, |m| m.start() as u64);
let end = captured.map_or(0_u64, |m| m.end() as u64);
let name = capture_names.get(i).cloned().flatten().unwrap_or_default();
Item::new_from_element(text)
.with_attribute(ATTR_START, start)
.with_attribute(ATTR_END, end)
.with_attribute(ATTR_NAME, name)
})
.collect()
) -> Result<IList<(Lane<String>, Attr<'e, Start>, Attr<'e, End>, Attr<'e, Name>)>, Interrupt> {
let regex = compile_regex(&pattern, case_insensitive, multiline);
let (row, span) = locate_expanded(strings, ctx.index() as usize, |string| {
regex.as_ref().map_or_else(Vec::new, |regex| capture_spans(regex, string, match_index))
})
.ok_or_else(|| Interrupt::from(GraphError::past_end()))?;
let (name, _) = ctx.arena().alloc(span.name).ok_or_else(|| Interrupt::from(GraphError::new("the arena is exhausted")))?;
Ok((strings.lane(row).map_element(span.text), Attr(span.start), Attr(span.end), Attr(name.as_str())))
}
/// Finds all non-overlapping matches of a regular expression pattern in the string, returning a list of the matched substrings.
/// The level holds every string's captures in order.
fn regex_find_extent(
strings: ListIn<'_, String>,
pattern: ValueIn<'_, String>,
match_index: ValueIn<'_, f64>,
case_insensitive: ValueIn<'_, bool>,
multiline: ValueIn<'_, bool>,
level: LevelIn,
) -> GPoll<Extent> {
match level.top() {
true => strings
.get()
.zip(pattern.get())
.zip(match_index.get())
.zip(case_insensitive.get())
.zip(multiline.get())
.map(|((((strings, pattern), match_index), case_insensitive), multiline)| {
let regex = compile_regex(&pattern, case_insensitive, multiline);
expanded_count(strings, |string| regex.as_ref().map_or(0, |regex| capture_spans(regex, string, match_index).len()))
}),
false => GPoll::Final(Extent::Exactly(1)),
}
}
/// Finds all non-overlapping matches of a regular expression pattern in each string, returning one flat list of the matched substrings.
///
/// Each item carries `start` and `end` byte-offset attributes pointing into the original string.
#[node_macro::node(category("Text: Regex"))]
fn regex_find_all(
_: impl Ctx,
/// The string to search within.
string: String,
/// Each item carries `start` and `end` byte-offset attributes pointing into its original string.
#[node_macro::node(category("Text: Regex"), extent(regex_find_all_extent))]
fn regex_find_all<'e>(
ctx: impl Ctx + ExtractArena<'e> + ExtractIndex + InjectIndex + Copy,
/// The strings to search within.
strings: IList<String>,
/// The regular expression pattern to search for.
pattern: String,
/// Match letters regardless of case.
case_insensitive: bool,
/// Make `^` and `$` match the start and end of each line, not just the whole string.
multiline: bool,
) -> List<String> {
if pattern.is_empty() {
return List::new();
}
let flags = match (case_insensitive, multiline) {
(false, false) => "",
(true, false) => "(?i)",
(false, true) => "(?m)",
(true, true) => "(?im)",
};
let full_pattern = format!("{flags}{pattern}");
let Ok(regex) = fancy_regex::Regex::new(&full_pattern) else {
log::error!("Invalid regex pattern: {pattern}");
return List::new();
};
regex
.find_iter(&string)
.filter_map(|m| m.ok())
.map(|m| {
Item::new_from_element(m.as_str().to_string())
.with_attribute(ATTR_START, m.start() as u64)
.with_attribute(ATTR_END, m.end() as u64)
})
.collect()
) -> Result<IList<(Lane<String>, Attr<'e, Start>, Attr<'e, End>)>, Interrupt> {
let regex = compile_regex(&pattern, case_insensitive, multiline);
let (row, span) =
locate_expanded(strings, ctx.index() as usize, |string| regex.as_ref().map_or_else(Vec::new, |regex| match_spans(regex, string))).ok_or_else(|| Interrupt::from(GraphError::past_end()))?;
Ok((strings.lane(row).map_element(span.text), Attr(span.start), Attr(span.end)))
}
/// Splits a string into a list of substrings pulled from between separator characters as matched by a regular expression.
/// The level holds every string's matches in order.
fn regex_find_all_extent(strings: ListIn<'_, String>, pattern: ValueIn<'_, String>, case_insensitive: ValueIn<'_, bool>, multiline: ValueIn<'_, bool>, level: LevelIn) -> GPoll<Extent> {
match level.top() {
true => strings
.get()
.zip(pattern.get())
.zip(case_insensitive.get())
.zip(multiline.get())
.map(|(((strings, pattern), case_insensitive), multiline)| {
let regex = compile_regex(&pattern, case_insensitive, multiline);
expanded_count(strings, |string| regex.as_ref().map_or(0, |regex| match_spans(regex, string).len()))
}),
false => GPoll::Final(Extent::Exactly(1)),
}
}
/// Splits each string into substrings pulled from between separator characters as matched by a regular expression, producing one flat list of all the substrings.
///
/// For example, splitting "Three, two, one... LIFTOFF" with pattern `\W+` (non-word characters) produces `["Three", "two", "one", "LIFTOFF"]`.
#[node_macro::node(category("Text: Regex"))]
#[node_macro::node(category("Text: Regex"), extent(regex_split_extent))]
fn regex_split(
_: impl Ctx,
/// The string to split into substrings.
string: String,
ctx: impl Ctx + ExtractIndex + InjectIndex + Copy,
/// The strings to split into substrings.
strings: IList<String>,
/// The regular expression pattern to split on. Matches are consumed and not included in the output.
pattern: String,
/// Match letters regardless of case.
case_insensitive: bool,
/// Make `^` and `$` match the start and end of each line, not just the whole string.
multiline: bool,
) -> List<String> {
if pattern.is_empty() {
return List::new_from_element(string);
}
let flags = match (case_insensitive, multiline) {
(false, false) => "",
(true, false) => "(?i)",
(false, true) => "(?m)",
(true, true) => "(?im)",
};
let full_pattern = format!("{flags}{pattern}");
let Ok(regex) = fancy_regex::Regex::new(&full_pattern) else {
log::error!("Invalid regex pattern: {pattern}");
return List::new_from_element(string);
};
regex.split(&string).filter_map(|s| s.ok()).map(|s| s.to_string()).map(Item::new_from_element).collect()
) -> Result<IList<Lane<String>>, Interrupt> {
let regex = compile_regex(&pattern, case_insensitive, multiline);
let (row, part) = locate_expanded(strings, ctx.index() as usize, |string| split_parts(regex.as_ref(), string)).ok_or_else(|| Interrupt::from(GraphError::past_end()))?;
Ok(strings.lane(row).map_element(part))
}
/// The level holds every string's parts in order.
fn regex_split_extent(strings: ListIn<'_, String>, pattern: ValueIn<'_, String>, case_insensitive: ValueIn<'_, bool>, multiline: ValueIn<'_, bool>, level: LevelIn) -> GPoll<Extent> {
match level.top() {
true => strings
.get()
.zip(pattern.get())
.zip(case_insensitive.get())
.zip(multiline.get())
.map(|(((strings, pattern), case_insensitive), multiline)| {
let regex = compile_regex(&pattern, case_insensitive, multiline);
expanded_count(strings, |string| split_parts(regex.as_ref(), string).len())
}),
false => GPoll::Final(Extent::Exactly(1)),
}
}