feat: Bases query pipeline: filter, sort, group, limit, summaries with lazy cells

run(Base, view, NoteSource, Env) with base and view filters (and/or/not), stable multi-key sort evaluating only the sort keys, groupBy and groupOrder (null group, hidden groups), limit across groups, built-in and custom summaries (Average, Min, Max, Sum, Range, Median, Stddev, Earliest, Latest, Checked, Unchecked, Empty, Filled, Unique, expressions over values), lazily memoised cells, dependency tracking so unused properties are never read; list methods sum, mean, median, min, max. 15000 rows with one filter and one sort run in well under the 500 ms bound. Bases task 7 (Claude Sonnet). Known gap: an empty groupOrder shows all groups (needs Option<Vec> in the model).

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>

crates/bases/src/eval/funcs_list.rs

@@ -87,6 +87,7 @@ pub(super) fn method(recv: &Value, name: &str, args: &[Value]) -> Option<Value>
             Ok((from, to)) => Value::List(items.iter().skip(from).take(to - from).cloned().collect()),
             Err(e) => e,
         },
+        ("sum" | "mean" | "median" | "min" | "max", []) => crate::query::number_stat(name, items)?,
         ("join", [other]) => err(format!("join() does not accept {}", other.type_name())),
         ("contains" | "isEmpty" | "flat" | "reverse" | "sort" | "unique" | "join" | "slice", _) => {
             err(format!("{name}() called with {} arguments", args.len()))

crates/bases/src/lib.rs

@@ -15,6 +15,7 @@ mod eval;
 pub mod expr;
 pub mod map;
 mod model;
+pub mod query;
 mod row;
 mod value;
 mod yaml;
@@ -24,6 +25,7 @@ pub use datetime::{Date, DateError, DateParts, Duration, add_duration, parse_dur
 pub use error::{Error, Result};
 pub use eval::{Env, Formulas, eval};
 pub use model::{Base, Direction, Filter, GroupBy, PropertyCfg, Sort, View, ViewKind};
+pub use query::{Column, Group, QueryResult, Reads, RowId, ViewSel, run};
 pub use row::{NoteSource, RowRef};
 pub use value::Value;
 pub use yaml::{Node, emit, parse};

crates/bases/src/query/columns.rs

@@ -0,0 +1,76 @@
+//! The columns of a view and the properties a query reads.
+
+use crate::expr::{Expr, parse_expr};
+use crate::model::{Base, View};
+
+/// The column shown when a view lists none.
+const DEFAULT_COLUMN: &str = "file.name";
+
+/// One column of a result.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct Column {
+    /// The property id: `file.name`, `note.year`, `formula.ppu` or a bare note property.
+    pub id: String,
+    /// The title: the `displayName` of the property, else the id without its `note.` / `file.` /
+    /// `formula.` prefix.
+    pub name: String,
+    /// The summary of the view for this column, a default or a custom name.
+    pub summary: Option<String>,
+}
+
+/// An expression that parsed, or the parse message.
+pub(super) type Parsed = Result<Expr, String>;
+
+pub(super) fn parse(src: &str) -> Parsed {
+    parse_expr(src).map_err(|e| e.to_string())
+}
+
+/// The columns of `view` with their parsed expressions (a property id is an expression).
+pub(super) fn columns(base: &Base, view: &View) -> (Vec<Column>, Vec<Parsed>) {
+    let ids: Vec<&str> = if view.order.is_empty() { vec![DEFAULT_COLUMN] } else { view.order.iter().map(String::as_str).collect() };
+    let column = |id: &&str| {
+        let display = base.properties.iter().find(|(p, _)| p == id).and_then(|(_, cfg)| cfg.display_name.clone());
+        let short = ["note.", "file.", "formula."].iter().find_map(|p| id.strip_prefix(p)).unwrap_or(id);
+        let summary = view.summaries.iter().find(|(p, _)| p == id).map(|(_, s)| s.clone());
+        Column { id: (*id).to_string(), name: display.unwrap_or_else(|| short.to_string()), summary }
+    };
+    (ids.iter().map(column).collect(), ids.iter().map(|id| parse(id)).collect())
+}
+
+/// What a query reads: the properties of a row it touches. Nothing else is read.
+#[derive(Debug, Clone, Default, PartialEq, Eq)]
+pub struct Reads {
+    /// Note properties, formulas expanded.
+    pub note: Vec<String>,
+    /// `file.*` fields, formulas expanded.
+    pub file: Vec<String>,
+    /// Whether `this` is read.
+    pub uses_this: bool,
+}
+
+impl Reads {
+    /// Adds the dependencies of `expr` and, transitively, of the formulas it names.
+    pub(super) fn add(&mut self, expr: &Expr, formulas: &[(String, String)], seen: &mut Vec<String>) {
+        let deps = expr.deps();
+        self.note.extend(deps.note);
+        self.file.extend(deps.file);
+        self.uses_this |= deps.uses_this;
+        for dep in deps.formula {
+            let name = dep.split('.').next().unwrap_or(&dep).to_string();
+            if seen.contains(&name) {
+                continue;
+            }
+            seen.push(name.clone());
+            if let Some(Ok(inner)) = formulas.iter().find(|(n, _)| *n == name).map(|(_, src)| parse(src)) {
+                self.add(&inner, formulas, seen);
+            }
+        }
+    }
+
+    pub(super) fn finish(&mut self) {
+        self.note.sort();
+        self.note.dedup();
+        self.file.sort();
+        self.file.dedup();
+    }
+}

crates/bases/src/query/mod.rs

@@ -0,0 +1,301 @@
+//! The query pipeline of a base view: filter, sort, group, limit, summaries.
+//!
+//! [`run`] walks the source once for the filters, evaluates only the sort and group keys, and
+//! returns [`RowId`]s. A cell is computed on demand by [`QueryResult::cell`] and memoised, so a
+//! row nobody asks for costs nothing. Rows and values of the source are never copied.
+
+mod columns;
+mod summary;
+
+use std::cell::{Cell, OnceCell};
+use std::cmp::Ordering;
+use std::collections::HashMap;
+
+pub use columns::{Column, Reads};
+pub(crate) use summary::number_stat;
+
+use self::columns::{Parsed, columns, parse};
+use self::summary::{Custom, summarize};
+use crate::compare::{compare, loose_eq, truthy};
+use crate::error::{Error, Result};
+use crate::eval::{Env, Formulas, err, eval};
+use crate::model::{Base, Direction, Filter, View};
+use crate::row::{NoteSource, RowRef};
+use crate::value::Value;
+use crate::yaml::Node;
+
+/// A row of the result: its index in the [`NoteSource`], stable while the source does not change.
+pub type RowId = usize;
+
+static NULL: Value = Value::Null;
+
+/// Which view to run: its index or its name.
+#[derive(Debug, Clone, Copy)]
+pub enum ViewSel<'a> {
+    /// Position in `Base::views`.
+    Index(usize),
+    /// The `name` of the view.
+    Name(&'a str),
+}
+
+impl From<usize> for ViewSel<'_> {
+    fn from(i: usize) -> Self {
+        ViewSel::Index(i)
+    }
+}
+
+impl<'a> From<&'a str> for ViewSel<'a> {
+    fn from(name: &'a str) -> Self {
+        ViewSel::Name(name)
+    }
+}
+
+/// One group of rows. Without `groupBy` there is one group with a `Null` key.
+#[derive(Debug, Clone, PartialEq)]
+pub struct Group {
+    /// The value of the `groupBy` property shared by the rows (`Null`: no value).
+    pub key: Value,
+    /// The rows, in sort order.
+    pub rows: Vec<RowId>,
+    /// The summary of each column over this group; `Null` for a column without one.
+    pub summaries: Vec<Value>,
+}
+
+/// What evaluates expressions: the environment, the formulas and the error count.
+struct Eval<'a> {
+    env: Env<'a>,
+    formulas: Formulas,
+    errors: Cell<usize>,
+}
+
+impl Eval<'_> {
+    fn value(&self, expr: &Parsed, row: &RowRef<'_>) -> Value {
+        let value = match expr {
+            Ok(expr) => eval(expr, &self.env, row, &self.formulas),
+            Err(message) => err(message.clone()),
+        };
+        if matches!(value, Value::Error(_)) {
+            self.errors.set(self.errors.get().saturating_add(1));
+        }
+        value
+    }
+}
+
+/// A filter with its expressions parsed.
+enum Compiled {
+    Expr(Parsed),
+    And(Vec<Compiled>),
+    Or(Vec<Compiled>),
+    Not(Vec<Compiled>),
+}
+
+fn compile(filter: &Filter, track: &mut dyn FnMut(&Parsed)) -> Compiled {
+    let mut all = |members: &[Filter]| members.iter().map(|m| compile(m, track)).collect::<Vec<_>>();
+    match filter {
+        Filter::Expr(src) => {
+            let parsed = parse(src);
+            track(&parsed);
+            Compiled::Expr(parsed)
+        }
+        Filter::And(m) => Compiled::And(all(m)),
+        Filter::Or(m) => Compiled::Or(all(m)),
+        Filter::Not(m) => Compiled::Not(all(m)),
+    }
+}
+
+impl Eval<'_> {
+    /// Whether the row passes; an empty group imposes nothing, an error counts as false.
+    fn passes(&self, filter: &Compiled, row: &RowRef<'_>) -> bool {
+        match filter {
+            Compiled::Expr(e) => {
+                let v = self.value(e, row);
+                !matches!(v, Value::Error(_)) && truthy(&v)
+            }
+            Compiled::And(m) => m.iter().all(|f| self.passes(f, row)),
+            Compiled::Or(m) => m.is_empty() || m.iter().any(|f| self.passes(f, row)),
+            Compiled::Not(m) => !m.iter().any(|f| self.passes(f, row)),
+        }
+    }
+}
+
+/// The outcome of [`run`].
+pub struct QueryResult<'a> {
+    /// The columns of the view.
+    pub columns: Vec<Column>,
+    /// The groups, with the limit applied.
+    pub groups: Vec<Group>,
+    /// Rows that matched the filters, before the limit.
+    pub total: usize,
+    /// The summary of each column over every shown row; `Null` for a column without one.
+    pub summaries: Vec<Value>,
+    /// The properties the query reads; every other property is never touched.
+    pub reads: Reads,
+    source: &'a dyn NoteSource,
+    eval: Eval<'a>,
+    exprs: Vec<Parsed>,
+    slots: HashMap<RowId, usize>,
+    cells: Vec<OnceCell<Value>>,
+    computed: Cell<usize>,
+}
+
+impl QueryResult<'_> {
+    /// The value of column `col` for `row`, computed on first use. `Null` for a row that is not in
+    /// the result or a column that does not exist.
+    #[must_use]
+    pub fn cell(&self, row: RowId, col: usize) -> &Value {
+        let (Some(slot), Some(expr)) = (self.slots.get(&row), self.exprs.get(col)) else { return &NULL };
+        let Some(cell) = self.cells.get(slot * self.exprs.len() + col) else { return &NULL };
+        cell.get_or_init(|| {
+            self.computed.set(self.computed.get().saturating_add(1));
+            self.source.row(row).map_or(Value::Null, |r| self.eval.value(expr, &r))
+        })
+    }
+
+    /// How many cells have been computed so far, summaries included.
+    #[must_use]
+    pub fn cells_computed(&self) -> usize {
+        self.computed.get()
+    }
+
+    /// How many evaluations failed so far: filters, keys and cells.
+    #[must_use]
+    pub fn errors(&self) -> usize {
+        self.eval.errors.get()
+    }
+
+    /// The rows of every group in display order.
+    pub fn rows(&self) -> impl Iterator<Item = RowId> + '_ {
+        self.groups.iter().flat_map(|g| g.rows.iter().copied())
+    }
+
+    fn summarize(&self, name: &str, rows: &[RowId], col: usize, custom: &[Custom]) -> Value {
+        let values: Vec<&Value> = rows.iter().map(|r| self.cell(*r, col)).collect();
+        summarize(name, &values, custom, &self.eval.env, &self.eval.formulas)
+    }
+}
+
+fn find_view<'b>(base: &'b Base, sel: ViewSel<'_>) -> Result<&'b View> {
+    let found = match sel {
+        ViewSel::Index(i) => base.views().get(i),
+        ViewSel::Name(n) => base.views().iter().find(|v| v.name.as_deref() == Some(n)),
+    };
+    found.ok_or_else(|| Error::Invalid(format!("no such view: {sel:?}")))
+}
+
+fn ordered(a: &Value, b: &Value, dir: Direction) -> Ordering {
+    let o = compare(a, b);
+    if dir == Direction::Desc { o.reverse() } else { o }
+}
+
+/// Whether a `groupOrder` entry names the group `key`.
+fn names_group(entry: &Node, key: &Value) -> bool {
+    match entry {
+        Node::Null => matches!(key, Value::Null),
+        Node::Bool(b) => loose_eq(key, &Value::Bool(*b)),
+        Node::Num(n) => loose_eq(key, &Value::Number(*n)),
+        Node::Str(s) => !matches!(key, Value::Null) && (loose_eq(key, &Value::Str(s.clone())) || key.to_display_string() == *s),
+        Node::Seq(_) | Node::Map(_) => false,
+    }
+}
+
+/// Runs a view of `base` over `source`.
+///
+/// `env.source` defaults to `source`. Rows are filtered by the base filters AND the view filters,
+/// sorted (stable, multi-key), grouped, cut to the view `limit` and summarised.
+///
+/// # Errors
+/// [`Error::Invalid`] when the view does not exist.
+pub fn run<'a>(base: &'a Base, view: impl Into<ViewSel<'a>>, source: &'a dyn NoteSource, env: &Env<'a>) -> Result<QueryResult<'a>> {
+    let view = find_view(base, view.into())?;
+    let mut env = *env;
+    env.source = env.source.or(Some(source));
+    let eval = Eval { env, formulas: Formulas::new(&base.formulas), errors: Cell::new(0) };
+    let mut reads = Reads::default();
+    let mut track = |p: &Parsed| {
+        if let Ok(expr) = p {
+            reads.add(expr, &base.formulas, &mut Vec::new());
+        }
+    };
+    let (cols, exprs) = columns(base, view);
+    exprs.iter().for_each(&mut track);
+    let filters: Vec<Compiled> = [&base.filters, &view.filters].into_iter().flatten().map(|f| compile(f, &mut track)).collect();
+    let sorts: Vec<(Parsed, Direction)> = view.sort.iter().map(|s| (parse(&s.property), s.direction)).collect();
+    let group = view.group_by.as_ref().map(|g| (parse(&g.property), g.direction));
+    sorts.iter().map(|(p, _)| p).chain(group.iter().map(|(p, _)| p)).for_each(&mut track);
+    reads.finish();
+
+    let kept: Vec<(RowId, RowRef<'a>)> =
+        (0..source.len()).filter_map(|i| source.row(i).map(|r| (i, r))).filter(|(_, row)| filters.iter().all(|f| eval.passes(f, row))).collect();
+    let total = kept.len();
+
+    let keys: Vec<Vec<Value>> =
+        if sorts.is_empty() { Vec::new() } else { kept.iter().map(|(_, r)| sorts.iter().map(|(p, _)| eval.value(p, r)).collect()).collect() };
+    let mut order: Vec<usize> = (0..total).collect();
+    if !sorts.is_empty() {
+        order.sort_by(|a, b| {
+            for (k, (_, dir)) in sorts.iter().enumerate() {
+                let o = match (keys.get(*a).and_then(|r| r.get(k)), keys.get(*b).and_then(|r| r.get(k))) {
+                    (Some(x), Some(y)) => ordered(x, y, *dir),
+                    _ => Ordering::Equal,
+                };
+                if o != Ordering::Equal {
+                    return o;
+                }
+            }
+            Ordering::Equal
+        });
+    }
+
+    let mut groups: Vec<Group> = Vec::new();
+    if let Some((expr, dir)) = &group {
+        let gkeys: Vec<Value> = kept.iter().map(|(_, r)| eval.value(expr, r)).collect();
+        order.sort_by(|a, b| match (gkeys.get(*a), gkeys.get(*b)) {
+            (Some(x), Some(y)) => ordered(x, y, *dir),
+            _ => Ordering::Equal,
+        });
+        for pos in &order {
+            let (Some(key), Some((id, _))) = (gkeys.get(*pos), kept.get(*pos)) else { continue };
+            match groups.last_mut() {
+                Some(g) if compare(&g.key, key) == Ordering::Equal => g.rows.push(*id),
+                _ => groups.push(Group { key: key.clone(), rows: vec![*id], summaries: Vec::new() }),
+            }
+        }
+        if !view.group_order.is_empty() {
+            let mut pool: Vec<Option<Group>> = groups.into_iter().map(Some).collect();
+            groups =
+                view.group_order.iter().filter_map(|entry| pool.iter_mut().find(|g| g.as_ref().is_some_and(|g| names_group(entry, &g.key)))?.take()).collect();
+        }
+    } else {
+        let rows: Vec<RowId> = order.iter().filter_map(|p| kept.get(*p).map(|(id, _)| *id)).collect();
+        if !rows.is_empty() {
+            groups.push(Group { key: Value::Null, rows, summaries: Vec::new() });
+        }
+    }
+
+    if let Some(limit) = view.limit {
+        let mut left = limit as usize;
+        for g in &mut groups {
+            g.rows.truncate(left);
+            left = left.saturating_sub(g.rows.len());
+        }
+        groups.retain(|g| !g.rows.is_empty());
+    }
+
+    let shown: Vec<RowId> = groups.iter().flat_map(|g| g.rows.iter().copied()).collect();
+    let slots = shown.iter().enumerate().map(|(slot, id)| (*id, slot)).collect();
+    let cells = (0..shown.len().saturating_mul(exprs.len())).map(|_| OnceCell::new()).collect();
+    let mut result =
+        QueryResult { columns: cols, groups: Vec::new(), total, summaries: Vec::new(), reads, source, eval, exprs, slots, cells, computed: Cell::new(0) };
+
+    let custom: Vec<Custom> = base.summaries.iter().map(|(n, src)| (n.clone(), parse(src))).collect();
+    let column_summaries = |rows: &[RowId]| -> Vec<Value> {
+        result.columns.iter().enumerate().map(|(col, c)| c.summary.as_deref().map_or(Value::Null, |name| result.summarize(name, rows, col, &custom))).collect()
+    };
+    for g in &mut groups {
+        g.summaries = column_summaries(&g.rows);
+    }
+    let overall = column_summaries(&shown);
+    result.groups = groups;
+    result.summaries = overall;
+    Ok(result)
+}

crates/bases/src/query/summary.rs

@@ -0,0 +1,128 @@
+//! Summaries: the default ones of the docs (implemented here, not as formulas) and the custom
+//! ones, which are expressions over `values`.
+
+use std::cmp::Ordering;
+
+use crate::compare::compare;
+use crate::datetime::Duration;
+use crate::eval::{Env, Formulas, err, eval, int_to_f64};
+use crate::expr::Expr;
+use crate::row::RowRef;
+use crate::value::Value;
+
+/// A custom summary: name and parsed expression (or the parse message).
+pub(super) type Custom = (String, Result<Expr, String>);
+
+/// The result of the named summary over `values`, `Error` when the name is unknown.
+pub(super) fn summarize(name: &str, values: &[&Value], custom: &[Custom], env: &Env<'_>, formulas: &Formulas) -> Value {
+    if let Some(done) = builtin(name, values) {
+        return done;
+    }
+    match custom.iter().find(|(n, _)| n == name) {
+        None => err(format!("unknown summary `{name}`")),
+        Some((_, Err(message))) => err(message.clone()),
+        Some((_, Ok(expr))) => {
+            let props = [("values".to_string(), Value::List(values.iter().map(|v| (*v).clone()).collect()))];
+            let row = RowRef { props: &props, ..RowRef::new("") };
+            eval(expr, env, &row, formulas)
+        }
+    }
+}
+
+/// `sum`, `mean`, `median`, `min`, `max` over the numbers of a list, `Null` when there are none
+/// (`sum` is 0). Used by the summaries and by the list methods of the same names.
+pub(crate) fn number_stat(name: &str, items: &[Value]) -> Option<Value> {
+    let nums = numbers(items.iter());
+    Some(match name {
+        "sum" => Value::Number(nums.iter().sum()),
+        "mean" => mean(&nums),
+        "median" => median(nums),
+        "min" => fold(&nums, f64::min),
+        "max" => fold(&nums, f64::max),
+        _ => return None,
+    })
+}
+
+fn numbers<'a>(values: impl Iterator<Item = &'a Value>) -> Vec<f64> {
+    values.filter_map(|v| if let Value::Number(n) = v { Some(*n).filter(|n| !n.is_nan()) } else { None }).collect()
+}
+
+fn mean(nums: &[f64]) -> Value {
+    if nums.is_empty() { Value::Null } else { Value::Number(nums.iter().sum::<f64>() / int_to_f64(nums.len())) }
+}
+
+fn fold(nums: &[f64], f: fn(f64, f64) -> f64) -> Value {
+    nums.iter().copied().reduce(f).map_or(Value::Null, Value::Number)
+}
+
+fn median(mut nums: Vec<f64>) -> Value {
+    nums.sort_by(f64::total_cmp);
+    let mid = nums.len() / 2;
+    match (nums.get(mid), mid.checked_sub(1).and_then(|i| nums.get(i))) {
+        (None, _) => Value::Null,
+        (Some(hi), Some(lo)) if nums.len().is_multiple_of(2) => Value::Number((lo + hi) / 2.0),
+        (Some(m), _) => Value::Number(*m),
+    }
+}
+
+/// Population standard deviation.
+fn stddev(nums: &[f64]) -> Value {
+    let Value::Number(m) = mean(nums) else { return Value::Null };
+    let var = nums.iter().map(|x| (x - m).powi(2)).sum::<f64>() / int_to_f64(nums.len());
+    Value::Number(var.sqrt())
+}
+
+fn dates(values: &[&Value]) -> Vec<crate::datetime::Date> {
+    values.iter().filter_map(|v| if let Value::Date(d) = v { Some(*d) } else { None }).collect()
+}
+
+/// Whether a value counts as empty: null, `""`, an empty list or object.
+fn is_empty(v: &Value) -> bool {
+    match v {
+        Value::Null => true,
+        Value::Str(s) => s.is_empty(),
+        Value::List(l) => l.is_empty(),
+        Value::Object(o) => o.is_empty(),
+        _ => false,
+    }
+}
+
+fn count(n: usize) -> Value {
+    Value::Number(int_to_f64(n))
+}
+
+fn builtin(name: &str, values: &[&Value]) -> Option<Value> {
+    let nums = || numbers(values.iter().copied());
+    Some(match name {
+        "Average" => mean(&nums()),
+        "Sum" => Value::Number(nums().iter().sum()),
+        "Min" => fold(&nums(), f64::min),
+        "Max" => fold(&nums(), f64::max),
+        "Median" => median(nums()),
+        "Stddev" => stddev(&nums()),
+        "Range" => {
+            let n = nums();
+            if let (Value::Number(lo), Value::Number(hi)) = (fold(&n, f64::min), fold(&n, f64::max)) {
+                return Some(Value::Number(hi - lo));
+            }
+            let d = dates(values);
+            match (d.iter().map(|d| d.ms()).min(), d.iter().map(|d| d.ms()).max()) {
+                (Some(lo), Some(hi)) => hi.checked_sub(lo).map_or_else(|| err("date range out of range"), |ms| Value::Duration(Duration::new(0, ms))),
+                _ => Value::Null,
+            }
+        }
+        "Earliest" => dates(values).into_iter().min_by_key(|d| d.ms()).map_or(Value::Null, Value::Date),
+        "Latest" => dates(values).into_iter().max_by_key(|d| d.ms()).map_or(Value::Null, Value::Date),
+        "Checked" => count(values.iter().filter(|v| matches!(v, Value::Bool(true))).count()),
+        "Unchecked" => count(values.iter().filter(|v| matches!(v, Value::Bool(false))).count()),
+        "Empty" => count(values.iter().filter(|v| is_empty(v)).count()),
+        "Filled" => count(values.iter().filter(|v| !is_empty(v)).count()),
+        "Unique" => {
+            let mut sorted: Vec<&Value> = values.to_vec();
+            sorted.sort_by(|a, b| compare(a, b));
+            sorted.dedup_by(|a, b| compare(a, b) == Ordering::Equal);
+            count(sorted.len())
+        }
+        _ => return None,
+    })
+}

crates/bases/tests/query.rs

@@ -0,0 +1,293 @@
+//! The query pipeline: filter, sort, group, limit, summaries, laziness and speed.
+
+mod common;
+
+use std::cell::Cell;
+use std::time::Instant;
+
+use common::{NOW, Rows, TestRow, n, s};
+use notescraft_bases::{Base, Env, NoteSource, QueryResult, RowRef, Value, run};
+
+const BOOKS: [(&str, Option<&str>, f64, f64, bool); 12] = [
+    ("Alpha", Some("sf"), 2001.0, 300.0, true),
+    ("Bravo", Some("sf"), 1999.0, 150.0, false),
+    ("Charlie", Some("fantasy"), 2010.0, 400.0, true),
+    ("Delta", Some("fantasy"), 2010.0, 90.0, false),
+    ("Echo", None, 2005.0, 200.0, true),
+    ("Foxtrot", Some("sf"), 2015.0, 250.0, true),
+    ("Golf", Some("skip"), 2000.0, 500.0, true),
+    ("Hotel", Some("fantasy"), 1990.0, 120.0, false),
+    ("India", None, 1980.0, 110.0, false),
+    ("Juliet", Some("sf"), 2001.0, 180.0, true),
+    ("Kilo", Some("fantasy"), 2010.0, 310.0, true),
+    ("Lima", Some("sf"), 1999.0, 101.0, false),
+];
+
+fn books() -> Rows {
+    Rows(
+        BOOKS
+            .iter()
+            .map(|(name, genre, year, pages, read)| {
+                let row = TestRow::new(&format!("Books/{name}.md")).prop("year", n(*year)).prop("pages", n(*pages)).prop("read", Value::Bool(*read));
+                match genre {
+                    Some(g) => row.prop("genre", s(g)),
+                    None => row,
+                }
+            })
+            .collect(),
+    )
+}
+
+const BASE: &str = "
+filters:
+  and:
+    - pages > 100
+summaries:
+  customAverage: 'values.mean().round(3)'
+formulas:
+  bad: nope(1)
+  half: pages / 2
+views:
+  - type: table
+    name: Main
+    filters:
+      not:
+        - genre == \"skip\"
+    groupBy:
+      property: genre
+      direction: ASC
+    sort:
+      - property: year
+        direction: DESC
+    order:
+      - file.name
+      - year
+      - pages
+    summaries:
+      pages: Sum
+      year: customAverage
+";
+
+fn query<'a>(base: &'a Base, view: impl Into<notescraft_bases::ViewSel<'a>>, rows: &'a Rows) -> QueryResult<'a> {
+    run(base, view, rows, &Env::new(NOW, 7)).unwrap_or_else(|e| panic!("{e}"))
+}
+
+fn base(text: &str) -> Base {
+    Base::parse(text).unwrap_or_else(|e| panic!("{e}"))
+}
+
+/// `key: name name ...` per group.
+fn table(r: &QueryResult<'_>) -> Vec<String> {
+    r.groups
+        .iter()
+        .map(|g| {
+            let names: Vec<String> = g.rows.iter().map(|id| r.cell(*id, 0).to_display_string().replace("Books/", "").replace(".md", "")).collect();
+            format!("{}: {}", g.key.to_display_string(), names.join(" "))
+        })
+        .collect()
+}
+
+#[test]
+fn filter_sort_group_summary_golden() {
+    let (rows, b) = (books(), base(BASE));
+    let r = query(&b, "Main", &rows);
+    assert_eq!(r.total, 10);
+    assert_eq!(table(&r), [": Echo India", "fantasy: Charlie Kilo Hotel", "sf: Foxtrot Alpha Juliet Bravo Lima"]);
+    assert_eq!(r.columns.iter().map(|c| c.name.as_str()).collect::<Vec<_>>(), ["name", "year", "pages"]);
+    let sums: Vec<_> = r.groups.iter().map(|g| g.summaries.clone()).collect();
+    assert_eq!(sums[0], [Value::Null, n(1992.5), n(310.0)]);
+    assert_eq!(sums[1], [Value::Null, n(2003.333), n(830.0)]);
+    assert_eq!(sums[2][2], n(981.0));
+    assert_eq!(r.summaries, [Value::Null, n(2001.0), n(2121.0)]);
+    assert_eq!(r.errors(), 0);
+}
+
+#[test]
+fn group_order_hides_and_places_null() {
+    let (rows, b) = (books(), base(&BASE.replace("    sort:", "    groupOrder:\n      - sf\n      - null\n    sort:")));
+    assert_eq!(table(&query(&b, 0, &rows)), ["sf: Foxtrot Alpha Juliet Bravo Lima", ": Echo India"]);
+}
+
+#[test]
+fn desc_groups_limit_and_default_column() {
+    let rows = books();
+    let b = base(
+        "views:\n  - type: table\n    limit: 3\n    groupBy:\n      property: genre\n      direction: DESC\n    sort:\n      - property: pages\n        direction: ASC\n",
+    );
+    let r = query(&b, 0, &rows);
+    assert_eq!(r.total, 12);
+    assert_eq!(r.columns.len(), 1);
+    assert_eq!(r.columns[0].id, "file.name");
+    assert_eq!(table(&r), ["skip: Golf", "sf: Lima Bravo"]);
+}
+
+#[test]
+fn multi_key_sort_is_stable_and_missing_view_is_an_error() {
+    let rows = books();
+    let b = base(
+        "views:\n  - type: table\n    filters: genre == \"sf\"\n    sort:\n      - property: year\n        direction: ASC\n      - property: pages\n        direction: DESC\n",
+    );
+    let r = query(&b, 0, &rows);
+    assert_eq!(table(&r), [": Bravo Lima Alpha Juliet Foxtrot"]);
+    assert!(run(&b, 5, &rows, &Env::new(NOW, 7)).is_err());
+    assert!(run(&b, "nope", &rows, &Env::new(NOW, 7)).is_err());
+}
+
+#[test]
+fn default_summaries() {
+    let rows = books();
+    let b = base(
+        "views:\n  - type: table\n    order: [pages, read, genre, year]\n    summaries:\n      pages: Median\n      read: Checked\n      genre: Unique\n      year: Range\n",
+    );
+    let r = query(&b, 0, &rows);
+    assert_eq!(r.summaries[0], n(190.0)); // median of 12 values: (180 + 200) / 2
+    assert_eq!(r.summaries[1], n(7.0));
+    assert_eq!(r.summaries[2], n(4.0)); // null, fantasy, sf, skip
+    assert_eq!(r.summaries[3], n(35.0));
+    for (name, want) in [("Min", n(90.0)), ("Max", n(500.0)), ("Unchecked", n(5.0)), ("Filled", n(12.0)), ("Empty", n(0.0)), ("Nope", Value::Null)] {
+        let col = if name == "Unchecked" { "read" } else { "pages" };
+        let b = base(&format!("views:\n  - type: table\n    order: [{col}]\n    summaries:\n      {col}: {name}\n"));
+        let got = query(&b, 0, &rows).summaries[0].clone();
+        if name == "Nope" {
+            assert!(matches!(got, Value::Error(_)));
+        } else {
+            assert_eq!(got, want, "{name}");
+        }
+    }
+    let b = base("views:\n  - type: table\n    order: [genre, pages]\n    summaries:\n      genre: Empty\n      pages: Stddev\n");
+    let r = query(&b, 0, &rows);
+    assert_eq!(r.summaries[0], n(2.0));
+    assert!(matches!(&r.summaries[1], Value::Number(x) if (x - 123.0).abs() < 50.0));
+}
+
+#[test]
+fn date_summaries() {
+    let rows = Rows(vec![
+        TestRow::new("a.md").prop("d", Value::Date(notescraft_bases::Date::new(0, false))),
+        TestRow::new("b.md").prop("d", Value::Date(notescraft_bases::Date::new(86_400_000, false))),
+    ]);
+    let b = base("views:\n  - type: table\n    order: [d]\n    summaries:\n      d: Range\n");
+    assert_eq!(query(&b, 0, &rows).summaries[0].to_display_string(), "1d");
+    let b = base("views:\n  - type: table\n    order: [d]\n    summaries:\n      d: Latest\n");
+    assert_eq!(query(&b, 0, &rows).summaries[0], Value::Date(notescraft_bases::Date::new(86_400_000, false)));
+}
+
+#[test]
+fn errors_are_counted_and_do_not_stop_the_table() {
+    let rows = books();
+    let b = base(&format!("{BASE}\n").replace("      - pages\n    summaries", "      - formula.bad\n      - formula.half\n    summaries"));
+    let r = query(&b, 0, &rows);
+    let id = r.groups[0].rows[0];
+    assert!(matches!(r.cell(id, 2), Value::Error(_)));
+    assert_eq!(r.cell(id, 3), &n(100.0));
+    assert!(r.errors() >= 1);
+    let b = base("filters: nope(1)\nviews:\n  - type: table\n");
+    let r = query(&b, 0, &rows);
+    assert_eq!(r.total, 0);
+    assert_eq!(r.errors(), 12);
+}
+
+#[test]
+fn display_name_and_formula_dependencies() {
+    let rows = books();
+    let b = base(
+        "properties:\n  year:\n    displayName: Year\nformulas:\n  half: pages / 2\nviews:\n  - type: table\n    order: [year, formula.half]\n    sort:\n      - property: file.name\n        direction: ASC\n",
+    );
+    let r = query(&b, 0, &rows);
+    assert_eq!(r.columns[0].name, "Year");
+    assert_eq!(r.columns[1].name, "half");
+    assert_eq!(r.reads.note, ["pages", "year"]);
+    assert_eq!(r.reads.file, ["name"]);
+    assert!(!r.reads.uses_this);
+}
+
+struct Counted {
+    rows: Rows,
+    backlinks: Box<dyn Fn() -> Vec<String>>,
+}
+
+impl NoteSource for Counted {
+    fn len(&self) -> usize {
+        self.rows.len()
+    }
+    fn row(&self, i: usize) -> Option<RowRef<'_>> {
+        self.rows.row(i).map(|r| RowRef { backlinks: Some(&*self.backlinks), ..r })
+    }
+    fn find(&self, path: &str) -> Option<RowRef<'_>> {
+        self.rows.find(path)
+    }
+}
+
+#[test]
+fn unreferenced_properties_are_never_read() {
+    let calls = std::rc::Rc::new(Cell::new(0));
+    let c = calls.clone();
+    let src = Counted {
+        rows: books(),
+        backlinks: Box::new(move || {
+            c.set(c.get() + 1);
+            vec![]
+        }),
+    };
+    let b = base(
+        "filters: year > 1990\nviews:\n  - type: table\n    order: [file.name]\n    sort:\n      - property: year\n        direction: ASC\n  - type: table\n    order: [file.backlinks.length]\n",
+    );
+    let env = Env::new(NOW, 7);
+    let r = run(&b, 0, &src, &env).unwrap_or_else(|e| panic!("{e}"));
+    for id in r.rows() {
+        let _ = r.cell(id, 0);
+    }
+    assert_eq!(calls.get(), 0);
+    let r = run(&b, 1, &src, &env).unwrap_or_else(|e| panic!("{e}"));
+    assert_eq!(r.cells_computed(), 0);
+    assert_eq!(r.cell(0, 0), &n(0.0));
+    assert_eq!(calls.get(), 1);
+    assert_eq!(r.reads.file, ["backlinks.length"]);
+}
+
+fn synthetic(count: usize) -> Rows {
+    Rows(
+        (0..count)
+            .map(|i| {
+                TestRow::new(&format!("Notes/n{i}.md"))
+                    .prop("year", n(f64::from(u16::try_from(i % 60).unwrap_or(0)) + 1970.0))
+                    .prop("rank", n(f64::from(u32::try_from(i.wrapping_mul(7919) % 10_007).unwrap_or(0))))
+            })
+            .collect(),
+    )
+}
+
+const PERF_BASE: &str =
+    "views:\n  - type: table\n    filters: year >= 1990\n    order: [file.name, year]\n    sort:\n      - property: rank\n        direction: DESC\n";
+
+#[test]
+fn no_cell_is_computed_for_rows_not_asked_for() {
+    let rows = synthetic(15_000);
+    let b = base(PERF_BASE);
+    let start = Instant::now();
+    let r = query(&b, 0, &rows);
+    let took = start.elapsed();
+    assert!(took.as_millis() < 500, "15 000 rows took {took:?}");
+    assert_eq!(r.total, 10_000);
+    assert_eq!(r.cells_computed(), 0);
+    let top: Vec<usize> = r.rows().take(5).collect();
+    for id in &top {
+        let _ = r.cell(*id, 0);
+        let _ = r.cell(*id, 0);
+    }
+    assert_eq!(r.cells_computed(), 5);
+    assert_eq!(r.cell(top[0], 1), &r.cell(top[0], 1).clone());
+    assert_eq!(r.cell(999_999, 0), &Value::Null);
+}
+
+#[test]
+#[ignore = "perf: run with --release -- --ignored"]
+fn perf_15000_rows_filter_and_sort_under_60ms() {
+    let rows = synthetic(15_000);
+    let b = base(PERF_BASE);
+    let start = Instant::now();
+    let r = query(&b, 0, &rows);
+    let took = start.elapsed();
+    assert_eq!(r.total, 10_000);
+    assert!(took.as_millis() < 60, "took {took:?}");
+}

specs/plugins-parity/bases/tasks.md

@@ -53,7 +53,7 @@ Create the empty crate `crates/bases` (package `notescraft-bases`, `[lints] work
   GOAL: list `contains containsAll containsAny filter flat join map reduce reverse slice sort unique length isEmpty` (lambda variables `value index acc`); date `year..millisecond format time relative date isEmpty` (format tokens `YYYY MM DD HH mm ss` plus the Moment subset of `notescraft-templates::format` semantics, re-implemented); link `asFile linksTo`; file `asLink hasLink hasProperty hasTag inFolder` and the `file.*` fields through `RowRef`; regex `matches`, string `replace` with a regex and `$1`.
   TESTS: `tests/eval_lists.rs`, `eval_dates.rs`, `eval_files.rs`: each function per the docs; `hasTag` matches nested tags; `inFolder` matches descendants; `relative` gives "3 days ago"; `list(type)[0].asFile().properties.icon` with a test source; a pathological regex (`(a+)+$`) is stopped by the regex crate's step limit.
   VERIFY: standing.  DO NOT: touch `eval/mod.rs` beyond the single registration line it names.
-- [ ] 7. Query pipeline: filter, sort, group, limit, summaries.
+- [x] 7. Query pipeline: filter, sort, group, limit, summaries.
   OWNS: `crates/bases/src/query/mod.rs`, `crates/bases/src/query/summary.rs`, `crates/bases/src/query/columns.rs`.
   FILES TO OPEN: `crates/bases/src/eval/mod.rs`, `crates/bases/src/model.rs`, `spec.md` D7.
   GOAL: `run(&Base, view, &dyn NoteSource, &Env) -> QueryResult` with `RowId`s, columns (`order` or defaults `file.name`), multi-key stable sort evaluating only sort keys, `groupBy` + `groupOrder` (null key), `limit`, default summaries of D6 and custom summaries over `values`; `QueryResult::cell(i, col)` lazy; `errors` counted; dependency analysis so unused properties are never read.