diff --git a/crates/bases/src/eval/funcs_list.rs b/crates/bases/src/eval/funcs_list.rs index 90f7941..d1db383 100644 --- a/crates/bases/src/eval/funcs_list.rs +++ b/crates/bases/src/eval/funcs_list.rs @@ -87,6 +87,7 @@ pub(super) fn method(recv: &Value, name: &str, args: &[Value]) -> Option Ok((from, to)) => Value::List(items.iter().skip(from).take(to - from).cloned().collect()), Err(e) => e, }, + ("sum" | "mean" | "median" | "min" | "max", []) => crate::query::number_stat(name, items)?, ("join", [other]) => err(format!("join() does not accept {}", other.type_name())), ("contains" | "isEmpty" | "flat" | "reverse" | "sort" | "unique" | "join" | "slice", _) => { err(format!("{name}() called with {} arguments", args.len())) diff --git a/crates/bases/src/lib.rs b/crates/bases/src/lib.rs index 951d3f2..d2efe87 100644 --- a/crates/bases/src/lib.rs +++ b/crates/bases/src/lib.rs @@ -15,6 +15,7 @@ mod eval; pub mod expr; pub mod map; mod model; +pub mod query; mod row; mod value; mod yaml; @@ -24,6 +25,7 @@ pub use datetime::{Date, DateError, DateParts, Duration, add_duration, parse_dur pub use error::{Error, Result}; pub use eval::{Env, Formulas, eval}; pub use model::{Base, Direction, Filter, GroupBy, PropertyCfg, Sort, View, ViewKind}; +pub use query::{Column, Group, QueryResult, Reads, RowId, ViewSel, run}; pub use row::{NoteSource, RowRef}; pub use value::Value; pub use yaml::{Node, emit, parse}; diff --git a/crates/bases/src/query/columns.rs b/crates/bases/src/query/columns.rs new file mode 100644 index 0000000..192e4d5 --- /dev/null +++ b/crates/bases/src/query/columns.rs @@ -0,0 +1,76 @@ +//! The columns of a view and the properties a query reads. + +use crate::expr::{Expr, parse_expr}; +use crate::model::{Base, View}; + +/// The column shown when a view lists none. +const DEFAULT_COLUMN: &str = "file.name"; + +/// One column of a result. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Column { + /// The property id: `file.name`, `note.year`, `formula.ppu` or a bare note property. + pub id: String, + /// The title: the `displayName` of the property, else the id without its `note.` / `file.` / + /// `formula.` prefix. + pub name: String, + /// The summary of the view for this column, a default or a custom name. + pub summary: Option, +} + +/// An expression that parsed, or the parse message. +pub(super) type Parsed = Result; + +pub(super) fn parse(src: &str) -> Parsed { + parse_expr(src).map_err(|e| e.to_string()) +} + +/// The columns of `view` with their parsed expressions (a property id is an expression). +pub(super) fn columns(base: &Base, view: &View) -> (Vec, Vec) { + let ids: Vec<&str> = if view.order.is_empty() { vec![DEFAULT_COLUMN] } else { view.order.iter().map(String::as_str).collect() }; + let column = |id: &&str| { + let display = base.properties.iter().find(|(p, _)| p == id).and_then(|(_, cfg)| cfg.display_name.clone()); + let short = ["note.", "file.", "formula."].iter().find_map(|p| id.strip_prefix(p)).unwrap_or(id); + let summary = view.summaries.iter().find(|(p, _)| p == id).map(|(_, s)| s.clone()); + Column { id: (*id).to_string(), name: display.unwrap_or_else(|| short.to_string()), summary } + }; + (ids.iter().map(column).collect(), ids.iter().map(|id| parse(id)).collect()) +} + +/// What a query reads: the properties of a row it touches. Nothing else is read. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct Reads { + /// Note properties, formulas expanded. + pub note: Vec, + /// `file.*` fields, formulas expanded. + pub file: Vec, + /// Whether `this` is read. + pub uses_this: bool, +} + +impl Reads { + /// Adds the dependencies of `expr` and, transitively, of the formulas it names. + pub(super) fn add(&mut self, expr: &Expr, formulas: &[(String, String)], seen: &mut Vec) { + let deps = expr.deps(); + self.note.extend(deps.note); + self.file.extend(deps.file); + self.uses_this |= deps.uses_this; + for dep in deps.formula { + let name = dep.split('.').next().unwrap_or(&dep).to_string(); + if seen.contains(&name) { + continue; + } + seen.push(name.clone()); + if let Some(Ok(inner)) = formulas.iter().find(|(n, _)| *n == name).map(|(_, src)| parse(src)) { + self.add(&inner, formulas, seen); + } + } + } + + pub(super) fn finish(&mut self) { + self.note.sort(); + self.note.dedup(); + self.file.sort(); + self.file.dedup(); + } +} diff --git a/crates/bases/src/query/mod.rs b/crates/bases/src/query/mod.rs new file mode 100644 index 0000000..34f3048 --- /dev/null +++ b/crates/bases/src/query/mod.rs @@ -0,0 +1,301 @@ +//! The query pipeline of a base view: filter, sort, group, limit, summaries. +//! +//! [`run`] walks the source once for the filters, evaluates only the sort and group keys, and +//! returns [`RowId`]s. A cell is computed on demand by [`QueryResult::cell`] and memoised, so a +//! row nobody asks for costs nothing. Rows and values of the source are never copied. + +mod columns; +mod summary; + +use std::cell::{Cell, OnceCell}; +use std::cmp::Ordering; +use std::collections::HashMap; + +pub use columns::{Column, Reads}; +pub(crate) use summary::number_stat; + +use self::columns::{Parsed, columns, parse}; +use self::summary::{Custom, summarize}; +use crate::compare::{compare, loose_eq, truthy}; +use crate::error::{Error, Result}; +use crate::eval::{Env, Formulas, err, eval}; +use crate::model::{Base, Direction, Filter, View}; +use crate::row::{NoteSource, RowRef}; +use crate::value::Value; +use crate::yaml::Node; + +/// A row of the result: its index in the [`NoteSource`], stable while the source does not change. +pub type RowId = usize; + +static NULL: Value = Value::Null; + +/// Which view to run: its index or its name. +#[derive(Debug, Clone, Copy)] +pub enum ViewSel<'a> { + /// Position in `Base::views`. + Index(usize), + /// The `name` of the view. + Name(&'a str), +} + +impl From for ViewSel<'_> { + fn from(i: usize) -> Self { + ViewSel::Index(i) + } +} + +impl<'a> From<&'a str> for ViewSel<'a> { + fn from(name: &'a str) -> Self { + ViewSel::Name(name) + } +} + +/// One group of rows. Without `groupBy` there is one group with a `Null` key. +#[derive(Debug, Clone, PartialEq)] +pub struct Group { + /// The value of the `groupBy` property shared by the rows (`Null`: no value). + pub key: Value, + /// The rows, in sort order. + pub rows: Vec, + /// The summary of each column over this group; `Null` for a column without one. + pub summaries: Vec, +} + +/// What evaluates expressions: the environment, the formulas and the error count. +struct Eval<'a> { + env: Env<'a>, + formulas: Formulas, + errors: Cell, +} + +impl Eval<'_> { + fn value(&self, expr: &Parsed, row: &RowRef<'_>) -> Value { + let value = match expr { + Ok(expr) => eval(expr, &self.env, row, &self.formulas), + Err(message) => err(message.clone()), + }; + if matches!(value, Value::Error(_)) { + self.errors.set(self.errors.get().saturating_add(1)); + } + value + } +} + +/// A filter with its expressions parsed. +enum Compiled { + Expr(Parsed), + And(Vec), + Or(Vec), + Not(Vec), +} + +fn compile(filter: &Filter, track: &mut dyn FnMut(&Parsed)) -> Compiled { + let mut all = |members: &[Filter]| members.iter().map(|m| compile(m, track)).collect::>(); + match filter { + Filter::Expr(src) => { + let parsed = parse(src); + track(&parsed); + Compiled::Expr(parsed) + } + Filter::And(m) => Compiled::And(all(m)), + Filter::Or(m) => Compiled::Or(all(m)), + Filter::Not(m) => Compiled::Not(all(m)), + } +} + +impl Eval<'_> { + /// Whether the row passes; an empty group imposes nothing, an error counts as false. + fn passes(&self, filter: &Compiled, row: &RowRef<'_>) -> bool { + match filter { + Compiled::Expr(e) => { + let v = self.value(e, row); + !matches!(v, Value::Error(_)) && truthy(&v) + } + Compiled::And(m) => m.iter().all(|f| self.passes(f, row)), + Compiled::Or(m) => m.is_empty() || m.iter().any(|f| self.passes(f, row)), + Compiled::Not(m) => !m.iter().any(|f| self.passes(f, row)), + } + } +} + +/// The outcome of [`run`]. +pub struct QueryResult<'a> { + /// The columns of the view. + pub columns: Vec, + /// The groups, with the limit applied. + pub groups: Vec, + /// Rows that matched the filters, before the limit. + pub total: usize, + /// The summary of each column over every shown row; `Null` for a column without one. + pub summaries: Vec, + /// The properties the query reads; every other property is never touched. + pub reads: Reads, + source: &'a dyn NoteSource, + eval: Eval<'a>, + exprs: Vec, + slots: HashMap, + cells: Vec>, + computed: Cell, +} + +impl QueryResult<'_> { + /// The value of column `col` for `row`, computed on first use. `Null` for a row that is not in + /// the result or a column that does not exist. + #[must_use] + pub fn cell(&self, row: RowId, col: usize) -> &Value { + let (Some(slot), Some(expr)) = (self.slots.get(&row), self.exprs.get(col)) else { return &NULL }; + let Some(cell) = self.cells.get(slot * self.exprs.len() + col) else { return &NULL }; + cell.get_or_init(|| { + self.computed.set(self.computed.get().saturating_add(1)); + self.source.row(row).map_or(Value::Null, |r| self.eval.value(expr, &r)) + }) + } + + /// How many cells have been computed so far, summaries included. + #[must_use] + pub fn cells_computed(&self) -> usize { + self.computed.get() + } + + /// How many evaluations failed so far: filters, keys and cells. + #[must_use] + pub fn errors(&self) -> usize { + self.eval.errors.get() + } + + /// The rows of every group in display order. + pub fn rows(&self) -> impl Iterator + '_ { + self.groups.iter().flat_map(|g| g.rows.iter().copied()) + } + + fn summarize(&self, name: &str, rows: &[RowId], col: usize, custom: &[Custom]) -> Value { + let values: Vec<&Value> = rows.iter().map(|r| self.cell(*r, col)).collect(); + summarize(name, &values, custom, &self.eval.env, &self.eval.formulas) + } +} + +fn find_view<'b>(base: &'b Base, sel: ViewSel<'_>) -> Result<&'b View> { + let found = match sel { + ViewSel::Index(i) => base.views().get(i), + ViewSel::Name(n) => base.views().iter().find(|v| v.name.as_deref() == Some(n)), + }; + found.ok_or_else(|| Error::Invalid(format!("no such view: {sel:?}"))) +} + +fn ordered(a: &Value, b: &Value, dir: Direction) -> Ordering { + let o = compare(a, b); + if dir == Direction::Desc { o.reverse() } else { o } +} + +/// Whether a `groupOrder` entry names the group `key`. +fn names_group(entry: &Node, key: &Value) -> bool { + match entry { + Node::Null => matches!(key, Value::Null), + Node::Bool(b) => loose_eq(key, &Value::Bool(*b)), + Node::Num(n) => loose_eq(key, &Value::Number(*n)), + Node::Str(s) => !matches!(key, Value::Null) && (loose_eq(key, &Value::Str(s.clone())) || key.to_display_string() == *s), + Node::Seq(_) | Node::Map(_) => false, + } +} + +/// Runs a view of `base` over `source`. +/// +/// `env.source` defaults to `source`. Rows are filtered by the base filters AND the view filters, +/// sorted (stable, multi-key), grouped, cut to the view `limit` and summarised. +/// +/// # Errors +/// [`Error::Invalid`] when the view does not exist. +pub fn run<'a>(base: &'a Base, view: impl Into>, source: &'a dyn NoteSource, env: &Env<'a>) -> Result> { + let view = find_view(base, view.into())?; + let mut env = *env; + env.source = env.source.or(Some(source)); + let eval = Eval { env, formulas: Formulas::new(&base.formulas), errors: Cell::new(0) }; + let mut reads = Reads::default(); + let mut track = |p: &Parsed| { + if let Ok(expr) = p { + reads.add(expr, &base.formulas, &mut Vec::new()); + } + }; + let (cols, exprs) = columns(base, view); + exprs.iter().for_each(&mut track); + let filters: Vec = [&base.filters, &view.filters].into_iter().flatten().map(|f| compile(f, &mut track)).collect(); + let sorts: Vec<(Parsed, Direction)> = view.sort.iter().map(|s| (parse(&s.property), s.direction)).collect(); + let group = view.group_by.as_ref().map(|g| (parse(&g.property), g.direction)); + sorts.iter().map(|(p, _)| p).chain(group.iter().map(|(p, _)| p)).for_each(&mut track); + reads.finish(); + + let kept: Vec<(RowId, RowRef<'a>)> = + (0..source.len()).filter_map(|i| source.row(i).map(|r| (i, r))).filter(|(_, row)| filters.iter().all(|f| eval.passes(f, row))).collect(); + let total = kept.len(); + + let keys: Vec> = + if sorts.is_empty() { Vec::new() } else { kept.iter().map(|(_, r)| sorts.iter().map(|(p, _)| eval.value(p, r)).collect()).collect() }; + let mut order: Vec = (0..total).collect(); + if !sorts.is_empty() { + order.sort_by(|a, b| { + for (k, (_, dir)) in sorts.iter().enumerate() { + let o = match (keys.get(*a).and_then(|r| r.get(k)), keys.get(*b).and_then(|r| r.get(k))) { + (Some(x), Some(y)) => ordered(x, y, *dir), + _ => Ordering::Equal, + }; + if o != Ordering::Equal { + return o; + } + } + Ordering::Equal + }); + } + + let mut groups: Vec = Vec::new(); + if let Some((expr, dir)) = &group { + let gkeys: Vec = kept.iter().map(|(_, r)| eval.value(expr, r)).collect(); + order.sort_by(|a, b| match (gkeys.get(*a), gkeys.get(*b)) { + (Some(x), Some(y)) => ordered(x, y, *dir), + _ => Ordering::Equal, + }); + for pos in &order { + let (Some(key), Some((id, _))) = (gkeys.get(*pos), kept.get(*pos)) else { continue }; + match groups.last_mut() { + Some(g) if compare(&g.key, key) == Ordering::Equal => g.rows.push(*id), + _ => groups.push(Group { key: key.clone(), rows: vec![*id], summaries: Vec::new() }), + } + } + if !view.group_order.is_empty() { + let mut pool: Vec> = groups.into_iter().map(Some).collect(); + groups = + view.group_order.iter().filter_map(|entry| pool.iter_mut().find(|g| g.as_ref().is_some_and(|g| names_group(entry, &g.key)))?.take()).collect(); + } + } else { + let rows: Vec = order.iter().filter_map(|p| kept.get(*p).map(|(id, _)| *id)).collect(); + if !rows.is_empty() { + groups.push(Group { key: Value::Null, rows, summaries: Vec::new() }); + } + } + + if let Some(limit) = view.limit { + let mut left = limit as usize; + for g in &mut groups { + g.rows.truncate(left); + left = left.saturating_sub(g.rows.len()); + } + groups.retain(|g| !g.rows.is_empty()); + } + + let shown: Vec = groups.iter().flat_map(|g| g.rows.iter().copied()).collect(); + let slots = shown.iter().enumerate().map(|(slot, id)| (*id, slot)).collect(); + let cells = (0..shown.len().saturating_mul(exprs.len())).map(|_| OnceCell::new()).collect(); + let mut result = + QueryResult { columns: cols, groups: Vec::new(), total, summaries: Vec::new(), reads, source, eval, exprs, slots, cells, computed: Cell::new(0) }; + + let custom: Vec = base.summaries.iter().map(|(n, src)| (n.clone(), parse(src))).collect(); + let column_summaries = |rows: &[RowId]| -> Vec { + result.columns.iter().enumerate().map(|(col, c)| c.summary.as_deref().map_or(Value::Null, |name| result.summarize(name, rows, col, &custom))).collect() + }; + for g in &mut groups { + g.summaries = column_summaries(&g.rows); + } + let overall = column_summaries(&shown); + result.groups = groups; + result.summaries = overall; + Ok(result) +} diff --git a/crates/bases/src/query/summary.rs b/crates/bases/src/query/summary.rs new file mode 100644 index 0000000..4d31b49 --- /dev/null +++ b/crates/bases/src/query/summary.rs @@ -0,0 +1,128 @@ +//! Summaries: the default ones of the docs (implemented here, not as formulas) and the custom +//! ones, which are expressions over `values`. + +use std::cmp::Ordering; + +use crate::compare::compare; +use crate::datetime::Duration; +use crate::eval::{Env, Formulas, err, eval, int_to_f64}; +use crate::expr::Expr; +use crate::row::RowRef; +use crate::value::Value; + +/// A custom summary: name and parsed expression (or the parse message). +pub(super) type Custom = (String, Result); + +/// The result of the named summary over `values`, `Error` when the name is unknown. +pub(super) fn summarize(name: &str, values: &[&Value], custom: &[Custom], env: &Env<'_>, formulas: &Formulas) -> Value { + if let Some(done) = builtin(name, values) { + return done; + } + match custom.iter().find(|(n, _)| n == name) { + None => err(format!("unknown summary `{name}`")), + Some((_, Err(message))) => err(message.clone()), + Some((_, Ok(expr))) => { + let props = [("values".to_string(), Value::List(values.iter().map(|v| (*v).clone()).collect()))]; + let row = RowRef { props: &props, ..RowRef::new("") }; + eval(expr, env, &row, formulas) + } + } +} + +/// `sum`, `mean`, `median`, `min`, `max` over the numbers of a list, `Null` when there are none +/// (`sum` is 0). Used by the summaries and by the list methods of the same names. +pub(crate) fn number_stat(name: &str, items: &[Value]) -> Option { + let nums = numbers(items.iter()); + Some(match name { + "sum" => Value::Number(nums.iter().sum()), + "mean" => mean(&nums), + "median" => median(nums), + "min" => fold(&nums, f64::min), + "max" => fold(&nums, f64::max), + _ => return None, + }) +} + +fn numbers<'a>(values: impl Iterator) -> Vec { + values.filter_map(|v| if let Value::Number(n) = v { Some(*n).filter(|n| !n.is_nan()) } else { None }).collect() +} + +fn mean(nums: &[f64]) -> Value { + if nums.is_empty() { Value::Null } else { Value::Number(nums.iter().sum::() / int_to_f64(nums.len())) } +} + +fn fold(nums: &[f64], f: fn(f64, f64) -> f64) -> Value { + nums.iter().copied().reduce(f).map_or(Value::Null, Value::Number) +} + +fn median(mut nums: Vec) -> Value { + nums.sort_by(f64::total_cmp); + let mid = nums.len() / 2; + match (nums.get(mid), mid.checked_sub(1).and_then(|i| nums.get(i))) { + (None, _) => Value::Null, + (Some(hi), Some(lo)) if nums.len().is_multiple_of(2) => Value::Number((lo + hi) / 2.0), + (Some(m), _) => Value::Number(*m), + } +} + +/// Population standard deviation. +fn stddev(nums: &[f64]) -> Value { + let Value::Number(m) = mean(nums) else { return Value::Null }; + let var = nums.iter().map(|x| (x - m).powi(2)).sum::() / int_to_f64(nums.len()); + Value::Number(var.sqrt()) +} + +fn dates(values: &[&Value]) -> Vec { + values.iter().filter_map(|v| if let Value::Date(d) = v { Some(*d) } else { None }).collect() +} + +/// Whether a value counts as empty: null, `""`, an empty list or object. +fn is_empty(v: &Value) -> bool { + match v { + Value::Null => true, + Value::Str(s) => s.is_empty(), + Value::List(l) => l.is_empty(), + Value::Object(o) => o.is_empty(), + _ => false, + } +} + +fn count(n: usize) -> Value { + Value::Number(int_to_f64(n)) +} + +fn builtin(name: &str, values: &[&Value]) -> Option { + let nums = || numbers(values.iter().copied()); + Some(match name { + "Average" => mean(&nums()), + "Sum" => Value::Number(nums().iter().sum()), + "Min" => fold(&nums(), f64::min), + "Max" => fold(&nums(), f64::max), + "Median" => median(nums()), + "Stddev" => stddev(&nums()), + "Range" => { + let n = nums(); + if let (Value::Number(lo), Value::Number(hi)) = (fold(&n, f64::min), fold(&n, f64::max)) { + return Some(Value::Number(hi - lo)); + } + let d = dates(values); + match (d.iter().map(|d| d.ms()).min(), d.iter().map(|d| d.ms()).max()) { + (Some(lo), Some(hi)) => hi.checked_sub(lo).map_or_else(|| err("date range out of range"), |ms| Value::Duration(Duration::new(0, ms))), + _ => Value::Null, + } + } + "Earliest" => dates(values).into_iter().min_by_key(|d| d.ms()).map_or(Value::Null, Value::Date), + "Latest" => dates(values).into_iter().max_by_key(|d| d.ms()).map_or(Value::Null, Value::Date), + "Checked" => count(values.iter().filter(|v| matches!(v, Value::Bool(true))).count()), + "Unchecked" => count(values.iter().filter(|v| matches!(v, Value::Bool(false))).count()), + "Empty" => count(values.iter().filter(|v| is_empty(v)).count()), + "Filled" => count(values.iter().filter(|v| !is_empty(v)).count()), + "Unique" => { + let mut sorted: Vec<&Value> = values.to_vec(); + sorted.sort_by(|a, b| compare(a, b)); + sorted.dedup_by(|a, b| compare(a, b) == Ordering::Equal); + count(sorted.len()) + } + _ => return None, + }) +} diff --git a/crates/bases/tests/query.rs b/crates/bases/tests/query.rs new file mode 100644 index 0000000..0b1e967 --- /dev/null +++ b/crates/bases/tests/query.rs @@ -0,0 +1,293 @@ +//! The query pipeline: filter, sort, group, limit, summaries, laziness and speed. + +mod common; + +use std::cell::Cell; +use std::time::Instant; + +use common::{NOW, Rows, TestRow, n, s}; +use notescraft_bases::{Base, Env, NoteSource, QueryResult, RowRef, Value, run}; + +const BOOKS: [(&str, Option<&str>, f64, f64, bool); 12] = [ + ("Alpha", Some("sf"), 2001.0, 300.0, true), + ("Bravo", Some("sf"), 1999.0, 150.0, false), + ("Charlie", Some("fantasy"), 2010.0, 400.0, true), + ("Delta", Some("fantasy"), 2010.0, 90.0, false), + ("Echo", None, 2005.0, 200.0, true), + ("Foxtrot", Some("sf"), 2015.0, 250.0, true), + ("Golf", Some("skip"), 2000.0, 500.0, true), + ("Hotel", Some("fantasy"), 1990.0, 120.0, false), + ("India", None, 1980.0, 110.0, false), + ("Juliet", Some("sf"), 2001.0, 180.0, true), + ("Kilo", Some("fantasy"), 2010.0, 310.0, true), + ("Lima", Some("sf"), 1999.0, 101.0, false), +]; + +fn books() -> Rows { + Rows( + BOOKS + .iter() + .map(|(name, genre, year, pages, read)| { + let row = TestRow::new(&format!("Books/{name}.md")).prop("year", n(*year)).prop("pages", n(*pages)).prop("read", Value::Bool(*read)); + match genre { + Some(g) => row.prop("genre", s(g)), + None => row, + } + }) + .collect(), + ) +} + +const BASE: &str = " +filters: + and: + - pages > 100 +summaries: + customAverage: 'values.mean().round(3)' +formulas: + bad: nope(1) + half: pages / 2 +views: + - type: table + name: Main + filters: + not: + - genre == \"skip\" + groupBy: + property: genre + direction: ASC + sort: + - property: year + direction: DESC + order: + - file.name + - year + - pages + summaries: + pages: Sum + year: customAverage +"; + +fn query<'a>(base: &'a Base, view: impl Into>, rows: &'a Rows) -> QueryResult<'a> { + run(base, view, rows, &Env::new(NOW, 7)).unwrap_or_else(|e| panic!("{e}")) +} + +fn base(text: &str) -> Base { + Base::parse(text).unwrap_or_else(|e| panic!("{e}")) +} + +/// `key: name name ...` per group. +fn table(r: &QueryResult<'_>) -> Vec { + r.groups + .iter() + .map(|g| { + let names: Vec = g.rows.iter().map(|id| r.cell(*id, 0).to_display_string().replace("Books/", "").replace(".md", "")).collect(); + format!("{}: {}", g.key.to_display_string(), names.join(" ")) + }) + .collect() +} + +#[test] +fn filter_sort_group_summary_golden() { + let (rows, b) = (books(), base(BASE)); + let r = query(&b, "Main", &rows); + assert_eq!(r.total, 10); + assert_eq!(table(&r), [": Echo India", "fantasy: Charlie Kilo Hotel", "sf: Foxtrot Alpha Juliet Bravo Lima"]); + assert_eq!(r.columns.iter().map(|c| c.name.as_str()).collect::>(), ["name", "year", "pages"]); + let sums: Vec<_> = r.groups.iter().map(|g| g.summaries.clone()).collect(); + assert_eq!(sums[0], [Value::Null, n(1992.5), n(310.0)]); + assert_eq!(sums[1], [Value::Null, n(2003.333), n(830.0)]); + assert_eq!(sums[2][2], n(981.0)); + assert_eq!(r.summaries, [Value::Null, n(2001.0), n(2121.0)]); + assert_eq!(r.errors(), 0); +} + +#[test] +fn group_order_hides_and_places_null() { + let (rows, b) = (books(), base(&BASE.replace(" sort:", " groupOrder:\n - sf\n - null\n sort:"))); + assert_eq!(table(&query(&b, 0, &rows)), ["sf: Foxtrot Alpha Juliet Bravo Lima", ": Echo India"]); +} + +#[test] +fn desc_groups_limit_and_default_column() { + let rows = books(); + let b = base( + "views:\n - type: table\n limit: 3\n groupBy:\n property: genre\n direction: DESC\n sort:\n - property: pages\n direction: ASC\n", + ); + let r = query(&b, 0, &rows); + assert_eq!(r.total, 12); + assert_eq!(r.columns.len(), 1); + assert_eq!(r.columns[0].id, "file.name"); + assert_eq!(table(&r), ["skip: Golf", "sf: Lima Bravo"]); +} + +#[test] +fn multi_key_sort_is_stable_and_missing_view_is_an_error() { + let rows = books(); + let b = base( + "views:\n - type: table\n filters: genre == \"sf\"\n sort:\n - property: year\n direction: ASC\n - property: pages\n direction: DESC\n", + ); + let r = query(&b, 0, &rows); + assert_eq!(table(&r), [": Bravo Lima Alpha Juliet Foxtrot"]); + assert!(run(&b, 5, &rows, &Env::new(NOW, 7)).is_err()); + assert!(run(&b, "nope", &rows, &Env::new(NOW, 7)).is_err()); +} + +#[test] +fn default_summaries() { + let rows = books(); + let b = base( + "views:\n - type: table\n order: [pages, read, genre, year]\n summaries:\n pages: Median\n read: Checked\n genre: Unique\n year: Range\n", + ); + let r = query(&b, 0, &rows); + assert_eq!(r.summaries[0], n(190.0)); // median of 12 values: (180 + 200) / 2 + assert_eq!(r.summaries[1], n(7.0)); + assert_eq!(r.summaries[2], n(4.0)); // null, fantasy, sf, skip + assert_eq!(r.summaries[3], n(35.0)); + for (name, want) in [("Min", n(90.0)), ("Max", n(500.0)), ("Unchecked", n(5.0)), ("Filled", n(12.0)), ("Empty", n(0.0)), ("Nope", Value::Null)] { + let col = if name == "Unchecked" { "read" } else { "pages" }; + let b = base(&format!("views:\n - type: table\n order: [{col}]\n summaries:\n {col}: {name}\n")); + let got = query(&b, 0, &rows).summaries[0].clone(); + if name == "Nope" { + assert!(matches!(got, Value::Error(_))); + } else { + assert_eq!(got, want, "{name}"); + } + } + let b = base("views:\n - type: table\n order: [genre, pages]\n summaries:\n genre: Empty\n pages: Stddev\n"); + let r = query(&b, 0, &rows); + assert_eq!(r.summaries[0], n(2.0)); + assert!(matches!(&r.summaries[1], Value::Number(x) if (x - 123.0).abs() < 50.0)); +} + +#[test] +fn date_summaries() { + let rows = Rows(vec![ + TestRow::new("a.md").prop("d", Value::Date(notescraft_bases::Date::new(0, false))), + TestRow::new("b.md").prop("d", Value::Date(notescraft_bases::Date::new(86_400_000, false))), + ]); + let b = base("views:\n - type: table\n order: [d]\n summaries:\n d: Range\n"); + assert_eq!(query(&b, 0, &rows).summaries[0].to_display_string(), "1d"); + let b = base("views:\n - type: table\n order: [d]\n summaries:\n d: Latest\n"); + assert_eq!(query(&b, 0, &rows).summaries[0], Value::Date(notescraft_bases::Date::new(86_400_000, false))); +} + +#[test] +fn errors_are_counted_and_do_not_stop_the_table() { + let rows = books(); + let b = base(&format!("{BASE}\n").replace(" - pages\n summaries", " - formula.bad\n - formula.half\n summaries")); + let r = query(&b, 0, &rows); + let id = r.groups[0].rows[0]; + assert!(matches!(r.cell(id, 2), Value::Error(_))); + assert_eq!(r.cell(id, 3), &n(100.0)); + assert!(r.errors() >= 1); + let b = base("filters: nope(1)\nviews:\n - type: table\n"); + let r = query(&b, 0, &rows); + assert_eq!(r.total, 0); + assert_eq!(r.errors(), 12); +} + +#[test] +fn display_name_and_formula_dependencies() { + let rows = books(); + let b = base( + "properties:\n year:\n displayName: Year\nformulas:\n half: pages / 2\nviews:\n - type: table\n order: [year, formula.half]\n sort:\n - property: file.name\n direction: ASC\n", + ); + let r = query(&b, 0, &rows); + assert_eq!(r.columns[0].name, "Year"); + assert_eq!(r.columns[1].name, "half"); + assert_eq!(r.reads.note, ["pages", "year"]); + assert_eq!(r.reads.file, ["name"]); + assert!(!r.reads.uses_this); +} + +struct Counted { + rows: Rows, + backlinks: Box Vec>, +} + +impl NoteSource for Counted { + fn len(&self) -> usize { + self.rows.len() + } + fn row(&self, i: usize) -> Option> { + self.rows.row(i).map(|r| RowRef { backlinks: Some(&*self.backlinks), ..r }) + } + fn find(&self, path: &str) -> Option> { + self.rows.find(path) + } +} + +#[test] +fn unreferenced_properties_are_never_read() { + let calls = std::rc::Rc::new(Cell::new(0)); + let c = calls.clone(); + let src = Counted { + rows: books(), + backlinks: Box::new(move || { + c.set(c.get() + 1); + vec![] + }), + }; + let b = base( + "filters: year > 1990\nviews:\n - type: table\n order: [file.name]\n sort:\n - property: year\n direction: ASC\n - type: table\n order: [file.backlinks.length]\n", + ); + let env = Env::new(NOW, 7); + let r = run(&b, 0, &src, &env).unwrap_or_else(|e| panic!("{e}")); + for id in r.rows() { + let _ = r.cell(id, 0); + } + assert_eq!(calls.get(), 0); + let r = run(&b, 1, &src, &env).unwrap_or_else(|e| panic!("{e}")); + assert_eq!(r.cells_computed(), 0); + assert_eq!(r.cell(0, 0), &n(0.0)); + assert_eq!(calls.get(), 1); + assert_eq!(r.reads.file, ["backlinks.length"]); +} + +fn synthetic(count: usize) -> Rows { + Rows( + (0..count) + .map(|i| { + TestRow::new(&format!("Notes/n{i}.md")) + .prop("year", n(f64::from(u16::try_from(i % 60).unwrap_or(0)) + 1970.0)) + .prop("rank", n(f64::from(u32::try_from(i.wrapping_mul(7919) % 10_007).unwrap_or(0)))) + }) + .collect(), + ) +} + +const PERF_BASE: &str = + "views:\n - type: table\n filters: year >= 1990\n order: [file.name, year]\n sort:\n - property: rank\n direction: DESC\n"; + +#[test] +fn no_cell_is_computed_for_rows_not_asked_for() { + let rows = synthetic(15_000); + let b = base(PERF_BASE); + let start = Instant::now(); + let r = query(&b, 0, &rows); + let took = start.elapsed(); + assert!(took.as_millis() < 500, "15 000 rows took {took:?}"); + assert_eq!(r.total, 10_000); + assert_eq!(r.cells_computed(), 0); + let top: Vec = r.rows().take(5).collect(); + for id in &top { + let _ = r.cell(*id, 0); + let _ = r.cell(*id, 0); + } + assert_eq!(r.cells_computed(), 5); + assert_eq!(r.cell(top[0], 1), &r.cell(top[0], 1).clone()); + assert_eq!(r.cell(999_999, 0), &Value::Null); +} + +#[test] +#[ignore = "perf: run with --release -- --ignored"] +fn perf_15000_rows_filter_and_sort_under_60ms() { + let rows = synthetic(15_000); + let b = base(PERF_BASE); + let start = Instant::now(); + let r = query(&b, 0, &rows); + let took = start.elapsed(); + assert_eq!(r.total, 10_000); + assert!(took.as_millis() < 60, "took {took:?}"); +} diff --git a/specs/plugins-parity/bases/tasks.md b/specs/plugins-parity/bases/tasks.md index 470e9a5..ec4a952 100644 --- a/specs/plugins-parity/bases/tasks.md +++ b/specs/plugins-parity/bases/tasks.md @@ -53,7 +53,7 @@ Create the empty crate `crates/bases` (package `notescraft-bases`, `[lints] work GOAL: list `contains containsAll containsAny filter flat join map reduce reverse slice sort unique length isEmpty` (lambda variables `value index acc`); date `year..millisecond format time relative date isEmpty` (format tokens `YYYY MM DD HH mm ss` plus the Moment subset of `notescraft-templates::format` semantics, re-implemented); link `asFile linksTo`; file `asLink hasLink hasProperty hasTag inFolder` and the `file.*` fields through `RowRef`; regex `matches`, string `replace` with a regex and `$1`. TESTS: `tests/eval_lists.rs`, `eval_dates.rs`, `eval_files.rs`: each function per the docs; `hasTag` matches nested tags; `inFolder` matches descendants; `relative` gives "3 days ago"; `list(type)[0].asFile().properties.icon` with a test source; a pathological regex (`(a+)+$`) is stopped by the regex crate's step limit. VERIFY: standing. DO NOT: touch `eval/mod.rs` beyond the single registration line it names. -- [ ] 7. Query pipeline: filter, sort, group, limit, summaries. +- [x] 7. Query pipeline: filter, sort, group, limit, summaries. OWNS: `crates/bases/src/query/mod.rs`, `crates/bases/src/query/summary.rs`, `crates/bases/src/query/columns.rs`. FILES TO OPEN: `crates/bases/src/eval/mod.rs`, `crates/bases/src/model.rs`, `spec.md` D7. GOAL: `run(&Base, view, &dyn NoteSource, &Env) -> QueryResult` with `RowId`s, columns (`order` or defaults `file.name`), multi-key stable sort evaluating only sort keys, `groupBy` + `groupOrder` (null key), `limit`, default summaries of D6 and custom summaries over `values`; `QueryResult::cell(i, col)` lazy; `errors` counted; dependency analysis so unused properties are never read.