notescraft/crates/bases/tests/query.rs

//! The query pipeline: filter, sort, group, limit, summaries, laziness and speed.

mod common;

use std::cell::Cell;
use std::time::Instant;

use common::{NOW, Rows, TestRow, n, s};
use notescraft_bases::{Base, Env, NoteSource, QueryResult, RowRef, Value, run};

const BOOKS: [(&str, Option<&str>, f64, f64, bool); 12] = [
    ("Alpha", Some("sf"), 2001.0, 300.0, true),
    ("Bravo", Some("sf"), 1999.0, 150.0, false),
    ("Charlie", Some("fantasy"), 2010.0, 400.0, true),
    ("Delta", Some("fantasy"), 2010.0, 90.0, false),
    ("Echo", None, 2005.0, 200.0, true),
    ("Foxtrot", Some("sf"), 2015.0, 250.0, true),
    ("Golf", Some("skip"), 2000.0, 500.0, true),
    ("Hotel", Some("fantasy"), 1990.0, 120.0, false),
    ("India", None, 1980.0, 110.0, false),
    ("Juliet", Some("sf"), 2001.0, 180.0, true),
    ("Kilo", Some("fantasy"), 2010.0, 310.0, true),
    ("Lima", Some("sf"), 1999.0, 101.0, false),
];

fn books() -> Rows {
    Rows(
        BOOKS
            .iter()
            .map(|(name, genre, year, pages, read)| {
                let row = TestRow::new(&format!("Books/{name}.md")).prop("year", n(*year)).prop("pages", n(*pages)).prop("read", Value::Bool(*read));
                match genre {
                    Some(g) => row.prop("genre", s(g)),
                    None => row,
                }
            })
            .collect(),
    )
}

const BASE: &str = "
filters:
  and:
    - pages > 100
summaries:
  customAverage: 'values.mean().round(3)'
formulas:
  bad: nope(1)
  half: pages / 2
views:
  - type: table
    name: Main
    filters:
      not:
        - genre == \"skip\"
    groupBy:
      property: genre
      direction: ASC
    sort:
      - property: year
        direction: DESC
    order:
      - file.name
      - year
      - pages
    summaries:
      pages: Sum
      year: customAverage
";

fn query<'a>(base: &'a Base, view: impl Into<notescraft_bases::ViewSel<'a>>, rows: &'a Rows) -> QueryResult<'a> {
    run(base, view, rows, &Env::new(NOW, 7)).unwrap_or_else(|e| panic!("{e}"))
}

fn base(text: &str) -> Base {
    Base::parse(text).unwrap_or_else(|e| panic!("{e}"))
}

/// `key: name name ...` per group.
fn table(r: &QueryResult<'_>) -> Vec<String> {
    r.groups
        .iter()
        .map(|g| {
            let names: Vec<String> = g.rows.iter().map(|id| r.cell(*id, 0).to_display_string().replace("Books/", "").replace(".md", "")).collect();
            format!("{}: {}", g.key.to_display_string(), names.join(" "))
        })
        .collect()
}

#[test]
fn filter_sort_group_summary_golden() {
    let (rows, b) = (books(), base(BASE));
    let r = query(&b, "Main", &rows);
    assert_eq!(r.total, 10);
    assert_eq!(table(&r), [": Echo India", "fantasy: Charlie Kilo Hotel", "sf: Foxtrot Alpha Juliet Bravo Lima"]);
    assert_eq!(r.columns.iter().map(|c| c.name.as_str()).collect::<Vec<_>>(), ["name", "year", "pages"]);
    let sums: Vec<_> = r.groups.iter().map(|g| g.summaries.clone()).collect();
    assert_eq!(sums[0], [Value::Null, n(1992.5), n(310.0)]);
    assert_eq!(sums[1], [Value::Null, n(2003.333), n(830.0)]);
    assert_eq!(sums[2][2], n(981.0));
    assert_eq!(r.summaries, [Value::Null, n(2001.0), n(2121.0)]);
    assert_eq!(r.errors(), 0);
}

#[test]
fn group_order_hides_and_places_null() {
    let (rows, b) = (books(), base(&BASE.replace("    sort:", "    groupOrder:\n      - sf\n      - null\n    sort:")));
    assert_eq!(table(&query(&b, 0, &rows)), ["sf: Foxtrot Alpha Juliet Bravo Lima", ": Echo India"]);
}

#[test]
fn desc_groups_limit_and_default_column() {
    let rows = books();
    let b = base(
        "views:\n  - type: table\n    limit: 3\n    groupBy:\n      property: genre\n      direction: DESC\n    sort:\n      - property: pages\n        direction: ASC\n",
    );
    let r = query(&b, 0, &rows);
    assert_eq!(r.total, 12);
    assert_eq!(r.columns.len(), 1);
    assert_eq!(r.columns[0].id, "file.name");
    assert_eq!(table(&r), ["skip: Golf", "sf: Lima Bravo"]);
}

#[test]
fn multi_key_sort_is_stable_and_missing_view_is_an_error() {
    let rows = books();
    let b = base(
        "views:\n  - type: table\n    filters: genre == \"sf\"\n    sort:\n      - property: year\n        direction: ASC\n      - property: pages\n        direction: DESC\n",
    );
    let r = query(&b, 0, &rows);
    assert_eq!(table(&r), [": Bravo Lima Alpha Juliet Foxtrot"]);
    assert!(run(&b, 5, &rows, &Env::new(NOW, 7)).is_err());
    assert!(run(&b, "nope", &rows, &Env::new(NOW, 7)).is_err());
}

#[test]
fn default_summaries() {
    let rows = books();
    let b = base(
        "views:\n  - type: table\n    order: [pages, read, genre, year]\n    summaries:\n      pages: Median\n      read: Checked\n      genre: Unique\n      year: Range\n",
    );
    let r = query(&b, 0, &rows);
    assert_eq!(r.summaries[0], n(190.0)); // median of 12 values: (180 + 200) / 2
    assert_eq!(r.summaries[1], n(7.0));
    assert_eq!(r.summaries[2], n(4.0)); // null, fantasy, sf, skip
    assert_eq!(r.summaries[3], n(35.0));
    for (name, want) in [("Min", n(90.0)), ("Max", n(500.0)), ("Unchecked", n(5.0)), ("Filled", n(12.0)), ("Empty", n(0.0)), ("Nope", Value::Null)] {
        let col = if name == "Unchecked" { "read" } else { "pages" };
        let b = base(&format!("views:\n  - type: table\n    order: [{col}]\n    summaries:\n      {col}: {name}\n"));
        let got = query(&b, 0, &rows).summaries[0].clone();
        if name == "Nope" {
            assert!(matches!(got, Value::Error(_)));
        } else {
            assert_eq!(got, want, "{name}");
        }
    }
    let b = base("views:\n  - type: table\n    order: [genre, pages]\n    summaries:\n      genre: Empty\n      pages: Stddev\n");
    let r = query(&b, 0, &rows);
    assert_eq!(r.summaries[0], n(2.0));
    assert!(matches!(&r.summaries[1], Value::Number(x) if (x - 123.0).abs() < 50.0));
}

#[test]
fn date_summaries() {
    let rows = Rows(vec![
        TestRow::new("a.md").prop("d", Value::Date(notescraft_bases::Date::new(0, false))),
        TestRow::new("b.md").prop("d", Value::Date(notescraft_bases::Date::new(86_400_000, false))),
    ]);
    let b = base("views:\n  - type: table\n    order: [d]\n    summaries:\n      d: Range\n");
    assert_eq!(query(&b, 0, &rows).summaries[0].to_display_string(), "1d");
    let b = base("views:\n  - type: table\n    order: [d]\n    summaries:\n      d: Latest\n");
    assert_eq!(query(&b, 0, &rows).summaries[0], Value::Date(notescraft_bases::Date::new(86_400_000, false)));
}

#[test]
fn errors_are_counted_and_do_not_stop_the_table() {
    let rows = books();
    let b = base(&format!("{BASE}\n").replace("      - pages\n    summaries", "      - formula.bad\n      - formula.half\n    summaries"));
    let r = query(&b, 0, &rows);
    let id = r.groups[0].rows[0];
    assert!(matches!(r.cell(id, 2), Value::Error(_)));
    assert_eq!(r.cell(id, 3), &n(100.0));
    assert!(r.errors() >= 1);
    let b = base("filters: nope(1)\nviews:\n  - type: table\n");
    let r = query(&b, 0, &rows);
    assert_eq!(r.total, 0);
    assert_eq!(r.errors(), 12);
}

#[test]
fn display_name_and_formula_dependencies() {
    let rows = books();
    let b = base(
        "properties:\n  year:\n    displayName: Year\nformulas:\n  half: pages / 2\nviews:\n  - type: table\n    order: [year, formula.half]\n    sort:\n      - property: file.name\n        direction: ASC\n",
    );
    let r = query(&b, 0, &rows);
    assert_eq!(r.columns[0].name, "Year");
    assert_eq!(r.columns[1].name, "half");
    assert_eq!(r.reads.note, ["pages", "year"]);
    assert_eq!(r.reads.file, ["name"]);
    assert!(!r.reads.uses_this);
}

struct Counted {
    rows: Rows,
    backlinks: Box<dyn Fn() -> Vec<String>>,
}

impl NoteSource for Counted {
    fn len(&self) -> usize {
        self.rows.len()
    }
    fn row(&self, i: usize) -> Option<RowRef<'_>> {
        self.rows.row(i).map(|r| RowRef { backlinks: Some(&*self.backlinks), ..r })
    }
    fn find(&self, path: &str) -> Option<RowRef<'_>> {
        self.rows.find(path)
    }
}

#[test]
fn unreferenced_properties_are_never_read() {
    let calls = std::rc::Rc::new(Cell::new(0));
    let c = calls.clone();
    let src = Counted {
        rows: books(),
        backlinks: Box::new(move || {
            c.set(c.get() + 1);
            vec![]
        }),
    };
    let b = base(
        "filters: year > 1990\nviews:\n  - type: table\n    order: [file.name]\n    sort:\n      - property: year\n        direction: ASC\n  - type: table\n    order: [file.backlinks.length]\n",
    );
    let env = Env::new(NOW, 7);
    let r = run(&b, 0, &src, &env).unwrap_or_else(|e| panic!("{e}"));
    for id in r.rows() {
        let _ = r.cell(id, 0);
    }
    assert_eq!(calls.get(), 0);
    let r = run(&b, 1, &src, &env).unwrap_or_else(|e| panic!("{e}"));
    assert_eq!(r.cells_computed(), 0);
    assert_eq!(r.cell(0, 0), &n(0.0));
    assert_eq!(calls.get(), 1);
    assert_eq!(r.reads.file, ["backlinks.length"]);
}

fn synthetic(count: usize) -> Rows {
    Rows(
        (0..count)
            .map(|i| {
                TestRow::new(&format!("Notes/n{i}.md"))
                    .prop("year", n(f64::from(u16::try_from(i % 60).unwrap_or(0)) + 1970.0))
                    .prop("rank", n(f64::from(u32::try_from(i.wrapping_mul(7919) % 10_007).unwrap_or(0))))
            })
            .collect(),
    )
}

const PERF_BASE: &str =
    "views:\n  - type: table\n    filters: year >= 1990\n    order: [file.name, year]\n    sort:\n      - property: rank\n        direction: DESC\n";

#[test]
fn no_cell_is_computed_for_rows_not_asked_for() {
    let rows = synthetic(15_000);
    let b = base(PERF_BASE);
    let start = Instant::now();
    let r = query(&b, 0, &rows);
    let took = start.elapsed();
    assert!(took.as_millis() < 500, "15 000 rows took {took:?}");
    assert_eq!(r.total, 10_000);
    assert_eq!(r.cells_computed(), 0);
    let top: Vec<usize> = r.rows().take(5).collect();
    for id in &top {
        let _ = r.cell(*id, 0);
        let _ = r.cell(*id, 0);
    }
    assert_eq!(r.cells_computed(), 5);
    assert_eq!(r.cell(top[0], 1), &r.cell(top[0], 1).clone());
    assert_eq!(r.cell(999_999, 0), &Value::Null);
}

#[test]
#[ignore = "perf: run with --release -- --ignored"]
fn perf_15000_rows_filter_and_sort_under_60ms() {
    let rows = synthetic(15_000);
    let b = base(PERF_BASE);
    let start = Instant::now();
    let r = query(&b, 0, &rows);
    let took = start.elapsed();
    assert_eq!(r.total, 10_000);
    assert!(took.as_millis() < 60, "took {took:?}");
}