notescraft/crates/engine/tests/index_cache.rs

//! The on-disk index cache: a second open gives the index a cold open gives, reads no unchanged
//! note, notices every edit, and ignores a cache it cannot trust.

mod common;

use std::path::Path;
use std::time::{Duration, SystemTime, UNIX_EPOCH};

use common::{seed, temp_vault};
use notescraft_engine::{Registry, Session, register_core};
use serde_json::{Value, json};

const CACHE: &str = ".notescraft/cache/index-v1.bin";
const OLD: Duration = Duration::from_secs(1_000_000);

const NOTES: [(&str, &str); 6] = [
    (
        "Home.md",
        "---\naliases: [Start]\ntags: [hub]\nyear: 1999\ndone: true\nrel: [a, b]\nwhen: 2026-01-02\n---\n# Home\n\nSee [[Trip]] and [[Soup#Steps]] and [[Missing]] #root\n\n- [ ] book the flight\n",
    ),
    ("Trips/Trip.md", "# Trip\n\nBook the flight to Lisbon and reserve the hotel near the river. [[Home]] #travel/pt\n\n## Plan\n\nzeppelin hangar ^block1\n"),
    ("Other/Soup.md", "# Soup\n\nBoil the onions and the carrots slowly.\n\n## Steps\n\n1. chop\n2. boil\n"),
    ("Other/Stew.md", "# Stew\n\nBraise the beef with the carrots. ![[pic.png]] [see](Soup.md)\n"),
    ("Private/Diary.md", "# Diary\n\nThe lisbon flight password is hunter2.\n"),
    ("Plain.md", "no heading at all, only words like quokka and wombat\n"),
];

fn old_vault(name: &str) -> common::TempVault {
    let vault = temp_vault(name);
    seed(vault.path(), &NOTES);
    seed(vault.path(), &[("pic.png", "x")]);
    for (path, _) in NOTES {
        set_mtime(&vault.path().join(path), UNIX_EPOCH + OLD);
    }
    vault
}

fn set_mtime(path: &Path, time: SystemTime) {
    std::fs::File::options().write(true).open(path).expect("open").set_modified(time).expect("set the time");
}

/// Writes `text` into `path` and puts the modified time back, so only the content changed.
fn rewrite_keeping_time(vault: &Path, path: &str, text: &str) {
    let target = vault.join(path);
    let time = std::fs::metadata(&target).and_then(|meta| meta.modified()).expect("mtime");
    std::fs::write(&target, text).expect("write");
    set_mtime(&target, time);
}

fn run(session: &mut Session, id: &str, params: &Value) -> Value {
    let mut registry = Registry::new();
    register_core(&mut registry).expect("register");
    registry.run(id, session, params).unwrap_or_else(|error| panic!("{id}: {error:?}"))
}

/// Everything the indexes answer, as one comparable string.
fn snapshot(session: &mut Session) -> String {
    let mut out: Vec<String> = Vec::new();
    let paths: Vec<String> = session.index().notes().into_iter().map(str::to_owned).collect();
    for path in &paths {
        out.push(format!("{:?}\n{:?}\n", session.index().note(path), session.index().backlinks(path)));
    }
    out.push(format!("{:?}\n{:?}\n{:?}\n", session.index().files(), session.index().unresolved(), session.index().orphans()));
    for query in ["flight", "carrots", "zeppelin", "tag:#travel", "quokka wombat", "\"lisbon flight\"", "task:book", "section:plan"] {
        out.push(format!("{query}: {:?}\n", session.search(query, 20).expect("search")));
    }
    for query in ["flight lisbon", "carrots onions", "quokka"] {
        out.push(format!("{query}: {}\n", run(session, "knowledge.search", &json!({ "query": query }))));
    }
    let status = run(session, "knowledge.status", &Value::Null);
    out.push(format!("{} {} {}\n", status["notes"], status["passages"], status["vocab_terms"]));
    out.push(format!("{}\n{}\n", run(session, "tags.list", &Value::Null), run(session, "lint.style", &Value::Null)));
    out.join("\n")
}

/// What a scan of the vault as it is gives: the session is closed and the cache removed first.
fn cold_snapshot(vault: &Path, session: Session) -> String {
    drop(session);
    std::fs::remove_dir_all(vault.join(".notescraft")).expect("remove the private folder");
    snapshot(&mut Session::open_read_only(vault).expect("read only"))
}

fn open(vault: &Path) -> Session {
    Session::open(vault).expect("open")
}

#[test]
fn a_cached_open_gives_the_index_a_cold_open_gives() {
    let vault = old_vault("ic_same");
    let mut cold = open(vault.path());
    let want = snapshot(&mut cold);
    drop(cold);
    assert!(vault.path().join(CACHE).is_file(), "the first open writes the cache");
    let mut cached = open(vault.path());
    assert_eq!(snapshot(&mut cached), want);
    drop(cached);
    let mut lazy = Session::open_lazy(vault.path()).expect("lazy");
    while !lazy.index_step(Duration::from_secs(30)).finished {}
    assert_eq!(snapshot(&mut lazy), want);
    assert_eq!(lazy.index_progress().done, lazy.index_progress().total);
}

#[test]
fn unchanged_notes_are_not_read_again() {
    let vault = old_vault("ic_noread");
    drop(open(vault.path()));
    // Same length, same time: only the cache can still know the old words.
    rewrite_keeping_time(vault.path(), "Plain.md", "no heading at all, only words like wombat and quokka\n");
    let session = open(vault.path());
    assert_eq!(session.search("\"like quokka\"", 5).expect("search").len(), 1, "the cached text is still what the index knows");
    let new = session.search("\"like wombat\"", 5).expect("search");
    assert!(new.is_empty(), "the phrase exists only in the new file text, which was not read: {new:?}");
}

#[test]
fn an_edited_note_is_read_again() {
    let vault = old_vault("ic_edit");
    drop(open(vault.path()));
    std::fs::write(vault.path().join("Plain.md"), "now about zeppelins and [[Soup]] #fresh\n").expect("edit");
    rewrite_keeping_time(vault.path(), "Other/Stew.md", "# Stew\n\nBraise the beef with the carrots. ![[pic.png]] [see](Soup.md)\n");
    let mut session = open(vault.path());
    assert_eq!(session.search("quokka", 5).expect("search").len(), 0);
    assert_eq!(session.search("zeppelins", 5).expect("search").len(), 1);
    assert!(session.index().backlinks("Other/Soup.md").iter().any(|link| link.source == "Plain.md"));
    assert_eq!(snapshot(&mut session), cold_snapshot(vault.path(), session), "same as a scan of the vault as it is");
}

#[test]
fn a_note_written_moments_before_the_cache_is_checked_by_content() {
    let vault = temp_vault("ic_recent");
    seed(vault.path(), &NOTES);
    drop(open(vault.path()));
    rewrite_keeping_time(vault.path(), "Plain.md", "no heading at all, only words like wombat and quokka\n");
    let session = open(vault.path());
    assert_eq!(session.search("\"like wombat\"", 5).expect("search").len(), 1, "equal stamps right after a write are not trusted");
}

#[test]
fn deleted_and_new_notes_follow_the_vault() {
    let vault = old_vault("ic_delete");
    drop(open(vault.path()));
    std::fs::remove_file(vault.path().join("Trips/Trip.md")).expect("delete");
    seed(vault.path(), &[("Extra.md", "# Extra\n\nlinks to [[Home]] and mentions zeppelin\n")]);
    let mut session = open(vault.path());
    assert!(!session.index().notes().contains(&"Trips/Trip.md"));
    assert!(session.index().notes().contains(&"Extra.md"));
    assert_eq!(session.search("zeppelin", 5).expect("search").iter().map(|hit| hit.path.as_str()).collect::<Vec<_>>(), ["Extra.md"]);
    assert!(session.index().unresolved().iter().any(|link| link.target == "Trip"), "Home.md now points at nothing");
    assert_eq!(snapshot(&mut session), cold_snapshot(vault.path(), session));
}

#[test]
fn a_cache_that_cannot_be_trusted_is_replaced_by_a_full_scan() {
    let vault = old_vault("ic_corrupt");
    let mut cold = open(vault.path());
    let want = snapshot(&mut cold);
    drop(cold);
    let path = vault.path().join(CACHE);
    let good = std::fs::read(&path).expect("cache");
    let mut flipped = good.clone();
    let middle = flipped.len() / 2;
    flipped[middle] ^= 0xff;
    let mut other_version = good.clone();
    other_version[4] = other_version[4].wrapping_add(1);
    let bad: [Vec<u8>; 6] = [Vec::new(), b"garbage".to_vec(), good[..good.len() / 2].to_vec(), flipped, other_version, vec![0xff; good.len()]];
    for bytes in bad {
        std::fs::write(&path, &bytes).expect("break the cache");
        let mut session = open(vault.path());
        assert_eq!(snapshot(&mut session), want);
        drop(session);
        let replaced = std::fs::read(&path).expect("cache");
        assert!(replaced.starts_with(b"NCIX") && replaced != bytes, "the broken cache was replaced");
    }
}

#[test]
fn a_cache_whose_vocabulary_changed_is_ignored() {
    let vault = old_vault("ic_vocab");
    drop(open(vault.path()));
    std::fs::write(vault.path().join(".notescraft/knowledge-vocab.json"), "{}").expect("break the vocabulary");
    rewrite_keeping_time(vault.path(), "Plain.md", "no heading at all, only words like wombat and quokka\n");
    let session = open(vault.path());
    assert_eq!(session.search("\"like wombat\"", 5).expect("search").len(), 1, "everything was read again");
}

#[test]
fn properties_size_and_times_survive_the_cache() {
    let vault = old_vault("ic_props");
    drop(open(vault.path()));
    let session = open(vault.path());
    let home = session.index().note("Home.md").expect("home").clone();
    assert_eq!(home.props.len(), 6);
    assert_eq!(home.props[2], ("year".to_string(), notescraft_markdown::Value::Number(1999.0)));
    assert_eq!(home.size, NOTES[0].1.len() as u64);
    assert_eq!(home.mtime_ms, i64::try_from(OLD.as_millis()).expect("ms"));
    assert!(home.ctime_ms > 0);
}

#[test]
fn a_cache_of_the_previous_version_is_replaced() {
    use std::hash::Hasher;
    let vault = old_vault("ic_oldver");
    drop(open(vault.path()));
    let path = vault.path().join(CACHE);
    let mut old = std::fs::read(&path).expect("cache");
    old[4..8].copy_from_slice(&1_u32.to_le_bytes());
    let body = old.len() - 8;
    let mut hasher = std::collections::hash_map::DefaultHasher::new();
    hasher.write(&old[..body]);
    old[body..].copy_from_slice(&hasher.finish().to_le_bytes());
    std::fs::write(&path, &old).expect("write the old cache");
    let session = open(vault.path());
    assert_eq!(session.index().note("Home.md").expect("home").props.len(), 6);
    drop(session);
    let replaced = std::fs::read(&path).expect("cache");
    assert_eq!(replaced[4..8], 2_u32.to_le_bytes(), "rewritten with the current version");
}