notescraft/crates/markdown/tests/corpus.rs

//! Real-vault corpus (ignored by default): parse every note of the Obsidian
//! help vault (`en`) and samples of `mdn-docs` and `k8s-docs`, never panic,
//! stay fast, and agree with `pulldown-cmark` (the oracle) on how many lines
//! of every note are fenced code content.

use std::path::{Path, PathBuf};
use std::time::{Duration, Instant};

use notescraft_markdown::{fenced_blocks, parse};
use pulldown_cmark::{CodeBlockKind, Event, Options, Parser, Tag, TagEnd};

/// Slowest parse a single note may take.
const MAX_PARSE: Duration = Duration::from_secs(2);
/// Every Nth note of the big vaults is sampled.
const SAMPLE_STRIDE: usize = 10;
/// Most paths a report lists.
const REPORT_MAX: usize = 15;

fn vault(name: &str) -> PathBuf {
    PathBuf::from(std::env::var("HOME").unwrap_or_default()).join("Documents/vaults").join(name)
}

fn notes(dir: &Path, stride: usize) -> Vec<PathBuf> {
    let mut out = Vec::new();
    let mut stack = vec![dir.to_path_buf()];
    while let Some(next) = stack.pop() {
        let Ok(entries) = std::fs::read_dir(&next) else { continue };
        for entry in entries.flatten() {
            let path = entry.path();
            if path.is_dir() {
                stack.push(path);
            } else if path.extension().is_some_and(|e| e == "md") {
                out.push(path);
            }
        }
    }
    out.sort();
    out.into_iter().step_by(stride).collect()
}

/// Content lines of fenced code blocks according to the oracle.
fn oracle(src: &str) -> usize {
    let mut inside = false;
    let mut lines = 0;
    for event in Parser::new_ext(src, Options::ENABLE_TABLES | Options::ENABLE_FOOTNOTES | Options::ENABLE_TASKLISTS) {
        match event {
            Event::Start(Tag::CodeBlock(CodeBlockKind::Fenced(_))) => inside = true,
            Event::End(TagEnd::CodeBlock) => inside = false,
            Event::Text(text) if inside => lines += text.matches('\n').count() + usize::from(!text.ends_with('\n') && !text.is_empty()),
            _ => {}
        }
    }
    lines
}

/// The scanner this repo used before the fence rule: character only, closes at the first line of the same character.
fn legacy(src: &str) -> usize {
    let mut open: Option<char> = None;
    let mut lines = 0;
    for line in src.lines() {
        let t = line.trim_start();
        let token = t.chars().next().filter(|c| ['`', '~'].contains(c)).filter(|c| t.chars().take_while(|x| x == c).count() >= 3);
        match (open, token) {
            (Some(o), Some(c)) if o == c => open = None,
            (Some(_), _) => lines += 1,
            (None, Some(c)) => open = Some(c),
            (None, None) => {}
        }
    }
    lines
}

fn check(name: &str, files: &[PathBuf]) {
    let (mut now_bad, mut old_bad, mut slowest) = (Vec::new(), 0, Duration::ZERO);
    for path in files {
        let Ok(src) = std::fs::read_to_string(path) else { continue };
        let started = Instant::now();
        let doc = parse(&src);
        let took = started.elapsed();
        slowest = slowest.max(took);
        assert!(took < MAX_PARSE, "{} took {took:?}", path.display());
        assert_eq!(doc.source, src);
        let mine: usize = fenced_blocks(&src).iter().map(|b| b.body_lines).sum();
        let want = oracle(&src);
        if mine != want {
            now_bad.push(format!("{} mine={mine} oracle={want}", path.display()));
        }
        old_bad += usize::from(legacy(&src) != want);
    }
    println!("{name}: {} notes, slowest parse {slowest:?}, fence disagreements with the oracle: before {old_bad}, now {}", files.len(), now_bad.len());
    for line in now_bad.iter().take(REPORT_MAX) {
        println!("  {line}");
    }
    assert!(now_bad.len() * 100 <= files.len(), "{name}: more than 1% of the notes disagree");
}

#[test]
#[ignore = "walks ~/Documents/vaults"]
fn real_vaults_parse_and_agree_on_fences() {
    for (name, dir, stride) in [
        ("obsidian-help-all/en", vault("obsidian-help-all/en"), 1),
        ("mdn-docs", vault("mdn-docs"), SAMPLE_STRIDE),
        ("k8s-docs", vault("k8s-docs"), SAMPLE_STRIDE),
    ] {
        if !dir.is_dir() {
            println!("{name}: missing, skipped");
            continue;
        }
        check(name, &notes(&dir, stride));
    }
}