notescraft/crates/textdiff/tests/word_diff.rs

//! Word-level diff of two texts.

use notescraft_textdiff::{Error, Limits, Op, Span, stats, word_diff};

/// A small linear congruential generator, so the cases are the same on every
/// run.
struct Lcg(u64);

impl Lcg {
    fn new(seed: u64) -> Self {
        Self(seed)
    }

    fn next(&mut self) -> u64 {
        self.0 = self.0.wrapping_mul(6_364_136_223_846_793_005).wrapping_add(1_442_695_040_888_963_407);
        self.0 >> 33
    }

    fn below(&mut self, limit: usize) -> usize {
        let limit = u64::try_from(limit).unwrap_or(1);
        usize::try_from(self.next() % limit).unwrap_or(0)
    }
}

const WORDS: [&str; 8] = ["alpha", "beta", "gamma", "delta", "eps", "zeta", "eta", "theta"];

fn random_text(rng: &mut Lcg) -> String {
    let count = rng.below(12);
    let mut out = String::new();
    for index in 0..count {
        if index > 0 {
            out.push(' ');
        }
        out.push_str(WORDS.get(rng.below(WORDS.len())).copied().unwrap_or("word"));
        if rng.below(6) == 0 {
            out.push(',');
        }
    }
    out
}

/// The old text of a diff: the same spans and the deletions, in order.
fn old_text(spans: &[Span]) -> String {
    let mut out = String::new();
    for span in spans {
        match span.op {
            Op::Same | Op::Del => out.push_str(&span.text),
            Op::Add => {}
        }
    }
    out
}

/// The new text of a diff: the same spans and the additions, in order.
fn new_text(spans: &[Span]) -> String {
    let mut out = String::new();
    for span in spans {
        match span.op {
            Op::Same | Op::Add => out.push_str(&span.text),
            Op::Del => {}
        }
    }
    out
}

fn count(spans: &[Span], op: Op) -> usize {
    spans.iter().filter(|span| span.op == op).count()
}

#[test]
fn reproduces_both_texts_for_a_thousand_pairs() {
    let mut rng = Lcg::new(0x5EED_1234);
    for _ in 0..1000 {
        let old = random_text(&mut rng);
        let new = if rng.below(2) == 0 { random_text(&mut rng) } else { old.clone() };
        let spans = word_diff(&old, &new, &Limits::default()).expect("a diff");
        assert_eq!(old_text(&spans), old, "the old text must be reproduced");
        assert_eq!(new_text(&spans), new, "the new text must be reproduced");
    }
}

#[test]
fn one_changed_word_gives_one_deletion_and_one_addition() {
    let old = "the quick brown fox jumps over the lazy dog and keeps running until the end of the paragraph";
    let new = old.replace("brown", "red");
    let spans = word_diff(old, &new, &Limits::default()).expect("a diff");
    assert_eq!(count(&spans, Op::Del), 1);
    assert_eq!(count(&spans, Op::Add), 1);
    assert_eq!(old_text(&spans), old);
    assert_eq!(new_text(&spans), new);
    assert_eq!(stats(&spans), (1, 1));
}

#[test]
fn inserts_at_the_start_and_at_the_end() {
    let old = "middle";
    let new = "start middle end";
    let spans = word_diff(old, new, &Limits::default()).expect("a diff");
    assert_eq!(old_text(&spans), old);
    assert_eq!(new_text(&spans), new);
    assert_eq!(count(&spans, Op::Add), 2);
    assert_eq!(count(&spans, Op::Same), 1);
    assert_eq!(stats(&spans), (2, 0));
}

#[test]
fn an_empty_old_text_is_all_additions() {
    let spans = word_diff("", "hello world", &Limits::default()).expect("a diff");
    assert_eq!(count(&spans, Op::Add), 1);
    assert_eq!(count(&spans, Op::Del), 0);
    assert_eq!(new_text(&spans), "hello world");
    assert_eq!(old_text(&spans), "");
}

#[test]
fn an_empty_new_text_is_all_deletions() {
    let spans = word_diff("hello world", "", &Limits::default()).expect("a diff");
    assert_eq!(count(&spans, Op::Del), 1);
    assert_eq!(count(&spans, Op::Add), 0);
    assert_eq!(old_text(&spans), "hello world");
    assert_eq!(new_text(&spans), "");
}

#[test]
fn two_empty_texts_give_no_spans() {
    let spans = word_diff("", "", &Limits::default()).expect("a diff");
    assert_eq!(spans, Vec::<Span>::new());
    assert_eq!(stats(&spans), (0, 0));
}

#[test]
fn carriage_returns_stay_in_the_whitespace() {
    let old = "first\r\nsecond\r\n";
    let new = "first\r\nthird\r\n";
    let spans = word_diff(old, new, &Limits::default()).expect("a diff");
    assert_eq!(old_text(&spans), old);
    assert_eq!(new_text(&spans), new);
    assert_eq!(count(&spans, Op::Del), 1);
    assert_eq!(count(&spans, Op::Add), 1);
}

#[test]
fn emoji_and_combining_marks_are_never_lost() {
    let old = "cafe\u{0301} \u{1F600} e\u{0301}";
    let new = "caf\u{e9} \u{1F600} \u{e8}";
    let spans = word_diff(old, new, &Limits::default()).expect("a diff");
    assert_eq!(old_text(&spans), old);
    assert_eq!(new_text(&spans), new);
}

#[test]
fn too_many_bytes_is_refused() {
    let limits = Limits { max_bytes: 8, ..Limits::default() };
    let result = word_diff("a longer old text", "a longer new text", &limits);
    assert!(matches!(result, Err(Error::TooLarge)));
}

#[test]
fn too_many_tokens_is_refused() {
    let limits = Limits { max_tokens: 4, ..Limits::default() };
    let result = word_diff("one two three four five", "one two three four five", &limits);
    assert!(matches!(result, Err(Error::TooLarge)));
}

#[test]
fn too_many_cells_is_refused() {
    let limits = Limits { max_cells: 0, ..Limits::default() };
    let result = word_diff("one two", "three four", &limits);
    assert!(matches!(result, Err(Error::TooLarge)));
}

#[test]
fn the_default_limits_accept_a_normal_note() {
    let note = "a note with a few hundred words, repeated for a while to look like a real note in a vault";
    let spans = word_diff(note, note, &Limits::default()).expect("a diff");
    assert_eq!(count(&spans, Op::Same), 1);
    assert_eq!(count(&spans, Op::Add), 0);
    assert_eq!(count(&spans, Op::Del), 0);
}

#[test]
#[ignore = "slow: two texts of 2 MiB"]
fn two_big_texts_are_fast_or_too_large() {
    let big = "word ".repeat(400_000);
    let start = std::time::Instant::now();
    let result = word_diff(big.as_str(), big.as_str(), &Limits::default());
    let elapsed = start.elapsed();
    match result {
        Ok(spans) => {
            assert_eq!(count(&spans, Op::Same), 1);
            assert!(elapsed.as_millis() < 500, "the diff took {elapsed:?}");
        }
        Err(Error::TooLarge) => {}
    }
}