notescraft/crates/htmlmd/tests/tokens.rs

//! Tests of the forgiving HTML tokenizer.

use notescraft_htmlmd::{Error, Token, decode_entities, tokenize};

/// Build an open token.
fn open(name: &str, attrs: &[(&str, &str)], self_closing: bool) -> Token {
    Token::Open { name: name.to_string(), attrs: attrs.iter().map(|(n, v)| (n.to_string(), v.to_string())).collect(), self_closing }
}

#[test]
fn tags_attributes_and_text() {
    let tokens = tokenize("<p class=\"a b\">Hi &amp; bye</p>", 64).expect("tokens");
    assert_eq!(tokens, vec![open("p", &[("class", "a b")], false), Token::Text("Hi & bye".to_string()), Token::Close("p".to_string()),]);
}

#[test]
fn attribute_values_may_be_unquoted_or_single_quoted() {
    let tokens = tokenize("<a href=/foo title='x y'>t</a>", 64).expect("tokens");
    assert_eq!(tokens[0], open("a", &[("href", "/foo"), ("title", "x y")], false));
}

#[test]
fn attribute_values_are_decoded() {
    let tokens = tokenize("<a href=\"?a=1&amp;b=2\">t</a>", 64).expect("tokens");
    assert_eq!(tokens[0], open("a", &[("href", "?a=1&b=2")], false));
}

#[test]
fn a_duplicate_attribute_keeps_the_first_value() {
    let tokens = tokenize("<p class=\"a\" class=\"b\">x</p>", 64).expect("tokens");
    assert_eq!(tokens[0], open("p", &[("class", "a")], false));
}

#[test]
fn names_are_lowercased() {
    let tokens = tokenize("<P CLASS=\"A\">x</P>", 64).expect("tokens");
    assert_eq!(tokens[0], open("p", &[("class", "A")], false));
    assert_eq!(tokens[2], Token::Close("p".to_string()));
}

#[test]
fn void_elements_are_self_closing() {
    let tokens = tokenize("<br><img src=\"x\"><hr/>", 64).expect("tokens");
    assert_eq!(tokens[0], open("br", &[], true));
    assert_eq!(tokens[1], open("img", &[("src", "x")], true));
    assert_eq!(tokens[2], open("hr", &[], true));
}

#[test]
fn script_content_is_raw() {
    let tokens = tokenize("<script>if (a < b) { x(); }</script>", 64).expect("tokens");
    assert_eq!(tokens, vec![open("script", &[], false), Token::Text("if (a < b) { x(); }".to_string()), Token::Close("script".to_string()),]);
}

#[test]
fn style_content_is_raw() {
    let tokens = tokenize("<style>a > b { color: red }</style>", 64).expect("tokens");
    assert_eq!(tokens[1], Token::Text("a > b { color: red }".to_string()));
}

#[test]
fn comments_are_returned() {
    let tokens = tokenize("<!-- hi -->", 64).expect("tokens");
    assert_eq!(tokens, vec![Token::Comment(" hi ".to_string())]);
}

#[test]
fn conditional_comments_are_returned() {
    let html = "<!--[if gte mso 9]><p>x</p><![endif]-->";
    let tokens = tokenize(html, 64).expect("tokens");
    assert_eq!(tokens, vec![Token::Comment("[if gte mso 9]><p>x</p><![endif]".to_string())]);
}

#[test]
fn declarations_are_dropped() {
    let html = "<!DOCTYPE html><?xml version=\"1.0\"?><![CDATA[ raw ]]>text";
    let tokens = tokenize(html, 64).expect("tokens");
    assert_eq!(tokens, vec![Token::Text("text".to_string())]);
}

#[test]
fn unterminated_markup_is_text() {
    let tokens = tokenize("<p class=\"abc", 64).expect("tokens");
    assert_eq!(tokens, vec![Token::Text("<p class=\"abc".to_string())]);

    let tokens = tokenize("<p class=x", 64).expect("tokens");
    assert_eq!(tokens, vec![Token::Text("<p class=x".to_string())]);

    let tokens = tokenize("<!-- oops", 64).expect("tokens");
    assert_eq!(tokens, vec![Token::Text("<!-- oops".to_string())]);
}

#[test]
fn a_bare_angle_bracket_is_text() {
    let tokens = tokenize("a < b", 64).expect("tokens");
    assert_eq!(tokens, vec![Token::Text("a < b".to_string())]);

    let tokens = tokenize("<3", 64).expect("tokens");
    assert_eq!(tokens, vec![Token::Text("<3".to_string())]);
}

#[test]
fn named_entities_are_decoded() {
    let tokens = tokenize("&amp; &lt; &gt; &quot; &apos; &nbsp; &hellip; &mdash; &copy;", 64).expect("tokens");
    assert_eq!(tokens, vec![Token::Text("& < > \" ' \u{a0} … — ©".to_string())]);
}

#[test]
fn numeric_entities_are_decoded() {
    let tokens = tokenize("&#39; &#x27; &#128512;", 64).expect("tokens");
    assert_eq!(tokens, vec![Token::Text("' ' 😀".to_string())]);
}

#[test]
fn invalid_and_unknown_entities_stay_verbatim() {
    let tokens = tokenize("&#0; &#xD800; &#x110000; &nope;", 64).expect("tokens");
    assert_eq!(tokens, vec![Token::Text("&#0; &#xD800; &#x110000; &nope;".to_string())]);
}

#[test]
fn emoji_survive() {
    let tokens = tokenize("hello 😀", 64).expect("tokens");
    assert_eq!(tokens, vec![Token::Text("hello 😀".to_string())]);
}

#[test]
fn decode_entities_is_public() {
    assert_eq!(decode_entities("a &amp; b"), "a & b");
}

#[test]
fn too_large_input_is_rejected() {
    let html = "a".repeat(3 * 1024 * 1024);
    let err = tokenize(&html, 64).expect_err("too large");
    assert!(matches!(err, Error::TooLarge));
}

#[test]
fn the_token_cap_is_enforced() {
    let err = tokenize("<p>a</p>", 2).expect_err("too many tokens");
    assert!(matches!(err, Error::TooLarge));
    let tokens = tokenize("<p>a</p>", 3).expect("three tokens");
    assert_eq!(tokens.len(), 3);
}

/// A tiny linear congruential generator, so the soups are reproducible.
struct Lcg(u64);

impl Lcg {
    fn next(&mut self) -> u64 {
        self.0 = self.0.wrapping_mul(6_364_136_223_846_793_005).wrapping_add(1_442_695_040_888_963_407);
        self.0 >> 33
    }
}

#[test]
fn random_soups_never_panic() {
    const ALPHABET: &[char] = &['<', '>', '/', '=', '"', '\'', ' ', 'a', '&', ';', '!', '-'];
    let mut rng = Lcg(0x5eed);
    for _ in 0..5000 {
        let len = usize::try_from(rng.next() % 64).unwrap_or(0);
        let mut html = String::new();
        for _ in 0..len {
            let index = usize::try_from(rng.next()).unwrap_or(0) % ALPHABET.len();
            if let Some(ch) = ALPHABET.get(index) {
                html.push(*ch);
            }
        }
        let _ = tokenize(&html, 4096);
    }
}