notescraft/crates/syntax/tests/hostile.rs

//! Hostile input for the syntax crate: random BMP text, huge blocks, deep nesting and nasty links.
//!
//! Nothing here may panic, escape its documented caps, or hand back a span outside its own line.

use notescraft_syntax::fetch::curl_args;
use notescraft_syntax::highlight::{MAX_BLOCK_BYTES, MAX_BLOCK_LINES};
use notescraft_syntax::load::is_known;
use notescraft_syntax::validate::{check_grammar, check_sha256, check_url};
use notescraft_syntax::{Error, Highlighted, Highlighter, TokenKind, scopes::classify, suggest_language};

/// Strings in the random sweep.
const RANDOM_CASES: usize = 2000;
/// Longest random string, in chars.
const RANDOM_MAX: usize = 400;
/// Smallest valid grammar, so the gate has one thing it must accept.
const TINY: &str = "%YAML 1.2\n---\nname: Tiny\nfile_extensions: [tiny]\nscope: source.tiny\ncontexts:\n  main:\n    - match: '\\bfoo\\b'\n      scope: keyword.control.tiny\n";

/// xorshift64: deterministic, tiny, no dependency.
struct Rng(u64);

impl Rng {
    fn next(&mut self) -> u64 {
        let mut x = self.0;
        x ^= x << 13;
        x ^= x >> 7;
        x ^= x << 17;
        self.0 = x;
        x
    }

    /// One char from the whole BMP (NUL, combining marks, unpaired-ish shapes included).
    fn bmp_char(&mut self) -> char {
        let mut n = u32::try_from(self.next() % 0x1_0000).unwrap_or(0x20);
        if (0xd800..0xe000).contains(&n) {
            n = 0x20;
        }
        char::from_u32(n).unwrap_or('?')
    }

    fn text(&mut self, len: usize) -> String {
        (0..len).map(|_| self.bmp_char()).collect()
    }
}

/// Lines as the highlighter counts them (`split_inclusive`).
fn line_count(body: &str) -> usize {
    body.split_inclusive('\n').count()
}

/// Every span must sit inside its line, on char boundaries, sorted and non-overlapping.
fn assert_spans_sane(body: &str, highlighted: &Highlighted) {
    for (offset, spans) in highlighted.lines.iter().enumerate() {
        let Some(line) = body.split_inclusive('\n').nth(highlighted.first_line.saturating_add(offset)) else {
            assert_eq!(offset, 0, "more highlighted lines than the body has");
            continue;
        };
        let content = line.trim_end_matches(['\n', '\r']);
        let mut previous_end = 0;
        for span in spans {
            assert!(span.start < span.end, "empty span in {content:?}");
            assert!(span.end <= content.len(), "span past the line in {content:?}");
            assert!(content.is_char_boundary(span.start) && content.is_char_boundary(span.end), "span not on char boundaries in {content:?}");
            assert!(span.start >= previous_end, "unsorted or overlapping spans in {content:?}: {spans:?}");
            assert!(TokenKind::ALL.contains(&span.kind), "unknown kind {:?}", span.kind);
            previous_end = span.end;
        }
    }
}

/// `rust` blocks are refused only over the documented caps; anything returned is sane.
fn assert_highlight_is_capped_or_sane(body: &str) {
    let over_caps = body.len() > MAX_BLOCK_BYTES || line_count(body) > MAX_BLOCK_LINES;
    match Highlighter::new("rust", body) {
        None => assert!(over_caps, "a block inside the caps was refused ({body:.40})"),
        Some(mut highlighter) => {
            assert!(!over_caps, "a block over the caps was accepted");
            let out = highlighter.lines(0..line_count(body));
            assert_eq!(out.lines.len(), line_count(body));
            assert_spans_sane(body, &out);
        }
    }
}

fn assert_suggestion_is_known(label: &str) {
    if let Some(name) = suggest_language(label) {
        assert!(is_known(&name), "suggestion {name:?} for {label:.40?} is not a known language");
    }
}

#[test]
fn random_bmp_text_never_panics_and_keeps_every_invariant() {
    let mut rng = Rng(0x2545_f491_4f6c_dd1d);
    for _ in 0..RANDOM_CASES {
        let cap = u64::try_from(RANDOM_MAX).unwrap_or(u64::MAX).saturating_add(1);
        let len = usize::try_from(rng.next() % cap).unwrap_or(0);
        let text = rng.text(len);
        let _ = check_url(&text, None);
        let _ = check_url(&text, Some("git.example.dev"));
        let _ = check_sha256(text.as_bytes(), &text);
        let _ = check_grammar(&text, &text);
        let _ = curl_args(&text, 1024);
        assert_suggestion_is_known(&text);
        if let Some(kind) = classify(&text) {
            assert!(TokenKind::ALL.contains(&kind));
        }
        assert_highlight_is_capped_or_sane(&text);
    }
}

#[test]
fn huge_blocks_long_lines_and_deep_nesting_are_capped_not_crashed() {
    // A single line exactly at the byte cap is accepted and stays plain (over MAX_LINE_BYTES).
    let megabyte = "a".repeat(MAX_BLOCK_BYTES);
    assert!(Highlighter::new("rust", &megabyte).is_some());
    assert_highlight_is_capped_or_sane(&megabyte);
    assert!(Highlighter::new("rust", &format!("{megabyte}a")).is_none(), "one byte over the cap is refused");

    // 60k newlines: over the line cap, refused. Exactly the cap: accepted.
    let flood = "\n".repeat(MAX_BLOCK_LINES + 10_000);
    assert!(flood.len() <= MAX_BLOCK_BYTES && flood.len() == 60_000);
    assert!(Highlighter::new("rust", &flood).is_none(), "60k lines is over the line cap");
    let at_cap = "\n".repeat(MAX_BLOCK_LINES);
    assert!(Highlighter::new("rust", &at_cap).is_some());
    assert_highlight_is_capped_or_sane(&at_cap);

    // Both caps at once: small bytes, too many lines.
    let lines_under_bytes: String = "a\n".repeat(MAX_BLOCK_LINES + 10_000);
    assert!(lines_under_bytes.len() <= MAX_BLOCK_BYTES);
    assert!(Highlighter::new("rust", &lines_under_bytes).is_none());

    // 10k nested brackets in one line, and the same spread over many lines.
    let deep = format!("{}x{}", "(".repeat(10_000), ")".repeat(10_000));
    assert_highlight_is_capped_or_sane(&deep);
    let deep_lines = "(".repeat(10_000).replace('(', "(\n");
    assert_highlight_is_capped_or_sane(&deep_lines);
    let mixed = format!("{}{}", "\u{0}\u{feff}".repeat(500), "\n".repeat(1_000));
    assert_highlight_is_capped_or_sane(&mixed);

    // Windows newlines and a body with no trailing newline.
    assert_highlight_is_capped_or_sane("let a = 1;\r\nfn b() {}\r\n");
    assert_highlight_is_capped_or_sane("fn tail() {}");
    assert_highlight_is_capped_or_sane("");
}

#[test]
fn weird_labels_never_panic_and_suggestions_are_always_known() {
    let mut rng = Rng(0x9e37_79b9_7f4a_7c15);
    let mut labels: Vec<String> = vec![
        String::new(),
        " ".to_owned(),
        "\0".to_owned(),
        "   \u{200b}\u{feff}".to_owned(),
        "\u{202e}rust".to_owned(),
        "rust\u{202e}".to_owned(),
        "\u{1f600}".to_owned(),
        "a".repeat(10 * 1024),
        "rsut".to_owned(),
        "RUST".to_owned(),
        "\t\n".to_owned(),
    ];
    for _ in 0..200 {
        labels.push(rng.text(64));
    }
    let body = "fn main() { let s = \"x\"; 1 }\n";
    for label in &labels {
        assert_suggestion_is_known(label);
        if let Some(kind) = classify(label) {
            assert!(TokenKind::ALL.contains(&kind));
        }
        if let Some(mut highlighter) = Highlighter::new(label, body) {
            assert_spans_sane(body, &highlighter.lines(0..line_count(body)));
        }
        assert_eq!(curl_args(label, 10).first().map(String::as_str), Some("--disable"), "curl locks down first");
    }
    assert_eq!(suggest_language(""), None);
    assert_eq!(suggest_language(&"a".repeat(10 * 1024)), None);
}

#[test]
fn credential_ip_localhost_and_non_https_links_are_refused() {
    for url in [
        "https://good.com@evil.com/",
        "https://evil.com\\@good.com/",
        "https://2130706433/",
        "https://127.0.0.1/",
        "https://[::1]/",
        "https://localhost/",
        "https://app.localhost/",
        "http://example.com/a",
        "ftp://example.com/a",
        "HTTPS://example.com/a",
        "https://example.com:8443/",
        "https:///a",
        "https://intranet/a",
        "",
        "https://example.com/a b",
        "https://example.com/\u{0}",
    ] {
        assert!(check_url(url, None).is_err(), "{url} must be refused");
    }
    assert_eq!(check_url("https://git.example.dev/a/raw.json", None), Ok("git.example.dev"));
    assert_eq!(check_url("https://git.example.dev/a", Some("GIT.EXAMPLE.DEV")), Ok("git.example.dev"));

    assert!(check_url(&format!("https://a.b/{}", "%".repeat(5000)), None).is_err(), "percent flood");
    assert!(check_url(&format!("https://a.b/{}", "a".repeat(3000)), None).is_err(), "long path");
    assert!(check_url("https://other.dev/a", Some("git.example.dev")).is_err(), "host mismatch");
    assert_eq!(check_url("https://a.b/", Some("a.b")), Ok("a.b"));
}

#[test]
fn unicode_hosts_and_hostile_urls_reach_the_gate_without_panicking() {
    let mut rng = Rng(0xdead_beef_cafe_f00d);
    for _ in 0..500 {
        let host: String = rng.text(8);
        let url = format!("https://{host}.example.dev/a");
        let _ = check_url(&url, None);
        let _ = check_url(&url, Some("x.example.dev"));
        let args = curl_args(&url, 4096);
        assert_eq!(args.last().map(String::as_str), Some(url.as_str()), "the url is one argument");
    }
    for url in ["https://пример.рф/a", "https://exämple.com/a", "https://xn--e1afmkfd.xn--p1ai/a", "https://a.b/\u{202e}"] {
        let _ = check_url(url, None);
        assert_eq!(curl_args(url, 1).first().map(String::as_str), Some("--disable"), "curl locks down first");
    }
}

#[test]
fn curl_args_stay_locked_down_for_hostile_urls() {
    let flood = format!("https://a.b/{}", "%".repeat(5000));
    for url in [flood.clone(), "https://good.com@evil.com/".to_owned(), "\u{0}\n--next".to_owned(), "https://a.b/x".to_owned()] {
        let args = curl_args(&url, 4096);
        let joined = args.join(" ");
        for flag in
            ["--disable", "-sS", "--fail", "--proto =https", "--proto-redir =https", "--max-redirs 0", "--max-filesize 4096", "--connect-timeout", "--max-time"]
        {
            assert!(joined.contains(flag), "{flag} missing from {joined}");
        }
        assert_eq!(args.iter().filter(|arg| *arg == "--url").count(), 1, "exactly one --url");
        assert_eq!(args.last().map(String::as_str), Some(url.as_str()), "the url is the last argument, unsplit");
    }
}

#[test]
fn the_grammar_and_checksum_gates_survive_hostile_text() {
    assert_eq!(check_grammar(TINY, "foo bar\n"), Ok("Tiny".to_owned()), "the one good grammar");
    assert!(check_grammar(&" ".repeat(600 * 1024), "").is_err(), "over the size cap");
    assert!(check_grammar(&"(".repeat(10_000), &"(".repeat(10_000)).is_err(), "deep junk");
    for text in ["\u{0}\u{1}\u{feff}", "<?xml version=\"1.0\"?><plist/>", "- -\n- -", "\t", "{}", "contexts:\n  main:\n    - match: '('"] {
        let _ = check_grammar(text, &"x\n".repeat(70_000));
    }
    assert_eq!(check_sha256(b"abc", "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad"), Ok(()));
    let bad_hashes: Vec<String> = vec![String::new(), " ".to_owned(), "\u{0}".to_owned(), "g".repeat(64), "0".repeat(64)];
    for bad in &bad_hashes {
        assert_eq!(check_sha256(b"abc", bad), Err(Error::Checksum), "{bad:.8}");
    }
    let mut rng = Rng(0x0123_4567_89ab_cdef);
    for _ in 0..200 {
        let hex: String = (0..64).map(|_| char::from_digit((rng.next() % 16) as u32, 16).unwrap_or('0')).collect();
        let _ = check_sha256(b"abc", &hex);
        let bytes = rng.text(16).into_bytes();
        let _ = check_sha256(&bytes, &hex);
    }
}
#[test]
fn trailing_dot_hides_localhost() {
    assert!(check_url("https://localhost./", None).is_err(), "https://localhost./ must be refused like https://localhost/");
}

#[test]
fn malformed_hosts_are_accepted() {
    for url in ["https://.example.com/a", "https://a..b/c", "https://.localhost/"] {
        assert!(check_url(url, None).is_err(), "{url} is not a valid host");
    }
}

#[test]
fn control_characters_in_a_link_are_refused_and_path_dots_are_the_servers_business() {
    assert!(check_url("https://example.com/a#\nhttps://evil.com/", None).is_err());
    // `..` and `%2e%2e` in the path only name a file on that one host: curl sends them as given, so they are not an SSRF vector.
    for url in ["https://example.com/%2e%2e/a", "https://example.com/../../etc/passwd"] {
        assert!(check_url(url, None).is_ok(), "{url}");
    }
}