//! Hostile input for the syntax crate: random BMP text, huge blocks, deep nesting and nasty links. //! //! Nothing here may panic, escape its documented caps, or hand back a span outside its own line. use notescraft_syntax::fetch::curl_args; use notescraft_syntax::highlight::{MAX_BLOCK_BYTES, MAX_BLOCK_LINES}; use notescraft_syntax::load::is_known; use notescraft_syntax::validate::{check_grammar, check_sha256, check_url}; use notescraft_syntax::{Error, Highlighted, Highlighter, TokenKind, scopes::classify, suggest_language}; /// Strings in the random sweep. const RANDOM_CASES: usize = 2000; /// Longest random string, in chars. const RANDOM_MAX: usize = 400; /// Smallest valid grammar, so the gate has one thing it must accept. const TINY: &str = "%YAML 1.2\n---\nname: Tiny\nfile_extensions: [tiny]\nscope: source.tiny\ncontexts:\n main:\n - match: '\\bfoo\\b'\n scope: keyword.control.tiny\n"; /// xorshift64: deterministic, tiny, no dependency. struct Rng(u64); impl Rng { fn next(&mut self) -> u64 { let mut x = self.0; x ^= x << 13; x ^= x >> 7; x ^= x << 17; self.0 = x; x } /// One char from the whole BMP (NUL, combining marks, unpaired-ish shapes included). fn bmp_char(&mut self) -> char { let mut n = u32::try_from(self.next() % 0x1_0000).unwrap_or(0x20); if (0xd800..0xe000).contains(&n) { n = 0x20; } char::from_u32(n).unwrap_or('?') } fn text(&mut self, len: usize) -> String { (0..len).map(|_| self.bmp_char()).collect() } } /// Lines as the highlighter counts them (`split_inclusive`). fn line_count(body: &str) -> usize { body.split_inclusive('\n').count() } /// Every span must sit inside its line, on char boundaries, sorted and non-overlapping. fn assert_spans_sane(body: &str, highlighted: &Highlighted) { for (offset, spans) in highlighted.lines.iter().enumerate() { let Some(line) = body.split_inclusive('\n').nth(highlighted.first_line.saturating_add(offset)) else { assert_eq!(offset, 0, "more highlighted lines than the body has"); continue; }; let content = line.trim_end_matches(['\n', '\r']); let mut previous_end = 0; for span in spans { assert!(span.start < span.end, "empty span in {content:?}"); assert!(span.end <= content.len(), "span past the line in {content:?}"); assert!(content.is_char_boundary(span.start) && content.is_char_boundary(span.end), "span not on char boundaries in {content:?}"); assert!(span.start >= previous_end, "unsorted or overlapping spans in {content:?}: {spans:?}"); assert!(TokenKind::ALL.contains(&span.kind), "unknown kind {:?}", span.kind); previous_end = span.end; } } } /// `rust` blocks are refused only over the documented caps; anything returned is sane. fn assert_highlight_is_capped_or_sane(body: &str) { let over_caps = body.len() > MAX_BLOCK_BYTES || line_count(body) > MAX_BLOCK_LINES; match Highlighter::new("rust", body) { None => assert!(over_caps, "a block inside the caps was refused ({body:.40})"), Some(mut highlighter) => { assert!(!over_caps, "a block over the caps was accepted"); let out = highlighter.lines(0..line_count(body)); assert_eq!(out.lines.len(), line_count(body)); assert_spans_sane(body, &out); } } } fn assert_suggestion_is_known(label: &str) { if let Some(name) = suggest_language(label) { assert!(is_known(&name), "suggestion {name:?} for {label:.40?} is not a known language"); } } #[test] fn random_bmp_text_never_panics_and_keeps_every_invariant() { let mut rng = Rng(0x2545_f491_4f6c_dd1d); for _ in 0..RANDOM_CASES { let cap = u64::try_from(RANDOM_MAX).unwrap_or(u64::MAX).saturating_add(1); let len = usize::try_from(rng.next() % cap).unwrap_or(0); let text = rng.text(len); let _ = check_url(&text, None); let _ = check_url(&text, Some("git.example.dev")); let _ = check_sha256(text.as_bytes(), &text); let _ = check_grammar(&text, &text); let _ = curl_args(&text, 1024); assert_suggestion_is_known(&text); if let Some(kind) = classify(&text) { assert!(TokenKind::ALL.contains(&kind)); } assert_highlight_is_capped_or_sane(&text); } } #[test] fn huge_blocks_long_lines_and_deep_nesting_are_capped_not_crashed() { // A single line exactly at the byte cap is accepted and stays plain (over MAX_LINE_BYTES). let megabyte = "a".repeat(MAX_BLOCK_BYTES); assert!(Highlighter::new("rust", &megabyte).is_some()); assert_highlight_is_capped_or_sane(&megabyte); assert!(Highlighter::new("rust", &format!("{megabyte}a")).is_none(), "one byte over the cap is refused"); // 60k newlines: over the line cap, refused. Exactly the cap: accepted. let flood = "\n".repeat(MAX_BLOCK_LINES + 10_000); assert!(flood.len() <= MAX_BLOCK_BYTES && flood.len() == 60_000); assert!(Highlighter::new("rust", &flood).is_none(), "60k lines is over the line cap"); let at_cap = "\n".repeat(MAX_BLOCK_LINES); assert!(Highlighter::new("rust", &at_cap).is_some()); assert_highlight_is_capped_or_sane(&at_cap); // Both caps at once: small bytes, too many lines. let lines_under_bytes: String = "a\n".repeat(MAX_BLOCK_LINES + 10_000); assert!(lines_under_bytes.len() <= MAX_BLOCK_BYTES); assert!(Highlighter::new("rust", &lines_under_bytes).is_none()); // 10k nested brackets in one line, and the same spread over many lines. let deep = format!("{}x{}", "(".repeat(10_000), ")".repeat(10_000)); assert_highlight_is_capped_or_sane(&deep); let deep_lines = "(".repeat(10_000).replace('(', "(\n"); assert_highlight_is_capped_or_sane(&deep_lines); let mixed = format!("{}{}", "\u{0}\u{feff}".repeat(500), "\n".repeat(1_000)); assert_highlight_is_capped_or_sane(&mixed); // Windows newlines and a body with no trailing newline. assert_highlight_is_capped_or_sane("let a = 1;\r\nfn b() {}\r\n"); assert_highlight_is_capped_or_sane("fn tail() {}"); assert_highlight_is_capped_or_sane(""); } #[test] fn weird_labels_never_panic_and_suggestions_are_always_known() { let mut rng = Rng(0x9e37_79b9_7f4a_7c15); let mut labels: Vec = vec![ String::new(), " ".to_owned(), "\0".to_owned(), " \u{200b}\u{feff}".to_owned(), "\u{202e}rust".to_owned(), "rust\u{202e}".to_owned(), "\u{1f600}".to_owned(), "a".repeat(10 * 1024), "rsut".to_owned(), "RUST".to_owned(), "\t\n".to_owned(), ]; for _ in 0..200 { labels.push(rng.text(64)); } let body = "fn main() { let s = \"x\"; 1 }\n"; for label in &labels { assert_suggestion_is_known(label); if let Some(kind) = classify(label) { assert!(TokenKind::ALL.contains(&kind)); } if let Some(mut highlighter) = Highlighter::new(label, body) { assert_spans_sane(body, &highlighter.lines(0..line_count(body))); } assert_eq!(curl_args(label, 10).first().map(String::as_str), Some("--disable"), "curl locks down first"); } assert_eq!(suggest_language(""), None); assert_eq!(suggest_language(&"a".repeat(10 * 1024)), None); } #[test] fn credential_ip_localhost_and_non_https_links_are_refused() { for url in [ "https://good.com@evil.com/", "https://evil.com\\@good.com/", "https://2130706433/", "https://127.0.0.1/", "https://[::1]/", "https://localhost/", "https://app.localhost/", "http://example.com/a", "ftp://example.com/a", "HTTPS://example.com/a", "https://example.com:8443/", "https:///a", "https://intranet/a", "", "https://example.com/a b", "https://example.com/\u{0}", ] { assert!(check_url(url, None).is_err(), "{url} must be refused"); } assert_eq!(check_url("https://git.example.dev/a/raw.json", None), Ok("git.example.dev")); assert_eq!(check_url("https://git.example.dev/a", Some("GIT.EXAMPLE.DEV")), Ok("git.example.dev")); assert!(check_url(&format!("https://a.b/{}", "%".repeat(5000)), None).is_err(), "percent flood"); assert!(check_url(&format!("https://a.b/{}", "a".repeat(3000)), None).is_err(), "long path"); assert!(check_url("https://other.dev/a", Some("git.example.dev")).is_err(), "host mismatch"); assert_eq!(check_url("https://a.b/", Some("a.b")), Ok("a.b")); } #[test] fn unicode_hosts_and_hostile_urls_reach_the_gate_without_panicking() { let mut rng = Rng(0xdead_beef_cafe_f00d); for _ in 0..500 { let host: String = rng.text(8); let url = format!("https://{host}.example.dev/a"); let _ = check_url(&url, None); let _ = check_url(&url, Some("x.example.dev")); let args = curl_args(&url, 4096); assert_eq!(args.last().map(String::as_str), Some(url.as_str()), "the url is one argument"); } for url in ["https://пример.рф/a", "https://exämple.com/a", "https://xn--e1afmkfd.xn--p1ai/a", "https://a.b/\u{202e}"] { let _ = check_url(url, None); assert_eq!(curl_args(url, 1).first().map(String::as_str), Some("--disable"), "curl locks down first"); } } #[test] fn curl_args_stay_locked_down_for_hostile_urls() { let flood = format!("https://a.b/{}", "%".repeat(5000)); for url in [flood.clone(), "https://good.com@evil.com/".to_owned(), "\u{0}\n--next".to_owned(), "https://a.b/x".to_owned()] { let args = curl_args(&url, 4096); let joined = args.join(" "); for flag in ["--disable", "-sS", "--fail", "--proto =https", "--proto-redir =https", "--max-redirs 0", "--max-filesize 4096", "--connect-timeout", "--max-time"] { assert!(joined.contains(flag), "{flag} missing from {joined}"); } assert_eq!(args.iter().filter(|arg| *arg == "--url").count(), 1, "exactly one --url"); assert_eq!(args.last().map(String::as_str), Some(url.as_str()), "the url is the last argument, unsplit"); } } #[test] fn the_grammar_and_checksum_gates_survive_hostile_text() { assert_eq!(check_grammar(TINY, "foo bar\n"), Ok("Tiny".to_owned()), "the one good grammar"); assert!(check_grammar(&" ".repeat(600 * 1024), "").is_err(), "over the size cap"); assert!(check_grammar(&"(".repeat(10_000), &"(".repeat(10_000)).is_err(), "deep junk"); for text in ["\u{0}\u{1}\u{feff}", "", "- -\n- -", "\t", "{}", "contexts:\n main:\n - match: '('"] { let _ = check_grammar(text, &"x\n".repeat(70_000)); } assert_eq!(check_sha256(b"abc", "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad"), Ok(())); let bad_hashes: Vec = vec![String::new(), " ".to_owned(), "\u{0}".to_owned(), "g".repeat(64), "0".repeat(64)]; for bad in &bad_hashes { assert_eq!(check_sha256(b"abc", bad), Err(Error::Checksum), "{bad:.8}"); } let mut rng = Rng(0x0123_4567_89ab_cdef); for _ in 0..200 { let hex: String = (0..64).map(|_| char::from_digit((rng.next() % 16) as u32, 16).unwrap_or('0')).collect(); let _ = check_sha256(b"abc", &hex); let bytes = rng.text(16).into_bytes(); let _ = check_sha256(&bytes, &hex); } } #[test] fn trailing_dot_hides_localhost() { assert!(check_url("https://localhost./", None).is_err(), "https://localhost./ must be refused like https://localhost/"); } #[test] fn malformed_hosts_are_accepted() { for url in ["https://.example.com/a", "https://a..b/c", "https://.localhost/"] { assert!(check_url(url, None).is_err(), "{url} is not a valid host"); } } #[test] fn control_characters_in_a_link_are_refused_and_path_dots_are_the_servers_business() { assert!(check_url("https://example.com/a#\nhttps://evil.com/", None).is_err()); // `..` and `%2e%2e` in the path only name a file on that one host: curl sends them as given, so they are not an SSRF vector. for url in ["https://example.com/%2e%2e/a", "https://example.com/../../etc/passwd"] { assert!(check_url(url, None).is_ok(), "{url}"); } }