Offline, privacy-first grammar checker. Fast, open-source, Rust-powered
feat: lex regish-like things found in code comments
hippietrailf277857bdd9d
diff --git a/harper-core/src/lexing/mod.rs b/harper-core/src/lexing/mod.rsindex 2b40adf70..7fac0e7b8 100644--- a/harper-core/src/lexing/mod.rs+++ b/harper-core/src/lexing/mod.rs@@ -21,6 +21,7 @@ pub struct FoundToken {pub fn lex_token(source: &[char]) -> Option<FoundToken> {let lexers = [+ lex_regexish,lex_punctuation,lex_tabs,lex_spaces,@@ -98,6 +99,47 @@ pub fn lex_number(source: &[char]) -> Option<FoundToken> {None}+// Often in comments we mention partial- or pseudo- regexes. Here's an example from Ghidra:+// ([a-z0-9]+ only) - We previously flagged just the z0 in the middle of it.+pub fn lex_regexish(src: &[char]) -> Option<FoundToken> {+ eprintln!("** lex_regexish: {}", src.iter().collect::<String>());++ let l = src.len();+ let mut i = 0;++ if i >= l || src[i] != '[' {+ return None;+ }+ i += 1;++ loop {+ if i >= l || !src[i].is_alphanumeric() {+ return None;+ }+ i += 1;+ if i < l && src[i] == '-' {+ i += 1;+ if i >= l || !src[i].is_alphanumeric() {+ return None;+ }+ i += 1;+ }++ if i >= l || src[i] != ']' {+ continue;+ }+ i += 1;+ break;+ }++ return Some(FoundToken {+ token: TokenKind::Regexish,+ next_index: i + 1,+ });++ // return None;+}+pub fn lex_hex_number(source: &[char]) -> Option<FoundToken> {// < 3 to avoid accepting 0x aloneif source.len() < 3 || source[0] != '0' || source[1] != 'x' || !source[2].is_ascii_hexdigit() {@@ -238,6 +280,151 @@ mod tests {));}+ #[test]+ fn doesnt_lex_regex_mini_range() {+ let source: Vec<_> = "[]".chars().collect();+ assert!(!matches!(+ lex_token(&source),+ Some(FoundToken {+ token: TokenKind::Regexish,+ ..+ })+ ))+ }++ #[test]+ fn lexes_regex_one_letter() {+ let source: Vec<_> = "[a]".chars().collect();+ assert!(matches!(+ lex_token(&source),+ Some(FoundToken {+ token: TokenKind::Regexish,+ ..+ })+ ));+ }++ #[test]+ fn lexes_regex_two_letters() {+ let source: Vec<_> = "[az]".chars().collect();+ assert!(matches!(+ lex_token(&source),+ Some(FoundToken {+ token: TokenKind::Regexish,+ ..+ })+ ));+ }++ #[test]+ fn lexes_regex_digits() {+ let source: Vec<_> = "[123]".chars().collect();+ assert!(matches!(+ lex_token(&source),+ Some(FoundToken {+ token: TokenKind::Regexish,+ ..+ })+ ));+ }++ #[test]+ fn lexes_regex_two_alphanumeric() {+ let source: Vec<_> = "[a0b1c2]".chars().collect();+ assert!(matches!(+ lex_token(&source),+ Some(FoundToken {+ token: TokenKind::Regexish,+ ..+ })+ ));+ }++ #[test]+ fn lexes_regex_one_range() {+ let source: Vec<_> = "[a-z]".chars().collect();+ assert!(matches!(+ lex_token(&source),+ Some(FoundToken {+ token: TokenKind::Regexish,+ ..+ })+ ));+ }++ #[test]+ fn lexes_regex_letter_plus_range() {+ let source: Vec<_> = "[ax-z]".chars().collect();+ assert!(matches!(+ lex_token(&source),+ Some(FoundToken {+ token: TokenKind::Regexish,+ ..+ })+ ));+ }++ #[test]+ fn lexes_regex_range_plus_letter() {+ let source: Vec<_> = "[a-cz]".chars().collect();+ assert!(matches!(+ lex_token(&source),+ Some(FoundToken {+ token: TokenKind::Regexish,+ ..+ })+ ));+ }++ #[test]+ fn lexes_regex_two_ranges() {+ let source: Vec<_> = "[a-cx-z]".chars().collect();+ assert!(matches!(+ lex_token(&source),+ Some(FoundToken {+ token: TokenKind::Regexish,+ ..+ })+ ));+ }++ #[test]+ fn doesnt_lex_regex_broken_two_ranges() {+ // you can't end a range and start a range with a single letter+ let source: Vec<_> = "[a-x-z]".chars().collect();+ assert!(!matches!(+ lex_token(&source),+ Some(FoundToken {+ token: TokenKind::Regexish,+ ..+ })+ ));+ }++ #[test]+ fn doesnt_lex_regex_hyphen_at_start() {+ let source: Vec<_> = "[a-]".chars().collect();+ assert!(!matches!(+ lex_token(&source),+ Some(FoundToken {+ token: TokenKind::Regexish,+ ..+ })+ ));+ }++ #[test]+ fn doesnt_lex_regex_hyphen_at_end() {+ let source: Vec<_> = "[-z]".chars().collect();+ assert!(!matches!(+ lex_token(&source),+ Some(FoundToken {+ token: TokenKind::Regexish,+ ..+ })+ ));+ }+#[test]fn lexes_good_hex_numeric() {let source: Vec<_> = "0x0".chars().collect();diff --git a/harper-core/src/token_kind.rs b/harper-core/src/token_kind.rsindex 0e1acd857..02c6a27fd 100644--- a/harper-core/src/token_kind.rs+++ b/harper-core/src/token_kind.rs@@ -23,6 +23,7 @@ pub enum TokenKind {#[default]Unlintable,ParagraphBreak,+ Regexish,}impl TokenKind {