Offline, privacy-first grammar checker. Fast, open-source, Rust-powered
fix: better ascii and hex digit checks
hippietrail10854ff3765e
diff --git a/harper-core/src/lexing/mod.rs b/harper-core/src/lexing/mod.rsindex 05912924f..790f5a900 100644--- a/harper-core/src/lexing/mod.rs+++ b/harper-core/src/lexing/mod.rs@@ -25,8 +25,8 @@ pub fn lex_token(source: &[char]) -> Option<FoundToken> {lex_tabs,lex_spaces,lex_newlines,- lex_hex_number, // before lex_number, which would match the initial 0- lex_long_decade, // before lex_number, which would match the digits up to the -s+ lex_hex_number, // Before lex_number, which would match the initial 0+ lex_long_decade, // Before lex_number, which would match the digits up to the -slex_number,lex_url,lex_email_address,@@ -47,7 +47,7 @@ pub fn lex_token(source: &[char]) -> Option<FoundToken> {fn lex_word(source: &[char]) -> Option<FoundToken> {let end = source.iter()- .position(|c| !c.is_english_lingual() && !c.is_numeric())+ .position(|c| !c.is_english_lingual() && !c.is_ascii_digit()).unwrap_or(source.len());if end == 0 {@@ -73,7 +73,7 @@ pub fn lex_number(source: &[char]) -> Option<FoundToken> {.iter().enumerate().rev()- .find_map(|(i, v)| v.is_numeric().then_some(i))?;+ .find_map(|(i, v)| v.is_ascii_digit().then_some(i))?;let mut s: String = source[0..end + 1].iter().collect();@@ -245,10 +245,108 @@ fn lex_catch(_source: &[char]) -> Option<FoundToken> {mod tests {use super::lex_hex_number;use super::lex_long_decade;+ use super::lex_number;use super::lex_token;use super::lex_word;use super::{FoundToken, TokenKind};+ // test various kinds of number+ #[test]+ fn lexes_0() {+ let source: Vec<_> = "0".chars().collect();+ assert!(matches!(+ lex_number(&source),+ Some(FoundToken {+ token: TokenKind::Number(_),+ ..+ })+ ));+ }++ #[test]+ fn lexes_0_point_0() {+ let source: Vec<_> = "0.0".chars().collect();+ assert!(matches!(+ lex_number(&source),+ Some(FoundToken {+ token: TokenKind::Number(_),+ ..+ })+ ));+ }++ #[test]+ fn lexes_00() {+ let source: Vec<_> = "00".chars().collect();+ assert!(matches!(+ lex_number(&source),+ Some(FoundToken {+ token: TokenKind::Number(_),+ ..+ })+ ));+ }++ // #[test]+ // fn lexes_negative_1() {+ // let source: Vec<_> = "-1".chars().collect();+ // assert!(matches!(+ // lex_number(&source),+ // Some(FoundToken {+ // token: TokenKind::Number(_),+ // ..+ // })+ // ));+ // }++ // #[test]+ // fn lexes_positive_1() {+ // let source: Vec<_> = "+1".chars().collect();+ // assert!(matches!(+ // lex_number(&source),+ // Some(FoundToken {+ // token: TokenKind::Number(_),+ // ..+ // })+ // ));+ // }++ #[test]+ fn lexes_pi() {+ let source: Vec<_> = "3.1415926535897932384626433832795028841971693993751058209749445923078164062862089986280348253421170679".chars().collect();+ assert!(matches!(+ lex_number(&source),+ Some(FoundToken {+ token: TokenKind::Number(_),+ ..+ })+ ));+ }++ #[test]+ fn lexes_speed_of_light() {+ let source: Vec<_> = "3.00e8".chars().collect();+ assert!(matches!(+ lex_number(&source),+ Some(FoundToken {+ token: TokenKind::Number(_),+ ..+ })+ ));+ }++ #[test]+ fn doesnt_lex_cjk_numeral() {+ let source: Vec<_> = "二".chars().collect();+ assert!(lex_number(&source).is_none());+ }++ #[test]+ fn doesnt_lex_thai_digit() {+ let source: Vec<_> = "๑".chars().collect();+ assert!(lex_number(&source).is_none());+ }+#[test]fn lexes_cjk_as_unlintable() {let source: Vec<_> = "世".chars().collect();diff --git a/harper-core/src/lexing/url.rs b/harper-core/src/lexing/url.rsindex c028f8e1a..b3ee8c6ea 100644--- a/harper-core/src/lexing/url.rs+++ b/harper-core/src/lexing/url.rs@@ -120,7 +120,7 @@ fn is_unreserved(c: char) -> bool {}fn is_hex(c: char) -> bool {- c.is_ascii_digit() || matches!(c, 'A'..='F' | 'a'..='f')+ c.is_ascii_hexdigit()}/// Lex an escaped hex code, returning the subsequent index