Skip to content

Offline, privacy-first grammar checker. Fast, open-source, Rust-powered

fix: better ascii and hex digit checks

hippietrail10854ff3765e
diff --git a/harper-core/src/lexing/mod.rs b/harper-core/src/lexing/mod.rs
index 05912924f..790f5a900 100644
--- a/harper-core/src/lexing/mod.rs
+++ b/harper-core/src/lexing/mod.rs
@@ -25,8 +25,8 @@ pub fn lex_token(source: &[char]) -> Option<FoundToken> {
lex_tabs,
lex_spaces,
lex_newlines,
- lex_hex_number, // before lex_number, which would match the initial 0
- lex_long_decade, // before lex_number, which would match the digits up to the -s
+ lex_hex_number, // Before lex_number, which would match the initial 0
+ lex_long_decade, // Before lex_number, which would match the digits up to the -s
lex_number,
lex_url,
lex_email_address,
@@ -47,7 +47,7 @@ pub fn lex_token(source: &[char]) -> Option<FoundToken> {
fn lex_word(source: &[char]) -> Option<FoundToken> {
let end = source
.iter()
- .position(|c| !c.is_english_lingual() && !c.is_numeric())
+ .position(|c| !c.is_english_lingual() && !c.is_ascii_digit())
.unwrap_or(source.len());
if end == 0 {
@@ -73,7 +73,7 @@ pub fn lex_number(source: &[char]) -> Option<FoundToken> {
.iter()
.enumerate()
.rev()
- .find_map(|(i, v)| v.is_numeric().then_some(i))?;
+ .find_map(|(i, v)| v.is_ascii_digit().then_some(i))?;
let mut s: String = source[0..end + 1].iter().collect();
@@ -245,10 +245,108 @@ fn lex_catch(_source: &[char]) -> Option<FoundToken> {
mod tests {
use super::lex_hex_number;
use super::lex_long_decade;
+ use super::lex_number;
use super::lex_token;
use super::lex_word;
use super::{FoundToken, TokenKind};
+ // test various kinds of number
+ #[test]
+ fn lexes_0() {
+ let source: Vec<_> = "0".chars().collect();
+ assert!(matches!(
+ lex_number(&source),
+ Some(FoundToken {
+ token: TokenKind::Number(_),
+ ..
+ })
+ ));
+ }
+
+ #[test]
+ fn lexes_0_point_0() {
+ let source: Vec<_> = "0.0".chars().collect();
+ assert!(matches!(
+ lex_number(&source),
+ Some(FoundToken {
+ token: TokenKind::Number(_),
+ ..
+ })
+ ));
+ }
+
+ #[test]
+ fn lexes_00() {
+ let source: Vec<_> = "00".chars().collect();
+ assert!(matches!(
+ lex_number(&source),
+ Some(FoundToken {
+ token: TokenKind::Number(_),
+ ..
+ })
+ ));
+ }
+
+ // #[test]
+ // fn lexes_negative_1() {
+ // let source: Vec<_> = "-1".chars().collect();
+ // assert!(matches!(
+ // lex_number(&source),
+ // Some(FoundToken {
+ // token: TokenKind::Number(_),
+ // ..
+ // })
+ // ));
+ // }
+
+ // #[test]
+ // fn lexes_positive_1() {
+ // let source: Vec<_> = "+1".chars().collect();
+ // assert!(matches!(
+ // lex_number(&source),
+ // Some(FoundToken {
+ // token: TokenKind::Number(_),
+ // ..
+ // })
+ // ));
+ // }
+
+ #[test]
+ fn lexes_pi() {
+ let source: Vec<_> = "3.1415926535897932384626433832795028841971693993751058209749445923078164062862089986280348253421170679".chars().collect();
+ assert!(matches!(
+ lex_number(&source),
+ Some(FoundToken {
+ token: TokenKind::Number(_),
+ ..
+ })
+ ));
+ }
+
+ #[test]
+ fn lexes_speed_of_light() {
+ let source: Vec<_> = "3.00e8".chars().collect();
+ assert!(matches!(
+ lex_number(&source),
+ Some(FoundToken {
+ token: TokenKind::Number(_),
+ ..
+ })
+ ));
+ }
+
+ #[test]
+ fn doesnt_lex_cjk_numeral() {
+ let source: Vec<_> = "二".chars().collect();
+ assert!(lex_number(&source).is_none());
+ }
+
+ #[test]
+ fn doesnt_lex_thai_digit() {
+ let source: Vec<_> = "๑".chars().collect();
+ assert!(lex_number(&source).is_none());
+ }
+
#[test]
fn lexes_cjk_as_unlintable() {
let source: Vec<_> = "世".chars().collect();
diff --git a/harper-core/src/lexing/url.rs b/harper-core/src/lexing/url.rs
index c028f8e1a..b3ee8c6ea 100644
--- a/harper-core/src/lexing/url.rs
+++ b/harper-core/src/lexing/url.rs
@@ -120,7 +120,7 @@ fn is_unreserved(c: char) -> bool {
}
fn is_hex(c: char) -> bool {
- c.is_ascii_digit() || matches!(c, 'A'..='F' | 'a'..='f')
+ c.is_ascii_hexdigit()
}
/// Lex an escaped hex code, returning the subsequent index