Offline, privacy-first grammar checker. Fast, open-source, Rust-powered
refactor(core): oldschool lexing logic for hex
hippietrail5f273451be05
diff --git a/harper-core/src/lexing/mod.rs b/harper-core/src/lexing/mod.rsindex 2aee8d5b5..9d272c05a 100644--- a/harper-core/src/lexing/mod.rs+++ b/harper-core/src/lexing/mod.rs@@ -25,7 +25,7 @@ pub fn lex_token(source: &[char]) -> Option<FoundToken> {lex_tabs,lex_spaces,lex_newlines,- lex_hex_number,+ lex_hex_number, // before lex_number, which would match the initial 0lex_number,lex_url,lex_email_address,@@ -92,28 +92,36 @@ pub fn lex_number(source: &[char]) -> Option<FoundToken> {}pub fn lex_hex_number(source: &[char]) -> Option<FoundToken> {- if source.len() < 3 || source[0] != '0' || source[1] != 'x' {+ // < 3 to avoid accepting 0x alone+ if source.len() < 3 || source[0] != '0' || source[1] != 'x' || !source[2].is_ascii_hexdigit() {return None;}- let end = &source[2..]- .iter()- .enumerate()- .rev()- .find_map(|(i, v)| v.is_ascii_hexdigit().then_some(i))?;+ let mut i = 2;+ let len = source.len();- let mut s: String = source[2..end + 2 + 1].iter().collect();+ while i < len {+ let next = source[i];- // Find the longest possible valid number- while !s.is_empty() {- if let Ok(n) = u64::from_str_radix(&s, 16) {- return Some(FoundToken {- token: TokenKind::Number(OrderedFloat(n as f64), None),- next_index: s.len() + 2,- });+ if !next.is_ascii_hexdigit() {+ if !next.is_alphanumeric() {+ break;+ } else {+ return None;+ }}- s.pop();+ i += 1;+ }++ let s: String = source[2..i].iter().collect();++ // Should always succeed unless the logic above is broken+ if let Ok(n) = u64::from_str_radix(&s, 16) {+ return Some(FoundToken {+ token: TokenKind::Number(OrderedFloat(n as f64), None),+ next_index: s.len() + 2,+ });}None@@ -197,6 +205,7 @@ fn lex_catch(_source: &[char]) -> Option<FoundToken> {mod tests {use super::lex_token;use super::lex_word;+ use super::lex_hex_number;use super::{FoundToken, TokenKind};#[test]@@ -216,4 +225,42 @@ mod tests {})));}++ #[test]+ fn lexes_good_hex() {+ let cases = [+ "0x0",+ "0xa",+ "0xF",+ "0xaF",+ "0x0123456789abcdef",+ "0xAbCdEf9876543210",+ ];++ for case in cases {+ let source: Vec<_> = case.chars().collect();+ assert!(matches!(+ lex_hex_number(&source),+ Some(FoundToken {+ token: TokenKind::Number(_, None),+ ..+ })+ ));+ }+ }++ #[test]+ fn lexes_bad_hex() {+ let cases = [+ "0x",+ "0xg",+ "0x123g",+ "0Xf00d",+ ];++ for &case in &cases {+ let source: Vec<_> = case.chars().collect();+ assert!(lex_hex_number(&source).is_none());+ }+ }}