Offline, privacy-first grammar checker. Fast, open-source, Rust-powered
refactor(core): adjustments in lexing code (#2618)
Steve Kurcha82fb978c37e
diff --git a/harper-core/src/lexing/mod.rs b/harper-core/src/lexing/mod.rsindex 2189f067a..0bb093149 100644--- a/harper-core/src/lexing/mod.rs+++ b/harper-core/src/lexing/mod.rs@@ -9,7 +9,7 @@ use url::lex_url;use self::email_address::lex_email_address;use crate::char_ext::CharExt;use crate::punctuation::{Punctuation, Quote};-use crate::{Number, TokenKind};+use crate::{Number, Span, Token, TokenKind};#[derive(Debug, Eq, PartialEq)]pub struct FoundToken {@@ -19,8 +19,31 @@ pub struct FoundToken {pub token: TokenKind,}-pub fn lex_weir_token(source: &[char]) -> Option<FoundToken> {- let lexers = [+/// Lex `source` with the provided `lex_fn`.+///+/// `lex_fn` should be a function that takes a subslice of the source, and returns the first found+/// token.+pub fn lex_with(source: &[char], lex_fn: fn(&[char]) -> FoundToken) -> Vec<Token> {+ let mut cursor = 0;+ let mut tokens = Vec::new();++ loop {+ if cursor >= source.len() {+ return tokens;+ }++ let FoundToken { token, next_index } = lex_fn(&source[cursor..]);++ tokens.push(Token {+ span: Span::new(cursor, cursor + next_index),+ kind: token,+ });+ cursor += next_index;+ }+}++pub fn lex_weir_token(source: &[char]) -> FoundToken {+ [lex_punctuation,lex_tabs,lex_spaces,@@ -33,20 +56,14 @@ pub fn lex_weir_token(source: &[char]) -> Option<FoundToken> {lex_email_address,lex_hostname_token,lex_word,- lex_catch,- ];-- for lexer in lexers {- if let Some(f) = lexer(source) {- return Some(f);- }- }-- None+ ]+ .into_iter()+ .find_map(|lexer| lexer(source))+ .unwrap_or_else(lex_catch)}-pub fn lex_english_token(source: &[char]) -> Option<FoundToken> {- let lexers = [+pub fn lex_english_token(source: &[char]) -> FoundToken {+ [lex_regexish,lex_punctuation,lex_tabs,@@ -60,16 +77,10 @@ pub fn lex_english_token(source: &[char]) -> Option<FoundToken> {lex_email_address,lex_hostname_token,lex_word,- lex_catch,- ];-- for lexer in lexers {- if let Some(f) = lexer(source) {- return Some(f);- }- }-- None+ ]+ .into_iter()+ .find_map(|lexer| lexer(source))+ .unwrap_or_else(lex_catch)}fn lex_word(source: &[char]) -> Option<FoundToken> {@@ -333,11 +344,11 @@ fn lex_quote(source: &[char]) -> Option<FoundToken> {}/// Covers cases not covered by the other lints.-fn lex_catch(_source: &[char]) -> Option<FoundToken> {- Some(FoundToken {+fn lex_catch() -> FoundToken {+ FoundToken {next_index: 1,token: TokenKind::Unlintable,- })+ }}#[cfg(test)]@@ -461,10 +472,10 @@ mod tests {let source: Vec<_> = "youtube.com".chars().collect();assert_eq!(lex_english_token(&source),- Some(FoundToken {+ FoundToken {token: TokenKind::Hostname,next_index: source.len()- })+ });}@@ -473,10 +484,10 @@ mod tests {let source: Vec<_> = "[]".chars().collect();assert!(!matches!(lex_english_token(&source),- Some(FoundToken {+ FoundToken {token: TokenKind::Regexish,next_index: 2- })+ }))}@@ -485,10 +496,10 @@ mod tests {let source: Vec<_> = "[a]".chars().collect();assert_eq!(lex_english_token(&source),- Some(FoundToken {+ FoundToken {token: TokenKind::Regexish,next_index: 3- })+ });}@@ -497,10 +508,10 @@ mod tests {let source: Vec<_> = "[az]".chars().collect();assert_eq!(lex_english_token(&source),- Some(FoundToken {+ FoundToken {token: TokenKind::Regexish,next_index: 4- })+ });}@@ -509,10 +520,10 @@ mod tests {let source: Vec<_> = "[123]".chars().collect();assert_eq!(lex_english_token(&source),- Some(FoundToken {+ FoundToken {token: TokenKind::Regexish,next_index: 5- })+ });}@@ -521,10 +532,10 @@ mod tests {let source: Vec<_> = "[a0b1c2]".chars().collect();assert_eq!(lex_english_token(&source),- Some(FoundToken {+ FoundToken {token: TokenKind::Regexish,next_index: 8- })+ });}@@ -533,10 +544,10 @@ mod tests {let source: Vec<_> = "[a-z]".chars().collect();assert_eq!(lex_english_token(&source),- Some(FoundToken {+ FoundToken {token: TokenKind::Regexish,next_index: 5- })+ });}@@ -545,10 +556,10 @@ mod tests {let source: Vec<_> = "[ax-z]".chars().collect();assert_eq!(lex_english_token(&source),- Some(FoundToken {+ FoundToken {token: TokenKind::Regexish,next_index: 6- })+ });}@@ -557,10 +568,10 @@ mod tests {let source: Vec<_> = "[a-cz]".chars().collect();assert_eq!(lex_english_token(&source),- Some(FoundToken {+ FoundToken {token: TokenKind::Regexish,next_index: 6- })+ });}@@ -569,10 +580,10 @@ mod tests {let source: Vec<_> = "[a-cx-z]".chars().collect();assert_eq!(lex_english_token(&source),- Some(FoundToken {+ FoundToken {token: TokenKind::Regexish,next_index: 8- })+ });}@@ -582,10 +593,10 @@ mod tests {let source: Vec<_> = "[a-x-z]".chars().collect();assert_eq!(lex_english_token(&source),- Some(FoundToken {+ FoundToken {token: TokenKind::Punctuation(Punctuation::OpenSquare),next_index: 1- })+ });}@@ -594,10 +605,10 @@ mod tests {let source: Vec<_> = "[a-]".chars().collect();assert!(!matches!(lex_english_token(&source),- Some(FoundToken {+ FoundToken {token: TokenKind::Regexish,..- })+ }));}@@ -606,10 +617,10 @@ mod tests {let source: Vec<_> = "[-z]".chars().collect();assert!(!matches!(lex_english_token(&source),- Some(FoundToken {+ FoundToken {token: TokenKind::Regexish,..- })+ }));}@@ -814,10 +825,10 @@ mod tests {let source: Vec<_> = "late 1980s".chars().collect();assert!(matches!(lex_english_token(&source),- Some(FoundToken {+ FoundToken {token: TokenKind::Word(_),..- })+ }));}@@ -826,10 +837,10 @@ mod tests {let source: Vec<_> = "1980s and".chars().collect();assert!(matches!(lex_english_token(&source),- Some(FoundToken {+ FoundToken {token: TokenKind::Decade,..- })+ }));}@@ -901,7 +912,7 @@ mod tests {break; // Exit if we've processed the entire source}- let token = lex_english_token(&sentence[next_index..]).expect("Failed to lex token");+ let token = lex_english_token(&sentence[next_index..]);assert_eq!(token.token, *expected_token);next_index += token.next_index;}@@ -927,7 +938,7 @@ mod tests {break; // Exit if we've processed the entire source}- let token = lex_english_token(&sentence[next_index..]).expect("Failed to lex token");+ let token = lex_english_token(&sentence[next_index..]);if i < 6 {assert_eq!(token.token, *expected_token);diff --git a/harper-core/src/parsers/plain_english.rs b/harper-core/src/parsers/plain_english.rsindex f36f15b59..fce1b40d1 100644--- a/harper-core/src/parsers/plain_english.rs+++ b/harper-core/src/parsers/plain_english.rs@@ -1,6 +1,6 @@use super::Parser;-use crate::lexing::{FoundToken, lex_english_token};-use crate::{Span, Token};+use crate::Token;+use crate::lexing::{lex_english_token, lex_with};/// A parser that will attempt to lex as many tokens as possible,/// without discrimination and until the end of input.@@ -9,25 +9,6 @@ pub struct PlainEnglish;impl Parser for PlainEnglish {fn parse(&self, source: &[char]) -> Vec<Token> {- let mut cursor = 0;-- // Lex tokens- let mut tokens = Vec::new();-- loop {- if cursor >= source.len() {- return tokens;- }-- if let Some(FoundToken { token, next_index }) = lex_english_token(&source[cursor..]) {- tokens.push(Token {- span: Span::new(cursor, cursor + next_index),- kind: token,- });- cursor += next_index;- } else {- panic!()- }- }+ lex_with(source, lex_english_token)}}diff --git a/harper-core/src/weir/parsing/mod.rs b/harper-core/src/weir/parsing/mod.rsindex fb63b0a06..2be8b9bd7 100644--- a/harper-core/src/weir/parsing/mod.rs+++ b/harper-core/src/weir/parsing/mod.rs@@ -7,8 +7,8 @@ use ast::{Ast, AstExprNode, AstStmtNode};pub use expr::parse_expr_str;pub use stmt::parse_str;-use crate::lexing::{FoundToken, lex_weir_token};-use crate::{Span, Token, TokenKind};+use crate::lexing::{lex_weir_token, lex_with};+use crate::{Token, TokenKind};use super::{ast,@@ -17,25 +17,7 @@ use super::{/// Lex the entirety of a Weir document.fn lex(source: &[char]) -> Vec<Token> {- let mut cursor = 0;-- let mut tokens = Vec::new();-- loop {- if cursor >= source.len() {- return tokens;- }-- if let Some(FoundToken { token, next_index }) = lex_weir_token(&source[cursor..]) {- tokens.push(Token {- span: Span::new(cursor, cursor + next_index),- kind: token,- });- cursor += next_index;- } else {- panic!()- }- }+ lex_with(source, lex_weir_token)}#[derive(Debug)]