Skip to content

Offline, privacy-first grammar checker. Fast, open-source, Rust-powered

refactor(core): adjustments in lexing code (#2618)

Steve Kurcha82fb978c37e
diff --git a/harper-core/src/lexing/mod.rs b/harper-core/src/lexing/mod.rs
index 2189f067a..0bb093149 100644
--- a/harper-core/src/lexing/mod.rs
+++ b/harper-core/src/lexing/mod.rs
@@ -9,7 +9,7 @@ use url::lex_url;
use self::email_address::lex_email_address;
use crate::char_ext::CharExt;
use crate::punctuation::{Punctuation, Quote};
-use crate::{Number, TokenKind};
+use crate::{Number, Span, Token, TokenKind};
#[derive(Debug, Eq, PartialEq)]
pub struct FoundToken {
@@ -19,8 +19,31 @@ pub struct FoundToken {
pub token: TokenKind,
}
-pub fn lex_weir_token(source: &[char]) -> Option<FoundToken> {
- let lexers = [
+/// Lex `source` with the provided `lex_fn`.
+///
+/// `lex_fn` should be a function that takes a subslice of the source, and returns the first found
+/// token.
+pub fn lex_with(source: &[char], lex_fn: fn(&[char]) -> FoundToken) -> Vec<Token> {
+ let mut cursor = 0;
+ let mut tokens = Vec::new();
+
+ loop {
+ if cursor >= source.len() {
+ return tokens;
+ }
+
+ let FoundToken { token, next_index } = lex_fn(&source[cursor..]);
+
+ tokens.push(Token {
+ span: Span::new(cursor, cursor + next_index),
+ kind: token,
+ });
+ cursor += next_index;
+ }
+}
+
+pub fn lex_weir_token(source: &[char]) -> FoundToken {
+ [
lex_punctuation,
lex_tabs,
lex_spaces,
@@ -33,20 +56,14 @@ pub fn lex_weir_token(source: &[char]) -> Option<FoundToken> {
lex_email_address,
lex_hostname_token,
lex_word,
- lex_catch,
- ];
-
- for lexer in lexers {
- if let Some(f) = lexer(source) {
- return Some(f);
- }
- }
-
- None
+ ]
+ .into_iter()
+ .find_map(|lexer| lexer(source))
+ .unwrap_or_else(lex_catch)
}
-pub fn lex_english_token(source: &[char]) -> Option<FoundToken> {
- let lexers = [
+pub fn lex_english_token(source: &[char]) -> FoundToken {
+ [
lex_regexish,
lex_punctuation,
lex_tabs,
@@ -60,16 +77,10 @@ pub fn lex_english_token(source: &[char]) -> Option<FoundToken> {
lex_email_address,
lex_hostname_token,
lex_word,
- lex_catch,
- ];
-
- for lexer in lexers {
- if let Some(f) = lexer(source) {
- return Some(f);
- }
- }
-
- None
+ ]
+ .into_iter()
+ .find_map(|lexer| lexer(source))
+ .unwrap_or_else(lex_catch)
}
fn lex_word(source: &[char]) -> Option<FoundToken> {
@@ -333,11 +344,11 @@ fn lex_quote(source: &[char]) -> Option<FoundToken> {
}
/// Covers cases not covered by the other lints.
-fn lex_catch(_source: &[char]) -> Option<FoundToken> {
- Some(FoundToken {
+fn lex_catch() -> FoundToken {
+ FoundToken {
next_index: 1,
token: TokenKind::Unlintable,
- })
+ }
}
#[cfg(test)]
@@ -461,10 +472,10 @@ mod tests {
let source: Vec<_> = "youtube.com".chars().collect();
assert_eq!(
lex_english_token(&source),
- Some(FoundToken {
+ FoundToken {
token: TokenKind::Hostname,
next_index: source.len()
- })
+ }
);
}
@@ -473,10 +484,10 @@ mod tests {
let source: Vec<_> = "[]".chars().collect();
assert!(!matches!(
lex_english_token(&source),
- Some(FoundToken {
+ FoundToken {
token: TokenKind::Regexish,
next_index: 2
- })
+ }
))
}
@@ -485,10 +496,10 @@ mod tests {
let source: Vec<_> = "[a]".chars().collect();
assert_eq!(
lex_english_token(&source),
- Some(FoundToken {
+ FoundToken {
token: TokenKind::Regexish,
next_index: 3
- })
+ }
);
}
@@ -497,10 +508,10 @@ mod tests {
let source: Vec<_> = "[az]".chars().collect();
assert_eq!(
lex_english_token(&source),
- Some(FoundToken {
+ FoundToken {
token: TokenKind::Regexish,
next_index: 4
- })
+ }
);
}
@@ -509,10 +520,10 @@ mod tests {
let source: Vec<_> = "[123]".chars().collect();
assert_eq!(
lex_english_token(&source),
- Some(FoundToken {
+ FoundToken {
token: TokenKind::Regexish,
next_index: 5
- })
+ }
);
}
@@ -521,10 +532,10 @@ mod tests {
let source: Vec<_> = "[a0b1c2]".chars().collect();
assert_eq!(
lex_english_token(&source),
- Some(FoundToken {
+ FoundToken {
token: TokenKind::Regexish,
next_index: 8
- })
+ }
);
}
@@ -533,10 +544,10 @@ mod tests {
let source: Vec<_> = "[a-z]".chars().collect();
assert_eq!(
lex_english_token(&source),
- Some(FoundToken {
+ FoundToken {
token: TokenKind::Regexish,
next_index: 5
- })
+ }
);
}
@@ -545,10 +556,10 @@ mod tests {
let source: Vec<_> = "[ax-z]".chars().collect();
assert_eq!(
lex_english_token(&source),
- Some(FoundToken {
+ FoundToken {
token: TokenKind::Regexish,
next_index: 6
- })
+ }
);
}
@@ -557,10 +568,10 @@ mod tests {
let source: Vec<_> = "[a-cz]".chars().collect();
assert_eq!(
lex_english_token(&source),
- Some(FoundToken {
+ FoundToken {
token: TokenKind::Regexish,
next_index: 6
- })
+ }
);
}
@@ -569,10 +580,10 @@ mod tests {
let source: Vec<_> = "[a-cx-z]".chars().collect();
assert_eq!(
lex_english_token(&source),
- Some(FoundToken {
+ FoundToken {
token: TokenKind::Regexish,
next_index: 8
- })
+ }
);
}
@@ -582,10 +593,10 @@ mod tests {
let source: Vec<_> = "[a-x-z]".chars().collect();
assert_eq!(
lex_english_token(&source),
- Some(FoundToken {
+ FoundToken {
token: TokenKind::Punctuation(Punctuation::OpenSquare),
next_index: 1
- })
+ }
);
}
@@ -594,10 +605,10 @@ mod tests {
let source: Vec<_> = "[a-]".chars().collect();
assert!(!matches!(
lex_english_token(&source),
- Some(FoundToken {
+ FoundToken {
token: TokenKind::Regexish,
..
- })
+ }
));
}
@@ -606,10 +617,10 @@ mod tests {
let source: Vec<_> = "[-z]".chars().collect();
assert!(!matches!(
lex_english_token(&source),
- Some(FoundToken {
+ FoundToken {
token: TokenKind::Regexish,
..
- })
+ }
));
}
@@ -814,10 +825,10 @@ mod tests {
let source: Vec<_> = "late 1980s".chars().collect();
assert!(matches!(
lex_english_token(&source),
- Some(FoundToken {
+ FoundToken {
token: TokenKind::Word(_),
..
- })
+ }
));
}
@@ -826,10 +837,10 @@ mod tests {
let source: Vec<_> = "1980s and".chars().collect();
assert!(matches!(
lex_english_token(&source),
- Some(FoundToken {
+ FoundToken {
token: TokenKind::Decade,
..
- })
+ }
));
}
@@ -901,7 +912,7 @@ mod tests {
break; // Exit if we've processed the entire source
}
- let token = lex_english_token(&sentence[next_index..]).expect("Failed to lex token");
+ let token = lex_english_token(&sentence[next_index..]);
assert_eq!(token.token, *expected_token);
next_index += token.next_index;
}
@@ -927,7 +938,7 @@ mod tests {
break; // Exit if we've processed the entire source
}
- let token = lex_english_token(&sentence[next_index..]).expect("Failed to lex token");
+ let token = lex_english_token(&sentence[next_index..]);
if i < 6 {
assert_eq!(token.token, *expected_token);
diff --git a/harper-core/src/parsers/plain_english.rs b/harper-core/src/parsers/plain_english.rs
index f36f15b59..fce1b40d1 100644
--- a/harper-core/src/parsers/plain_english.rs
+++ b/harper-core/src/parsers/plain_english.rs
@@ -1,6 +1,6 @@
use super::Parser;
-use crate::lexing::{FoundToken, lex_english_token};
-use crate::{Span, Token};
+use crate::Token;
+use crate::lexing::{lex_english_token, lex_with};
/// A parser that will attempt to lex as many tokens as possible,
/// without discrimination and until the end of input.
@@ -9,25 +9,6 @@ pub struct PlainEnglish;
impl Parser for PlainEnglish {
fn parse(&self, source: &[char]) -> Vec<Token> {
- let mut cursor = 0;
-
- // Lex tokens
- let mut tokens = Vec::new();
-
- loop {
- if cursor >= source.len() {
- return tokens;
- }
-
- if let Some(FoundToken { token, next_index }) = lex_english_token(&source[cursor..]) {
- tokens.push(Token {
- span: Span::new(cursor, cursor + next_index),
- kind: token,
- });
- cursor += next_index;
- } else {
- panic!()
- }
- }
+ lex_with(source, lex_english_token)
}
}
diff --git a/harper-core/src/weir/parsing/mod.rs b/harper-core/src/weir/parsing/mod.rs
index fb63b0a06..2be8b9bd7 100644
--- a/harper-core/src/weir/parsing/mod.rs
+++ b/harper-core/src/weir/parsing/mod.rs
@@ -7,8 +7,8 @@ use ast::{Ast, AstExprNode, AstStmtNode};
pub use expr::parse_expr_str;
pub use stmt::parse_str;
-use crate::lexing::{FoundToken, lex_weir_token};
-use crate::{Span, Token, TokenKind};
+use crate::lexing::{lex_weir_token, lex_with};
+use crate::{Token, TokenKind};
use super::{
ast,
@@ -17,25 +17,7 @@ use super::{
/// Lex the entirety of a Weir document.
fn lex(source: &[char]) -> Vec<Token> {
- let mut cursor = 0;
-
- let mut tokens = Vec::new();
-
- loop {
- if cursor >= source.len() {
- return tokens;
- }
-
- if let Some(FoundToken { token, next_index }) = lex_weir_token(&source[cursor..]) {
- tokens.push(Token {
- span: Span::new(cursor, cursor + next_index),
- kind: token,
- });
- cursor += next_index;
- } else {
- panic!()
- }
- }
+ lex_with(source, lex_weir_token)
}
#[derive(Debug)]