Offline, privacy-first grammar checker. Fast, open-source, Rust-powered
fix(core): indexing problem in Regexish work
Elijah Potterde297eddd7a5
diff --git a/.github/workflows/precommit.yml b/.github/workflows/precommit.ymlindex d6666cce7..f3f324477 100644--- a/.github/workflows/precommit.yml+++ b/.github/workflows/precommit.yml@@ -8,8 +8,8 @@ on:env:CARGO_TERM_COLOR: always- # Run for 100 times the default- QUICKCHECK_TESTS: 10000+ # Run for 300 times the default+ QUICKCHECK_TESTS: 30000jobs:precommit:diff --git a/harper-core/src/lexing/mod.rs b/harper-core/src/lexing/mod.rsindex 716c145c6..62aea5d7e 100644--- a/harper-core/src/lexing/mod.rs+++ b/harper-core/src/lexing/mod.rs@@ -11,7 +11,7 @@ use crate::char_ext::CharExt;use crate::punctuation::{Punctuation, Quote};use crate::{Number, TokenKind};-#[derive(Debug)]+#[derive(Debug, Eq, PartialEq)]pub struct FoundToken {/// The index of the character __after__ the lexed tokenpub next_index: usize,@@ -128,7 +128,6 @@ pub fn lex_regexish(src: &[char]) -> Option<FoundToken> {if i >= l || src[i] != ']' {continue;}- i += 1;break;}@@ -424,13 +423,13 @@ mod tests {#[test]fn lexes_youtube_as_hostname() {let source: Vec<_> = "YouTube.com".chars().collect();- assert!(matches!(+ assert_eq!(lex_token(&source),Some(FoundToken {token: TokenKind::Hostname,- ..+ next_index: source.len()})- ));+ );}#[test]@@ -440,7 +439,7 @@ mod tests {lex_token(&source),Some(FoundToken {token: TokenKind::Regexish,- ..+ next_index: 2})))}@@ -448,110 +447,110 @@ mod tests {#[test]fn lexes_regex_one_letter() {let source: Vec<_> = "[a]".chars().collect();- assert!(matches!(+ assert_eq!(lex_token(&source),Some(FoundToken {token: TokenKind::Regexish,- ..+ next_index: 3})- ));+ );}#[test]fn lexes_regex_two_letters() {let source: Vec<_> = "[az]".chars().collect();- assert!(matches!(+ assert_eq!(lex_token(&source),Some(FoundToken {token: TokenKind::Regexish,- ..+ next_index: 4})- ));+ );}#[test]fn lexes_regex_digits() {let source: Vec<_> = "[123]".chars().collect();- assert!(matches!(+ assert_eq!(lex_token(&source),Some(FoundToken {token: TokenKind::Regexish,- ..+ next_index: 5})- ));+ );}#[test]fn lexes_regex_two_alphanumeric() {let source: Vec<_> = "[a0b1c2]".chars().collect();- assert!(matches!(+ assert_eq!(lex_token(&source),Some(FoundToken {token: TokenKind::Regexish,- ..+ next_index: 8})- ));+ );}#[test]fn lexes_regex_one_range() {let source: Vec<_> = "[a-z]".chars().collect();- assert!(matches!(+ assert_eq!(lex_token(&source),Some(FoundToken {token: TokenKind::Regexish,- ..+ next_index: 5})- ));+ );}#[test]fn lexes_regex_letter_plus_range() {let source: Vec<_> = "[ax-z]".chars().collect();- assert!(matches!(+ assert_eq!(lex_token(&source),Some(FoundToken {token: TokenKind::Regexish,- ..+ next_index: 6})- ));+ );}#[test]fn lexes_regex_range_plus_letter() {let source: Vec<_> = "[a-cz]".chars().collect();- assert!(matches!(+ assert_eq!(lex_token(&source),Some(FoundToken {token: TokenKind::Regexish,- ..+ next_index: 6})- ));+ );}#[test]fn lexes_regex_two_ranges() {let source: Vec<_> = "[a-cx-z]".chars().collect();- assert!(matches!(+ assert_eq!(lex_token(&source),Some(FoundToken {token: TokenKind::Regexish,- ..+ next_index: 8})- ));+ );}#[test]fn doesnt_lex_regex_broken_two_ranges() {- // you can't end a range and start a range with a single letter+ // You can't end a range and start a range with a single letterlet source: Vec<_> = "[a-x-z]".chars().collect();- assert!(!matches!(+ assert_eq!(lex_token(&source),Some(FoundToken {token: TokenKind::Regexish,- ..+ next_index: 7})- ));+ );}#[test]diff --git a/harper-core/src/parsers/plain_english.rs b/harper-core/src/parsers/plain_english.rsindex 6c110f29a..8f6acd0f5 100644--- a/harper-core/src/parsers/plain_english.rs+++ b/harper-core/src/parsers/plain_english.rs@@ -15,10 +15,7 @@ impl Parser for PlainEnglish {let mut tokens = Vec::new();loop {- if cursor > source.len() {- panic!()- }- if cursor == source.len() {+ if cursor >= source.len() {return tokens;}