Skip to content

Offline, privacy-first grammar checker. Fast, open-source, Rust-powered

fix(core): indexing problem in Regexish work

Elijah Potterde297eddd7a5
diff --git a/.github/workflows/precommit.yml b/.github/workflows/precommit.yml
index d6666cce7..f3f324477 100644
--- a/.github/workflows/precommit.yml
+++ b/.github/workflows/precommit.yml
@@ -8,8 +8,8 @@ on:
env:
CARGO_TERM_COLOR: always
- # Run for 100 times the default
- QUICKCHECK_TESTS: 10000
+ # Run for 300 times the default
+ QUICKCHECK_TESTS: 30000
jobs:
precommit:
diff --git a/harper-core/src/lexing/mod.rs b/harper-core/src/lexing/mod.rs
index 716c145c6..62aea5d7e 100644
--- a/harper-core/src/lexing/mod.rs
+++ b/harper-core/src/lexing/mod.rs
@@ -11,7 +11,7 @@ use crate::char_ext::CharExt;
use crate::punctuation::{Punctuation, Quote};
use crate::{Number, TokenKind};
-#[derive(Debug)]
+#[derive(Debug, Eq, PartialEq)]
pub struct FoundToken {
/// The index of the character __after__ the lexed token
pub next_index: usize,
@@ -128,7 +128,6 @@ pub fn lex_regexish(src: &[char]) -> Option<FoundToken> {
if i >= l || src[i] != ']' {
continue;
}
- i += 1;
break;
}
@@ -424,13 +423,13 @@ mod tests {
#[test]
fn lexes_youtube_as_hostname() {
let source: Vec<_> = "YouTube.com".chars().collect();
- assert!(matches!(
+ assert_eq!(
lex_token(&source),
Some(FoundToken {
token: TokenKind::Hostname,
- ..
+ next_index: source.len()
})
- ));
+ );
}
#[test]
@@ -440,7 +439,7 @@ mod tests {
lex_token(&source),
Some(FoundToken {
token: TokenKind::Regexish,
- ..
+ next_index: 2
})
))
}
@@ -448,110 +447,110 @@ mod tests {
#[test]
fn lexes_regex_one_letter() {
let source: Vec<_> = "[a]".chars().collect();
- assert!(matches!(
+ assert_eq!(
lex_token(&source),
Some(FoundToken {
token: TokenKind::Regexish,
- ..
+ next_index: 3
})
- ));
+ );
}
#[test]
fn lexes_regex_two_letters() {
let source: Vec<_> = "[az]".chars().collect();
- assert!(matches!(
+ assert_eq!(
lex_token(&source),
Some(FoundToken {
token: TokenKind::Regexish,
- ..
+ next_index: 4
})
- ));
+ );
}
#[test]
fn lexes_regex_digits() {
let source: Vec<_> = "[123]".chars().collect();
- assert!(matches!(
+ assert_eq!(
lex_token(&source),
Some(FoundToken {
token: TokenKind::Regexish,
- ..
+ next_index: 5
})
- ));
+ );
}
#[test]
fn lexes_regex_two_alphanumeric() {
let source: Vec<_> = "[a0b1c2]".chars().collect();
- assert!(matches!(
+ assert_eq!(
lex_token(&source),
Some(FoundToken {
token: TokenKind::Regexish,
- ..
+ next_index: 8
})
- ));
+ );
}
#[test]
fn lexes_regex_one_range() {
let source: Vec<_> = "[a-z]".chars().collect();
- assert!(matches!(
+ assert_eq!(
lex_token(&source),
Some(FoundToken {
token: TokenKind::Regexish,
- ..
+ next_index: 5
})
- ));
+ );
}
#[test]
fn lexes_regex_letter_plus_range() {
let source: Vec<_> = "[ax-z]".chars().collect();
- assert!(matches!(
+ assert_eq!(
lex_token(&source),
Some(FoundToken {
token: TokenKind::Regexish,
- ..
+ next_index: 6
})
- ));
+ );
}
#[test]
fn lexes_regex_range_plus_letter() {
let source: Vec<_> = "[a-cz]".chars().collect();
- assert!(matches!(
+ assert_eq!(
lex_token(&source),
Some(FoundToken {
token: TokenKind::Regexish,
- ..
+ next_index: 6
})
- ));
+ );
}
#[test]
fn lexes_regex_two_ranges() {
let source: Vec<_> = "[a-cx-z]".chars().collect();
- assert!(matches!(
+ assert_eq!(
lex_token(&source),
Some(FoundToken {
token: TokenKind::Regexish,
- ..
+ next_index: 8
})
- ));
+ );
}
#[test]
fn doesnt_lex_regex_broken_two_ranges() {
- // you can't end a range and start a range with a single letter
+ // You can't end a range and start a range with a single letter
let source: Vec<_> = "[a-x-z]".chars().collect();
- assert!(!matches!(
+ assert_eq!(
lex_token(&source),
Some(FoundToken {
token: TokenKind::Regexish,
- ..
+ next_index: 7
})
- ));
+ );
}
#[test]
diff --git a/harper-core/src/parsers/plain_english.rs b/harper-core/src/parsers/plain_english.rs
index 6c110f29a..8f6acd0f5 100644
--- a/harper-core/src/parsers/plain_english.rs
+++ b/harper-core/src/parsers/plain_english.rs
@@ -15,10 +15,7 @@ impl Parser for PlainEnglish {
let mut tokens = Vec::new();
loop {
- if cursor > source.len() {
- panic!()
- }
- if cursor == source.len() {
+ if cursor >= source.len() {
return tokens;
}