Offline, privacy-first grammar checker. Fast, open-source, Rust-powered
feat: lex 4-digit plural decades
hippietrail507fb2c7108d
diff --git a/harper-core/src/lexing/mod.rs b/harper-core/src/lexing/mod.rsindex 2b40adf70..ce1e24f1c 100644--- a/harper-core/src/lexing/mod.rs+++ b/harper-core/src/lexing/mod.rs@@ -25,7 +25,8 @@ pub fn lex_token(source: &[char]) -> Option<FoundToken> {lex_tabs,lex_spaces,lex_newlines,- lex_hex_number, // before lex_number, which would match the initial 0+ lex_hex_number, // before lex_number, which would match the initial 0+ lex_long_decade, // before lex_number, which would match the digits up to the -slex_number,lex_url,lex_email_address,@@ -139,6 +140,33 @@ pub fn lex_hex_number(source: &[char]) -> Option<FoundToken> {None}+pub fn lex_long_decade(source: &[char]) -> Option<FoundToken> {+ // lex 4-digit decades in their plural such as: 1980s 1990s 2000s 2020s+ if source.len() != 5 {+ return None;+ }+ if source[0] != '1' && source[0] != '2' {+ return None;+ }+ if !source[1].is_numeric() {+ return None;+ }+ if !source[2].is_numeric() {+ return None;+ }+ if source[3] != '0' {+ return None;+ }+ if source[4] != 's' {+ return None;+ }++ Some(FoundToken {+ token: TokenKind::Decade,+ next_index: 5,+ })+}+fn lex_newlines(source: &[char]) -> Option<FoundToken> {let count = source.iter().take_while(|c| **c == '\n').count();@@ -216,6 +244,7 @@ fn lex_catch(_source: &[char]) -> Option<FoundToken> {#[cfg(test)]mod tests {use super::lex_hex_number;+ use super::lex_long_decade;use super::lex_token;use super::lex_word;use super::{FoundToken, TokenKind};@@ -333,4 +362,88 @@ mod tests {let source: Vec<_> = "0Xf00d".chars().collect();assert!(lex_hex_number(&source).is_none());}++ #[test]+ fn lexes_20c_decade() {+ let source: Vec<_> = "1980s".chars().collect();+ assert!(matches!(+ lex_long_decade(&source),+ Some(FoundToken {+ token: TokenKind::Decade,+ ..+ })+ ));+ }++ #[test]+ fn lexes_21c_decade() {+ let source: Vec<_> = "2020s".chars().collect();+ assert!(matches!(+ lex_long_decade(&source),+ Some(FoundToken {+ token: TokenKind::Decade,+ ..+ })+ ));+ }++ #[test]+ fn lexes_ancient_decade() {+ let source: Vec<_> = "1010s".chars().collect();+ assert!(matches!(+ lex_long_decade(&source),+ Some(FoundToken {+ token: TokenKind::Decade,+ ..+ })+ ));+ }++ #[test]+ fn doesnt_lex_far_future_decade() {+ let source: Vec<_> = "3190s".chars().collect();+ assert!(lex_long_decade(&source).is_none());+ }++ #[test]+ fn doesnt_lex_too_ancient_decade() {+ let source: Vec<_> = "100s".chars().collect();+ assert!(lex_long_decade(&source).is_none());+ }++ #[test]+ fn doesnt_lex_0_prefixed_decade() {+ let source: Vec<_> = "0100s".chars().collect();+ assert!(lex_long_decade(&source).is_none());+ }++ #[test]+ fn doesnt_lex_uppercase_decade() {+ let source: Vec<_> = "2000S".chars().collect();+ assert!(lex_long_decade(&source).is_none());+ }++ #[test]+ fn doesnt_lex_overlong_decade() {+ let source: Vec<_> = "20000s".chars().collect();+ assert!(lex_long_decade(&source).is_none());+ }++ #[test]+ fn doesnt_lex_apostrophe_long_decade() {+ let source: Vec<_> = "2020's".chars().collect();+ assert!(lex_long_decade(&source).is_none());+ }++ #[test]+ fn doesnt_lex_bad_apostrophe_short_decade() {+ let source: Vec<_> = "80's".chars().collect();+ assert!(lex_long_decade(&source).is_none());+ }++ #[test]+ fn doesnt_lex_good_apostrophe_short_decade() {+ let source: Vec<_> = "'90s".chars().collect();+ assert!(lex_long_decade(&source).is_none());+ }}diff --git a/harper-core/src/token_kind.rs b/harper-core/src/token_kind.rsindex 0e1acd857..3e7a5fc77 100644--- a/harper-core/src/token_kind.rs+++ b/harper-core/src/token_kind.rs@@ -10,6 +10,7 @@ use crate::{ConjunctionData, NounData, Number, Punctuation, Quote, WordMetadata}pub enum TokenKind {Word(WordMetadata),Punctuation(Punctuation),+ Decade,Number(Number),/// A sequence of " " spaces.Space(usize),@@ -46,6 +47,7 @@ impl TokenKind {TokenKind::Word(..)| TokenKind::EmailAddress| TokenKind::Hostname+ | TokenKind::Decade| TokenKind::Number(..))}