Offline, privacy-first grammar checker. Fast, open-source, Rust-powered
refactor(core): move `TokenStringExt` into its own module
Elijah Potterf7b48c4ce8d2
diff --git a/harper-core/src/lib.rs b/harper-core/src/lib.rsindex 462a4c6a2..ccf9df6cb 100644--- a/harper-core/src/lib.rs+++ b/harper-core/src/lib.rs@@ -21,6 +21,7 @@ mod sync;mod title_case;mod token;mod token_kind;+mod token_string_ext;mod vec_ext;mod word_metadata;@@ -38,8 +39,9 @@ pub use span::Span;pub use spell::{Dictionary, FstDictionary, FullDictionary, MergedDictionary};pub use sync::Lrc;pub use title_case::{make_title_case, make_title_case_str};-pub use token::{Token, TokenStringExt};+pub use token::Token;pub use token_kind::{NumberSuffix, TokenKind};+pub use token_string_ext::TokenStringExt;pub use vec_ext::VecExt;pub use word_metadata::{AdverbData, ConjunctionData, NounData, Tense, VerbData, WordMetadata};diff --git a/harper-core/src/linting/correct_number_suffix.rs b/harper-core/src/linting/correct_number_suffix.rsindex 7237b49c6..a2fbfb0a3 100644--- a/harper-core/src/linting/correct_number_suffix.rs+++ b/harper-core/src/linting/correct_number_suffix.rs@@ -1,5 +1,5 @@use super::{Lint, LintKind, Linter, Suggestion};-use crate::token::TokenStringExt;+use crate::TokenStringExt;use crate::{Document, NumberSuffix, Span, TokenKind};/// Detect and warn that the sentence is too long.diff --git a/harper-core/src/linting/linking_verbs.rs b/harper-core/src/linting/linking_verbs.rsindex 60fea3cad..ad89c4e2f 100644--- a/harper-core/src/linting/linking_verbs.rs+++ b/harper-core/src/linting/linking_verbs.rs@@ -1,6 +1,6 @@use super::{Lint, LintKind, Linter};-use crate::token::TokenStringExt;use crate::Document;+use crate::TokenStringExt;/// Detect and warn that the sentence is too long.#[derive(Debug, Clone, Copy, Default)]diff --git a/harper-core/src/linting/long_sentences.rs b/harper-core/src/linting/long_sentences.rsindex 666d018e0..fac2e8fcd 100644--- a/harper-core/src/linting/long_sentences.rs+++ b/harper-core/src/linting/long_sentences.rs@@ -1,5 +1,5 @@use super::{Lint, LintKind, Linter};-use crate::token::TokenStringExt;+use crate::TokenStringExt;use crate::{Document, Span};/// Detect and warn that the sentence is too long.diff --git a/harper-core/src/linting/number_suffix_capitalization.rs b/harper-core/src/linting/number_suffix_capitalization.rsindex 7c46950f4..b7763a670 100644--- a/harper-core/src/linting/number_suffix_capitalization.rs+++ b/harper-core/src/linting/number_suffix_capitalization.rs@@ -1,5 +1,5 @@use super::{Lint, LintKind, Linter, Suggestion};-use crate::token::TokenStringExt;+use crate::TokenStringExt;use crate::{Document, Span, TokenKind};/// Detect and warn that the sentence is too long.diff --git a/harper-core/src/linting/repeated_words.rs b/harper-core/src/linting/repeated_words.rsindex 61dd1bc24..1d10f4abc 100644--- a/harper-core/src/linting/repeated_words.rs+++ b/harper-core/src/linting/repeated_words.rs@@ -1,7 +1,7 @@use smallvec::smallvec;use super::{Lint, LintKind, Linter, Suggestion};-use crate::token::TokenStringExt;+use crate::TokenStringExt;use crate::{CharString, CharStringExt, Document, Span};#[derive(Debug, Clone)]diff --git a/harper-core/src/linting/spaces.rs b/harper-core/src/linting/spaces.rsindex f9faa0872..28fc5803e 100644--- a/harper-core/src/linting/spaces.rs+++ b/harper-core/src/linting/spaces.rs@@ -1,5 +1,5 @@use super::{Lint, LintKind, Linter, Suggestion};-use crate::token::TokenStringExt;+use crate::TokenStringExt;use crate::{Document, Token, TokenKind};#[derive(Debug, Default)]diff --git a/harper-core/src/token.rs b/harper-core/src/token.rsindex 83eb246f1..7c93f2ce1 100644--- a/harper-core/src/token.rs+++ b/harper-core/src/token.rs@@ -1,8 +1,6 @@-use itertools::Itertools;-use paste::paste;use serde::{Deserialize, Serialize};-use crate::{FatToken, Span, TokenKind};+use crate::{FatToken, Span, TokenKind, TokenStringExt};#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Default)]pub struct Token {@@ -26,237 +24,6 @@ impl Token {}}-macro_rules! create_decl_for {- ($thing:ident) => {- paste! {- fn [< first_ $thing >](&self) -> Option<Token>;-- fn [< last_ $thing >](&self) -> Option<Token>;-- fn [< last_ $thing _index >](&self) -> Option<usize>;-- fn [<iter_ $thing _indices>](&self) -> impl Iterator<Item = usize> + '_;-- fn [<iter_ $thing s>](&self) -> impl Iterator<Item = Token> + '_;- }- };-}--macro_rules! create_fns_for {- ($thing:ident) => {- paste! {- fn [< first_ $thing >](&self) -> Option<Token> {- self.iter().find(|v| v.kind.[<is_ $thing>]()).copied()- }-- fn [< last_ $thing >](&self) -> Option<Token> {- self.iter().rev().find(|v| v.kind.[<is_ $thing>]()).copied()- }-- fn [< last_ $thing _index >](&self) -> Option<usize> {- self.iter().rev().position(|v| v.kind.[<is_ $thing>]()).map(|i| self.len() - i - 1)- }-- fn [<iter_ $thing _indices>](&self) -> impl Iterator<Item = usize> + '_ {- self.iter()- .enumerate()- .filter(|(_, t)| t.kind.[<is_ $thing>]())- .map(|(i, _)| i)- }-- fn [<iter_ $thing s>](&self) -> impl Iterator<Item = Token> + '_ {- self.[<iter_ $thing _indices>]().map(|i| self[i])- }- }- };-}--/// Extension methods for [`Token`] sequences that make them easier to wrangle and query.-pub trait TokenStringExt {- fn first_sentence_word(&self) -> Option<Token>;- fn first_non_whitespace(&self) -> Option<Token>;- /// Grab the span that represents the beginning of the first element and the- /// end of the last element.- fn span(&self) -> Option<Span>;-- create_decl_for!(word);- create_decl_for!(word_like);- create_decl_for!(conjunction);- create_decl_for!(space);- create_decl_for!(apostrophe);- create_decl_for!(pipe);- create_decl_for!(quote);- create_decl_for!(number);- create_decl_for!(at);- create_decl_for!(ellipsis);- create_decl_for!(unlintable);- create_decl_for!(sentence_terminator);- create_decl_for!(paragraph_break);- create_decl_for!(chunk_terminator);- create_decl_for!(punctuation);- create_decl_for!(currency);- create_decl_for!(likely_homograph);-- fn iter_linking_verb_indices(&self) -> impl Iterator<Item = usize> + '_;- fn iter_linking_verbs(&self) -> impl Iterator<Item = Token> + '_;-- /// Iterate over chunks.- ///- /// For example, the following sentence contains two chunks separated by a- /// comma:- ///- /// ```text- /// Here is an example, it is short.- /// ```- fn iter_chunks(&self) -> impl Iterator<Item = &'_ [Token]> + '_;-- /// Get an iterator over token slices that represent the individual- /// paragraphs in a document.- fn iter_paragraphs(&self) -> impl Iterator<Item = &'_ [Token]> + '_;-- /// Get an iterator over token slices that represent the individual- /// sentences in a document.- fn iter_sentences(&self) -> impl Iterator<Item = &'_ [Token]> + '_;-}--impl TokenStringExt for [Token] {- create_fns_for!(word);- create_fns_for!(word_like);- create_fns_for!(conjunction);- create_fns_for!(space);- create_fns_for!(apostrophe);- create_fns_for!(pipe);- create_fns_for!(quote);- create_fns_for!(number);- create_fns_for!(at);- create_fns_for!(punctuation);- create_fns_for!(ellipsis);- create_fns_for!(unlintable);- create_fns_for!(sentence_terminator);- create_fns_for!(paragraph_break);- create_fns_for!(chunk_terminator);- create_fns_for!(currency);- create_fns_for!(likely_homograph);-- fn first_non_whitespace(&self) -> Option<Token> {- self.iter().find(|t| !t.kind.is_whitespace()).copied()- }-- fn first_sentence_word(&self) -> Option<Token> {- let (w_idx, word) = self.iter().find_position(|v| v.kind.is_word())?;-- let Some(u_idx) = self.iter().position(|v| v.kind.is_unlintable()) else {- return Some(*word);- };-- if w_idx < u_idx {- Some(*word)- } else {- None- }- }-- fn span(&self) -> Option<Span> {- let min_max = self- .iter()- .flat_map(|v| [v.span.start, v.span.end].into_iter())- .minmax();-- match min_max {- itertools::MinMaxResult::NoElements => None,- itertools::MinMaxResult::OneElement(min) => Some(Span::new(min, min)),- itertools::MinMaxResult::MinMax(min, max) => Some(Span::new(min, max)),- }- }-- fn iter_linking_verb_indices(&self) -> impl Iterator<Item = usize> + '_ {- self.iter_word_indices().filter(|idx| {- let word = self[*idx];- let TokenKind::Word(word) = word.kind else {- panic!("Should be unreachable.");- };-- word.is_linking_verb()- })- }-- fn iter_linking_verbs(&self) -> impl Iterator<Item = Token> + '_ {- self.iter_linking_verb_indices().map(|idx| self[idx])- }-- fn iter_chunks(&self) -> impl Iterator<Item = &'_ [Token]> + '_ {- let first_chunk = self- .iter_chunk_terminator_indices()- .next()- .map(|first_term| &self[0..=first_term]);-- let rest = self- .iter_chunk_terminator_indices()- .tuple_windows()- .map(move |(a, b)| &self[a + 1..=b]);-- let last = if let Some(last_i) = self.last_chunk_terminator_index() {- if last_i + 1 < self.len() {- Some(&self[last_i + 1..])- } else {- None- }- } else {- Some(self)- };-- first_chunk.into_iter().chain(rest).chain(last)- }-- fn iter_paragraphs(&self) -> impl Iterator<Item = &'_ [Token]> + '_ {- let first_pg = self- .iter_paragraph_break_indices()- .next()- .map(|first_term| &self[0..=first_term]);-- let rest = self- .iter_paragraph_break_indices()- .tuple_windows()- .map(move |(a, b)| &self[a + 1..=b]);-- let last_pg = if let Some(last_i) = self.last_paragraph_break_index() {- if last_i + 1 < self.len() {- Some(&self[last_i + 1..])- } else {- None- }- } else {- Some(self)- };-- first_pg.into_iter().chain(rest).chain(last_pg)- }-- fn iter_sentences(&self) -> impl Iterator<Item = &'_ [Token]> + '_ {- let first_sentence = self- .iter_sentence_terminator_indices()- .next()- .map(|first_term| &self[0..=first_term]);-- let rest = self- .iter_sentence_terminator_indices()- .tuple_windows()- .map(move |(a, b)| &self[a + 1..=b]);-- let last_sentence = if let Some(last_i) = self.last_sentence_terminator_index() {- if last_i + 1 < self.len() {- Some(&self[last_i + 1..])- } else {- None- }- } else {- Some(self)- };-- first_sentence.into_iter().chain(rest).chain(last_sentence)- }-}-#[cfg(test)]mod tests {use crate::{diff --git a/harper-core/src/token_string_ext.rs b/harper-core/src/token_string_ext.rsnew file mode 100644index 000000000..d3ceda72d--- /dev/null+++ b/harper-core/src/token_string_ext.rs@@ -0,0 +1,234 @@+use crate::{Span, Token, TokenKind};+use itertools::Itertools;+use paste::paste;++macro_rules! create_decl_for {+ ($thing:ident) => {+ paste! {+ fn [< first_ $thing >](&self) -> Option<Token>;++ fn [< last_ $thing >](&self) -> Option<Token>;++ fn [< last_ $thing _index >](&self) -> Option<usize>;++ fn [<iter_ $thing _indices>](&self) -> impl Iterator<Item = usize> + '_;++ fn [<iter_ $thing s>](&self) -> impl Iterator<Item = Token> + '_;+ }+ };+}++macro_rules! create_fns_for {+ ($thing:ident) => {+ paste! {+ fn [< first_ $thing >](&self) -> Option<Token> {+ self.iter().find(|v| v.kind.[<is_ $thing>]()).copied()+ }++ fn [< last_ $thing >](&self) -> Option<Token> {+ self.iter().rev().find(|v| v.kind.[<is_ $thing>]()).copied()+ }++ fn [< last_ $thing _index >](&self) -> Option<usize> {+ self.iter().rev().position(|v| v.kind.[<is_ $thing>]()).map(|i| self.len() - i - 1)+ }++ fn [<iter_ $thing _indices>](&self) -> impl Iterator<Item = usize> + '_ {+ self.iter()+ .enumerate()+ .filter(|(_, t)| t.kind.[<is_ $thing>]())+ .map(|(i, _)| i)+ }++ fn [<iter_ $thing s>](&self) -> impl Iterator<Item = Token> + '_ {+ self.[<iter_ $thing _indices>]().map(|i| self[i])+ }+ }+ };+}++/// Extension methods for [`Token`] sequences that make them easier to wrangle and query.+pub trait TokenStringExt {+ fn first_sentence_word(&self) -> Option<Token>;+ fn first_non_whitespace(&self) -> Option<Token>;+ /// Grab the span that represents the beginning of the first element and the+ /// end of the last element.+ fn span(&self) -> Option<Span>;++ create_decl_for!(word);+ create_decl_for!(word_like);+ create_decl_for!(conjunction);+ create_decl_for!(space);+ create_decl_for!(apostrophe);+ create_decl_for!(pipe);+ create_decl_for!(quote);+ create_decl_for!(number);+ create_decl_for!(at);+ create_decl_for!(ellipsis);+ create_decl_for!(unlintable);+ create_decl_for!(sentence_terminator);+ create_decl_for!(paragraph_break);+ create_decl_for!(chunk_terminator);+ create_decl_for!(punctuation);+ create_decl_for!(currency);+ create_decl_for!(likely_homograph);++ fn iter_linking_verb_indices(&self) -> impl Iterator<Item = usize> + '_;+ fn iter_linking_verbs(&self) -> impl Iterator<Item = Token> + '_;++ /// Iterate over chunks.+ ///+ /// For example, the following sentence contains two chunks separated by a+ /// comma:+ ///+ /// ```text+ /// Here is an example, it is short.+ /// ```+ fn iter_chunks(&self) -> impl Iterator<Item = &'_ [Token]> + '_;++ /// Get an iterator over token slices that represent the individual+ /// paragraphs in a document.+ fn iter_paragraphs(&self) -> impl Iterator<Item = &'_ [Token]> + '_;++ /// Get an iterator over token slices that represent the individual+ /// sentences in a document.+ fn iter_sentences(&self) -> impl Iterator<Item = &'_ [Token]> + '_;+}++impl TokenStringExt for [Token] {+ create_fns_for!(word);+ create_fns_for!(word_like);+ create_fns_for!(conjunction);+ create_fns_for!(space);+ create_fns_for!(apostrophe);+ create_fns_for!(pipe);+ create_fns_for!(quote);+ create_fns_for!(number);+ create_fns_for!(at);+ create_fns_for!(punctuation);+ create_fns_for!(ellipsis);+ create_fns_for!(unlintable);+ create_fns_for!(sentence_terminator);+ create_fns_for!(paragraph_break);+ create_fns_for!(chunk_terminator);+ create_fns_for!(currency);+ create_fns_for!(likely_homograph);++ fn first_non_whitespace(&self) -> Option<Token> {+ self.iter().find(|t| !t.kind.is_whitespace()).copied()+ }++ fn first_sentence_word(&self) -> Option<Token> {+ let (w_idx, word) = self.iter().find_position(|v| v.kind.is_word())?;++ let Some(u_idx) = self.iter().position(|v| v.kind.is_unlintable()) else {+ return Some(*word);+ };++ if w_idx < u_idx {+ Some(*word)+ } else {+ None+ }+ }++ fn span(&self) -> Option<Span> {+ let min_max = self+ .iter()+ .flat_map(|v| [v.span.start, v.span.end].into_iter())+ .minmax();++ match min_max {+ itertools::MinMaxResult::NoElements => None,+ itertools::MinMaxResult::OneElement(min) => Some(Span::new(min, min)),+ itertools::MinMaxResult::MinMax(min, max) => Some(Span::new(min, max)),+ }+ }++ fn iter_linking_verb_indices(&self) -> impl Iterator<Item = usize> + '_ {+ self.iter_word_indices().filter(|idx| {+ let word = self[*idx];+ let TokenKind::Word(word) = word.kind else {+ panic!("Should be unreachable.");+ };++ word.is_linking_verb()+ })+ }++ fn iter_linking_verbs(&self) -> impl Iterator<Item = Token> + '_ {+ self.iter_linking_verb_indices().map(|idx| self[idx])+ }++ fn iter_chunks(&self) -> impl Iterator<Item = &'_ [Token]> + '_ {+ let first_chunk = self+ .iter_chunk_terminator_indices()+ .next()+ .map(|first_term| &self[0..=first_term]);++ let rest = self+ .iter_chunk_terminator_indices()+ .tuple_windows()+ .map(move |(a, b)| &self[a + 1..=b]);++ let last = if let Some(last_i) = self.last_chunk_terminator_index() {+ if last_i + 1 < self.len() {+ Some(&self[last_i + 1..])+ } else {+ None+ }+ } else {+ Some(self)+ };++ first_chunk.into_iter().chain(rest).chain(last)+ }++ fn iter_paragraphs(&self) -> impl Iterator<Item = &'_ [Token]> + '_ {+ let first_pg = self+ .iter_paragraph_break_indices()+ .next()+ .map(|first_term| &self[0..=first_term]);++ let rest = self+ .iter_paragraph_break_indices()+ .tuple_windows()+ .map(move |(a, b)| &self[a + 1..=b]);++ let last_pg = if let Some(last_i) = self.last_paragraph_break_index() {+ if last_i + 1 < self.len() {+ Some(&self[last_i + 1..])+ } else {+ None+ }+ } else {+ Some(self)+ };++ first_pg.into_iter().chain(rest).chain(last_pg)+ }++ fn iter_sentences(&self) -> impl Iterator<Item = &'_ [Token]> + '_ {+ let first_sentence = self+ .iter_sentence_terminator_indices()+ .next()+ .map(|first_term| &self[0..=first_term]);++ let rest = self+ .iter_sentence_terminator_indices()+ .tuple_windows()+ .map(move |(a, b)| &self[a + 1..=b]);++ let last_sentence = if let Some(last_i) = self.last_sentence_terminator_index() {+ if last_i + 1 < self.len() {+ Some(&self[last_i + 1..])+ } else {+ None+ }+ } else {+ Some(self)+ };++ first_sentence.into_iter().chain(rest).chain(last_sentence)+ }+}