Offline, privacy-first grammar checker. Fast, open-source, Rust-powered
fix: moved sentence and chunk tools to `TokenStringExt`
Elijah Potter46b1cbf76ef0
diff --git a/harper-core/src/document.rs b/harper-core/src/document.rsindex ed9096c69..a6b8c8b8b 100644--- a/harper-core/src/document.rs+++ b/harper-core/src/document.rs@@ -2,7 +2,6 @@ use std::cmp::Ordering;use std::collections::VecDeque;use std::fmt::Display;-use itertools::Itertools;use paste::paste;use crate::parsers::{Markdown, Parser, PlainEnglish};@@ -187,112 +186,6 @@ impl Document {self.tokens().map(|token| token.to_fat(&self.source))}- /// Iterate over the locations of punctuation the separates chunks.- fn chunk_terminators(&self) -> impl Iterator<Item = usize> + '_ {- self.tokens.iter().enumerate().filter_map(|(index, token)| {- if is_chunk_terminator(&token.kind) {- return Some(index);- }- None- })- }-- /// Get the index of the last chunk terminator.- fn last_chunk_terminator(&self) -> Option<usize> {- self.tokens- .iter()- .enumerate()- .rev()- .find_map(|(index, token)| {- if is_chunk_terminator(&token.kind) {- return Some(index);- }- None- })- }-- /// Iterate over chunks.- ///- /// For example, the following sentence contains two chunks separated by a- /// comma:- ///- /// ```text- /// Here is an example, it is short.- /// ```- pub fn chunks(&self) -> impl Iterator<Item = &'_ [Token]> + '_ {- let first_chunk = self- .chunk_terminators()- .next()- .map(|first_term| &self.tokens[0..=first_term]);-- let rest = self- .chunk_terminators()- .tuple_windows()- .map(move |(a, b)| &self.tokens[a + 1..=b]);-- let last = if let Some(last_i) = self.last_chunk_terminator() {- if last_i + 1 < self.tokens.len() {- Some(&self.tokens[last_i + 1..])- } else {- None- }- } else {- Some(self.tokens.as_slice())- };-- first_chunk.into_iter().chain(rest).chain(last)- }-- /// Iterate over the locations of the sentence terminators in the document.- fn sentence_terminators(&self) -> impl Iterator<Item = usize> + '_ {- self.tokens.iter().enumerate().filter_map(|(index, token)| {- if is_sentence_terminator(&token.kind) {- return Some(index);- }- None- })- }-- /// Get the index of the last sentence terminator in the document.- fn last_sentence_terminator(&self) -> Option<usize> {- self.tokens- .iter()- .enumerate()- .rev()- .find_map(|(index, token)| {- if is_sentence_terminator(&token.kind) {- return Some(index);- }- None- })- }-- /// Get an iterator over token slices that represent the individual- /// sentences in a document.- pub fn sentences(&self) -> impl Iterator<Item = &'_ [Token]> + '_ {- let first_sentence = self- .sentence_terminators()- .next()- .map(|first_term| &self.tokens[0..=first_term]);-- let rest = self- .sentence_terminators()- .tuple_windows()- .map(move |(a, b)| &self.tokens[a + 1..=b]);-- let last = if let Some(last_i) = self.last_sentence_terminator() {- if last_i + 1 < self.tokens.len() {- Some(&self.tokens[last_i + 1..])- } else {- None- }- } else {- Some(self.tokens.as_slice())- };-- first_sentence.into_iter().chain(rest).chain(last)- }-pub fn get_span_content(&self, span: Span) -> &[char] {span.get_content(&self.source)}@@ -581,6 +474,10 @@ macro_rules! create_fns_on_doc {self.tokens.[< last_ $thing >]()}+ fn [< last_ $thing _index>](&self) -> Option<usize> {+ self.tokens.[< last_ $thing _index >]()+ }+fn [<iter_ $thing _indices>](&self) -> impl Iterator<Item = usize> + '_ {self.tokens.[< iter_ $thing _indices >]()}@@ -602,6 +499,8 @@ impl TokenStringExt for Document {create_fns_on_doc!(at);create_fns_on_doc!(ellipsis);create_fns_on_doc!(unlintable);+ create_fns_on_doc!(sentence_terminator);+ create_fns_on_doc!(chunk_terminator);fn first_sentence_word(&self) -> Option<Token> {self.tokens.first_sentence_word()@@ -622,6 +521,14 @@ impl TokenStringExt for Document {fn iter_linking_verbs(&self) -> impl Iterator<Item = Token> + '_ {self.tokens.iter_linking_verbs()}++ fn iter_chunks(&self) -> impl Iterator<Item = &'_ [Token]> + '_ {+ self.tokens.iter_chunks()+ }++ fn iter_sentences(&self) -> impl Iterator<Item = &'_ [Token]> + '_ {+ self.tokens.iter_sentences()+ }}impl Display for Document {@@ -634,62 +541,13 @@ impl Display for Document {}}-fn is_chunk_terminator(token: &TokenKind) -> bool {- if is_sentence_terminator(token) {- return true;- }-- match token {- TokenKind::Punctuation(punct) => {- matches!(punct, Punctuation::Comma | Punctuation::Quote { .. })- }- _ => false,- }-}--fn is_sentence_terminator(token: &TokenKind) -> bool {- match token {- TokenKind::Punctuation(punct) => [- Punctuation::Period,- Punctuation::Bang,- Punctuation::Question,- ]- .contains(punct),- TokenKind::ParagraphBreak => true,- _ => false,- }-}-#[cfg(test)]mod tests {use itertools::Itertools;use super::Document;- use crate::token::TokenStringExt;use crate::Span;- #[test]- fn parses_sentences_correctly() {- let text = "There were three little pigs. They built three little homes.";- let document = Document::new_plain_english_curated(text);-- let mut sentence_strs = vec![];-- for sentence in document.sentences() {- if let Some(span) = sentence.span() {- sentence_strs.push(document.get_span_content_str(span));- }- }-- assert_eq!(- sentence_strs,- vec![- "There were three little pigs.",- " They built three little homes."- ]- )- }-fn assert_condensed_contractions(text: &str, final_tok_count: usize) {let document = Document::new_plain_english_curated(text);diff --git a/harper-core/src/language_detection.rs b/harper-core/src/language_detection.rsindex 4e2f19a96..fb9a15092 100644--- a/harper-core/src/language_detection.rs+++ b/harper-core/src/language_detection.rs@@ -1,18 +1,22 @@-use crate::{Dictionary, Document, TokenKind};+use crate::{Dictionary, Document, Token, TokenKind};++pub fn is_doc_likely_english(doc: &Document, dict: &impl Dictionary) -> bool {+ is_likely_english(doc.get_tokens(), doc.get_source(), dict)+}/// Check if the contents of the document are likely intended to represent/// English.-pub fn is_likely_english(doc: &Document, dict: &impl Dictionary) -> bool {+pub fn is_likely_english(toks: &[Token], source: &[char], dict: &impl Dictionary) -> bool {let mut total_words = 0;let mut valid_words = 0;let mut punctuation = 0;- for token in doc.tokens() {+ for token in toks {match token.kind {TokenKind::Word(_) => {total_words += 1;- let word_content = doc.get_span_content(token.span);+ let word_content = token.span.get_content(source);if dict.contains_word(word_content) {valid_words += 1;}@@ -35,13 +39,13 @@ pub fn is_likely_english(doc: &Document, dict: &impl Dictionary) -> bool {#[cfg(test)]mod tests {- use super::is_likely_english;+ use super::is_doc_likely_english;use crate::{Document, FullDictionary};fn assert_not_english(source: &'static str) {let dict = FullDictionary::curated();let doc = Document::new_plain_english(source, &dict);- let is_likely_english = is_likely_english(&doc, &dict);+ let is_likely_english = is_doc_likely_english(&doc, &dict);dbg!(source);assert!(!is_likely_english);}@@ -49,7 +53,7 @@ mod tests {fn assert_english(source: &'static str) {let dict = FullDictionary::curated();let doc = Document::new_plain_english(source, &dict);- let is_likely_english = is_likely_english(&doc, &dict);+ let is_likely_english = is_doc_likely_english(&doc, &dict);dbg!(source);assert!(is_likely_english);}diff --git a/harper-core/src/linting/an_a.rs b/harper-core/src/linting/an_a.rsindex fc82a4ded..d28301ee6 100644--- a/harper-core/src/linting/an_a.rs+++ b/harper-core/src/linting/an_a.rs@@ -12,7 +12,7 @@ impl Linter for AnA {fn lint(&mut self, document: &Document) -> Vec<Lint> {let mut lints = Vec::new();- for chunk in document.chunks() {+ for chunk in document.iter_chunks() {for (first_idx, second_idx) in chunk.iter_word_indices().tuple_windows() {// [`TokenKind::Unlintable`] might be semantic words.if chunk[first_idx..second_idx].iter_unlintables().count() > 0 {diff --git a/harper-core/src/linting/linking_verbs.rs b/harper-core/src/linting/linking_verbs.rsindex c355b1318..dadb4fded 100644--- a/harper-core/src/linting/linking_verbs.rs+++ b/harper-core/src/linting/linking_verbs.rs@@ -10,7 +10,7 @@ impl Linter for LinkingVerbs {fn lint(&mut self, document: &Document) -> Vec<Lint> {let mut output = Vec::new();- for chunk in document.chunks() {+ for chunk in document.iter_chunks() {// The word prior to "is" must be a noun.for idx in chunk.iter_linking_verb_indices() {let linking_verb = chunk[idx];diff --git a/harper-core/src/linting/long_sentences.rs b/harper-core/src/linting/long_sentences.rsindex c6ba801f0..005a842f0 100644--- a/harper-core/src/linting/long_sentences.rs+++ b/harper-core/src/linting/long_sentences.rs@@ -10,7 +10,7 @@ impl Linter for LongSentences {fn lint(&mut self, document: &Document) -> Vec<Lint> {let mut output = Vec::new();- for sentence in document.sentences() {+ for sentence in document.iter_sentences() {let word_count = sentence.iter_words().count();if word_count > 40 {diff --git a/harper-core/src/linting/pattern_linter.rs b/harper-core/src/linting/pattern_linter.rsindex 7026cde9f..fc2cb26fe 100644--- a/harper-core/src/linting/pattern_linter.rs+++ b/harper-core/src/linting/pattern_linter.rs@@ -1,6 +1,6 @@use super::{Lint, Linter};use crate::patterns::Pattern;-use crate::Token;+use crate::{Token, TokenStringExt};#[cfg(not(feature = "concurrent"))]pub trait PatternLinter {@@ -24,7 +24,7 @@ wherelet mut lints = Vec::new();let source = document.get_source();- for chunk in document.chunks() {+ for chunk in document.iter_chunks() {let mut tok_cursor = 0;loop {diff --git a/harper-core/src/linting/sentence_capitalization.rs b/harper-core/src/linting/sentence_capitalization.rsindex 51d1093fa..90011e17b 100644--- a/harper-core/src/linting/sentence_capitalization.rs+++ b/harper-core/src/linting/sentence_capitalization.rs@@ -14,7 +14,7 @@ impl Linter for SentenceCapitalization {fn lint(&mut self, document: &Document) -> Vec<Lint> {let mut lints = Vec::new();- for sentence in document.sentences() {+ for sentence in document.iter_sentences() {if let Some(first_word) = sentence.first_non_whitespace() {if !first_word.kind.is_word() {continue;diff --git a/harper-core/src/linting/spaces.rs b/harper-core/src/linting/spaces.rsindex 09b469e87..5da5eac11 100644--- a/harper-core/src/linting/spaces.rs+++ b/harper-core/src/linting/spaces.rs@@ -9,7 +9,7 @@ impl Linter for Spaces {fn lint(&mut self, document: &Document) -> Vec<Lint> {let mut output = Vec::new();- for sentence in document.sentences() {+ for sentence in document.iter_sentences() {for space in sentence.iter_spaces() {let TokenKind::Space(count) = space.kind else {panic!("The space iterator should only return spaces.")diff --git a/harper-core/src/token.rs b/harper-core/src/token.rsindex 9f72ed30f..da33e7204 100644--- a/harper-core/src/token.rs+++ b/harper-core/src/token.rs@@ -97,6 +97,32 @@ impl TokenKind {)}+ fn is_chunk_terminator(&self) -> bool {+ if self.is_sentence_terminator() {+ return true;+ }++ match self {+ TokenKind::Punctuation(punct) => {+ matches!(punct, Punctuation::Comma | Punctuation::Quote { .. })+ }+ _ => false,+ }+ }++ fn is_sentence_terminator(&self) -> bool {+ match self {+ TokenKind::Punctuation(punct) => [+ Punctuation::Period,+ Punctuation::Bang,+ Punctuation::Question,+ ]+ .contains(punct),+ TokenKind::ParagraphBreak => true,+ _ => false,+ }+ }+pub fn is_ellipsis(&self) -> bool {matches!(self, TokenKind::Punctuation(Punctuation::Ellipsis))}@@ -287,9 +313,11 @@ impl TokenKind {macro_rules! create_decl_for {($thing:ident) => {paste! {- fn [< first_ $thing >](&self) -> Option<Token> ;+ fn [< first_ $thing >](&self) -> Option<Token>;- fn [< last_ $thing >](&self) -> Option<Token> ;+ fn [< last_ $thing >](&self) -> Option<Token>;++ fn [< last_ $thing _index >](&self) -> Option<usize>;fn [<iter_ $thing _indices>](&self) -> impl Iterator<Item = usize> + '_;@@ -309,6 +337,10 @@ macro_rules! create_fns_for {self.iter().rev().find(|v| v.kind.[<is_ $thing>]()).copied()}+ fn [< last_ $thing _index >](&self) -> Option<usize> {+ self.iter().rev().position(|v| v.kind.[<is_ $thing>]()).map(|i| self.len() - i - 1)+ }+fn [<iter_ $thing _indices>](&self) -> impl Iterator<Item = usize> + '_ {self.iter().enumerate()@@ -339,9 +371,25 @@ pub trait TokenStringExt {create_decl_for!(at);create_decl_for!(ellipsis);create_decl_for!(unlintable);+ create_decl_for!(sentence_terminator);+ create_decl_for!(chunk_terminator);fn iter_linking_verb_indices(&self) -> impl Iterator<Item = usize> + '_;fn iter_linking_verbs(&self) -> impl Iterator<Item = Token> + '_;++ /// Iterate over chunks.+ ///+ /// For example, the following sentence contains two chunks separated by a+ /// comma:+ ///+ /// ```text+ /// Here is an example, it is short.+ /// ```+ fn iter_chunks(&self) -> impl Iterator<Item = &'_ [Token]> + '_;++ /// Get an iterator over token slices that represent the individual+ /// sentences in a document.+ fn iter_sentences(&self) -> impl Iterator<Item = &'_ [Token]> + '_;}impl TokenStringExt for [Token] {@@ -354,6 +402,8 @@ impl TokenStringExt for [Token] {create_fns_for!(at);create_fns_for!(ellipsis);create_fns_for!(unlintable);+ create_fns_for!(sentence_terminator);+ create_fns_for!(chunk_terminator);fn first_non_whitespace(&self) -> Option<Token> {self.iter().find(|t| !t.kind.is_whitespace()).copied()@@ -391,4 +441,83 @@ impl TokenStringExt for [Token] {fn iter_linking_verbs(&self) -> impl Iterator<Item = Token> + '_ {self.iter_linking_verb_indices().map(|idx| self[idx])}++ fn iter_chunks(&self) -> impl Iterator<Item = &'_ [Token]> + '_ {+ let first_chunk = self+ .iter_chunk_terminator_indices()+ .next()+ .map(|first_term| &self[0..=first_term]);++ let rest = self+ .iter_chunk_terminator_indices()+ .tuple_windows()+ .map(move |(a, b)| &self[a + 1..=b]);++ let last = if let Some(last_i) = self.last_chunk_terminator_index() {+ if last_i + 1 < self.len() {+ Some(&self[last_i + 1..])+ } else {+ None+ }+ } else {+ Some(self)+ };++ first_chunk.into_iter().chain(rest).chain(last)+ }++ fn iter_sentences(&self) -> impl Iterator<Item = &'_ [Token]> + '_ {+ let first_sentence = self+ .iter_sentence_terminator_indices()+ .next()+ .map(|first_term| &self[0..=first_term]);++ let rest = self+ .iter_sentence_terminator_indices()+ .tuple_windows()+ .map(move |(a, b)| &self[a + 1..=b]);++ let last_sentence = if let Some(last_i) = self.last_sentence_terminator_index() {+ if last_i + 1 < self.len() {+ Some(&self[last_i + 1..])+ } else {+ None+ }+ } else {+ Some(self)+ };++ first_sentence.into_iter().chain(rest).chain(last_sentence)+ }+}++#[cfg(test)]+mod tests {+ use crate::{+ parsers::{Parser, PlainEnglish},+ TokenStringExt,+ };++ #[test]+ fn parses_sentences_correctly() {+ let text = "There were three little pigs. They built three little homes.";+ let chars: Vec<char> = text.chars().collect();+ let toks = PlainEnglish.parse(&chars);++ let mut sentence_strs = vec![];++ for sentence in toks.iter_sentences() {+ if let Some(span) = sentence.span() {+ sentence_strs.push(span.get_content_string(&chars));+ }+ }++ assert_eq!(+ sentence_strs,+ vec![+ "There were three little pigs.",+ " They built three little homes."+ ]+ )+ }}