Skip to content

Offline, privacy-first grammar checker. Fast, open-source, Rust-powered

fix: moved sentence and chunk tools to `TokenStringExt`

Elijah Potter46b1cbf76ef0
diff --git a/harper-core/src/document.rs b/harper-core/src/document.rs
index ed9096c69..a6b8c8b8b 100644
--- a/harper-core/src/document.rs
+++ b/harper-core/src/document.rs
@@ -2,7 +2,6 @@ use std::cmp::Ordering;
use std::collections::VecDeque;
use std::fmt::Display;
-use itertools::Itertools;
use paste::paste;
use crate::parsers::{Markdown, Parser, PlainEnglish};
@@ -187,112 +186,6 @@ impl Document {
self.tokens().map(|token| token.to_fat(&self.source))
}
- /// Iterate over the locations of punctuation the separates chunks.
- fn chunk_terminators(&self) -> impl Iterator<Item = usize> + '_ {
- self.tokens.iter().enumerate().filter_map(|(index, token)| {
- if is_chunk_terminator(&token.kind) {
- return Some(index);
- }
- None
- })
- }
-
- /// Get the index of the last chunk terminator.
- fn last_chunk_terminator(&self) -> Option<usize> {
- self.tokens
- .iter()
- .enumerate()
- .rev()
- .find_map(|(index, token)| {
- if is_chunk_terminator(&token.kind) {
- return Some(index);
- }
- None
- })
- }
-
- /// Iterate over chunks.
- ///
- /// For example, the following sentence contains two chunks separated by a
- /// comma:
- ///
- /// ```text
- /// Here is an example, it is short.
- /// ```
- pub fn chunks(&self) -> impl Iterator<Item = &'_ [Token]> + '_ {
- let first_chunk = self
- .chunk_terminators()
- .next()
- .map(|first_term| &self.tokens[0..=first_term]);
-
- let rest = self
- .chunk_terminators()
- .tuple_windows()
- .map(move |(a, b)| &self.tokens[a + 1..=b]);
-
- let last = if let Some(last_i) = self.last_chunk_terminator() {
- if last_i + 1 < self.tokens.len() {
- Some(&self.tokens[last_i + 1..])
- } else {
- None
- }
- } else {
- Some(self.tokens.as_slice())
- };
-
- first_chunk.into_iter().chain(rest).chain(last)
- }
-
- /// Iterate over the locations of the sentence terminators in the document.
- fn sentence_terminators(&self) -> impl Iterator<Item = usize> + '_ {
- self.tokens.iter().enumerate().filter_map(|(index, token)| {
- if is_sentence_terminator(&token.kind) {
- return Some(index);
- }
- None
- })
- }
-
- /// Get the index of the last sentence terminator in the document.
- fn last_sentence_terminator(&self) -> Option<usize> {
- self.tokens
- .iter()
- .enumerate()
- .rev()
- .find_map(|(index, token)| {
- if is_sentence_terminator(&token.kind) {
- return Some(index);
- }
- None
- })
- }
-
- /// Get an iterator over token slices that represent the individual
- /// sentences in a document.
- pub fn sentences(&self) -> impl Iterator<Item = &'_ [Token]> + '_ {
- let first_sentence = self
- .sentence_terminators()
- .next()
- .map(|first_term| &self.tokens[0..=first_term]);
-
- let rest = self
- .sentence_terminators()
- .tuple_windows()
- .map(move |(a, b)| &self.tokens[a + 1..=b]);
-
- let last = if let Some(last_i) = self.last_sentence_terminator() {
- if last_i + 1 < self.tokens.len() {
- Some(&self.tokens[last_i + 1..])
- } else {
- None
- }
- } else {
- Some(self.tokens.as_slice())
- };
-
- first_sentence.into_iter().chain(rest).chain(last)
- }
-
pub fn get_span_content(&self, span: Span) -> &[char] {
span.get_content(&self.source)
}
@@ -581,6 +474,10 @@ macro_rules! create_fns_on_doc {
self.tokens.[< last_ $thing >]()
}
+ fn [< last_ $thing _index>](&self) -> Option<usize> {
+ self.tokens.[< last_ $thing _index >]()
+ }
+
fn [<iter_ $thing _indices>](&self) -> impl Iterator<Item = usize> + '_ {
self.tokens.[< iter_ $thing _indices >]()
}
@@ -602,6 +499,8 @@ impl TokenStringExt for Document {
create_fns_on_doc!(at);
create_fns_on_doc!(ellipsis);
create_fns_on_doc!(unlintable);
+ create_fns_on_doc!(sentence_terminator);
+ create_fns_on_doc!(chunk_terminator);
fn first_sentence_word(&self) -> Option<Token> {
self.tokens.first_sentence_word()
@@ -622,6 +521,14 @@ impl TokenStringExt for Document {
fn iter_linking_verbs(&self) -> impl Iterator<Item = Token> + '_ {
self.tokens.iter_linking_verbs()
}
+
+ fn iter_chunks(&self) -> impl Iterator<Item = &'_ [Token]> + '_ {
+ self.tokens.iter_chunks()
+ }
+
+ fn iter_sentences(&self) -> impl Iterator<Item = &'_ [Token]> + '_ {
+ self.tokens.iter_sentences()
+ }
}
impl Display for Document {
@@ -634,62 +541,13 @@ impl Display for Document {
}
}
-fn is_chunk_terminator(token: &TokenKind) -> bool {
- if is_sentence_terminator(token) {
- return true;
- }
-
- match token {
- TokenKind::Punctuation(punct) => {
- matches!(punct, Punctuation::Comma | Punctuation::Quote { .. })
- }
- _ => false,
- }
-}
-
-fn is_sentence_terminator(token: &TokenKind) -> bool {
- match token {
- TokenKind::Punctuation(punct) => [
- Punctuation::Period,
- Punctuation::Bang,
- Punctuation::Question,
- ]
- .contains(punct),
- TokenKind::ParagraphBreak => true,
- _ => false,
- }
-}
-
#[cfg(test)]
mod tests {
use itertools::Itertools;
use super::Document;
- use crate::token::TokenStringExt;
use crate::Span;
- #[test]
- fn parses_sentences_correctly() {
- let text = "There were three little pigs. They built three little homes.";
- let document = Document::new_plain_english_curated(text);
-
- let mut sentence_strs = vec![];
-
- for sentence in document.sentences() {
- if let Some(span) = sentence.span() {
- sentence_strs.push(document.get_span_content_str(span));
- }
- }
-
- assert_eq!(
- sentence_strs,
- vec![
- "There were three little pigs.",
- " They built three little homes."
- ]
- )
- }
-
fn assert_condensed_contractions(text: &str, final_tok_count: usize) {
let document = Document::new_plain_english_curated(text);
diff --git a/harper-core/src/language_detection.rs b/harper-core/src/language_detection.rs
index 4e2f19a96..fb9a15092 100644
--- a/harper-core/src/language_detection.rs
+++ b/harper-core/src/language_detection.rs
@@ -1,18 +1,22 @@
-use crate::{Dictionary, Document, TokenKind};
+use crate::{Dictionary, Document, Token, TokenKind};
+
+pub fn is_doc_likely_english(doc: &Document, dict: &impl Dictionary) -> bool {
+ is_likely_english(doc.get_tokens(), doc.get_source(), dict)
+}
/// Check if the contents of the document are likely intended to represent
/// English.
-pub fn is_likely_english(doc: &Document, dict: &impl Dictionary) -> bool {
+pub fn is_likely_english(toks: &[Token], source: &[char], dict: &impl Dictionary) -> bool {
let mut total_words = 0;
let mut valid_words = 0;
let mut punctuation = 0;
- for token in doc.tokens() {
+ for token in toks {
match token.kind {
TokenKind::Word(_) => {
total_words += 1;
- let word_content = doc.get_span_content(token.span);
+ let word_content = token.span.get_content(source);
if dict.contains_word(word_content) {
valid_words += 1;
}
@@ -35,13 +39,13 @@ pub fn is_likely_english(doc: &Document, dict: &impl Dictionary) -> bool {
#[cfg(test)]
mod tests {
- use super::is_likely_english;
+ use super::is_doc_likely_english;
use crate::{Document, FullDictionary};
fn assert_not_english(source: &'static str) {
let dict = FullDictionary::curated();
let doc = Document::new_plain_english(source, &dict);
- let is_likely_english = is_likely_english(&doc, &dict);
+ let is_likely_english = is_doc_likely_english(&doc, &dict);
dbg!(source);
assert!(!is_likely_english);
}
@@ -49,7 +53,7 @@ mod tests {
fn assert_english(source: &'static str) {
let dict = FullDictionary::curated();
let doc = Document::new_plain_english(source, &dict);
- let is_likely_english = is_likely_english(&doc, &dict);
+ let is_likely_english = is_doc_likely_english(&doc, &dict);
dbg!(source);
assert!(is_likely_english);
}
diff --git a/harper-core/src/linting/an_a.rs b/harper-core/src/linting/an_a.rs
index fc82a4ded..d28301ee6 100644
--- a/harper-core/src/linting/an_a.rs
+++ b/harper-core/src/linting/an_a.rs
@@ -12,7 +12,7 @@ impl Linter for AnA {
fn lint(&mut self, document: &Document) -> Vec<Lint> {
let mut lints = Vec::new();
- for chunk in document.chunks() {
+ for chunk in document.iter_chunks() {
for (first_idx, second_idx) in chunk.iter_word_indices().tuple_windows() {
// [`TokenKind::Unlintable`] might be semantic words.
if chunk[first_idx..second_idx].iter_unlintables().count() > 0 {
diff --git a/harper-core/src/linting/linking_verbs.rs b/harper-core/src/linting/linking_verbs.rs
index c355b1318..dadb4fded 100644
--- a/harper-core/src/linting/linking_verbs.rs
+++ b/harper-core/src/linting/linking_verbs.rs
@@ -10,7 +10,7 @@ impl Linter for LinkingVerbs {
fn lint(&mut self, document: &Document) -> Vec<Lint> {
let mut output = Vec::new();
- for chunk in document.chunks() {
+ for chunk in document.iter_chunks() {
// The word prior to "is" must be a noun.
for idx in chunk.iter_linking_verb_indices() {
let linking_verb = chunk[idx];
diff --git a/harper-core/src/linting/long_sentences.rs b/harper-core/src/linting/long_sentences.rs
index c6ba801f0..005a842f0 100644
--- a/harper-core/src/linting/long_sentences.rs
+++ b/harper-core/src/linting/long_sentences.rs
@@ -10,7 +10,7 @@ impl Linter for LongSentences {
fn lint(&mut self, document: &Document) -> Vec<Lint> {
let mut output = Vec::new();
- for sentence in document.sentences() {
+ for sentence in document.iter_sentences() {
let word_count = sentence.iter_words().count();
if word_count > 40 {
diff --git a/harper-core/src/linting/pattern_linter.rs b/harper-core/src/linting/pattern_linter.rs
index 7026cde9f..fc2cb26fe 100644
--- a/harper-core/src/linting/pattern_linter.rs
+++ b/harper-core/src/linting/pattern_linter.rs
@@ -1,6 +1,6 @@
use super::{Lint, Linter};
use crate::patterns::Pattern;
-use crate::Token;
+use crate::{Token, TokenStringExt};
#[cfg(not(feature = "concurrent"))]
pub trait PatternLinter {
@@ -24,7 +24,7 @@ where
let mut lints = Vec::new();
let source = document.get_source();
- for chunk in document.chunks() {
+ for chunk in document.iter_chunks() {
let mut tok_cursor = 0;
loop {
diff --git a/harper-core/src/linting/sentence_capitalization.rs b/harper-core/src/linting/sentence_capitalization.rs
index 51d1093fa..90011e17b 100644
--- a/harper-core/src/linting/sentence_capitalization.rs
+++ b/harper-core/src/linting/sentence_capitalization.rs
@@ -14,7 +14,7 @@ impl Linter for SentenceCapitalization {
fn lint(&mut self, document: &Document) -> Vec<Lint> {
let mut lints = Vec::new();
- for sentence in document.sentences() {
+ for sentence in document.iter_sentences() {
if let Some(first_word) = sentence.first_non_whitespace() {
if !first_word.kind.is_word() {
continue;
diff --git a/harper-core/src/linting/spaces.rs b/harper-core/src/linting/spaces.rs
index 09b469e87..5da5eac11 100644
--- a/harper-core/src/linting/spaces.rs
+++ b/harper-core/src/linting/spaces.rs
@@ -9,7 +9,7 @@ impl Linter for Spaces {
fn lint(&mut self, document: &Document) -> Vec<Lint> {
let mut output = Vec::new();
- for sentence in document.sentences() {
+ for sentence in document.iter_sentences() {
for space in sentence.iter_spaces() {
let TokenKind::Space(count) = space.kind else {
panic!("The space iterator should only return spaces.")
diff --git a/harper-core/src/token.rs b/harper-core/src/token.rs
index 9f72ed30f..da33e7204 100644
--- a/harper-core/src/token.rs
+++ b/harper-core/src/token.rs
@@ -97,6 +97,32 @@ impl TokenKind {
)
}
+ fn is_chunk_terminator(&self) -> bool {
+ if self.is_sentence_terminator() {
+ return true;
+ }
+
+ match self {
+ TokenKind::Punctuation(punct) => {
+ matches!(punct, Punctuation::Comma | Punctuation::Quote { .. })
+ }
+ _ => false,
+ }
+ }
+
+ fn is_sentence_terminator(&self) -> bool {
+ match self {
+ TokenKind::Punctuation(punct) => [
+ Punctuation::Period,
+ Punctuation::Bang,
+ Punctuation::Question,
+ ]
+ .contains(punct),
+ TokenKind::ParagraphBreak => true,
+ _ => false,
+ }
+ }
+
pub fn is_ellipsis(&self) -> bool {
matches!(self, TokenKind::Punctuation(Punctuation::Ellipsis))
}
@@ -287,9 +313,11 @@ impl TokenKind {
macro_rules! create_decl_for {
($thing:ident) => {
paste! {
- fn [< first_ $thing >](&self) -> Option<Token> ;
+ fn [< first_ $thing >](&self) -> Option<Token>;
- fn [< last_ $thing >](&self) -> Option<Token> ;
+ fn [< last_ $thing >](&self) -> Option<Token>;
+
+ fn [< last_ $thing _index >](&self) -> Option<usize>;
fn [<iter_ $thing _indices>](&self) -> impl Iterator<Item = usize> + '_;
@@ -309,6 +337,10 @@ macro_rules! create_fns_for {
self.iter().rev().find(|v| v.kind.[<is_ $thing>]()).copied()
}
+ fn [< last_ $thing _index >](&self) -> Option<usize> {
+ self.iter().rev().position(|v| v.kind.[<is_ $thing>]()).map(|i| self.len() - i - 1)
+ }
+
fn [<iter_ $thing _indices>](&self) -> impl Iterator<Item = usize> + '_ {
self.iter()
.enumerate()
@@ -339,9 +371,25 @@ pub trait TokenStringExt {
create_decl_for!(at);
create_decl_for!(ellipsis);
create_decl_for!(unlintable);
+ create_decl_for!(sentence_terminator);
+ create_decl_for!(chunk_terminator);
fn iter_linking_verb_indices(&self) -> impl Iterator<Item = usize> + '_;
fn iter_linking_verbs(&self) -> impl Iterator<Item = Token> + '_;
+
+ /// Iterate over chunks.
+ ///
+ /// For example, the following sentence contains two chunks separated by a
+ /// comma:
+ ///
+ /// ```text
+ /// Here is an example, it is short.
+ /// ```
+ fn iter_chunks(&self) -> impl Iterator<Item = &'_ [Token]> + '_;
+
+ /// Get an iterator over token slices that represent the individual
+ /// sentences in a document.
+ fn iter_sentences(&self) -> impl Iterator<Item = &'_ [Token]> + '_;
}
impl TokenStringExt for [Token] {
@@ -354,6 +402,8 @@ impl TokenStringExt for [Token] {
create_fns_for!(at);
create_fns_for!(ellipsis);
create_fns_for!(unlintable);
+ create_fns_for!(sentence_terminator);
+ create_fns_for!(chunk_terminator);
fn first_non_whitespace(&self) -> Option<Token> {
self.iter().find(|t| !t.kind.is_whitespace()).copied()
@@ -391,4 +441,83 @@ impl TokenStringExt for [Token] {
fn iter_linking_verbs(&self) -> impl Iterator<Item = Token> + '_ {
self.iter_linking_verb_indices().map(|idx| self[idx])
}
+
+ fn iter_chunks(&self) -> impl Iterator<Item = &'_ [Token]> + '_ {
+ let first_chunk = self
+ .iter_chunk_terminator_indices()
+ .next()
+ .map(|first_term| &self[0..=first_term]);
+
+ let rest = self
+ .iter_chunk_terminator_indices()
+ .tuple_windows()
+ .map(move |(a, b)| &self[a + 1..=b]);
+
+ let last = if let Some(last_i) = self.last_chunk_terminator_index() {
+ if last_i + 1 < self.len() {
+ Some(&self[last_i + 1..])
+ } else {
+ None
+ }
+ } else {
+ Some(self)
+ };
+
+ first_chunk.into_iter().chain(rest).chain(last)
+ }
+
+ fn iter_sentences(&self) -> impl Iterator<Item = &'_ [Token]> + '_ {
+ let first_sentence = self
+ .iter_sentence_terminator_indices()
+ .next()
+ .map(|first_term| &self[0..=first_term]);
+
+ let rest = self
+ .iter_sentence_terminator_indices()
+ .tuple_windows()
+ .map(move |(a, b)| &self[a + 1..=b]);
+
+ let last_sentence = if let Some(last_i) = self.last_sentence_terminator_index() {
+ if last_i + 1 < self.len() {
+ Some(&self[last_i + 1..])
+ } else {
+ None
+ }
+ } else {
+ Some(self)
+ };
+
+ first_sentence.into_iter().chain(rest).chain(last_sentence)
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use crate::{
+ parsers::{Parser, PlainEnglish},
+ TokenStringExt,
+ };
+
+ #[test]
+ fn parses_sentences_correctly() {
+ let text = "There were three little pigs. They built three little homes.";
+ let chars: Vec<char> = text.chars().collect();
+ let toks = PlainEnglish.parse(&chars);
+
+ let mut sentence_strs = vec![];
+
+ for sentence in toks.iter_sentences() {
+ if let Some(span) = sentence.span() {
+ sentence_strs.push(span.get_content_string(&chars));
+ }
+ }
+
+ assert_eq!(
+ sentence_strs,
+ vec![
+ "There were three little pigs.",
+ " They built three little homes."
+ ]
+ )
+ }
}