Skip to content

Offline, privacy-first grammar checker. Fast, open-source, Rust-powered

Added spaces, tests, and fixed issue in sentence parsing

Elijah Potterdaea731f7492
diff --git a/harper-core/src/document.rs b/harper-core/src/document.rs
index 222403196..239b5a4ba 100644
--- a/harper-core/src/document.rs
+++ b/harper-core/src/document.rs
@@ -101,16 +101,33 @@ impl Document {
} = token
{
if is_sentence_terminator(punct) {
- Some(index)
- } else {
- None
+ return Some(index);
}
- } else {
- None
}
+ None
})
}
+ /// Get the index of the last sentence terminator.
+ fn last_sentence_terminator(&self) -> Option<usize> {
+ self.tokens
+ .iter()
+ .enumerate()
+ .rev()
+ .find_map(|(index, token)| {
+ if let Token {
+ kind: TokenKind::Punctuation(punct),
+ ..
+ } = token
+ {
+ if is_sentence_terminator(punct) {
+ return Some(index);
+ }
+ }
+ None
+ })
+ }
+
pub fn sentences(&self) -> impl Iterator<Item = &'_ [Token]> + '_ {
let first_sentence = self
.sentence_terminators()
@@ -122,7 +139,17 @@ impl Document {
.tuple_windows()
.map(move |(a, b)| &self.tokens[a + 1..=b]);
- first_sentence.into_iter().chain(rest)
+ let last = if let Some(last_i) = self.last_sentence_terminator() {
+ if last_i + 1 < self.tokens.len() {
+ Some(&self.tokens[last_i + 1..])
+ } else {
+ None
+ }
+ } else {
+ Some(self.tokens.as_slice())
+ };
+
+ first_sentence.into_iter().chain(rest).chain(last)
}
/** Returns all tokens whose `kind` is [`Punctuation::Word`] */
diff --git a/harper-core/src/linting/lint.rs b/harper-core/src/linting/lint.rs
index 3bc89178f..b7334e2ca 100644
--- a/harper-core/src/linting/lint.rs
+++ b/harper-core/src/linting/lint.rs
@@ -17,8 +17,7 @@ pub struct Lint {
pub enum LintKind {
Spelling,
Capitalization,
- UnmatchedQuote,
- WrongQuotes,
+ Formatting,
Repetition,
Readability,
#[default]
diff --git a/harper-core/src/linting/lint_set.rs b/harper-core/src/linting/lint_set.rs
index e3d31c39f..a5fea8f09 100644
--- a/harper-core/src/linting/lint_set.rs
+++ b/harper-core/src/linting/lint_set.rs
@@ -1,6 +1,6 @@
use crate::{Dictionary, Lint};
-use super::Linter;
+use super::{spaces::Spaces, Linter};
use paste::paste;
use super::{
@@ -39,7 +39,8 @@ impl LintSet {
.add_long_sentences()
.add_unclosed_quotes()
.add_sentence_capitalization()
- .add_spell_check(dictionary);
+ .add_spell_check(dictionary)
+ .add_spaces();
self
}
@@ -91,5 +92,6 @@ create_simple_builder_methods!(
UnclosedQuotes,
WrongQuotes,
LongSentences,
- RepeatedWords
+ RepeatedWords,
+ Spaces
);
diff --git a/harper-core/src/linting/mod.rs b/harper-core/src/linting/mod.rs
index 8c89ee144..687e3a3a5 100644
--- a/harper-core/src/linting/mod.rs
+++ b/harper-core/src/linting/mod.rs
@@ -3,6 +3,7 @@ mod lint_set;
mod long_sentences;
mod repeated_words;
mod sentence_capitalization;
+mod spaces;
mod spell_check;
mod unclosed_quotes;
mod wrong_quotes;
@@ -15,3 +16,14 @@ use crate::Document;
pub trait Linter: Send + Sync {
fn lint(&mut self, document: &Document) -> Vec<Lint>;
}
+
+#[cfg(test)]
+mod tests {
+ use crate::{Document, Linter};
+
+ pub fn assert_lint_count(text: &str, mut linter: impl Linter, count: usize) {
+ let test = Document::new(text, false);
+ let lints = linter.lint(&test);
+ assert_eq!(lints.len(), count);
+ }
+}
diff --git a/harper-core/src/linting/repeated_words.rs b/harper-core/src/linting/repeated_words.rs
index 329e62f41..d1828e8b3 100644
--- a/harper-core/src/linting/repeated_words.rs
+++ b/harper-core/src/linting/repeated_words.rs
@@ -94,15 +94,11 @@ impl Linter for RepeatedWords {
#[cfg(test)]
mod tests {
- use super::super::Linter;
+ use super::super::tests::assert_lint_count;
use super::RepeatedWords;
- use crate::Document;
#[test]
fn catches_basic() {
- let test = Document::new("I wanted the the banana.", false);
- let mut linter = RepeatedWords::new();
- let lints = linter.lint(&test);
- assert!(lints.len() == 1);
+ assert_lint_count("I wanted the the banana.", RepeatedWords::new(), 1)
}
}
diff --git a/harper-core/src/linting/sentence_capitalization.rs b/harper-core/src/linting/sentence_capitalization.rs
index c60049090..eacd97b33 100644
--- a/harper-core/src/linting/sentence_capitalization.rs
+++ b/harper-core/src/linting/sentence_capitalization.rs
@@ -14,6 +14,7 @@ impl Linter for SentenceCapitalization {
let mut lints = Vec::new();
for sentence in document.sentences() {
+ dbg!(sentence);
if let Some(first_word) = sentence.first_word() {
let letters = document.get_span_content(first_word.span);
@@ -36,3 +37,28 @@ impl Linter for SentenceCapitalization {
lints
}
}
+
+#[cfg(test)]
+mod tests {
+ use super::super::tests::assert_lint_count;
+ use super::SentenceCapitalization;
+
+ #[test]
+ fn catches_basic() {
+ assert_lint_count("there is no way.", SentenceCapitalization::default(), 1)
+ }
+
+ #[test]
+ fn no_period() {
+ assert_lint_count("there is no way", SentenceCapitalization::default(), 1)
+ }
+
+ #[test]
+ fn two_sentence() {
+ assert_lint_count(
+ "i have complete conviction. she is guilty",
+ SentenceCapitalization::default(),
+ 2,
+ )
+ }
+}
diff --git a/harper-core/src/linting/spaces.rs b/harper-core/src/linting/spaces.rs
new file mode 100644
index 000000000..2434e7ab8
--- /dev/null
+++ b/harper-core/src/linting/spaces.rs
@@ -0,0 +1,33 @@
+use super::{Lint, Linter};
+use crate::{parsing::TokenStringExt, Document, LintKind, Suggestion, TokenKind};
+
+#[derive(Debug, Default)]
+pub struct Spaces;
+
+impl Linter for Spaces {
+ fn lint(&mut self, document: &Document) -> Vec<Lint> {
+ let mut output = Vec::new();
+
+ for sentence in document.sentences() {
+ for space in sentence.iter_spaces() {
+ let TokenKind::Space(count) = space.kind else {
+ panic!("The space iterator should only return spaces.")
+ };
+
+ if count > 1 {
+ output.push(Lint {
+ span: space.span,
+ lint_kind: LintKind::Formatting,
+ suggestions: vec![Suggestion::ReplaceWith(vec![' '])],
+ message: format!(
+ "There are {} spaces where there should be only one.",
+ count
+ ),
+ })
+ }
+ }
+ }
+
+ output
+ }
+}
diff --git a/harper-core/src/linting/unclosed_quotes.rs b/harper-core/src/linting/unclosed_quotes.rs
index 67653aba1..c94da2819 100644
--- a/harper-core/src/linting/unclosed_quotes.rs
+++ b/harper-core/src/linting/unclosed_quotes.rs
@@ -15,7 +15,7 @@ impl Linter for UnclosedQuotes {
{
lints.push(Lint {
span: token.span,
- lint_kind: LintKind::UnmatchedQuote,
+ lint_kind: LintKind::Formatting,
suggestions: vec![],
message: "This quote has no termination.".to_string(),
})
diff --git a/harper-core/src/linting/wrong_quotes.rs b/harper-core/src/linting/wrong_quotes.rs
index b7a9eb12c..8446a083c 100644
--- a/harper-core/src/linting/wrong_quotes.rs
+++ b/harper-core/src/linting/wrong_quotes.rs
@@ -28,9 +28,9 @@ fn lint_quote(document: &Document, quote_idx: usize, quote_token: Token) -> Opti
if quote_char != should_be {
Some(Lint {
span: quote_token.span,
- lint_kind: LintKind::WrongQuotes,
suggestions: vec![Suggestion::ReplaceWith(vec![should_be])],
message: "Use the better-formatted quote character.".to_string(),
+ ..Default::default()
})
} else {
None
diff --git a/harper-core/src/parsing/token.rs b/harper-core/src/parsing/token.rs
index 85a908ec7..2da897866 100644
--- a/harper-core/src/parsing/token.rs
+++ b/harper-core/src/parsing/token.rs
@@ -93,6 +93,8 @@ pub trait TokenStringExt {
fn first_word(&self) -> Option<Token>;
fn iter_word_indices(&self) -> impl Iterator<Item = usize> + '_;
fn iter_words(&self) -> impl Iterator<Item = &Token> + '_;
+ fn iter_space_indices(&self) -> impl Iterator<Item = usize> + '_;
+ fn iter_spaces(&self) -> impl Iterator<Item = &Token> + '_;
}
impl TokenStringExt for [Token] {
@@ -108,6 +110,17 @@ impl TokenStringExt for [Token] {
}
fn iter_words(&self) -> impl Iterator<Item = &Token> + '_ {
- self.iter().filter(|t| t.kind.is_word())
+ self.iter_word_indices().map(|i| &self[i])
+ }
+
+ fn iter_space_indices(&self) -> impl Iterator<Item = usize> + '_ {
+ self.iter()
+ .enumerate()
+ .filter(|(_, t)| t.kind.is_space())
+ .map(|(i, _)| i)
+ }
+
+ fn iter_spaces(&self) -> impl Iterator<Item = &Token> + '_ {
+ self.iter_space_indices().map(|i| &self[i])
}
}
diff --git a/harper-ls/src/backend.rs b/harper-ls/src/backend.rs
index 1d052e96c..d17bf94f1 100644
--- a/harper-ls/src/backend.rs
+++ b/harper-ls/src/backend.rs
@@ -1,4 +1,4 @@
-use std::{borrow::BorrowMut, ops::DerefMut};
+use std::ops::DerefMut;
use harper_core::{Dictionary, LintSet};
use tokio::{sync::Mutex, time::Instant};
diff --git a/web/src/lib/analysis.ts b/web/src/lib/analysis.ts
index 8d9ed8a1e..a78e07e98 100644
--- a/web/src/lib/analysis.ts
+++ b/web/src/lib/analysis.ts
@@ -98,7 +98,9 @@ export async function lintText(text: string, useWasm = defaultUseWasm): Promise<
const res: LintResponse = await req.json();
// We only want to show fixable errors.
- return res.lints.filter((lint) => lint.suggestions.length > 0);
+ const lints = res.lints.filter((lint) => lint.suggestions.length > 0);
+ console.log(lints);
+ return lints;
}
}