Offline, privacy-first grammar checker. Fast, open-source, Rust-powered
Added spaces, tests, and fixed issue in sentence parsing
Elijah Potterdaea731f7492
diff --git a/harper-core/src/document.rs b/harper-core/src/document.rsindex 222403196..239b5a4ba 100644--- a/harper-core/src/document.rs+++ b/harper-core/src/document.rs@@ -101,16 +101,33 @@ impl Document {} = token{if is_sentence_terminator(punct) {- Some(index)- } else {- None+ return Some(index);}- } else {- None}+ None})}+ /// Get the index of the last sentence terminator.+ fn last_sentence_terminator(&self) -> Option<usize> {+ self.tokens+ .iter()+ .enumerate()+ .rev()+ .find_map(|(index, token)| {+ if let Token {+ kind: TokenKind::Punctuation(punct),+ ..+ } = token+ {+ if is_sentence_terminator(punct) {+ return Some(index);+ }+ }+ None+ })+ }+pub fn sentences(&self) -> impl Iterator<Item = &'_ [Token]> + '_ {let first_sentence = self.sentence_terminators()@@ -122,7 +139,17 @@ impl Document {.tuple_windows().map(move |(a, b)| &self.tokens[a + 1..=b]);- first_sentence.into_iter().chain(rest)+ let last = if let Some(last_i) = self.last_sentence_terminator() {+ if last_i + 1 < self.tokens.len() {+ Some(&self.tokens[last_i + 1..])+ } else {+ None+ }+ } else {+ Some(self.tokens.as_slice())+ };++ first_sentence.into_iter().chain(rest).chain(last)}/** Returns all tokens whose `kind` is [`Punctuation::Word`] */diff --git a/harper-core/src/linting/lint.rs b/harper-core/src/linting/lint.rsindex 3bc89178f..b7334e2ca 100644--- a/harper-core/src/linting/lint.rs+++ b/harper-core/src/linting/lint.rs@@ -17,8 +17,7 @@ pub struct Lint {pub enum LintKind {Spelling,Capitalization,- UnmatchedQuote,- WrongQuotes,+ Formatting,Repetition,Readability,#[default]diff --git a/harper-core/src/linting/lint_set.rs b/harper-core/src/linting/lint_set.rsindex e3d31c39f..a5fea8f09 100644--- a/harper-core/src/linting/lint_set.rs+++ b/harper-core/src/linting/lint_set.rs@@ -1,6 +1,6 @@use crate::{Dictionary, Lint};-use super::Linter;+use super::{spaces::Spaces, Linter};use paste::paste;use super::{@@ -39,7 +39,8 @@ impl LintSet {.add_long_sentences().add_unclosed_quotes().add_sentence_capitalization()- .add_spell_check(dictionary);+ .add_spell_check(dictionary)+ .add_spaces();self}@@ -91,5 +92,6 @@ create_simple_builder_methods!(UnclosedQuotes,WrongQuotes,LongSentences,- RepeatedWords+ RepeatedWords,+ Spaces);diff --git a/harper-core/src/linting/mod.rs b/harper-core/src/linting/mod.rsindex 8c89ee144..687e3a3a5 100644--- a/harper-core/src/linting/mod.rs+++ b/harper-core/src/linting/mod.rs@@ -3,6 +3,7 @@ mod lint_set;mod long_sentences;mod repeated_words;mod sentence_capitalization;+mod spaces;mod spell_check;mod unclosed_quotes;mod wrong_quotes;@@ -15,3 +16,14 @@ use crate::Document;pub trait Linter: Send + Sync {fn lint(&mut self, document: &Document) -> Vec<Lint>;}++#[cfg(test)]+mod tests {+ use crate::{Document, Linter};++ pub fn assert_lint_count(text: &str, mut linter: impl Linter, count: usize) {+ let test = Document::new(text, false);+ let lints = linter.lint(&test);+ assert_eq!(lints.len(), count);+ }+}diff --git a/harper-core/src/linting/repeated_words.rs b/harper-core/src/linting/repeated_words.rsindex 329e62f41..d1828e8b3 100644--- a/harper-core/src/linting/repeated_words.rs+++ b/harper-core/src/linting/repeated_words.rs@@ -94,15 +94,11 @@ impl Linter for RepeatedWords {#[cfg(test)]mod tests {- use super::super::Linter;+ use super::super::tests::assert_lint_count;use super::RepeatedWords;- use crate::Document;#[test]fn catches_basic() {- let test = Document::new("I wanted the the banana.", false);- let mut linter = RepeatedWords::new();- let lints = linter.lint(&test);- assert!(lints.len() == 1);+ assert_lint_count("I wanted the the banana.", RepeatedWords::new(), 1)}}diff --git a/harper-core/src/linting/sentence_capitalization.rs b/harper-core/src/linting/sentence_capitalization.rsindex c60049090..eacd97b33 100644--- a/harper-core/src/linting/sentence_capitalization.rs+++ b/harper-core/src/linting/sentence_capitalization.rs@@ -14,6 +14,7 @@ impl Linter for SentenceCapitalization {let mut lints = Vec::new();for sentence in document.sentences() {+ dbg!(sentence);if let Some(first_word) = sentence.first_word() {let letters = document.get_span_content(first_word.span);@@ -36,3 +37,28 @@ impl Linter for SentenceCapitalization {lints}}++#[cfg(test)]+mod tests {+ use super::super::tests::assert_lint_count;+ use super::SentenceCapitalization;++ #[test]+ fn catches_basic() {+ assert_lint_count("there is no way.", SentenceCapitalization::default(), 1)+ }++ #[test]+ fn no_period() {+ assert_lint_count("there is no way", SentenceCapitalization::default(), 1)+ }++ #[test]+ fn two_sentence() {+ assert_lint_count(+ "i have complete conviction. she is guilty",+ SentenceCapitalization::default(),+ 2,+ )+ }+}diff --git a/harper-core/src/linting/spaces.rs b/harper-core/src/linting/spaces.rsnew file mode 100644index 000000000..2434e7ab8--- /dev/null+++ b/harper-core/src/linting/spaces.rs@@ -0,0 +1,33 @@+use super::{Lint, Linter};+use crate::{parsing::TokenStringExt, Document, LintKind, Suggestion, TokenKind};++#[derive(Debug, Default)]+pub struct Spaces;++impl Linter for Spaces {+ fn lint(&mut self, document: &Document) -> Vec<Lint> {+ let mut output = Vec::new();++ for sentence in document.sentences() {+ for space in sentence.iter_spaces() {+ let TokenKind::Space(count) = space.kind else {+ panic!("The space iterator should only return spaces.")+ };++ if count > 1 {+ output.push(Lint {+ span: space.span,+ lint_kind: LintKind::Formatting,+ suggestions: vec![Suggestion::ReplaceWith(vec![' '])],+ message: format!(+ "There are {} spaces where there should be only one.",+ count+ ),+ })+ }+ }+ }++ output+ }+}diff --git a/harper-core/src/linting/unclosed_quotes.rs b/harper-core/src/linting/unclosed_quotes.rsindex 67653aba1..c94da2819 100644--- a/harper-core/src/linting/unclosed_quotes.rs+++ b/harper-core/src/linting/unclosed_quotes.rs@@ -15,7 +15,7 @@ impl Linter for UnclosedQuotes {{lints.push(Lint {span: token.span,- lint_kind: LintKind::UnmatchedQuote,+ lint_kind: LintKind::Formatting,suggestions: vec![],message: "This quote has no termination.".to_string(),})diff --git a/harper-core/src/linting/wrong_quotes.rs b/harper-core/src/linting/wrong_quotes.rsindex b7a9eb12c..8446a083c 100644--- a/harper-core/src/linting/wrong_quotes.rs+++ b/harper-core/src/linting/wrong_quotes.rs@@ -28,9 +28,9 @@ fn lint_quote(document: &Document, quote_idx: usize, quote_token: Token) -> Optiif quote_char != should_be {Some(Lint {span: quote_token.span,- lint_kind: LintKind::WrongQuotes,suggestions: vec![Suggestion::ReplaceWith(vec![should_be])],message: "Use the better-formatted quote character.".to_string(),+ ..Default::default()})} else {Nonediff --git a/harper-core/src/parsing/token.rs b/harper-core/src/parsing/token.rsindex 85a908ec7..2da897866 100644--- a/harper-core/src/parsing/token.rs+++ b/harper-core/src/parsing/token.rs@@ -93,6 +93,8 @@ pub trait TokenStringExt {fn first_word(&self) -> Option<Token>;fn iter_word_indices(&self) -> impl Iterator<Item = usize> + '_;fn iter_words(&self) -> impl Iterator<Item = &Token> + '_;+ fn iter_space_indices(&self) -> impl Iterator<Item = usize> + '_;+ fn iter_spaces(&self) -> impl Iterator<Item = &Token> + '_;}impl TokenStringExt for [Token] {@@ -108,6 +110,17 @@ impl TokenStringExt for [Token] {}fn iter_words(&self) -> impl Iterator<Item = &Token> + '_ {- self.iter().filter(|t| t.kind.is_word())+ self.iter_word_indices().map(|i| &self[i])+ }++ fn iter_space_indices(&self) -> impl Iterator<Item = usize> + '_ {+ self.iter()+ .enumerate()+ .filter(|(_, t)| t.kind.is_space())+ .map(|(i, _)| i)+ }++ fn iter_spaces(&self) -> impl Iterator<Item = &Token> + '_ {+ self.iter_space_indices().map(|i| &self[i])}}diff --git a/harper-ls/src/backend.rs b/harper-ls/src/backend.rsindex 1d052e96c..d17bf94f1 100644--- a/harper-ls/src/backend.rs+++ b/harper-ls/src/backend.rs@@ -1,4 +1,4 @@-use std::{borrow::BorrowMut, ops::DerefMut};+use std::ops::DerefMut;use harper_core::{Dictionary, LintSet};use tokio::{sync::Mutex, time::Instant};diff --git a/web/src/lib/analysis.ts b/web/src/lib/analysis.tsindex 8d9ed8a1e..a78e07e98 100644--- a/web/src/lib/analysis.ts+++ b/web/src/lib/analysis.ts@@ -98,7 +98,9 @@ export async function lintText(text: string, useWasm = defaultUseWasm): Promise<const res: LintResponse = await req.json();// We only want to show fixable errors.- return res.lints.filter((lint) => lint.suggestions.length > 0);+ const lints = res.lints.filter((lint) => lint.suggestions.length > 0);+ console.log(lints);+ return lints;}}