Offline, privacy-first grammar checker. Fast, open-source, Rust-powered
feat: created Mask pattern
Elijah Potter676527eae9ac
diff --git a/harper-comments/src/better.rs b/harper-comments/src/better.rsnew file mode 100644index 000000000..8b1378917--- /dev/null+++ b/harper-comments/src/better.rs@@ -0,0 +1 @@+diff --git a/harper-comments/src/comment_parsers/jsdoc.rs b/harper-comments/src/comment_parsers/jsdoc.rsindex e4f72a00c..4db6b908a 100644--- a/harper-comments/src/comment_parsers/jsdoc.rs+++ b/harper-comments/src/comment_parsers/jsdoc.rs@@ -1,5 +1,5 @@use harper_core::parsers::{Markdown, Parser};-use harper_core::{Punctuation, Token, TokenKind};+use harper_core::{Punctuation, Span, Token, TokenKind};use itertools::Itertools;use super::without_initiators;@@ -15,6 +15,11 @@ impl Parser for JsDoc {for line in source.split(|c| *c == '\n') {let mut new_tokens = parse_line(line);+ new_tokens.push(Token::new(+ Span::new_with_len(line.len(), 1),+ harper_core::TokenKind::Newline(1),+ ));+new_tokens.iter_mut().for_each(|t| t.span.push_by(chars_traversed));@@ -199,7 +204,7 @@ mod tests {TokenKind::Unlintable,TokenKind::Unlintable,TokenKind::Punctuation(Punctuation::Period),- TokenKind::Newline(1),+ TokenKind::Newline(2),]);}diff --git a/harper-comments/src/comment_parsers/unit.rs b/harper-comments/src/comment_parsers/unit.rsindex 44fb33a04..07eb144f6 100644--- a/harper-comments/src/comment_parsers/unit.rs+++ b/harper-comments/src/comment_parsers/unit.rs@@ -1,5 +1,5 @@use harper_core::parsers::{Markdown, Parser};-use harper_core::Token;+use harper_core::{Span, Token};use super::without_initiators;@@ -20,6 +20,11 @@ impl Parser for Unit {for line in source.split(|c| *c == '\n') {let mut new_tokens = parse_line(line);+ new_tokens.push(Token::new(+ Span::new_with_len(line.len(), 1),+ harper_core::TokenKind::Newline(1),+ ));+new_tokens.iter_mut().for_each(|t| t.span.push_by(chars_traversed));diff --git a/harper-comments/src/lib.rs b/harper-comments/src/lib.rsindex ed3eebd4b..f7b014eb2 100644--- a/harper-comments/src/lib.rs+++ b/harper-comments/src/lib.rs@@ -1,6 +1,7 @@#![doc = include_str!("../README.md")]+mod better;mod comment_parsers;+mod tree_sitter_masker;mod tree_sitter_parser;-pub use tree_sitter_parser::TreeSitterParser;diff --git a/harper-comments/src/tree_sitter_masker.rs b/harper-comments/src/tree_sitter_masker.rsnew file mode 100644index 000000000..09c0df740--- /dev/null+++ b/harper-comments/src/tree_sitter_masker.rs@@ -0,0 +1,145 @@+use std::collections::HashSet;++use harper_core::{FullDictionary, Span};+use harper_core::{Mask, Masker};+use tree_sitter::{Language, Node, Tree, TreeCursor};++/// A Harper parser that wraps various comment parsers that+/// exclusively parses comments in any language supported by [`tree_sitter`].+pub struct TreeSitterMasker {+ language: Language,+}++impl TreeSitterMasker {+ pub fn new(language: Language) -> Self {+ Self { language }+ }++ fn parse_root(&self, text: &str) -> Option<Tree> {+ let mut parser = tree_sitter::Parser::new();+ parser.set_language(self.language).unwrap();++ // TODO: Use incremental parsing+ parser.parse(text, None)+ }++ pub fn create_ident_dict(&self, source: &[char]) -> Option<FullDictionary> {+ let text: String = source.iter().collect();++ // Byte-indexed+ let mut ident_spans = Vec::new();++ let tree = self.parse_root(&text)?;+ Self::visit_nodes(&mut tree.walk(), &mut |node: &Node| {+ if node.child_count() == 0 && node.kind().contains("ident") {+ ident_spans.push(node.byte_range().into())+ }+ });++ byte_spans_to_char_spans(&mut ident_spans, &text);++ let mut idents = HashSet::new();++ for span in ident_spans {+ idents.insert(span.get_content(source));+ }++ let idents: Vec<_> = idents.into_iter().collect();++ let mut dictionary = FullDictionary::new();+ dictionary.extend_words(idents);++ Some(dictionary)+ }++ /// Visits the children of a TreeSitter node, searching for comments.+ ///+ /// Returns the BYTE spans of the comment position.+ fn extract_comments(cursor: &mut TreeCursor, comments: &mut Vec<Span>) {+ Self::visit_nodes(cursor, &mut |node: &Node| {+ if node.kind().contains("comment") {+ comments.push(node.byte_range().into());+ }+ });+ }++ fn visit_nodes(cursor: &mut TreeCursor, visit: &mut impl FnMut(&Node)) {+ if !cursor.goto_first_child() {+ return;+ }++ loop {+ let node = cursor.node();++ visit(&node);++ Self::visit_nodes(cursor, visit);++ if !cursor.goto_next_sibling() {+ break;+ }+ }++ cursor.goto_parent();+ }+}++impl Masker for TreeSitterMasker {+ fn create_mask(&mut self, source: &[char]) -> Mask {+ let text: String = source.iter().collect();++ let Some(root) = self.parse_root(&text) else {+ return Mask::new_blank();+ };++ let mut comments_spans = Vec::new();++ Self::extract_comments(&mut root.walk(), &mut comments_spans);+ byte_spans_to_char_spans(&mut comments_spans, &text);++ let mut mask = Mask::new_blank();++ for span in comments_spans {+ mask.push_allowed(span);+ }++ mask.merge_whitespace_sep(source);++ mask+ }+}++/// Converts a set of byte-indexed [`Span`]s to char-index Spans, in-place.+/// NOTE: Will sort the given slice by their [`Span::start`].+///+/// If any spans overlap, it will remove the second one.+fn byte_spans_to_char_spans(byte_spans: &mut Vec<Span>, source: &str) {+ byte_spans.sort_by_key(|s| s.start);++ let cloned = byte_spans.clone();++ let mut i: usize = 0;+ byte_spans.retain(|cur| {+ i += 1;+ if let Some(prev) = cloned.get(i.wrapping_sub(2)) {+ !cur.overlaps_with(*prev)+ } else {+ true+ }+ });++ let mut last_byte_pos = 0;+ let mut last_char_pos = 0;++ byte_spans.iter_mut().for_each(|span| {+ let byte_span = *span;++ last_char_pos += source[last_byte_pos..byte_span.start].chars().count();+ span.start = last_char_pos;++ last_char_pos += source[byte_span.start..byte_span.end].chars().count();+ span.end = last_char_pos;++ last_byte_pos = byte_span.end;+ })+}diff --git a/harper-comments/src/tree_sitter_parser.rs b/harper-comments/src/tree_sitter_parser.rsindex c17d06bbf..d6d1a3964 100644--- a/harper-comments/src/tree_sitter_parser.rs+++ b/harper-comments/src/tree_sitter_parser.rs@@ -1,21 +1,21 @@-use std::collections::HashSet;use std::path::Path;-use harper_core::parsers::Parser;-use harper_core::{FullDictionary, Span, Token, TokenKind};-use tree_sitter::{Language, Node, Tree, TreeCursor};+use crate::{comment_parsers, tree_sitter_masker::TreeSitterMasker};+use comment_parsers::{Go, JsDoc, Unit};+use harper_core::{+ parsers::{self, Parser},+ FullDictionary, Token,+};-use super::comment_parsers::{Go, Unit};-use crate::comment_parsers::JsDoc;--/// A Harper parser that wraps various comment parsers that-/// exclusively parses comments in any language supported by [`tree_sitter`].pub struct TreeSitterParser {- language: Language,- comment_parser: Box<dyn Parser>+ inner: parsers::Mask<TreeSitterMasker, Box<dyn Parser>>,}impl TreeSitterParser {+ pub fn create_ident_dict(&self, source: &[char]) -> Option<FullDictionary> {+ self.inner.masker.create_ident_dict(source)+ }+pub fn new_from_language_id(language_id: &str) -> Option<Self> {let language = match language_id {"rust" => tree_sitter_rust::language(),@@ -34,18 +34,17 @@ impl TreeSitterParser {"lua" => tree_sitter_lua::language(),"sh" => tree_sitter_bash::language(),"java" => tree_sitter_java::language(),- _ => return None+ _ => return None,};let comment_parser: Box<dyn Parser> = match language_id {"javascriptreact" | "typescript" | "typescriptreact" | "javascript" => Box::new(JsDoc),"go" => Box::new(Go),- _ => Box::new(Unit)+ _ => Box::new(Unit),};Some(Self {- language,- comment_parser+ inner: parsers::Mask::new(TreeSitterMasker::new(language), comment_parser),})}@@ -78,167 +77,13 @@ impl TreeSitterParser {"sh" => "sh","bash" => "sh","java" => "java",- _ => return None+ _ => return None,})}-- fn parse_root(&self, text: &str) -> Option<Tree> {- let mut parser = tree_sitter::Parser::new();- parser.set_language(self.language).unwrap();-- // TODO: Use incremental parsing- parser.parse(text, None)- }-- pub fn create_ident_dict(&self, source: &[char]) -> Option<FullDictionary> {- let text: String = source.iter().collect();-- // Byte-indexed- let mut ident_spans = Vec::new();-- let tree = self.parse_root(&text)?;- Self::visit_nodes(&mut tree.walk(), &mut |node: &Node| {- if node.child_count() == 0 && node.kind().contains("ident") {- ident_spans.push(node.byte_range().into())- }- });-- byte_spans_to_char_spans(&mut ident_spans, &text);-- let mut idents = HashSet::new();-- for span in ident_spans {- idents.insert(span.get_content(source));- }-- let idents: Vec<_> = idents.into_iter().collect();-- let mut dictionary = FullDictionary::new();- dictionary.extend_words(idents);-- Some(dictionary)- }-- /// Visits the children of a TreeSitter node, searching for comments.- ///- /// Returns the BYTE spans of the comment position.- fn extract_comments(cursor: &mut TreeCursor, comments: &mut Vec<Span>) {- Self::visit_nodes(cursor, &mut |node: &Node| {- if node.kind().contains("comment") {- comments.push(node.byte_range().into());- }- });- }-- fn visit_nodes(cursor: &mut TreeCursor, visit: &mut impl FnMut(&Node)) {- if !cursor.goto_first_child() {- return;- }-- loop {- let node = cursor.node();-- visit(&node);-- Self::visit_nodes(cursor, visit);-- if !cursor.goto_next_sibling() {- break;- }- }-- cursor.goto_parent();- }}impl Parser for TreeSitterParser {fn parse(&mut self, source: &[char]) -> Vec<Token> {- let text: String = source.iter().collect();-- let Some(root) = self.parse_root(&text) else {- return vec![];- };-- let mut comments_spans = Vec::new();-- Self::extract_comments(&mut root.walk(), &mut comments_spans);-- byte_spans_to_char_spans(&mut comments_spans, &text);-- let mut tokens = Vec::new();-- let mut span_iter = comments_spans.iter().peekable();-- while let Some(span) = span_iter.next() {- let mut new_tokens = self.comment_parser.parse(span.get_content(source));-- new_tokens- .iter_mut()- .for_each(|v| v.span.push_by(span.start));-- // Same goes for newlines within the comment- for t in new_tokens.iter_mut() {- if let TokenKind::Newline(n) = &mut t.kind {- *n = 1;- }- }-- tokens.append(&mut new_tokens);-- // Insert a newline manually since we didn't pass the last one (if it existed)- // to the comment parser.- if let Some(last) = tokens.last() {- let is_terminating = if let Some(next_span) = span_iter.peek() {- Span::new(last.span.end, next_span.start)- .get_content(source)- .iter()- .any(|c| !c.is_whitespace())- } else {- false- };-- tokens.push(Token::new(- Span::new_with_len(last.span.end, 1),- TokenKind::Newline(if is_terminating { 2 } else { 1 })- ));- }- }-- tokens+ self.inner.parse(source)}}--/// Converts a set of byte-indexed [`Span`]s to char-index Spans, in-place.-/// NOTE: Will sort the given slice by their [`Span::start`].-///-/// If any spans overlap, it will remove the second one.-fn byte_spans_to_char_spans(byte_spans: &mut Vec<Span>, source: &str) {- byte_spans.sort_by_key(|s| s.start);-- let cloned = byte_spans.clone();-- let mut i: usize = 0;- byte_spans.retain(|cur| {- i += 1;- if let Some(prev) = cloned.get(i.wrapping_sub(2)) {- !cur.overlaps_with(*prev)- } else {- true- }- });-- let mut last_byte_pos = 0;- let mut last_char_pos = 0;-- byte_spans.iter_mut().for_each(|span| {- let byte_span = *span;-- last_char_pos += source[last_byte_pos..byte_span.start].chars().count();- span.start = last_char_pos;-- last_char_pos += source[byte_span.start..byte_span.end].chars().count();- span.end = last_char_pos;-- last_byte_pos = byte_span.end;- })-}diff --git a/harper-comments/tests/language_support.rs b/harper-comments/tests/language_support.rsindex a2ab82aa6..b25c9c256 100644--- a/harper-comments/tests/language_support.rs+++ b/harper-comments/tests/language_support.rs@@ -40,3 +40,4 @@ create_test!(multiline_comments.ts, 3);create_test!(clean.rs, 0);create_test!(jsdoc.ts, 4);create_test!(issue_96.lua, 0);+create_test!(merged_lines.ts, 1);diff --git a/harper-comments/tests/language_support_sources/issue_96.rb b/harper-comments/tests/language_support_sources/issue_96.rbindex d2dbee230..38b904f9c 100644--- a/harper-comments/tests/language_support_sources/issue_96.rb+++ b/harper-comments/tests/language_support_sources/issue_96.rb@@ -10,6 +10,6 @@ alphabet = ["g", # This is a test"h", # This is a test"i", # This is a test-= "j", # This is a test-= "k", # This is a test+ "j", # This is a test+ "k", # This is a test]diff --git a/harper-comments/tests/language_support_sources/jsdoc.ts b/harper-comments/tests/language_support_sources/jsdoc.tsindex cf19f6fc1..f46178f33 100644--- a/harper-comments/tests/language_support_sources/jsdoc.ts+++ b/harper-comments/tests/language_support_sources/jsdoc.ts@@ -19,10 +19,6 @@ class Clazz { }* @return {string} ignor** This should not be ignor- *- * @example- *- * foo('hello')*/function foo(n, o, d) {diff --git a/harper-comments/tests/language_support_sources/merged_lines.ts b/harper-comments/tests/language_support_sources/merged_lines.tsnew file mode 100644index 000000000..32fd0a113--- /dev/null+++ b/harper-comments/tests/language_support_sources/merged_lines.ts@@ -0,0 +1,11 @@+// This is a test to make sure we don't split up paragraphs on newlines.+// This should be the same sentence,+// this should be the same sentence,+// this should be the same sentence,+// this should be the same sentence,+// this should be the same sentence,+// this should be the same sentence,+// this should be the same sentence,+// this should be the same sentence,+//+// And this should be a new sentence.diff --git a/harper-comments/tests/language_support_sources/multiline_comments.ts b/harper-comments/tests/language_support_sources/multiline_comments.tsindex 51d87c35e..07f3a8cfa 100644--- a/harper-comments/tests/language_support_sources/multiline_comments.ts+++ b/harper-comments/tests/language_support_sources/multiline_comments.ts@@ -1,5 +1,5 @@-/// This is an example of an problematic comment.-/// It should produce one error.+// This is an example of an problematic comment+// It should produce one errorfunction test() {}/***@@ -9,4 +9,5 @@ function test() {}*/function arbitrary() {}-/// Let's aadd a cuple spelling errors for good measure.+// Let's aadd a cuple spelling errors for good measure.+diff --git a/harper-core/src/lib.rs b/harper-core/src/lib.rsindex 61ed492c3..c6f18df42 100644--- a/harper-core/src/lib.rs+++ b/harper-core/src/lib.rs@@ -7,20 +7,22 @@ mod document;mod language_detection;mod lexing;mod linting;-mod sync;+mod mask;pub mod parsers;mod punctuation;mod span;mod spell;+mod sync;mod token;-pub use sync::Lrc;pub use char_string::{CharString, CharStringExt};pub use document::Document;pub use linting::{Lint, LintGroup, LintGroupConfig, LintKind, Linter, Suggestion};+pub use mask::{Mask, Masker};pub use punctuation::{Punctuation, Quote};pub use span::Span;pub use spell::{Dictionary, FullDictionary, MergedDictionary};+pub use sync::Lrc;pub use token::{FatToken, Token, TokenKind, TokenStringExt};/// A utility function that removes overlapping lints in a vector,diff --git a/harper-core/src/mask/mod.rs b/harper-core/src/mask/mod.rsnew file mode 100644index 000000000..2300d8304--- /dev/null+++ b/harper-core/src/mask/mod.rs@@ -0,0 +1,109 @@+use itertools::Itertools;++use crate::Span;++/// A Masker is a tool that can be composed to eliminate chunks of text from being parsed.+/// They can be composed to do things like isolate comments from a programming language or disable+/// linting for languages that have been determined to not be English.+pub trait Masker: Send + Sync {+ fn create_mask(&mut self, source: &[char]) -> Mask;+}++pub struct Mask {+ // Right now, there aren't any use-cases where we can't treat this as a stack.+ //+ // Assumed that no elements overlap and exist in sorted order.+ pub(self) allowed: Vec<Span>,+}++impl Mask {+ /// Create a new Mask for a given piece of text, marking all text as disallowed.+ pub fn new_blank() -> Self {+ Self {+ allowed: Vec::new(),+ }+ }++ pub fn iter_allowed<'a>(+ &'a self,+ source: &'a [char],+ ) -> impl Iterator<Item = (Span, &'a [char])> + '_ {+ self.allowed.iter().map(|s| (*s, s.get_content(source)))+ }++ /// Mark a span of the text as allowed.+ pub fn push_allowed(&mut self, allowed: Span) {+ if let Some(last) = self.allowed.last_mut() {+ assert!(allowed.start >= last.end);++ if allowed.start == last.end {+ last.end = allowed.end;+ return;+ }+ }++ self.allowed.push(allowed)+ }++ /// Merge chunks that are only separated by whitespace.+ pub fn merge_whitespace_sep(&mut self, source: &[char]) {+ let mut after = Vec::with_capacity(self.allowed.len());++ let mut iter = 0..self.allowed.len();++ while let Some(i) = iter.next() {+ let a = self.allowed[i];++ if let Some(b) = self.allowed.get(i + 1) {+ let sep = Span::new(a.end, b.start);+ let sep_content = sep.get_content(source);++ if sep_content.iter().all(|c| c.is_whitespace() || *c == '\n') {+ iter.next();+ after.push(Span::new(a.start, b.end));+ continue;+ }+ }++ after.push(a);+ }++ if self.allowed.len() != after.len() {+ self.allowed = after;+ self.merge_whitespace_sep(source);+ } else {+ self.allowed = after;+ }+ }+}++#[cfg(test)]+mod tests {+ use crate::{Mask, Span};++ #[test]+ fn bumps_existing() {+ let mut mask = Mask::new_blank();++ mask.push_allowed(Span::new_with_len(0, 1));+ mask.push_allowed(Span::new_with_len(1, 2));++ assert_eq!(mask.allowed.len(), 1)+ }++ #[test]+ fn merges_whitespace_sep() {+ let source: Vec<_> = "word word\nword".chars().collect();++ let mut mask = Mask::new_blank();+ mask.push_allowed(Span::new_with_len(0, 4));+ mask.push_allowed(Span::new_with_len(5, 4));+ mask.push_allowed(Span::new_with_len(10, 4));++ assert_eq!(mask.allowed.len(), 3);++ mask.merge_whitespace_sep(&source);++ assert_eq!(mask.allowed.len(), 1);+ }+}diff --git a/harper-core/src/parsers/mask.rs b/harper-core/src/parsers/mask.rsnew file mode 100644index 000000000..47a8af456--- /dev/null+++ b/harper-core/src/parsers/mask.rs@@ -0,0 +1,55 @@+use crate::{mask::Masker, Span, Token, TokenKind};++use super::Parser;++/// Composes a Masker and a Parser to parse only masked chunks of text.+pub struct Mask<M, P>+where+ M: Masker,+ P: Parser,+{+ pub masker: M,+ pub parser: P,+}++impl<M, P> Mask<M, P>+where+ M: Masker,+ P: Parser,+{+ pub fn new(masker: M, parser: P) -> Self {+ Self { masker, parser }+ }+}++impl<M, P> Parser for Mask<M, P>+where+ M: Masker,+ P: Parser,+{+ fn parse(&mut self, source: &[char]) -> Vec<Token> {+ let mask = self.masker.create_mask(source);++ let mut tokens = Vec::new();++ for (span, content) in mask.iter_allowed(source) {+ let new_tokens = &mut self.parser.parse(content);++ if let Some(last) = new_tokens.last_mut() {+ if let TokenKind::Newline(n) = &mut last.kind {+ if *n == 1 {+ *n = 2;+ }+ }+ }++ for token in new_tokens.iter_mut() {+ token.span.push_by(span.start);+ }++ tokens.append(new_tokens);+ }++ tokens+ }+}diff --git a/harper-core/src/parsers/mod.rs b/harper-core/src/parsers/mod.rsindex 17cbfa3cc..94fd5779d 100644--- a/harper-core/src/parsers/mod.rs+++ b/harper-core/src/parsers/mod.rs@@ -1,11 +1,15 @@mod markdown;+mod mask;mod plain_english;+use blanket::blanket;pub use markdown::Markdown;+pub use mask::Mask;pub use plain_english::PlainEnglish;pub use crate::token::{Token, TokenKind, TokenStringExt};+#[blanket(derive(Box))]pub trait Parser: Send + Sync {fn parse(&mut self, source: &[char]) -> Vec<Token>;}@@ -16,7 +20,7 @@ pub trait StrParser {impl<T> StrParser for Twhere- T: Parser+ T: Parser,{fn parse_str(&mut self, source: impl AsRef<str>) -> Vec<Token> {let source: Vec<_> = source.as_ref().chars().collect();@@ -33,7 +37,7 @@ mod tests {fn assert_tokens_eq(test_str: impl AsRef<str>,expected: &[TokenKind],- parser: &mut impl Parser+ parser: &mut impl Parser,) {let chars: Vec<_> = test_str.as_ref().chars().collect();let tokens = parser.parse(&chars);@@ -70,8 +74,8 @@ mod tests {Space(1),Word,Space(1),- Word- ]+ Word,+ ],)}@@ -87,8 +91,8 @@ mod tests {Space(1),Word,Space(1),- Word- ]+ Word,+ ],);}@@ -104,8 +108,8 @@ mod tests {Newline(2),Word,Space(1),- Word- ]+ Word,+ ],);}