Offline, privacy-first grammar checker. Fast, open-source, Rust-powered
feat: now includes tree-sitter identifiers in dictionary
Elijah Potter74caf75edfb9
diff --git a/harper-core/src/linting/lint_set.rs b/harper-core/src/linting/lint_set.rsindex 101175c04..86bc404bc 100644--- a/harper-core/src/linting/lint_set.rs+++ b/harper-core/src/linting/lint_set.rs@@ -1,4 +1,4 @@-use crate::{Dictionary, Document, FullDictionary, Lint};+use crate::{Dictionary, Document, Lint};use super::{spaces::Spaces, Linter};use paste::paste;diff --git a/harper-core/src/spell/full_dictionary.rs b/harper-core/src/spell/full_dictionary.rsindex 585fcb81d..15a11b250 100644--- a/harper-core/src/spell/full_dictionary.rs+++ b/harper-core/src/spell/full_dictionary.rs@@ -1,6 +1,6 @@use hashbrown::HashSet;use once_cell::sync::Lazy;-use smallvec::SmallVec;+use smallvec::{SmallVec, ToSmallVec};use super::{dictionary::Dictionary,@@ -44,6 +44,14 @@ fn uncached_inner_new() -> FullDictionary {static DICT: Lazy<FullDictionary> = Lazy::new(uncached_inner_new);impl FullDictionary {+ pub fn new() -> Self {+ Self {+ words: Vec::new(),+ word_len_starts: Vec::new(),+ word_set: HashSet::new(),+ }+ }+/// Create a dictionary from the curated Hunspell dictionary included/// in the Harper binary.pub fn create_from_curated() -> Self {@@ -53,9 +61,9 @@ impl FullDictionary {/// Appends words to the dictionary./// It is significantly faster to append many words with one call than many/// distinct calls to this function.- pub fn append_words(&mut self, words: &mut [DictWord]) {- self.words.extend(words.iter().cloned());- self.word_set.extend(words.iter().cloned());+ pub fn append_words(&mut self, words: &[&[char]]) {+ self.words.extend(words.iter().map(|v| v.to_smallvec()));+ self.word_set.extend(words.iter().map(|v| v.to_smallvec()));self.word_len_starts = Self::create_len_starts(&mut self.words);}@@ -76,6 +84,12 @@ impl FullDictionary {}}+impl Default for FullDictionary {+ fn default() -> Self {+ Self::new()+ }+}+impl Dictionary for FullDictionary {fn words_iter(&self) -> impl Iterator<Item = &'_ [char]> {self.words.iter().map(|v| v.as_slice())diff --git a/harper-core/src/spell/merged_dictionary.rs b/harper-core/src/spell/merged_dictionary.rsindex 63d66a234..234b8b99f 100644--- a/harper-core/src/spell/merged_dictionary.rs+++ b/harper-core/src/spell/merged_dictionary.rs@@ -22,8 +22,17 @@ where}}- pub fn add_dictionary(&mut self, dictionary: impl AsRef<Arc<T>>) {- self.children.push(dictionary.as_ref().clone());+ pub fn add_dictionary(&mut self, dictionary: Arc<T>) {+ self.children.push(dictionary.clone());+ }+}++impl<T> Default for MergedDictionary<T>+where+ T: Dictionary + Clone,+{+ fn default() -> Self {+ Self::new()}}diff --git a/harper-ls/src/backend.rs b/harper-ls/src/backend.rsindex a46b8e662..46c8a449f 100644--- a/harper-ls/src/backend.rs+++ b/harper-ls/src/backend.rs@@ -1,8 +1,8 @@-use std::{collections::HashMap, sync::Arc};+use std::{collections::HashMap, process::id, sync::Arc};use harper_core::{parsers::{Markdown, Parser},- Document, FullDictionary, LintSet, Linter,+ Document, FullDictionary, LintSet, Linter, MergedDictionary,};use tokio::sync::Mutex;use tower_lsp::{@@ -29,6 +29,8 @@ pub struct Backend {client: Client,global_dictionary: Arc<FullDictionary>,files: Mutex<HashMap<Url, Document>>,+ /// The identifiers extracted from each file by Tree-sitter.+ ident_dicts: Mutex<HashMap<Url, Arc<FullDictionary>>>,}impl Backend {@@ -38,21 +40,38 @@ impl Backend {}async fn update_document(&self, url: &Url, text: &str) {- let mut parser: Box<dyn Parser> = Box::new(Markdown);-- if let Some(extension) = url.to_file_path().unwrap().extension() {- parser = TreeSitterParser::new_from_extension(&extension.to_string_lossy())- .map::<Box<dyn Parser>, _>(|v| Box::new(v))- .unwrap_or(Box::new(Markdown));- }+ let doc = if let Some(extension) = url.to_file_path().unwrap().extension() {+ if let Some(ts_parser) =+ TreeSitterParser::new_from_extension(&extension.to_string_lossy())+ {+ let doc = Document::new(text, Box::new(ts_parser.clone()));++ if let Some(new_dict) = ts_parser.create_ident_dict(doc.get_full_content()) {+ let mut ident_dicts = self.ident_dicts.lock().await;+ ident_dicts.insert(url.clone(), new_dict.into());+ }++ doc+ } else {+ Document::new(text, Box::new(Markdown))+ }+ } else {+ Document::new(text, Box::new(Markdown))+ };- let doc = Document::new(text, parser);let mut files = self.files.lock().await;files.insert(url.clone(), doc);}- fn create_linter(&self) -> LintSet {- LintSet::new().with_standard(self.global_dictionary.clone())+ async fn create_linter(&self, url: &Url) -> LintSet {+ let mut dictionary = MergedDictionary::new();+ dictionary.add_dictionary(self.global_dictionary.clone());++ if let Some(ident_dict) = self.ident_dicts.lock().await.get(url) {+ dictionary.add_dictionary(ident_dict.clone());+ };++ LintSet::new().with_standard(dictionary)}async fn generate_code_actions(&self, url: &Url, range: Range) -> Result<Vec<CodeAction>> {@@ -61,7 +80,7 @@ impl Backend {return Ok(vec![]);};- let mut linter = self.create_linter();+ let mut linter = self.create_linter(url).await;let mut lints = linter.lint(document);lints.sort_by_key(|l| l.priority);@@ -86,6 +105,7 @@ impl Backend {client,global_dictionary: dictionary.into(),files: Mutex::new(HashMap::new()),+ ident_dicts: Mutex::new(HashMap::new()),}}@@ -96,7 +116,7 @@ impl Backend {return vec![];};- let mut linter = self.create_linter();+ let mut linter = self.create_linter(url).await;let lints = linter.lint(document);lints_to_diagnostics(document.get_full_content(), &lints)diff --git a/harper-ls/src/tree_sitter_parser.rs b/harper-ls/src/tree_sitter_parser.rsindex 63a3e569d..b73b84744 100644--- a/harper-ls/src/tree_sitter_parser.rs+++ b/harper-ls/src/tree_sitter_parser.rs@@ -1,11 +1,14 @@+use std::collections::HashSet;+use harper_core::{parsers::{Markdown, Parser},- Span,+ FullDictionary, Span,};-use tree_sitter::{Language, TreeCursor};+use tree_sitter::{Language, Node, Tree, TreeCursor};/// A Harper parser that wraps the standard [`Markdown`] parser that exclusively parses/// comments in any language supported by [`tree_sitter`].+#[derive(Debug, Clone)]pub struct TreeSitterParser {language: Language,}@@ -37,6 +40,72 @@ impl TreeSitterParser {Some(Self { language })}++ fn parse_root(&self, text: &str) -> Option<Tree> {+ let mut parser = tree_sitter::Parser::new();+ parser.set_language(self.language).unwrap();++ // TODO: Use incremental parsing+ parser.parse(text, None)+ }++ pub fn create_ident_dict(&self, source: &[char]) -> Option<FullDictionary> {+ let text: String = source.iter().collect();++ // Byte-indexed+ let mut ident_spans = Vec::new();++ let tree = self.parse_root(&text)?;+ Self::visit_nodes(+ &mut tree.walk(),+ &mut |node: &Node| match node.child_count() == 0 && node.kind().contains("ident") {+ true => ident_spans.push(node.byte_range().into()),+ false => (),+ },+ );++ byte_spans_to_char_spans(&mut ident_spans, &text);++ let mut idents = HashSet::new();++ for span in ident_spans {+ idents.insert(span.get_content(source));+ }++ let idents: Vec<_> = idents.into_iter().collect();++ let mut dictionary = FullDictionary::new();+ dictionary.append_words(idents.as_slice());++ Some(dictionary)+ }++ /// Visits the children of a TreeSitter node, searching for comments.+ ///+ /// Returns the BYTE spans of the comment position.+ fn extract_comments(cursor: &mut TreeCursor, comments: &mut Vec<Span>) {+ Self::visit_nodes(cursor, &mut |node: &Node| {+ if node.kind().contains("comment") {+ comments.push(node.byte_range().into());+ }+ });+ }++ fn visit_nodes(cursor: &mut TreeCursor, mut visit: &mut impl FnMut(&Node)) {+ if !cursor.goto_first_child() {+ return;+ }++ while cursor.goto_next_sibling() {+ let node = cursor.node();++ visit(&node);++ Self::visit_nodes(cursor, visit);+ }++ cursor.goto_parent();+ }}impl Parser for TreeSitterParser {@@ -44,17 +113,14 @@ impl Parser for TreeSitterParser {let text: String = source.iter().collect();let mut markdown_parser = Markdown;- let mut parser = tree_sitter::Parser::new();- parser.set_language(self.language).unwrap();- // TODO: Use incremental parsing- let Some(root) = parser.parse(&text, None) else {+ let Some(root) = self.parse_root(&text) else {return vec![];};let mut comments_spans = Vec::new();- extract_comments(&mut root.walk(), &mut comments_spans);+ Self::extract_comments(&mut root.walk(), &mut comments_spans);byte_spans_to_char_spans(&mut comments_spans, &text);let mut tokens = Vec::new();@@ -106,7 +172,7 @@ fn is_comment_character(c: char) -> bool {}/// Converts a set of byte-indexed [`Span`]s to char-index Spans, in-place.-/// NOTE: Will sort the given array by their [`Span::start`].+/// NOTE: Will sort the given slice by their [`Span::start`].////// Assumes that none of the Spans are overlapping.fn byte_spans_to_char_spans(byte_spans: &mut [Span], source: &str) {@@ -127,24 +193,3 @@ fn byte_spans_to_char_spans(byte_spans: &mut [Span], source: &str) {last_byte_pos = byte_span.end;})}--/// Visits the children of a TreeSitter node, searching for comments.-///-/// Returns the BYTE spans of the comment position.-fn extract_comments(cursor: &mut TreeCursor, comments: &mut Vec<Span>) {- if !cursor.goto_first_child() {- return;- }-- while cursor.goto_next_sibling() {- let node = cursor.node();-- if node.kind().contains("comment") {- comments.push(node.byte_range().into());- }-- extract_comments(cursor, comments);- }-- cursor.goto_parent();-}