Skip to content

Offline, privacy-first grammar checker. Fast, open-source, Rust-powered

feat: now includes tree-sitter identifiers in dictionary

Elijah Potter74caf75edfb9
diff --git a/harper-core/src/linting/lint_set.rs b/harper-core/src/linting/lint_set.rs
index 101175c04..86bc404bc 100644
--- a/harper-core/src/linting/lint_set.rs
+++ b/harper-core/src/linting/lint_set.rs
@@ -1,4 +1,4 @@
-use crate::{Dictionary, Document, FullDictionary, Lint};
+use crate::{Dictionary, Document, Lint};
use super::{spaces::Spaces, Linter};
use paste::paste;
diff --git a/harper-core/src/spell/full_dictionary.rs b/harper-core/src/spell/full_dictionary.rs
index 585fcb81d..15a11b250 100644
--- a/harper-core/src/spell/full_dictionary.rs
+++ b/harper-core/src/spell/full_dictionary.rs
@@ -1,6 +1,6 @@
use hashbrown::HashSet;
use once_cell::sync::Lazy;
-use smallvec::SmallVec;
+use smallvec::{SmallVec, ToSmallVec};
use super::{
dictionary::Dictionary,
@@ -44,6 +44,14 @@ fn uncached_inner_new() -> FullDictionary {
static DICT: Lazy<FullDictionary> = Lazy::new(uncached_inner_new);
impl FullDictionary {
+ pub fn new() -> Self {
+ Self {
+ words: Vec::new(),
+ word_len_starts: Vec::new(),
+ word_set: HashSet::new(),
+ }
+ }
+
/// Create a dictionary from the curated Hunspell dictionary included
/// in the Harper binary.
pub fn create_from_curated() -> Self {
@@ -53,9 +61,9 @@ impl FullDictionary {
/// Appends words to the dictionary.
/// It is significantly faster to append many words with one call than many
/// distinct calls to this function.
- pub fn append_words(&mut self, words: &mut [DictWord]) {
- self.words.extend(words.iter().cloned());
- self.word_set.extend(words.iter().cloned());
+ pub fn append_words(&mut self, words: &[&[char]]) {
+ self.words.extend(words.iter().map(|v| v.to_smallvec()));
+ self.word_set.extend(words.iter().map(|v| v.to_smallvec()));
self.word_len_starts = Self::create_len_starts(&mut self.words);
}
@@ -76,6 +84,12 @@ impl FullDictionary {
}
}
+impl Default for FullDictionary {
+ fn default() -> Self {
+ Self::new()
+ }
+}
+
impl Dictionary for FullDictionary {
fn words_iter(&self) -> impl Iterator<Item = &'_ [char]> {
self.words.iter().map(|v| v.as_slice())
diff --git a/harper-core/src/spell/merged_dictionary.rs b/harper-core/src/spell/merged_dictionary.rs
index 63d66a234..234b8b99f 100644
--- a/harper-core/src/spell/merged_dictionary.rs
+++ b/harper-core/src/spell/merged_dictionary.rs
@@ -22,8 +22,17 @@ where
}
}
- pub fn add_dictionary(&mut self, dictionary: impl AsRef<Arc<T>>) {
- self.children.push(dictionary.as_ref().clone());
+ pub fn add_dictionary(&mut self, dictionary: Arc<T>) {
+ self.children.push(dictionary.clone());
+ }
+}
+
+impl<T> Default for MergedDictionary<T>
+where
+ T: Dictionary + Clone,
+{
+ fn default() -> Self {
+ Self::new()
}
}
diff --git a/harper-ls/src/backend.rs b/harper-ls/src/backend.rs
index a46b8e662..46c8a449f 100644
--- a/harper-ls/src/backend.rs
+++ b/harper-ls/src/backend.rs
@@ -1,8 +1,8 @@
-use std::{collections::HashMap, sync::Arc};
+use std::{collections::HashMap, process::id, sync::Arc};
use harper_core::{
parsers::{Markdown, Parser},
- Document, FullDictionary, LintSet, Linter,
+ Document, FullDictionary, LintSet, Linter, MergedDictionary,
};
use tokio::sync::Mutex;
use tower_lsp::{
@@ -29,6 +29,8 @@ pub struct Backend {
client: Client,
global_dictionary: Arc<FullDictionary>,
files: Mutex<HashMap<Url, Document>>,
+ /// The identifiers extracted from each file by Tree-sitter.
+ ident_dicts: Mutex<HashMap<Url, Arc<FullDictionary>>>,
}
impl Backend {
@@ -38,21 +40,38 @@ impl Backend {
}
async fn update_document(&self, url: &Url, text: &str) {
- let mut parser: Box<dyn Parser> = Box::new(Markdown);
-
- if let Some(extension) = url.to_file_path().unwrap().extension() {
- parser = TreeSitterParser::new_from_extension(&extension.to_string_lossy())
- .map::<Box<dyn Parser>, _>(|v| Box::new(v))
- .unwrap_or(Box::new(Markdown));
- }
+ let doc = if let Some(extension) = url.to_file_path().unwrap().extension() {
+ if let Some(ts_parser) =
+ TreeSitterParser::new_from_extension(&extension.to_string_lossy())
+ {
+ let doc = Document::new(text, Box::new(ts_parser.clone()));
+
+ if let Some(new_dict) = ts_parser.create_ident_dict(doc.get_full_content()) {
+ let mut ident_dicts = self.ident_dicts.lock().await;
+ ident_dicts.insert(url.clone(), new_dict.into());
+ }
+
+ doc
+ } else {
+ Document::new(text, Box::new(Markdown))
+ }
+ } else {
+ Document::new(text, Box::new(Markdown))
+ };
- let doc = Document::new(text, parser);
let mut files = self.files.lock().await;
files.insert(url.clone(), doc);
}
- fn create_linter(&self) -> LintSet {
- LintSet::new().with_standard(self.global_dictionary.clone())
+ async fn create_linter(&self, url: &Url) -> LintSet {
+ let mut dictionary = MergedDictionary::new();
+ dictionary.add_dictionary(self.global_dictionary.clone());
+
+ if let Some(ident_dict) = self.ident_dicts.lock().await.get(url) {
+ dictionary.add_dictionary(ident_dict.clone());
+ };
+
+ LintSet::new().with_standard(dictionary)
}
async fn generate_code_actions(&self, url: &Url, range: Range) -> Result<Vec<CodeAction>> {
@@ -61,7 +80,7 @@ impl Backend {
return Ok(vec![]);
};
- let mut linter = self.create_linter();
+ let mut linter = self.create_linter(url).await;
let mut lints = linter.lint(document);
lints.sort_by_key(|l| l.priority);
@@ -86,6 +105,7 @@ impl Backend {
client,
global_dictionary: dictionary.into(),
files: Mutex::new(HashMap::new()),
+ ident_dicts: Mutex::new(HashMap::new()),
}
}
@@ -96,7 +116,7 @@ impl Backend {
return vec![];
};
- let mut linter = self.create_linter();
+ let mut linter = self.create_linter(url).await;
let lints = linter.lint(document);
lints_to_diagnostics(document.get_full_content(), &lints)
diff --git a/harper-ls/src/tree_sitter_parser.rs b/harper-ls/src/tree_sitter_parser.rs
index 63a3e569d..b73b84744 100644
--- a/harper-ls/src/tree_sitter_parser.rs
+++ b/harper-ls/src/tree_sitter_parser.rs
@@ -1,11 +1,14 @@
+use std::collections::HashSet;
+
use harper_core::{
parsers::{Markdown, Parser},
- Span,
+ FullDictionary, Span,
};
-use tree_sitter::{Language, TreeCursor};
+use tree_sitter::{Language, Node, Tree, TreeCursor};
/// A Harper parser that wraps the standard [`Markdown`] parser that exclusively parses
/// comments in any language supported by [`tree_sitter`].
+#[derive(Debug, Clone)]
pub struct TreeSitterParser {
language: Language,
}
@@ -37,6 +40,72 @@ impl TreeSitterParser {
Some(Self { language })
}
+
+ fn parse_root(&self, text: &str) -> Option<Tree> {
+ let mut parser = tree_sitter::Parser::new();
+ parser.set_language(self.language).unwrap();
+
+ // TODO: Use incremental parsing
+ parser.parse(text, None)
+ }
+
+ pub fn create_ident_dict(&self, source: &[char]) -> Option<FullDictionary> {
+ let text: String = source.iter().collect();
+
+ // Byte-indexed
+ let mut ident_spans = Vec::new();
+
+ let tree = self.parse_root(&text)?;
+ Self::visit_nodes(
+ &mut tree.walk(),
+ &mut |node: &Node| match node.child_count() == 0 && node.kind().contains("ident") {
+ true => ident_spans.push(node.byte_range().into()),
+ false => (),
+ },
+ );
+
+ byte_spans_to_char_spans(&mut ident_spans, &text);
+
+ let mut idents = HashSet::new();
+
+ for span in ident_spans {
+ idents.insert(span.get_content(source));
+ }
+
+ let idents: Vec<_> = idents.into_iter().collect();
+
+ let mut dictionary = FullDictionary::new();
+ dictionary.append_words(idents.as_slice());
+
+ Some(dictionary)
+ }
+
+ /// Visits the children of a TreeSitter node, searching for comments.
+ ///
+ /// Returns the BYTE spans of the comment position.
+ fn extract_comments(cursor: &mut TreeCursor, comments: &mut Vec<Span>) {
+ Self::visit_nodes(cursor, &mut |node: &Node| {
+ if node.kind().contains("comment") {
+ comments.push(node.byte_range().into());
+ }
+ });
+ }
+
+ fn visit_nodes(cursor: &mut TreeCursor, mut visit: &mut impl FnMut(&Node)) {
+ if !cursor.goto_first_child() {
+ return;
+ }
+
+ while cursor.goto_next_sibling() {
+ let node = cursor.node();
+
+ visit(&node);
+
+ Self::visit_nodes(cursor, visit);
+ }
+
+ cursor.goto_parent();
+ }
}
impl Parser for TreeSitterParser {
@@ -44,17 +113,14 @@ impl Parser for TreeSitterParser {
let text: String = source.iter().collect();
let mut markdown_parser = Markdown;
- let mut parser = tree_sitter::Parser::new();
- parser.set_language(self.language).unwrap();
- // TODO: Use incremental parsing
- let Some(root) = parser.parse(&text, None) else {
+ let Some(root) = self.parse_root(&text) else {
return vec![];
};
let mut comments_spans = Vec::new();
- extract_comments(&mut root.walk(), &mut comments_spans);
+ Self::extract_comments(&mut root.walk(), &mut comments_spans);
byte_spans_to_char_spans(&mut comments_spans, &text);
let mut tokens = Vec::new();
@@ -106,7 +172,7 @@ fn is_comment_character(c: char) -> bool {
}
/// Converts a set of byte-indexed [`Span`]s to char-index Spans, in-place.
-/// NOTE: Will sort the given array by their [`Span::start`].
+/// NOTE: Will sort the given slice by their [`Span::start`].
///
/// Assumes that none of the Spans are overlapping.
fn byte_spans_to_char_spans(byte_spans: &mut [Span], source: &str) {
@@ -127,24 +193,3 @@ fn byte_spans_to_char_spans(byte_spans: &mut [Span], source: &str) {
last_byte_pos = byte_span.end;
})
}
-
-/// Visits the children of a TreeSitter node, searching for comments.
-///
-/// Returns the BYTE spans of the comment position.
-fn extract_comments(cursor: &mut TreeCursor, comments: &mut Vec<Span>) {
- if !cursor.goto_first_child() {
- return;
- }
-
- while cursor.goto_next_sibling() {
- let node = cursor.node();
-
- if node.kind().contains("comment") {
- comments.push(node.byte_range().into());
- }
-
- extract_comments(cursor, comments);
- }
-
- cursor.goto_parent();
-}