Skip to content

Offline, privacy-first grammar checker. Fast, open-source, Rust-powered

Changed how language parsers work

Elijah Potter61dcf22b87b1
diff --git a/Cargo.lock b/Cargo.lock
index c287eef34..7624845c0 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -35,6 +35,15 @@ dependencies = [
"zerocopy",
]
+[[package]]
+name = "aho-corasick"
+version = "1.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b2969dcb958b36655471fc61f7e416fa76033bdd4bfed0678d8fee1e2d07a1f0"
+dependencies = [
+ "memchr",
+]
+
[[package]]
name = "allocator-api2"
version = "0.2.16"
@@ -434,6 +443,9 @@ dependencies = [
"once_cell",
"tokio",
"tower-lsp",
+ "tree-sitter",
+ "tree-sitter-rust",
+ "tree-sitter-typescript",
]
[[package]]
@@ -838,6 +850,35 @@ dependencies = [
"bitflags",
]
+[[package]]
+name = "regex"
+version = "1.10.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b62dbe01f0b06f9d8dc7d49e05a0785f153b00b2c227856282f671e0318c9b15"
+dependencies = [
+ "aho-corasick",
+ "memchr",
+ "regex-automata",
+ "regex-syntax",
+]
+
+[[package]]
+name = "regex-automata"
+version = "0.4.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3b7fa1134405e2ec9353fd416b17f8dacd46c473d7d3fd1cf202706a14eb792a"
+dependencies = [
+ "aho-corasick",
+ "memchr",
+ "regex-syntax",
+]
+
+[[package]]
+name = "regex-syntax"
+version = "0.8.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f"
+
[[package]]
name = "rustc-demangle"
version = "0.1.23"
@@ -1234,6 +1275,36 @@ dependencies = [
"wasm-bindgen",
]
+[[package]]
+name = "tree-sitter"
+version = "0.20.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e747b1f9b7b931ed39a548c1fae149101497de3c1fc8d9e18c62c1a66c683d3d"
+dependencies = [
+ "cc",
+ "regex",
+]
+
+[[package]]
+name = "tree-sitter-rust"
+version = "0.20.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b0832309b0b2b6d33760ce5c0e818cb47e1d72b468516bfe4134408926fa7594"
+dependencies = [
+ "cc",
+ "tree-sitter",
+]
+
+[[package]]
+name = "tree-sitter-typescript"
+version = "0.20.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a75049f0aafabb2aac205d7bb24da162b53dcd0cfb326785f25a2f32efa8071a"
+dependencies = [
+ "cc",
+ "tree-sitter",
+]
+
[[package]]
name = "try-lock"
version = "0.2.5"
diff --git a/demo.md b/demo.md
index df49aebe2..0367cc266 100644
--- a/demo.md
+++ b/demo.md
@@ -1,3 +1,5 @@
Harper is a language checker for artists. it can detect
improper capitalization and mispelled words. There are some cases,
-where the the standard grammar checkers don't cut it. That's where Harper comes in handy.
+where the the standard grammar checkers don't cut it.
+
+That's where Harper comes in handy.
diff --git a/harper-core/src/document.rs b/harper-core/src/document.rs
index 16c35e218..60c09a160 100644
--- a/harper-core/src/document.rs
+++ b/harper-core/src/document.rs
@@ -4,9 +4,8 @@ use itertools::Itertools;
use crate::{
linting::Suggestion,
- parsing::{lex_to_end, lex_to_end_md},
span::Span,
- FatToken,
+ FatToken, MarkdownParser, Parser, PlainEnglishParser,
Punctuation::{self},
Token, TokenKind,
};
@@ -14,35 +13,39 @@ use crate::{
pub struct Document {
source: Vec<char>,
tokens: Vec<Token>,
- markdown: bool,
+ parser: Box<dyn Parser>,
}
impl Document {
- // Lexes and parses text to produce a document.
- //
- // Choosing to parse with markdown may have a performance penalty
- pub fn new(text: &str, markdown: bool) -> Self {
+ /// Lexes and parses text to produce a document.
+ ///
+ /// Choosing to parse with markdown may have a performance penalty
+ pub fn new(text: &str, parser: Box<dyn Parser>) -> Self {
let source: Vec<_> = text.chars().collect();
let mut doc = Self {
source,
tokens: Vec::new(),
- markdown,
+ parser,
};
doc.parse();
doc
}
+ pub fn new_plain_english(text: &str) -> Self {
+ Self::new(text, Box::new(PlainEnglishParser))
+ }
+
+ pub fn new_markdown(text: &str) -> Self {
+ Self::new(text, Box::new(MarkdownParser))
+ }
+
/// Re-parse important language constructs.
///
/// Should be run after every change to the underlying [`Self::source`].
fn parse(&mut self) {
- if self.markdown {
- self.tokens = lex_to_end_md(&self.source);
- } else {
- self.tokens = lex_to_end(&self.source);
- }
+ self.tokens = self.parser.parse(&self.source);
self.condense_contractions();
// Since quote matches depend on token indices.
@@ -291,26 +294,19 @@ fn is_sentence_terminator(punctuation: &Punctuation) -> bool {
#[cfg(test)]
mod tests {
use super::Document;
- use crate::Token;
-
- impl Document {
- fn from_raw_parts(source: Vec<char>, tokens: Vec<Token>, markdown: bool) -> Self {
- Self {
- source,
- tokens,
- markdown,
- }
- }
- }
+ use crate::{
+ parsers::{MarkdownParser, PlainEnglishParser},
+ token::TokenStringExt,
+ };
fn assert_condensed_contractions(text: &str, final_tok_count: usize) {
- let mut document = Document::new(text, false);
+ let mut document = Document::new(text, Box::new(PlainEnglishParser));
dbg!(&document.tokens);
document.condense_contractions();
assert_eq!(document.tokens.len(), final_tok_count);
- let mut document = Document::new(text, true);
+ let mut document = Document::new(text, Box::new(MarkdownParser));
dbg!(&document.tokens);
document.condense_contractions();
@@ -345,15 +341,14 @@ mod tests {
#[test]
fn parses_sentences_correctly() {
let text = "There were three little pigs. They built three little homes.";
- let document = Document::new(text, false);
+ let document = Document::new(text, Box::new(PlainEnglishParser));
let mut sentence_strs = vec![];
for sentence in document.sentences() {
- sentence_strs.push(
- Document::from_raw_parts(document.source.clone(), sentence.to_vec(), false)
- .to_string(),
- );
+ if let Some(span) = sentence.span() {
+ sentence_strs.push(document.get_span_content_str(span));
+ }
}
assert_eq!(
diff --git a/harper-core/src/lexing/mod.rs b/harper-core/src/lexing/mod.rs
new file mode 100644
index 000000000..7ed16b598
--- /dev/null
+++ b/harper-core/src/lexing/mod.rs
@@ -0,0 +1,157 @@
+use crate::token::Quote;
+
+use crate::token::{Punctuation, TokenKind};
+
+#[derive(Debug)]
+pub struct FoundToken {
+ /// The index of the character __after__ the lexed token
+ pub next_index: usize,
+ /// Token lexed
+ pub token: TokenKind,
+}
+
+pub fn lex_token(source: &[char]) -> Option<FoundToken> {
+ let lexers = [
+ lex_punctuation,
+ lex_spaces,
+ lex_newlines,
+ lex_number,
+ lex_word,
+ ];
+
+ for lexer in lexers {
+ if let Some(f) = lexer(source) {
+ return Some(f);
+ }
+ }
+
+ None
+}
+
+fn lex_word(source: &[char]) -> Option<FoundToken> {
+ let mut end = 0;
+
+ while end < source.len() {
+ if lex_punctuation(&source[end + 1..]).is_none()
+ && lex_spaces(&source[end + 1..]).is_none()
+ && lex_newlines(&source[end + 1..]).is_none()
+ && end + 1 != source.len()
+ {
+ end += 1;
+ } else {
+ return Some(FoundToken {
+ next_index: end + 1,
+ token: TokenKind::Word,
+ });
+ }
+ }
+
+ None
+}
+
+pub fn lex_number(source: &[char]) -> Option<FoundToken> {
+ if source.is_empty() {
+ return None;
+ }
+
+ if !source[0].is_numeric() {
+ return None;
+ }
+
+ let Some(end) = source
+ .iter()
+ .enumerate()
+ .rev()
+ .find_map(|(i, v)| v.is_numeric().then_some(i))
+ else {
+ return None;
+ };
+
+ {
+ let s: String = source[0..end + 1].iter().collect();
+
+ if let Ok(n) = s.parse::<f64>() {
+ return Some(FoundToken {
+ token: TokenKind::Number(n),
+ next_index: end + 1,
+ });
+ }
+ }
+
+ lex_number(&source[0..end])
+}
+
+fn lex_newlines(source: &[char]) -> Option<FoundToken> {
+ let count = source.iter().take_while(|c| **c == '\n').count();
+
+ if count > 0 {
+ Some(FoundToken {
+ token: TokenKind::Newline(count),
+ next_index: count,
+ })
+ } else {
+ None
+ }
+}
+
+fn lex_spaces(source: &[char]) -> Option<FoundToken> {
+ let count = source.iter().take_while(|c| **c == ' ').count();
+
+ if count > 0 {
+ Some(FoundToken {
+ token: TokenKind::Space(count),
+ next_index: count,
+ })
+ } else {
+ None
+ }
+}
+
+fn lex_punctuation(source: &[char]) -> Option<FoundToken> {
+ if let Some(found) = lex_quote(source) {
+ return Some(found);
+ }
+
+ let c = source.first()?;
+
+ use Punctuation::*;
+
+ let punct = match c {
+ '/' => ForwardSlash,
+ '\\' => Backslash,
+ '%' => Percent,
+ '’' => Apostrophe,
+ '\'' => Apostrophe,
+ '.' => Period,
+ '!' => Bang,
+ '?' => Question,
+ ':' => Colon,
+ ';' => Semicolon,
+ ',' => Comma,
+ '-' => Hyphen,
+ '[' => OpenSquare,
+ ']' => CloseSquare,
+ '(' => OpenRound,
+ ')' => CloseRound,
+ '#' => Hash,
+ _ => return None,
+ };
+
+ Some(FoundToken {
+ next_index: 1,
+ token: TokenKind::Punctuation(punct),
+ })
+}
+
+fn lex_quote(source: &[char]) -> Option<FoundToken> {
+ let c = *source.first()?;
+
+ if c == '\"' || c == '“' || c == '”' {
+ Some(FoundToken {
+ next_index: 1,
+ token: TokenKind::Punctuation(Punctuation::Quote(Quote { twin_loc: None })),
+ })
+ } else {
+ None
+ }
+}
diff --git a/harper-core/src/lib.rs b/harper-core/src/lib.rs
index 08ec5fa9e..6a71d4400 100644
--- a/harper-core/src/lib.rs
+++ b/harper-core/src/lib.rs
@@ -1,14 +1,17 @@
#![allow(dead_code)]
mod document;
+mod lexing;
mod linting;
-mod parsing;
+mod parsers;
mod span;
mod spell;
+mod token;
pub use document::Document;
pub use linting::LintSet;
pub use linting::{Lint, LintKind, Linter, Suggestion};
-pub use parsing::{FatToken, Punctuation, Token, TokenKind};
+pub use parsers::{MarkdownParser, Parser, PlainEnglishParser};
pub use span::Span;
pub use spell::Dictionary;
+pub use token::{FatToken, Punctuation, Token, TokenKind, TokenStringExt};
diff --git a/harper-core/src/linting/long_sentences.rs b/harper-core/src/linting/long_sentences.rs
index 901e69072..0cdd115c8 100644
--- a/harper-core/src/linting/long_sentences.rs
+++ b/harper-core/src/linting/long_sentences.rs
@@ -1,5 +1,5 @@
use super::{Lint, LintKind, Linter};
-use crate::{parsing::TokenStringExt, Document, Span};
+use crate::{token::TokenStringExt, Document, Span};
/// Detect and warn that the sentence is too long.
#[derive(Debug, Clone, Copy, Default)]
diff --git a/harper-core/src/linting/mod.rs b/harper-core/src/linting/mod.rs
index 687e3a3a5..1f79c82df 100644
--- a/harper-core/src/linting/mod.rs
+++ b/harper-core/src/linting/mod.rs
@@ -19,10 +19,10 @@ pub trait Linter: Send + Sync {
#[cfg(test)]
mod tests {
- use crate::{Document, Linter};
+ use crate::{parsers::PlainEnglishParser, Document, Linter};
pub fn assert_lint_count(text: &str, mut linter: impl Linter, count: usize) {
- let test = Document::new(text, false);
+ let test = Document::new(text, Box::new(PlainEnglishParser));
let lints = linter.lint(&test);
assert_eq!(lints.len(), count);
}
diff --git a/harper-core/src/linting/repeated_words.rs b/harper-core/src/linting/repeated_words.rs
index 0a778e623..e86ff97a8 100644
--- a/harper-core/src/linting/repeated_words.rs
+++ b/harper-core/src/linting/repeated_words.rs
@@ -1,7 +1,7 @@
use hashbrown::HashSet;
use crate::{
- parsing::{Token, TokenKind, TokenStringExt},
+ token::{Token, TokenKind, TokenStringExt},
Document, Span, Suggestion,
};
diff --git a/harper-core/src/linting/sentence_capitalization.rs b/harper-core/src/linting/sentence_capitalization.rs
index 090e9d8a5..e578cc6a1 100644
--- a/harper-core/src/linting/sentence_capitalization.rs
+++ b/harper-core/src/linting/sentence_capitalization.rs
@@ -1,6 +1,6 @@
use itertools::Itertools;
-use crate::{document::Document, parsing::TokenStringExt};
+use crate::{document::Document, TokenStringExt};
use super::lint::Suggestion;
use super::{Lint, LintKind, Linter};
diff --git a/harper-core/src/linting/spaces.rs b/harper-core/src/linting/spaces.rs
index 2434e7ab8..2a3fda75e 100644
--- a/harper-core/src/linting/spaces.rs
+++ b/harper-core/src/linting/spaces.rs
@@ -1,5 +1,5 @@
use super::{Lint, Linter};
-use crate::{parsing::TokenStringExt, Document, LintKind, Suggestion, TokenKind};
+use crate::{token::TokenStringExt, Document, LintKind, Suggestion, TokenKind};
#[derive(Debug, Default)]
pub struct Spaces;
diff --git a/harper-core/src/linting/unclosed_quotes.rs b/harper-core/src/linting/unclosed_quotes.rs
index c94da2819..f723d440c 100644
--- a/harper-core/src/linting/unclosed_quotes.rs
+++ b/harper-core/src/linting/unclosed_quotes.rs
@@ -1,4 +1,4 @@
-use crate::{document::Document, parsing::Quote, Punctuation, TokenKind};
+use crate::{document::Document, token::Quote, Punctuation, TokenKind};
use super::{Lint, LintKind, Linter};
diff --git a/harper-core/src/parsers/markdown.rs b/harper-core/src/parsers/markdown.rs
new file mode 100644
index 000000000..280cc53cc
--- /dev/null
+++ b/harper-core/src/parsers/markdown.rs
@@ -0,0 +1,62 @@
+use super::{Parser, PlainEnglishParser, StrParser};
+use crate::Token;
+pub struct MarkdownParser;
+
+impl Parser for MarkdownParser {
+ fn parse(&mut self, source: &[char]) -> Vec<Token> {
+ let mut english_parser = PlainEnglishParser;
+
+ let source_str: String = source.iter().collect();
+ let md_parser =
+ pulldown_cmark::Parser::new_ext(&source_str, pulldown_cmark::Options::all());
+
+ let mut tokens = Vec::new();
+
+ let mut traversed_bytes = 0;
+ let mut traversed_chars = 0;
+
+ let mut stack = Vec::new();
+
+ // NOTE: the range spits out __byte__ indices, not char indices.
+ // This is why we keep track above.
+ for (event, range) in md_parser.into_offset_iter() {
+ match event {
+ pulldown_cmark::Event::Start(tag) => stack.push(tag),
+ pulldown_cmark::Event::End(_) => {
+ stack.pop();
+ }
+ pulldown_cmark::Event::Text(text) => {
+ traversed_chars += source_str[traversed_bytes..range.start].chars().count();
+ traversed_bytes = range.start;
+
+ if let Some(tag) = stack.last() {
+ use pulldown_cmark::Tag;
+
+ if !(matches!(tag, Tag::Paragraph)
+ || matches!(tag, Tag::Heading(_, _, _))
+ || matches!(tag, Tag::Item)
+ || matches!(tag, Tag::TableCell)
+ || matches!(tag, Tag::Emphasis)
+ || matches!(tag, Tag::Strong)
+ || matches!(tag, Tag::Link(..))
+ || matches!(tag, Tag::Strikethrough))
+ {
+ continue;
+ }
+ }
+
+ let mut new_tokens = english_parser.parse_str(text);
+
+ new_tokens
+ .iter_mut()
+ .for_each(|token| token.span.offset(traversed_chars));
+
+ tokens.append(&mut new_tokens);
+ }
+ _ => (),
+ }
+ }
+
+ tokens
+ }
+}
diff --git a/harper-core/src/parsers/mod.rs b/harper-core/src/parsers/mod.rs
new file mode 100644
index 000000000..b33219137
--- /dev/null
+++ b/harper-core/src/parsers/mod.rs
@@ -0,0 +1,95 @@
+mod markdown;
+mod plain_english;
+
+pub use crate::token::{Quote, Token, TokenKind, TokenStringExt};
+pub use markdown::MarkdownParser;
+pub use plain_english::PlainEnglishParser;
+
+pub trait Parser: Send + Sync {
+ fn parse(&mut self, source: &[char]) -> Vec<Token>;
+}
+
+pub trait StrParser {
+ fn parse_str(&mut self, source: impl AsRef<str>) -> Vec<Token>;
+}
+
+impl<T> StrParser for T
+where
+ T: Parser,
+{
+ fn parse_str(&mut self, source: impl AsRef<str>) -> Vec<Token> {
+ let source: Vec<_> = source.as_ref().chars().collect();
+ self.parse(&source)
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::{MarkdownParser, PlainEnglishParser};
+ use crate::{
+ Parser, Punctuation,
+ TokenKind::{self, *},
+ };
+
+ fn assert_tokens_eq(
+ test_str: impl AsRef<str>,
+ expected: &[TokenKind],
+ parser: &mut impl Parser,
+ ) {
+ let chars: Vec<_> = test_str.as_ref().chars().collect();
+ let tokens = parser.parse(&chars);
+ let kinds: Vec<_> = tokens.into_iter().map(|v| v.kind).collect();
+
+ assert_eq!(&kinds, expected)
+ }
+
+ fn assert_tokens_eq_plain(test_str: impl AsRef<str>, expected: &[TokenKind]) {
+ let mut parser = PlainEnglishParser;
+ assert_tokens_eq(test_str, expected, &mut parser);
+ }
+
+ fn assert_tokens_eq_md(test_str: impl AsRef<str>, expected: &[TokenKind]) {
+ let mut parser = MarkdownParser;
+
+ assert_tokens_eq(test_str, expected, &mut parser)
+ }
+
+ #[test]
+ fn single_letter() {
+ assert_tokens_eq_plain("a", &[Word])
+ }
+
+ #[test]
+ fn sentence() {
+ assert_tokens_eq_plain(
+ "hello world, my friend",
+ &[
+ Word,
+ Space(1),
+ Word,
+ Punctuation(Punctuation::Comma),
+ Space(1),
+ Word,
+ Space(1),
+ Word,
+ ],
+ )
+ }
+
+ #[test]
+ fn sentence_md() {
+ assert_tokens_eq_md(
+ "__hello__ world, [my]() friend",
+ &[
+ Word,
+ Space(1),
+ Word,
+ Punctuation(Punctuation::Comma),
+ Space(1),
+ Word,
+ Space(1),
+ Word,
+ ],
+ );
+ }
+}
diff --git a/harper-core/src/parsers/plain_english.rs b/harper-core/src/parsers/plain_english.rs
new file mode 100644
index 000000000..c81652e99
--- /dev/null
+++ b/harper-core/src/parsers/plain_english.rs
@@ -0,0 +1,30 @@
+use super::Parser;
+use crate::{
+ lexing::{lex_token, FoundToken},
+ Span, Token,
+};
+
+pub struct PlainEnglishParser;
+
+impl Parser for PlainEnglishParser {
+ fn parse(&mut self, source: &[char]) -> Vec<Token> {
+ let mut cursor = 0;
+ let mut tokens = Vec::new();
+
+ loop {
+ if cursor == source.len() {
+ return tokens;
+ }
+
+ if let Some(FoundToken { token, next_index }) = lex_token(&source[cursor..]) {
+ tokens.push(Token {
+ span: Span::new(cursor, cursor + next_index),
+ kind: token,
+ });
+ cursor += next_index;
+ } else {
+ panic!()
+ }
+ }
+ }
+}
diff --git a/harper-core/src/parsing/lexer.rs b/harper-core/src/parsing/lexer.rs
deleted file mode 100644
index 5b2a31707..000000000
--- a/harper-core/src/parsing/lexer.rs
+++ /dev/null
@@ -1,315 +0,0 @@
-use crate::span::Span;
-
-use super::{
- token::{Punctuation, Token, TokenKind},
- Quote,
-};
-
-#[derive(Debug)]
-pub struct FoundToken {
- /// The index of the character __after__ the lexed token
- pub next_index: usize,
- /// Token lexed
- pub token: TokenKind,
-}
-
-/// Same as [`lex_to_end`], but with additional infrastructure to intelligently ignore Markdown.
-pub fn lex_to_end_md(source: &[char]) -> Vec<Token> {
- let source_str: String = source.iter().collect();
- let md_parser = pulldown_cmark::Parser::new_ext(&source_str, pulldown_cmark::Options::all());
-
- let mut tokens = Vec::new();
-
- let mut traversed_bytes = 0;
- let mut traversed_chars = 0;
-
- let mut stack = Vec::new();
-
- // NOTE: the range spits out __byte__ indices, not char indices.
- // This is why we keep track above.
- for (event, range) in md_parser.into_offset_iter() {
- match event {
- pulldown_cmark::Event::Start(tag) => stack.push(tag),
- pulldown_cmark::Event::End(_) => {
- stack.pop();
- }
- pulldown_cmark::Event::Text(text) => {
- traversed_chars += source_str[traversed_bytes..range.start].chars().count();
- traversed_bytes = range.start;
-
- if let Some(tag) = stack.last() {
- use pulldown_cmark::Tag;
-
- if !(matches!(tag, Tag::Paragraph)
- || matches!(tag, Tag::Heading(_, _, _))
- || matches!(tag, Tag::Item))
- || matches!(tag, Tag::TableCell)
- {
- continue;
- }
- }
-
- let mut new_tokens = lex_to_end_str(text);
-
- new_tokens
- .iter_mut()
- .for_each(|token| token.span.offset(traversed_chars));
-
- tokens.append(&mut new_tokens);
- }
- _ => (),
- }
- }
-
- tokens
-}
-
-/// Same as [`lex_to_end_str`], but with additional infrastructure to intelligently ignore Markdown.
-///
-/// Yes, I am aware this implementation is doubly redundant, but I prefer to have a consistent API.
-/// If its an issue, we can use a different markdown parser.
-pub fn lex_to_end_md_str(source: impl AsRef<str>) -> Vec<Token> {
- let r = source.as_ref();
-
- let chars: Vec<_> = r.chars().collect();
-
- lex_to_end_md(&chars)
-}
-
-pub fn lex_to_end_str(source: impl AsRef<str>) -> Vec<Token> {
- let r = source.as_ref();
-
- let chars: Vec<_> = r.chars().collect();
-
- lex_to_end(&chars)
-}
-
-/// Lex all tokens, if possible.
-pub fn lex_to_end(source: &[char]) -> Vec<Token> {
- let mut cursor = 0;
- let mut tokens = Vec::new();
-
- loop {
- if cursor == source.len() {
- return tokens;
- }
-
- if let Some(FoundToken { token, next_index }) = lex_token(&source[cursor..]) {
- tokens.push(Token {
- span: Span::new(cursor, cursor + next_index),
- kind: token,
- });
- cursor += next_index;
- } else {
- panic!()
- }
- }
-}
-
-fn lex_token(source: &[char]) -> Option<FoundToken> {
- let lexers = [
- lex_punctuation,
- lex_spaces,
- lex_newlines,
- lex_number,
- lex_word,
- ];
-
- for lexer in lexers {
- if let Some(f) = lexer(source) {
- return Some(f);
- }
- }
-
- None
-}
-
-fn lex_word(source: &[char]) -> Option<FoundToken> {
- let mut end = 0;
-
- while end < source.len() {
- if lex_punctuation(&source[end + 1..]).is_none()
- && lex_spaces(&source[end + 1..]).is_none()
- && lex_newlines(&source[end + 1..]).is_none()
- && end + 1 != source.len()
- {
- end += 1;
- } else {
- return Some(FoundToken {
- next_index: end + 1,
- token: TokenKind::Word,
- });
- }
- }
-
- None
-}
-
-pub fn lex_number(source: &[char]) -> Option<FoundToken> {
- if source.is_empty() {
- return None;
- }
-
- if !source[0].is_numeric() {
- return None;
- }
-
- let Some(end) = source
- .iter()
- .enumerate()
- .rev()
- .find_map(|(i, v)| v.is_numeric().then_some(i))
- else {
- return None;
- };
-
- {
- let s: String = source[0..end + 1].iter().collect();
-
- if let Ok(n) = s.parse::<f64>() {
- return Some(FoundToken {
- token: TokenKind::Number(n),
- next_index: end + 1,
- });
- }
- }
-
- lex_number(&source[0..end])
-}
-
-fn lex_newlines(source: &[char]) -> Option<FoundToken> {
- let count = source.iter().take_while(|c| **c == '\n').count();
-
- if count > 0 {
- Some(FoundToken {
- token: TokenKind::Newline(count),
- next_index: count,
- })
- } else {
- None
- }
-}
-
-fn lex_spaces(source: &[char]) -> Option<FoundToken> {
- let count = source.iter().take_while(|c| **c == ' ').count();
-
- if count > 0 {
- Some(FoundToken {
- token: TokenKind::Space(count),
- next_index: count,
- })
- } else {
- None
- }
-}
-
-fn lex_punctuation(source: &[char]) -> Option<FoundToken> {
- if let Some(found) = lex_quote(source) {
- return Some(found);
- }
-
- let c = source.first()?;
-
- use Punctuation::*;
-
- let punct = match c {
- '/' => ForwardSlash,
- '\\' => Backslash,
- '%' => Percent,
- '’' => Apostrophe,
- '\'' => Apostrophe,
- '.' => Period,
- '!' => Bang,
- '?' => Question,
- ':' => Colon,
- ';' => Semicolon,
- ',' => Comma,
- '-' => Hyphen,
- '[' => OpenSquare,
- ']' => CloseSquare,
- '(' => OpenRound,
- ')' => CloseRound,
- '#' => Hash,
- _ => return None,
- };
-
- Some(FoundToken {
- next_index: 1,
- token: TokenKind::Punctuation(punct),
- })
-}
-
-fn lex_quote(source: &[char]) -> Option<FoundToken> {
- let c = *source.first()?;
-
- if c == '\"' || c == '“' || c == '”' {
- Some(FoundToken {
- next_index: 1,
- token: TokenKind::Punctuation(Punctuation::Quote(Quote { twin_loc: None })),
- })
- } else {
- None
- }
-}
-
-#[cfg(test)]
-mod tests {
- use super::{lex_to_end_md_str, lex_to_end_str};
- use crate::{
- Punctuation,
- TokenKind::{self, *},
- };
-
- fn assert_tokens_eq_plain(test_str: impl AsRef<str>, expected: &[TokenKind]) {
- let tokens = lex_to_end_str(test_str);
- let kinds: Vec<_> = tokens.into_iter().map(|v| v.kind).collect();
-
- assert_eq!(&kinds, expected)
- }
-
- fn assert_tokens_eq_md(test_str: impl AsRef<str>, expected: &[TokenKind]) {
- let tokens = lex_to_end_md_str(test_str);
- let kinds: Vec<_> = tokens.into_iter().map(|v| v.kind).collect();
-
- assert_eq!(&kinds, expected)
- }
-
- #[test]
- fn single_letter() {
- assert_tokens_eq_plain("a", &[Word])
- }
-
- #[test]
- fn sentence() {
- assert_tokens_eq_plain(
- "hello world, my friend",
- &[
- Word,
- Space(1),
- Word,
- Punctuation(Punctuation::Comma),
- Space(1),
- Word,
- Space(1),
- Word,
- ],
- )
- }
-
- #[test]
- fn sentence_md() {
- assert_tokens_eq_md(
- "__hello__ world, [my]() friend",
- &[
- Word,
- Space(1),
- Word,
- Punctuation(Punctuation::Comma),
- Space(1),
- Word,
- Space(1),
- Word,
- ],
- );
- }
-}
diff --git a/harper-core/src/parsing/mod.rs b/harper-core/src/parsing/mod.rs
deleted file mode 100644
index cb299dd9b..000000000
--- a/harper-core/src/parsing/mod.rs
+++ /dev/null
@@ -1,5 +0,0 @@
-mod lexer;
-mod token;
-
-pub use lexer::{lex_to_end, lex_to_end_md};
-pub use token::{FatToken, Punctuation, Quote, Token, TokenKind, TokenStringExt};
diff --git a/harper-core/src/span.rs b/harper-core/src/span.rs
index 9d6e1a1d7..0ec7e89fa 100644
--- a/harper-core/src/span.rs
+++ b/harper-core/src/span.rs
@@ -1,3 +1,5 @@
+use std::ops::Range;
+
use serde::{Deserialize, Serialize};
/// A window in a [char].
@@ -55,6 +57,12 @@ impl Span {
}
}
+impl From<Range<usize>> for Span {
+ fn from(value: Range<usize>) -> Self {
+ Self::new(value.start, value.end)
+ }
+}
+
#[cfg(test)]
mod tests {
use crate::Span;
diff --git a/harper-core/src/parsing/token.rs b/harper-core/src/token.rs
similarity index 93%
rename from harper-core/src/parsing/token.rs
rename to harper-core/src/token.rs
index b03610717..a5b15077e 100644
--- a/harper-core/src/parsing/token.rs
+++ b/harper-core/src/token.rs
@@ -108,6 +108,9 @@ pub trait TokenStringExt {
fn iter_spaces(&self) -> impl Iterator<Item = &Token> + '_;
fn iter_apostrophe_indices(&self) -> impl Iterator<Item = usize> + '_;
fn iter_apostrophes(&self) -> impl Iterator<Item = &Token> + '_;
+ /// Grab the span that represents the beginning of the first element and the end of the last
+ /// element.
+ fn span(&self) -> Option<Span>;
}
impl TokenStringExt for [Token] {
@@ -147,4 +150,8 @@ impl TokenStringExt for [Token] {
fn iter_apostrophes(&self) -> impl Iterator<Item = &Token> + '_ {
self.iter_apostrophe_indices().map(|i| &self[i])
}
+
+ fn span(&self) -> Option<Span> {
+ Some(Span::new(self.first()?.span.start, self.last()?.span.end))
+ }
}
diff --git a/harper-ls/Cargo.toml b/harper-ls/Cargo.toml
index 4fb234973..849fd0ecb 100644
--- a/harper-ls/Cargo.toml
+++ b/harper-ls/Cargo.toml
@@ -11,3 +11,6 @@ tower-lsp = "0.20.0"
tokio = { version = "1.35.1", features = ["full"] }
clap = { version = "4.4.18", features = ["derive"] }
once_cell = "1.19.0"
+tree-sitter = "0.20.10"
+tree-sitter-rust = "0.20.4"
+tree-sitter-typescript = "0.20.3"
diff --git a/harper-ls/src/backend.rs b/harper-ls/src/backend.rs
index 8e9b2d779..8d3003d8f 100644
--- a/harper-ls/src/backend.rs
+++ b/harper-ls/src/backend.rs
@@ -1,6 +1,6 @@
-use std::{collections::HashMap, fs};
+use std::{borrow::Borrow, collections::HashMap, fs};
-use harper_core::{Dictionary, Document, LintSet, Linter};
+use harper_core::{Dictionary, Document, Lint, LintSet, Linter, MarkdownParser};
use tokio::sync::Mutex;
use tower_lsp::{
jsonrpc::Result,
@@ -33,11 +33,19 @@ impl Backend {
}
async fn update_document(&self, url: &Url, text: &str) {
- let doc = Document::new(text, true);
+ let doc = Document::new(text, Box::new(MarkdownParser));
let mut files = self.files.lock().await;
files.insert(url.clone(), doc);
}
+ async fn generate_lints_for_url(&self, url: &Url) -> Option<Vec<Lint>> {
+ let files = self.files.lock().await;
+ let file_contents = files.get(url)?;
+
+ let mut linter = self.linter.lock().await;
+ Some(linter.lint(file_contents))
+ }
+
async fn generate_code_actions(&self, url: &Url, range: Range) -> Result<Vec<CodeAction>> {
let files = self.files.lock().await;
let Some(document) = files.get(url) else {
diff --git a/harper-ls/src/comments.rs b/harper-ls/src/comments.rs
new file mode 100644
index 000000000..0003fefef
--- /dev/null
+++ b/harper-ls/src/comments.rs
@@ -0,0 +1,37 @@
+use harper_core::Span;
+use tree_sitter::{Parser, TreeCursor};
+
+/// Extract each comment astris a seperate block.
+pub fn extract_comments_rust(text: &str) -> Vec<Span> {
+ let mut parser = Parser::new();
+ parser.set_language(tree_sitter_rust::language()).unwrap();
+
+ // TODO: Use incremental parsing
+ let Some(root) = parser.parse(text, None) else {
+ return vec![];
+ };
+
+ let mut comments = Vec::new();
+
+ extract_comments(&mut root.walk(), &mut comments);
+
+ comments
+}
+
+fn extract_comments(cursor: &mut TreeCursor, comments: &mut Vec<Span>) {
+ if !cursor.goto_first_child() {
+ return;
+ }
+
+ while cursor.goto_next_sibling() {
+ let node = cursor.node();
+
+ if node.kind().contains("comment") {
+ comments.push(node.byte_range().into());
+ }
+
+ extract_comments(cursor, comments);
+ }
+
+ cursor.goto_parent();
+}
diff --git a/harper-ls/src/main.rs b/harper-ls/src/main.rs
index 2e0870a29..36d0ad7d0 100644
--- a/harper-ls/src/main.rs
+++ b/harper-ls/src/main.rs
@@ -1,5 +1,6 @@
use tokio::net::TcpListener;
mod backend;
+mod comments;
mod diagnostics;
mod pos_conv;
diff --git a/harper-serve/src/main.rs b/harper-serve/src/main.rs
index dfb7e10e5..c4e2632ba 100644
--- a/harper-serve/src/main.rs
+++ b/harper-serve/src/main.rs
@@ -1,7 +1,7 @@
#![allow(dead_code)]
use harper_core::{Dictionary, Document, FatToken, Lint, LintSet, Linter, Span, Suggestion};
-use std::net::SocketAddr;
+use std::{marker, net::SocketAddr};
use tokio::time::Instant;
use tracing::{info, Level};
use tracing_subscriber::FmtSubscriber;
@@ -71,7 +71,7 @@ async fn root() -> &'static str {
async fn parse_text(Json(payload): Json<ParseRequest>) -> (StatusCode, Json<ParseResponse>) {
let text = payload.text;
- let document = Document::new(&text, true);
+ let document = Document::new_markdown(&text);
let tokens: Vec<_> = document.fat_tokens().collect();
(StatusCode::ACCEPTED, Json(ParseResponse { tokens }))
@@ -90,7 +90,7 @@ struct ParseResponse {
async fn lint(Json(payload): Json<LintRequest>) -> (StatusCode, Json<LintResponse>) {
let text = payload.text;
- let document = Document::new(&text, true);
+ let document = Document::new_markdown(&text);
let dictionary = Dictionary::new();
let mut linter = LintSet::new().with_standard(dictionary);
@@ -113,7 +113,7 @@ async fn apply_suggestion(
Json(payload): Json<ApplySuggestionRequest>,
) -> (StatusCode, Json<ApplySuggestionResponse>) {
let text = payload.text;
- let mut document = Document::new(&text, true);
+ let mut document = Document::new_markdown(&text);
document.apply_suggestion(&payload.suggestion, payload.span);
(
diff --git a/harper-wasm/src/lib.rs b/harper-wasm/src/lib.rs
index 3631ed357..9cfe30621 100644
--- a/harper-wasm/src/lib.rs
+++ b/harper-wasm/src/lib.rs
@@ -20,7 +20,7 @@ pub fn setup() {
#[wasm_bindgen]
pub fn lint(text: String) -> Vec<JsValue> {
let dictionary = Dictionary::new();
- let document = Document::new(&text, true);
+ let document = Document::new_markdown(&text);
let mut linter = LintSet::new().with_standard(dictionary);
let lints = linter.lint(&document);
@@ -33,7 +33,7 @@ pub fn lint(text: String) -> Vec<JsValue> {
#[wasm_bindgen]
pub fn parse(text: String) -> Vec<JsValue> {
- let document = Document::new(&text, true);
+ let document = Document::new_markdown(&text);
document
.fat_tokens()
@@ -50,7 +50,7 @@ pub fn apply_suggestion(
let span = serde_wasm_bindgen::from_value(span).map_err(|e| e.to_string())?;
let suggestion = serde_wasm_bindgen::from_value(suggestion).map_err(|e| e.to_string())?;
- let mut document = Document::new(&text, true);
+ let mut document = Document::new_markdown(&text);
document.apply_suggestion(&suggestion, span);
Ok(document.get_full_string())