Offline, privacy-first grammar checker. Fast, open-source, Rust-powered
Changed how language parsers work
Elijah Potter61dcf22b87b1
diff --git a/Cargo.lock b/Cargo.lockindex c287eef34..7624845c0 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -35,6 +35,15 @@ dependencies = ["zerocopy",]+[[package]]+name = "aho-corasick"+version = "1.1.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "b2969dcb958b36655471fc61f7e416fa76033bdd4bfed0678d8fee1e2d07a1f0"+dependencies = [+ "memchr",+]+[[package]]name = "allocator-api2"version = "0.2.16"@@ -434,6 +443,9 @@ dependencies = ["once_cell","tokio","tower-lsp",+ "tree-sitter",+ "tree-sitter-rust",+ "tree-sitter-typescript",][[package]]@@ -838,6 +850,35 @@ dependencies = ["bitflags",]+[[package]]+name = "regex"+version = "1.10.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "b62dbe01f0b06f9d8dc7d49e05a0785f153b00b2c227856282f671e0318c9b15"+dependencies = [+ "aho-corasick",+ "memchr",+ "regex-automata",+ "regex-syntax",+]++[[package]]+name = "regex-automata"+version = "0.4.4"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "3b7fa1134405e2ec9353fd416b17f8dacd46c473d7d3fd1cf202706a14eb792a"+dependencies = [+ "aho-corasick",+ "memchr",+ "regex-syntax",+]++[[package]]+name = "regex-syntax"+version = "0.8.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f"+[[package]]name = "rustc-demangle"version = "0.1.23"@@ -1234,6 +1275,36 @@ dependencies = ["wasm-bindgen",]+[[package]]+name = "tree-sitter"+version = "0.20.10"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "e747b1f9b7b931ed39a548c1fae149101497de3c1fc8d9e18c62c1a66c683d3d"+dependencies = [+ "cc",+ "regex",+]++[[package]]+name = "tree-sitter-rust"+version = "0.20.4"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "b0832309b0b2b6d33760ce5c0e818cb47e1d72b468516bfe4134408926fa7594"+dependencies = [+ "cc",+ "tree-sitter",+]++[[package]]+name = "tree-sitter-typescript"+version = "0.20.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "a75049f0aafabb2aac205d7bb24da162b53dcd0cfb326785f25a2f32efa8071a"+dependencies = [+ "cc",+ "tree-sitter",+]+[[package]]name = "try-lock"version = "0.2.5"diff --git a/demo.md b/demo.mdindex df49aebe2..0367cc266 100644--- a/demo.md+++ b/demo.md@@ -1,3 +1,5 @@Harper is a language checker for artists. it can detectimproper capitalization and mispelled words. There are some cases,-where the the standard grammar checkers don't cut it. That's where Harper comes in handy.+where the the standard grammar checkers don't cut it.++That's where Harper comes in handy.diff --git a/harper-core/src/document.rs b/harper-core/src/document.rsindex 16c35e218..60c09a160 100644--- a/harper-core/src/document.rs+++ b/harper-core/src/document.rs@@ -4,9 +4,8 @@ use itertools::Itertools;use crate::{linting::Suggestion,- parsing::{lex_to_end, lex_to_end_md},span::Span,- FatToken,+ FatToken, MarkdownParser, Parser, PlainEnglishParser,Punctuation::{self},Token, TokenKind,};@@ -14,35 +13,39 @@ use crate::{pub struct Document {source: Vec<char>,tokens: Vec<Token>,- markdown: bool,+ parser: Box<dyn Parser>,}impl Document {- // Lexes and parses text to produce a document.- //- // Choosing to parse with markdown may have a performance penalty- pub fn new(text: &str, markdown: bool) -> Self {+ /// Lexes and parses text to produce a document.+ ///+ /// Choosing to parse with markdown may have a performance penalty+ pub fn new(text: &str, parser: Box<dyn Parser>) -> Self {let source: Vec<_> = text.chars().collect();let mut doc = Self {source,tokens: Vec::new(),- markdown,+ parser,};doc.parse();doc}+ pub fn new_plain_english(text: &str) -> Self {+ Self::new(text, Box::new(PlainEnglishParser))+ }++ pub fn new_markdown(text: &str) -> Self {+ Self::new(text, Box::new(MarkdownParser))+ }+/// Re-parse important language constructs.////// Should be run after every change to the underlying [`Self::source`].fn parse(&mut self) {- if self.markdown {- self.tokens = lex_to_end_md(&self.source);- } else {- self.tokens = lex_to_end(&self.source);- }+ self.tokens = self.parser.parse(&self.source);self.condense_contractions();// Since quote matches depend on token indices.@@ -291,26 +294,19 @@ fn is_sentence_terminator(punctuation: &Punctuation) -> bool {#[cfg(test)]mod tests {use super::Document;- use crate::Token;-- impl Document {- fn from_raw_parts(source: Vec<char>, tokens: Vec<Token>, markdown: bool) -> Self {- Self {- source,- tokens,- markdown,- }- }- }+ use crate::{+ parsers::{MarkdownParser, PlainEnglishParser},+ token::TokenStringExt,+ };fn assert_condensed_contractions(text: &str, final_tok_count: usize) {- let mut document = Document::new(text, false);+ let mut document = Document::new(text, Box::new(PlainEnglishParser));dbg!(&document.tokens);document.condense_contractions();assert_eq!(document.tokens.len(), final_tok_count);- let mut document = Document::new(text, true);+ let mut document = Document::new(text, Box::new(MarkdownParser));dbg!(&document.tokens);document.condense_contractions();@@ -345,15 +341,14 @@ mod tests {#[test]fn parses_sentences_correctly() {let text = "There were three little pigs. They built three little homes.";- let document = Document::new(text, false);+ let document = Document::new(text, Box::new(PlainEnglishParser));let mut sentence_strs = vec![];for sentence in document.sentences() {- sentence_strs.push(- Document::from_raw_parts(document.source.clone(), sentence.to_vec(), false)- .to_string(),- );+ if let Some(span) = sentence.span() {+ sentence_strs.push(document.get_span_content_str(span));+ }}assert_eq!(diff --git a/harper-core/src/lexing/mod.rs b/harper-core/src/lexing/mod.rsnew file mode 100644index 000000000..7ed16b598--- /dev/null+++ b/harper-core/src/lexing/mod.rs@@ -0,0 +1,157 @@+use crate::token::Quote;++use crate::token::{Punctuation, TokenKind};++#[derive(Debug)]+pub struct FoundToken {+ /// The index of the character __after__ the lexed token+ pub next_index: usize,+ /// Token lexed+ pub token: TokenKind,+}++pub fn lex_token(source: &[char]) -> Option<FoundToken> {+ let lexers = [+ lex_punctuation,+ lex_spaces,+ lex_newlines,+ lex_number,+ lex_word,+ ];++ for lexer in lexers {+ if let Some(f) = lexer(source) {+ return Some(f);+ }+ }++ None+}++fn lex_word(source: &[char]) -> Option<FoundToken> {+ let mut end = 0;++ while end < source.len() {+ if lex_punctuation(&source[end + 1..]).is_none()+ && lex_spaces(&source[end + 1..]).is_none()+ && lex_newlines(&source[end + 1..]).is_none()+ && end + 1 != source.len()+ {+ end += 1;+ } else {+ return Some(FoundToken {+ next_index: end + 1,+ token: TokenKind::Word,+ });+ }+ }++ None+}++pub fn lex_number(source: &[char]) -> Option<FoundToken> {+ if source.is_empty() {+ return None;+ }++ if !source[0].is_numeric() {+ return None;+ }++ let Some(end) = source+ .iter()+ .enumerate()+ .rev()+ .find_map(|(i, v)| v.is_numeric().then_some(i))+ else {+ return None;+ };++ {+ let s: String = source[0..end + 1].iter().collect();++ if let Ok(n) = s.parse::<f64>() {+ return Some(FoundToken {+ token: TokenKind::Number(n),+ next_index: end + 1,+ });+ }+ }++ lex_number(&source[0..end])+}++fn lex_newlines(source: &[char]) -> Option<FoundToken> {+ let count = source.iter().take_while(|c| **c == '\n').count();++ if count > 0 {+ Some(FoundToken {+ token: TokenKind::Newline(count),+ next_index: count,+ })+ } else {+ None+ }+}++fn lex_spaces(source: &[char]) -> Option<FoundToken> {+ let count = source.iter().take_while(|c| **c == ' ').count();++ if count > 0 {+ Some(FoundToken {+ token: TokenKind::Space(count),+ next_index: count,+ })+ } else {+ None+ }+}++fn lex_punctuation(source: &[char]) -> Option<FoundToken> {+ if let Some(found) = lex_quote(source) {+ return Some(found);+ }++ let c = source.first()?;++ use Punctuation::*;++ let punct = match c {+ '/' => ForwardSlash,+ '\\' => Backslash,+ '%' => Percent,+ '’' => Apostrophe,+ '\'' => Apostrophe,+ '.' => Period,+ '!' => Bang,+ '?' => Question,+ ':' => Colon,+ ';' => Semicolon,+ ',' => Comma,+ '-' => Hyphen,+ '[' => OpenSquare,+ ']' => CloseSquare,+ '(' => OpenRound,+ ')' => CloseRound,+ '#' => Hash,+ _ => return None,+ };++ Some(FoundToken {+ next_index: 1,+ token: TokenKind::Punctuation(punct),+ })+}++fn lex_quote(source: &[char]) -> Option<FoundToken> {+ let c = *source.first()?;++ if c == '\"' || c == '“' || c == '”' {+ Some(FoundToken {+ next_index: 1,+ token: TokenKind::Punctuation(Punctuation::Quote(Quote { twin_loc: None })),+ })+ } else {+ None+ }+}diff --git a/harper-core/src/lib.rs b/harper-core/src/lib.rsindex 08ec5fa9e..6a71d4400 100644--- a/harper-core/src/lib.rs+++ b/harper-core/src/lib.rs@@ -1,14 +1,17 @@#![allow(dead_code)]mod document;+mod lexing;mod linting;-mod parsing;+mod parsers;mod span;mod spell;+mod token;pub use document::Document;pub use linting::LintSet;pub use linting::{Lint, LintKind, Linter, Suggestion};-pub use parsing::{FatToken, Punctuation, Token, TokenKind};+pub use parsers::{MarkdownParser, Parser, PlainEnglishParser};pub use span::Span;pub use spell::Dictionary;+pub use token::{FatToken, Punctuation, Token, TokenKind, TokenStringExt};diff --git a/harper-core/src/linting/long_sentences.rs b/harper-core/src/linting/long_sentences.rsindex 901e69072..0cdd115c8 100644--- a/harper-core/src/linting/long_sentences.rs+++ b/harper-core/src/linting/long_sentences.rs@@ -1,5 +1,5 @@use super::{Lint, LintKind, Linter};-use crate::{parsing::TokenStringExt, Document, Span};+use crate::{token::TokenStringExt, Document, Span};/// Detect and warn that the sentence is too long.#[derive(Debug, Clone, Copy, Default)]diff --git a/harper-core/src/linting/mod.rs b/harper-core/src/linting/mod.rsindex 687e3a3a5..1f79c82df 100644--- a/harper-core/src/linting/mod.rs+++ b/harper-core/src/linting/mod.rs@@ -19,10 +19,10 @@ pub trait Linter: Send + Sync {#[cfg(test)]mod tests {- use crate::{Document, Linter};+ use crate::{parsers::PlainEnglishParser, Document, Linter};pub fn assert_lint_count(text: &str, mut linter: impl Linter, count: usize) {- let test = Document::new(text, false);+ let test = Document::new(text, Box::new(PlainEnglishParser));let lints = linter.lint(&test);assert_eq!(lints.len(), count);}diff --git a/harper-core/src/linting/repeated_words.rs b/harper-core/src/linting/repeated_words.rsindex 0a778e623..e86ff97a8 100644--- a/harper-core/src/linting/repeated_words.rs+++ b/harper-core/src/linting/repeated_words.rs@@ -1,7 +1,7 @@use hashbrown::HashSet;use crate::{- parsing::{Token, TokenKind, TokenStringExt},+ token::{Token, TokenKind, TokenStringExt},Document, Span, Suggestion,};diff --git a/harper-core/src/linting/sentence_capitalization.rs b/harper-core/src/linting/sentence_capitalization.rsindex 090e9d8a5..e578cc6a1 100644--- a/harper-core/src/linting/sentence_capitalization.rs+++ b/harper-core/src/linting/sentence_capitalization.rs@@ -1,6 +1,6 @@use itertools::Itertools;-use crate::{document::Document, parsing::TokenStringExt};+use crate::{document::Document, TokenStringExt};use super::lint::Suggestion;use super::{Lint, LintKind, Linter};diff --git a/harper-core/src/linting/spaces.rs b/harper-core/src/linting/spaces.rsindex 2434e7ab8..2a3fda75e 100644--- a/harper-core/src/linting/spaces.rs+++ b/harper-core/src/linting/spaces.rs@@ -1,5 +1,5 @@use super::{Lint, Linter};-use crate::{parsing::TokenStringExt, Document, LintKind, Suggestion, TokenKind};+use crate::{token::TokenStringExt, Document, LintKind, Suggestion, TokenKind};#[derive(Debug, Default)]pub struct Spaces;diff --git a/harper-core/src/linting/unclosed_quotes.rs b/harper-core/src/linting/unclosed_quotes.rsindex c94da2819..f723d440c 100644--- a/harper-core/src/linting/unclosed_quotes.rs+++ b/harper-core/src/linting/unclosed_quotes.rs@@ -1,4 +1,4 @@-use crate::{document::Document, parsing::Quote, Punctuation, TokenKind};+use crate::{document::Document, token::Quote, Punctuation, TokenKind};use super::{Lint, LintKind, Linter};diff --git a/harper-core/src/parsers/markdown.rs b/harper-core/src/parsers/markdown.rsnew file mode 100644index 000000000..280cc53cc--- /dev/null+++ b/harper-core/src/parsers/markdown.rs@@ -0,0 +1,62 @@+use super::{Parser, PlainEnglishParser, StrParser};+use crate::Token;+pub struct MarkdownParser;++impl Parser for MarkdownParser {+ fn parse(&mut self, source: &[char]) -> Vec<Token> {+ let mut english_parser = PlainEnglishParser;++ let source_str: String = source.iter().collect();+ let md_parser =+ pulldown_cmark::Parser::new_ext(&source_str, pulldown_cmark::Options::all());++ let mut tokens = Vec::new();++ let mut traversed_bytes = 0;+ let mut traversed_chars = 0;++ let mut stack = Vec::new();++ // NOTE: the range spits out __byte__ indices, not char indices.+ // This is why we keep track above.+ for (event, range) in md_parser.into_offset_iter() {+ match event {+ pulldown_cmark::Event::Start(tag) => stack.push(tag),+ pulldown_cmark::Event::End(_) => {+ stack.pop();+ }+ pulldown_cmark::Event::Text(text) => {+ traversed_chars += source_str[traversed_bytes..range.start].chars().count();+ traversed_bytes = range.start;++ if let Some(tag) = stack.last() {+ use pulldown_cmark::Tag;++ if !(matches!(tag, Tag::Paragraph)+ || matches!(tag, Tag::Heading(_, _, _))+ || matches!(tag, Tag::Item)+ || matches!(tag, Tag::TableCell)+ || matches!(tag, Tag::Emphasis)+ || matches!(tag, Tag::Strong)+ || matches!(tag, Tag::Link(..))+ || matches!(tag, Tag::Strikethrough))+ {+ continue;+ }+ }++ let mut new_tokens = english_parser.parse_str(text);++ new_tokens+ .iter_mut()+ .for_each(|token| token.span.offset(traversed_chars));++ tokens.append(&mut new_tokens);+ }+ _ => (),+ }+ }++ tokens+ }+}diff --git a/harper-core/src/parsers/mod.rs b/harper-core/src/parsers/mod.rsnew file mode 100644index 000000000..b33219137--- /dev/null+++ b/harper-core/src/parsers/mod.rs@@ -0,0 +1,95 @@+mod markdown;+mod plain_english;++pub use crate::token::{Quote, Token, TokenKind, TokenStringExt};+pub use markdown::MarkdownParser;+pub use plain_english::PlainEnglishParser;++pub trait Parser: Send + Sync {+ fn parse(&mut self, source: &[char]) -> Vec<Token>;+}++pub trait StrParser {+ fn parse_str(&mut self, source: impl AsRef<str>) -> Vec<Token>;+}++impl<T> StrParser for T+where+ T: Parser,+{+ fn parse_str(&mut self, source: impl AsRef<str>) -> Vec<Token> {+ let source: Vec<_> = source.as_ref().chars().collect();+ self.parse(&source)+ }+}++#[cfg(test)]+mod tests {+ use super::{MarkdownParser, PlainEnglishParser};+ use crate::{+ Parser, Punctuation,+ TokenKind::{self, *},+ };++ fn assert_tokens_eq(+ test_str: impl AsRef<str>,+ expected: &[TokenKind],+ parser: &mut impl Parser,+ ) {+ let chars: Vec<_> = test_str.as_ref().chars().collect();+ let tokens = parser.parse(&chars);+ let kinds: Vec<_> = tokens.into_iter().map(|v| v.kind).collect();++ assert_eq!(&kinds, expected)+ }++ fn assert_tokens_eq_plain(test_str: impl AsRef<str>, expected: &[TokenKind]) {+ let mut parser = PlainEnglishParser;+ assert_tokens_eq(test_str, expected, &mut parser);+ }++ fn assert_tokens_eq_md(test_str: impl AsRef<str>, expected: &[TokenKind]) {+ let mut parser = MarkdownParser;++ assert_tokens_eq(test_str, expected, &mut parser)+ }++ #[test]+ fn single_letter() {+ assert_tokens_eq_plain("a", &[Word])+ }++ #[test]+ fn sentence() {+ assert_tokens_eq_plain(+ "hello world, my friend",+ &[+ Word,+ Space(1),+ Word,+ Punctuation(Punctuation::Comma),+ Space(1),+ Word,+ Space(1),+ Word,+ ],+ )+ }++ #[test]+ fn sentence_md() {+ assert_tokens_eq_md(+ "__hello__ world, [my]() friend",+ &[+ Word,+ Space(1),+ Word,+ Punctuation(Punctuation::Comma),+ Space(1),+ Word,+ Space(1),+ Word,+ ],+ );+ }+}diff --git a/harper-core/src/parsers/plain_english.rs b/harper-core/src/parsers/plain_english.rsnew file mode 100644index 000000000..c81652e99--- /dev/null+++ b/harper-core/src/parsers/plain_english.rs@@ -0,0 +1,30 @@+use super::Parser;+use crate::{+ lexing::{lex_token, FoundToken},+ Span, Token,+};++pub struct PlainEnglishParser;++impl Parser for PlainEnglishParser {+ fn parse(&mut self, source: &[char]) -> Vec<Token> {+ let mut cursor = 0;+ let mut tokens = Vec::new();++ loop {+ if cursor == source.len() {+ return tokens;+ }++ if let Some(FoundToken { token, next_index }) = lex_token(&source[cursor..]) {+ tokens.push(Token {+ span: Span::new(cursor, cursor + next_index),+ kind: token,+ });+ cursor += next_index;+ } else {+ panic!()+ }+ }+ }+}diff --git a/harper-core/src/parsing/lexer.rs b/harper-core/src/parsing/lexer.rsdeleted file mode 100644index 5b2a31707..000000000--- a/harper-core/src/parsing/lexer.rs+++ /dev/null@@ -1,315 +0,0 @@-use crate::span::Span;--use super::{- token::{Punctuation, Token, TokenKind},- Quote,-};--#[derive(Debug)]-pub struct FoundToken {- /// The index of the character __after__ the lexed token- pub next_index: usize,- /// Token lexed- pub token: TokenKind,-}--/// Same as [`lex_to_end`], but with additional infrastructure to intelligently ignore Markdown.-pub fn lex_to_end_md(source: &[char]) -> Vec<Token> {- let source_str: String = source.iter().collect();- let md_parser = pulldown_cmark::Parser::new_ext(&source_str, pulldown_cmark::Options::all());-- let mut tokens = Vec::new();-- let mut traversed_bytes = 0;- let mut traversed_chars = 0;-- let mut stack = Vec::new();-- // NOTE: the range spits out __byte__ indices, not char indices.- // This is why we keep track above.- for (event, range) in md_parser.into_offset_iter() {- match event {- pulldown_cmark::Event::Start(tag) => stack.push(tag),- pulldown_cmark::Event::End(_) => {- stack.pop();- }- pulldown_cmark::Event::Text(text) => {- traversed_chars += source_str[traversed_bytes..range.start].chars().count();- traversed_bytes = range.start;-- if let Some(tag) = stack.last() {- use pulldown_cmark::Tag;-- if !(matches!(tag, Tag::Paragraph)- || matches!(tag, Tag::Heading(_, _, _))- || matches!(tag, Tag::Item))- || matches!(tag, Tag::TableCell)- {- continue;- }- }-- let mut new_tokens = lex_to_end_str(text);-- new_tokens- .iter_mut()- .for_each(|token| token.span.offset(traversed_chars));-- tokens.append(&mut new_tokens);- }- _ => (),- }- }-- tokens-}--/// Same as [`lex_to_end_str`], but with additional infrastructure to intelligently ignore Markdown.-///-/// Yes, I am aware this implementation is doubly redundant, but I prefer to have a consistent API.-/// If its an issue, we can use a different markdown parser.-pub fn lex_to_end_md_str(source: impl AsRef<str>) -> Vec<Token> {- let r = source.as_ref();-- let chars: Vec<_> = r.chars().collect();-- lex_to_end_md(&chars)-}--pub fn lex_to_end_str(source: impl AsRef<str>) -> Vec<Token> {- let r = source.as_ref();-- let chars: Vec<_> = r.chars().collect();-- lex_to_end(&chars)-}--/// Lex all tokens, if possible.-pub fn lex_to_end(source: &[char]) -> Vec<Token> {- let mut cursor = 0;- let mut tokens = Vec::new();-- loop {- if cursor == source.len() {- return tokens;- }-- if let Some(FoundToken { token, next_index }) = lex_token(&source[cursor..]) {- tokens.push(Token {- span: Span::new(cursor, cursor + next_index),- kind: token,- });- cursor += next_index;- } else {- panic!()- }- }-}--fn lex_token(source: &[char]) -> Option<FoundToken> {- let lexers = [- lex_punctuation,- lex_spaces,- lex_newlines,- lex_number,- lex_word,- ];-- for lexer in lexers {- if let Some(f) = lexer(source) {- return Some(f);- }- }-- None-}--fn lex_word(source: &[char]) -> Option<FoundToken> {- let mut end = 0;-- while end < source.len() {- if lex_punctuation(&source[end + 1..]).is_none()- && lex_spaces(&source[end + 1..]).is_none()- && lex_newlines(&source[end + 1..]).is_none()- && end + 1 != source.len()- {- end += 1;- } else {- return Some(FoundToken {- next_index: end + 1,- token: TokenKind::Word,- });- }- }-- None-}--pub fn lex_number(source: &[char]) -> Option<FoundToken> {- if source.is_empty() {- return None;- }-- if !source[0].is_numeric() {- return None;- }-- let Some(end) = source- .iter()- .enumerate()- .rev()- .find_map(|(i, v)| v.is_numeric().then_some(i))- else {- return None;- };-- {- let s: String = source[0..end + 1].iter().collect();-- if let Ok(n) = s.parse::<f64>() {- return Some(FoundToken {- token: TokenKind::Number(n),- next_index: end + 1,- });- }- }-- lex_number(&source[0..end])-}--fn lex_newlines(source: &[char]) -> Option<FoundToken> {- let count = source.iter().take_while(|c| **c == '\n').count();-- if count > 0 {- Some(FoundToken {- token: TokenKind::Newline(count),- next_index: count,- })- } else {- None- }-}--fn lex_spaces(source: &[char]) -> Option<FoundToken> {- let count = source.iter().take_while(|c| **c == ' ').count();-- if count > 0 {- Some(FoundToken {- token: TokenKind::Space(count),- next_index: count,- })- } else {- None- }-}--fn lex_punctuation(source: &[char]) -> Option<FoundToken> {- if let Some(found) = lex_quote(source) {- return Some(found);- }-- let c = source.first()?;-- use Punctuation::*;-- let punct = match c {- '/' => ForwardSlash,- '\\' => Backslash,- '%' => Percent,- '’' => Apostrophe,- '\'' => Apostrophe,- '.' => Period,- '!' => Bang,- '?' => Question,- ':' => Colon,- ';' => Semicolon,- ',' => Comma,- '-' => Hyphen,- '[' => OpenSquare,- ']' => CloseSquare,- '(' => OpenRound,- ')' => CloseRound,- '#' => Hash,- _ => return None,- };-- Some(FoundToken {- next_index: 1,- token: TokenKind::Punctuation(punct),- })-}--fn lex_quote(source: &[char]) -> Option<FoundToken> {- let c = *source.first()?;-- if c == '\"' || c == '“' || c == '”' {- Some(FoundToken {- next_index: 1,- token: TokenKind::Punctuation(Punctuation::Quote(Quote { twin_loc: None })),- })- } else {- None- }-}--#[cfg(test)]-mod tests {- use super::{lex_to_end_md_str, lex_to_end_str};- use crate::{- Punctuation,- TokenKind::{self, *},- };-- fn assert_tokens_eq_plain(test_str: impl AsRef<str>, expected: &[TokenKind]) {- let tokens = lex_to_end_str(test_str);- let kinds: Vec<_> = tokens.into_iter().map(|v| v.kind).collect();-- assert_eq!(&kinds, expected)- }-- fn assert_tokens_eq_md(test_str: impl AsRef<str>, expected: &[TokenKind]) {- let tokens = lex_to_end_md_str(test_str);- let kinds: Vec<_> = tokens.into_iter().map(|v| v.kind).collect();-- assert_eq!(&kinds, expected)- }-- #[test]- fn single_letter() {- assert_tokens_eq_plain("a", &[Word])- }-- #[test]- fn sentence() {- assert_tokens_eq_plain(- "hello world, my friend",- &[- Word,- Space(1),- Word,- Punctuation(Punctuation::Comma),- Space(1),- Word,- Space(1),- Word,- ],- )- }-- #[test]- fn sentence_md() {- assert_tokens_eq_md(- "__hello__ world, [my]() friend",- &[- Word,- Space(1),- Word,- Punctuation(Punctuation::Comma),- Space(1),- Word,- Space(1),- Word,- ],- );- }-}diff --git a/harper-core/src/parsing/mod.rs b/harper-core/src/parsing/mod.rsdeleted file mode 100644index cb299dd9b..000000000--- a/harper-core/src/parsing/mod.rs+++ /dev/null@@ -1,5 +0,0 @@-mod lexer;-mod token;--pub use lexer::{lex_to_end, lex_to_end_md};-pub use token::{FatToken, Punctuation, Quote, Token, TokenKind, TokenStringExt};diff --git a/harper-core/src/span.rs b/harper-core/src/span.rsindex 9d6e1a1d7..0ec7e89fa 100644--- a/harper-core/src/span.rs+++ b/harper-core/src/span.rs@@ -1,3 +1,5 @@+use std::ops::Range;+use serde::{Deserialize, Serialize};/// A window in a [char].@@ -55,6 +57,12 @@ impl Span {}}+impl From<Range<usize>> for Span {+ fn from(value: Range<usize>) -> Self {+ Self::new(value.start, value.end)+ }+}+#[cfg(test)]mod tests {use crate::Span;diff --git a/harper-core/src/parsing/token.rs b/harper-core/src/token.rssimilarity index 93%rename from harper-core/src/parsing/token.rsrename to harper-core/src/token.rsindex b03610717..a5b15077e 100644--- a/harper-core/src/parsing/token.rs+++ b/harper-core/src/token.rs@@ -108,6 +108,9 @@ pub trait TokenStringExt {fn iter_spaces(&self) -> impl Iterator<Item = &Token> + '_;fn iter_apostrophe_indices(&self) -> impl Iterator<Item = usize> + '_;fn iter_apostrophes(&self) -> impl Iterator<Item = &Token> + '_;+ /// Grab the span that represents the beginning of the first element and the end of the last+ /// element.+ fn span(&self) -> Option<Span>;}impl TokenStringExt for [Token] {@@ -147,4 +150,8 @@ impl TokenStringExt for [Token] {fn iter_apostrophes(&self) -> impl Iterator<Item = &Token> + '_ {self.iter_apostrophe_indices().map(|i| &self[i])}++ fn span(&self) -> Option<Span> {+ Some(Span::new(self.first()?.span.start, self.last()?.span.end))+ }}diff --git a/harper-ls/Cargo.toml b/harper-ls/Cargo.tomlindex 4fb234973..849fd0ecb 100644--- a/harper-ls/Cargo.toml+++ b/harper-ls/Cargo.toml@@ -11,3 +11,6 @@ tower-lsp = "0.20.0"tokio = { version = "1.35.1", features = ["full"] }clap = { version = "4.4.18", features = ["derive"] }once_cell = "1.19.0"+tree-sitter = "0.20.10"+tree-sitter-rust = "0.20.4"+tree-sitter-typescript = "0.20.3"diff --git a/harper-ls/src/backend.rs b/harper-ls/src/backend.rsindex 8e9b2d779..8d3003d8f 100644--- a/harper-ls/src/backend.rs+++ b/harper-ls/src/backend.rs@@ -1,6 +1,6 @@-use std::{collections::HashMap, fs};+use std::{borrow::Borrow, collections::HashMap, fs};-use harper_core::{Dictionary, Document, LintSet, Linter};+use harper_core::{Dictionary, Document, Lint, LintSet, Linter, MarkdownParser};use tokio::sync::Mutex;use tower_lsp::{jsonrpc::Result,@@ -33,11 +33,19 @@ impl Backend {}async fn update_document(&self, url: &Url, text: &str) {- let doc = Document::new(text, true);+ let doc = Document::new(text, Box::new(MarkdownParser));let mut files = self.files.lock().await;files.insert(url.clone(), doc);}+ async fn generate_lints_for_url(&self, url: &Url) -> Option<Vec<Lint>> {+ let files = self.files.lock().await;+ let file_contents = files.get(url)?;++ let mut linter = self.linter.lock().await;+ Some(linter.lint(file_contents))+ }+async fn generate_code_actions(&self, url: &Url, range: Range) -> Result<Vec<CodeAction>> {let files = self.files.lock().await;let Some(document) = files.get(url) else {diff --git a/harper-ls/src/comments.rs b/harper-ls/src/comments.rsnew file mode 100644index 000000000..0003fefef--- /dev/null+++ b/harper-ls/src/comments.rs@@ -0,0 +1,37 @@+use harper_core::Span;+use tree_sitter::{Parser, TreeCursor};++/// Extract each comment astris a seperate block.+pub fn extract_comments_rust(text: &str) -> Vec<Span> {+ let mut parser = Parser::new();+ parser.set_language(tree_sitter_rust::language()).unwrap();++ // TODO: Use incremental parsing+ let Some(root) = parser.parse(text, None) else {+ return vec![];+ };++ let mut comments = Vec::new();++ extract_comments(&mut root.walk(), &mut comments);++ comments+}++fn extract_comments(cursor: &mut TreeCursor, comments: &mut Vec<Span>) {+ if !cursor.goto_first_child() {+ return;+ }++ while cursor.goto_next_sibling() {+ let node = cursor.node();++ if node.kind().contains("comment") {+ comments.push(node.byte_range().into());+ }++ extract_comments(cursor, comments);+ }++ cursor.goto_parent();+}diff --git a/harper-ls/src/main.rs b/harper-ls/src/main.rsindex 2e0870a29..36d0ad7d0 100644--- a/harper-ls/src/main.rs+++ b/harper-ls/src/main.rs@@ -1,5 +1,6 @@use tokio::net::TcpListener;mod backend;+mod comments;mod diagnostics;mod pos_conv;diff --git a/harper-serve/src/main.rs b/harper-serve/src/main.rsindex dfb7e10e5..c4e2632ba 100644--- a/harper-serve/src/main.rs+++ b/harper-serve/src/main.rs@@ -1,7 +1,7 @@#![allow(dead_code)]use harper_core::{Dictionary, Document, FatToken, Lint, LintSet, Linter, Span, Suggestion};-use std::net::SocketAddr;+use std::{marker, net::SocketAddr};use tokio::time::Instant;use tracing::{info, Level};use tracing_subscriber::FmtSubscriber;@@ -71,7 +71,7 @@ async fn root() -> &'static str {async fn parse_text(Json(payload): Json<ParseRequest>) -> (StatusCode, Json<ParseResponse>) {let text = payload.text;- let document = Document::new(&text, true);+ let document = Document::new_markdown(&text);let tokens: Vec<_> = document.fat_tokens().collect();(StatusCode::ACCEPTED, Json(ParseResponse { tokens }))@@ -90,7 +90,7 @@ struct ParseResponse {async fn lint(Json(payload): Json<LintRequest>) -> (StatusCode, Json<LintResponse>) {let text = payload.text;- let document = Document::new(&text, true);+ let document = Document::new_markdown(&text);let dictionary = Dictionary::new();let mut linter = LintSet::new().with_standard(dictionary);@@ -113,7 +113,7 @@ async fn apply_suggestion(Json(payload): Json<ApplySuggestionRequest>,) -> (StatusCode, Json<ApplySuggestionResponse>) {let text = payload.text;- let mut document = Document::new(&text, true);+ let mut document = Document::new_markdown(&text);document.apply_suggestion(&payload.suggestion, payload.span);(diff --git a/harper-wasm/src/lib.rs b/harper-wasm/src/lib.rsindex 3631ed357..9cfe30621 100644--- a/harper-wasm/src/lib.rs+++ b/harper-wasm/src/lib.rs@@ -20,7 +20,7 @@ pub fn setup() {#[wasm_bindgen]pub fn lint(text: String) -> Vec<JsValue> {let dictionary = Dictionary::new();- let document = Document::new(&text, true);+ let document = Document::new_markdown(&text);let mut linter = LintSet::new().with_standard(dictionary);let lints = linter.lint(&document);@@ -33,7 +33,7 @@ pub fn lint(text: String) -> Vec<JsValue> {#[wasm_bindgen]pub fn parse(text: String) -> Vec<JsValue> {- let document = Document::new(&text, true);+ let document = Document::new_markdown(&text);document.fat_tokens()@@ -50,7 +50,7 @@ pub fn apply_suggestion(let span = serde_wasm_bindgen::from_value(span).map_err(|e| e.to_string())?;let suggestion = serde_wasm_bindgen::from_value(suggestion).map_err(|e| e.to_string())?;- let mut document = Document::new(&text, true);+ let mut document = Document::new_markdown(&text);document.apply_suggestion(&suggestion, span);Ok(document.get_full_string())