Offline, privacy-first grammar checker. Fast, open-source, Rust-powered
Merge branch 'master' into ignore-link-title-cfg
Grant Lemons809cacdda36b
diff --cc harper-cli/src/main.rsindex 63ee32e12,0044ecde3..b1508df3a--- a/harper-cli/src/main.rs+++ b/harper-cli/src/main.rs@@@ -7,8 -7,9 +7,9 @@@ use ariadne::{Color, Label, Report, Repuse clap::Parser;use harper_comments::CommentParser;use harper_core::linting::{LintGroup, LintGroupConfig, Linter};-use harper_core::parsers::Markdown;+use harper_core::parsers::{Markdown, MarkdownOptions};- use harper_core::{remove_overlaps, Dictionary, Document, FstDictionary};+ use harper_core::{remove_overlaps, Dictionary, Document, FstDictionary, TokenKind};+ use harper_literate_haskell::LiterateHaskellParser;#[derive(Debug, Parser)]enum Args {@@@ -91,8 -95,11 +97,11 @@@ fn main() -> anyhow::Result<()>Ok(())}- Args::Spans { file } => {+ Args::Spans {+ file,+ include_newlines,+ } => {- let (doc, source) = load_file(&file)?;+ let (doc, source) = load_file(&file, markdown_options)?;let primary_color = Color::Blue;let secondary_color = Color::Magenta;@@@ -141,19 -166,20 +168,22 @@@}}-fn load_file(file: &Path) -> anyhow::Result<(Document, String)> {+fn load_file(file: &Path, markdown_options: MarkdownOptions) -> anyhow::Result<(Document, String)> {let source = std::fs::read_to_string(file)?;- let mut parser: Box<dyn harper_core::parsers::Parser> =- if let Some("md") = file.extension().map(|v| v.to_str().unwrap()) {- Box::new(Markdown::new(markdown_options))- } else {- Box::new(- CommentParser::new_from_filename(file, Markdown::new(markdown_options))+ let parser: Box<dyn harper_core::parsers::Parser> =+ match file.extension().map(|v| v.to_str().unwrap()) {- Some("md") => Box::new(Markdown),- Some("lhs") => Box::new(LiterateHaskellParser),++ Some("md") => Box::new(Markdown::default()),++ Some("lhs") => Box::new(LiterateHaskellParser::new_markdown(++ MarkdownOptions::default(),++ )),+ Some("typ") => Box::new(harper_typst::Typst),+ _ => Box::new(- CommentParser::new_from_filename(file)++ CommentParser::new_from_filename(file, markdown_options).map(Box::new).ok_or(format_err!("Could not detect language ID."))?,- )+ ),};- Ok((Document::new_curated(&source, &mut parser), source))+ Ok((Document::new_curated(&source, &parser), source))}diff --cc harper-comments/src/comment_parser.rsindex 5f93706ba,7c15942c1..bf631c7b1--- a/harper-comments/src/comment_parser.rs+++ b/harper-comments/src/comment_parser.rs@@@ -1,7 -1,7 +1,7 @@@use std::path::Path;use comment_parsers::{Go, JavaDoc, JsDoc, Unit};- use harper_core::parsers::{self, Markdown, Parser};-use harper_core::parsers::{self, Parser};++use harper_core::parsers::{self, MarkdownOptions, Parser};use harper_core::{FullDictionary, Token};use harper_tree_sitter::TreeSitterMasker;use tree_sitter::Node;@@@ -17,7 -17,7 +17,10 @@@ impl CommentParserself.inner.masker.create_ident_dict(source)}- pub fn new_from_language_id(language_id: &str, markdown_parser: Markdown) -> Option<Self> {- pub fn new_from_language_id(language_id: &str) -> Option<Self> {++ pub fn new_from_language_id(++ language_id: &str,++ markdown_options: MarkdownOptions,++ ) -> Option<Self> {let language = match language_id {"rust" => tree_sitter_rust::language(),"typescriptreact" => tree_sitter_typescript::language_tsx(),@@@ -40,12 -42,10 +45,12 @@@};let comment_parser: Box<dyn Parser> = match language_id {- "javascriptreact" | "typescript" | "typescriptreact" | "javascript" => Box::new(JsDoc),+ "javascriptreact" | "typescript" | "typescriptreact" | "javascript" => {- Box::new(JsDoc::new(markdown_parser))++ Box::new(JsDoc::new_markdown(markdown_options))+ }"java" => Box::new(JavaDoc::default()),- "go" => Box::new(Go::new(markdown_parser)),- _ => Box::new(Unit::new(markdown_parser)),- "go" => Box::new(Go),- _ => Box::new(Unit),++ "go" => Box::new(Go::new_markdown(markdown_options)),++ _ => Box::new(Unit::new_markdown(markdown_options)),};Some(Self {@@@ -57,8 -57,8 +62,8 @@@}/// Infer the programming language from a provided filename.- pub fn new_from_filename(filename: &Path, markdown_parser: Markdown) -> Option<Self> {- Self::new_from_language_id(Self::filename_to_filetype(filename)?, markdown_parser)- pub fn new_from_filename(filename: &Path) -> Option<Self> {- Self::new_from_language_id(Self::filename_to_filetype(filename)?)++ pub fn new_from_filename(filename: &Path, markdown_options: MarkdownOptions) -> Option<Self> {++ Self::new_from_language_id(Self::filename_to_filetype(filename)?, markdown_options)}/// Convert a provided path to a corresponding Language Server Protocol filediff --cc harper-comments/src/comment_parsers/go.rsindex 020df3d4c,f7f824dbd..71d499e9d--- a/harper-comments/src/comment_parsers/go.rs+++ b/harper-comments/src/comment_parsers/go.rs@@@ -1,21 -1,13 +1,29 @@@--use harper_core::parsers::{Markdown, Parser};++use harper_core::parsers::{Markdown, MarkdownOptions, Parser};++use harper_core::Lrc;use harper_core::Token;use super::without_initiators;-#[derive(Debug, Clone, Copy)]-pub struct Go;+#[derive(Clone)]+pub struct Go {- markdown_parser: Markdown,++ inner: Lrc<dyn Parser>,+}++impl Go {- pub fn new(markdown_parser: Markdown) -> Self {- Self { markdown_parser }++ #[allow(unused)]++ pub fn new(parser: Lrc<dyn Parser>) -> Self {++ Self { inner: parser }++ }++++ pub fn new_markdown(markdown_options: MarkdownOptions) -> Self {++ Self {++ inner: Lrc::new(Markdown::new(markdown_options)),++ }+ }+}impl Parser for Go {- fn parse(&mut self, source: &[char]) -> Vec<Token> {+ fn parse(&self, source: &[char]) -> Vec<Token> {let mut actual = without_initiators(source);let mut actual_source = actual.get_content(source);@@@ -33,7 -25,7 +41,7 @@@actual_source = new_source}- let mut new_tokens = self.markdown_parser.parse(actual_source);- let mut new_tokens = Markdown.parse(actual_source);++ let mut new_tokens = self.inner.parse(actual_source);new_tokens.iter_mut()diff --cc harper-comments/src/comment_parsers/jsdoc.rsindex 197661145,dcc56a0a8..e8a27820b--- a/harper-comments/src/comment_parsers/jsdoc.rs+++ b/harper-comments/src/comment_parsers/jsdoc.rs@@@ -1,28 -1,19 +1,36 @@@--use harper_core::parsers::{Markdown, Parser};++use harper_core::parsers::{Markdown, MarkdownOptions, Parser};++use harper_core::Lrc;use harper_core::{Punctuation, Span, Token, TokenKind};use itertools::Itertools;use super::without_initiators;-pub struct JsDoc;+#[derive(Clone)]+pub struct JsDoc {- markdown_parser: Markdown,++ inner: Lrc<dyn Parser>,+}++impl JsDoc {- pub fn new(markdown_parser: Markdown) -> Self {- Self { markdown_parser }++ #[allow(unused)]++ pub fn new(parser: Lrc<dyn Parser>) -> Self {++ Self { inner: parser }++ }++++ pub fn new_markdown(markdown_options: MarkdownOptions) -> Self {++ Self {++ inner: Lrc::new(Markdown::new(markdown_options)),++ }+ }+}impl Parser for JsDoc {- fn parse(&mut self, source: &[char]) -> Vec<Token> {+ fn parse(&self, source: &[char]) -> Vec<Token> {let mut tokens = Vec::new();let mut chars_traversed = 0;for line in source.split(|c| *c == '\n') {- let mut new_tokens = parse_line(line, &mut self.markdown_parser);- let mut new_tokens = parse_line(line);++ let mut new_tokens = parse_line(line, self.inner.clone());if chars_traversed + line.len() < source.len() {new_tokens.push(Token::new(@@@ -43,7 -34,7 +51,7 @@@}}- fn parse_line(source: &[char], markdown_parser: &mut Markdown) -> Vec<Token> {-fn parse_line(source: &[char]) -> Vec<Token> {++fn parse_line(source: &[char], parser: Lrc<dyn Parser>) -> Vec<Token> {let actual_line = without_initiators(source);if actual_line.is_empty() {@@@ -52,7 -43,7 +60,7 @@@let source_line = actual_line.get_content(source);- let mut new_tokens = markdown_parser.parse(source_line);- let mut new_tokens = Markdown.parse(source_line);++ let mut new_tokens = parser.parse(source_line);// Handle inline tagsmark_inline_tags(&mut new_tokens);@@@ -157,23 -148,21 +165,23 @@@ fn parse_inline_tag(tokens: &[Token]) -#[cfg(test)]mod tests {- use harper_core::{parsers::Markdown, Document, Punctuation, TokenKind};- use harper_core::{Document, Punctuation, TokenKind};++ use harper_core::{parsers::MarkdownOptions, Document, Punctuation, TokenKind};use crate::CommentParser;#[test]fn escapes_loop() {let source = "/** This should _not_cause an infinite loop: {@ */";- let mut parser = CommentParser::new_from_language_id("javascript").unwrap();+ let mut parser =- CommentParser::new_from_language_id("javascript", Markdown::default()).unwrap();++ CommentParser::new_from_language_id("javascript", MarkdownOptions::default()).unwrap();Document::new_curated(source, &mut parser);}#[test]fn handles_inline_link() {let source = "/** See {@link MyClass} and [MyClass's foo property]{@link MyClass#foo}. */";- let mut parser = CommentParser::new_from_language_id("javascript").unwrap();+ let mut parser =- CommentParser::new_from_language_id("javascript", Markdown::default()).unwrap();++ CommentParser::new_from_language_id("javascript", MarkdownOptions::default()).unwrap();let document = Document::new_curated(source, &mut parser);assert!(matches!(@@@ -217,8 -206,7 +225,8 @@@#[test]fn handles_class() {let source = "/** @class Circle representing a circle. */";- let mut parser = CommentParser::new_from_language_id("javascript").unwrap();+ let mut parser =- CommentParser::new_from_language_id("javascript", Markdown::default()).unwrap();++ CommentParser::new_from_language_id("javascript", MarkdownOptions::default()).unwrap();let document = Document::new_curated(source, &mut parser);assert!(documentdiff --cc harper-comments/src/comment_parsers/unit.rsindex 97af2869a,3185a1725..905a2a4b1--- a/harper-comments/src/comment_parsers/unit.rs+++ b/harper-comments/src/comment_parsers/unit.rs@@@ -1,4 -1,4 +1,5 @@@--use harper_core::parsers::{Markdown, Parser};++use harper_core::parsers::{Markdown, MarkdownOptions, Parser};++use harper_core::Lrc;use harper_core::{Span, Token};use super::without_initiators;@@@ -9,19 -9,10 +10,26 @@@////// It assumes it is being provided a single line of comment at a time,/// including the comment initiation characters.-pub struct Unit;+#[derive(Clone)]+pub struct Unit {- markdown_parser: Markdown,++ inner: Lrc<dyn Parser>,+}++impl Unit {- pub fn new(markdown_parser: Markdown) -> Self {- Self { markdown_parser }++ #[allow(unused)]++ pub fn new(parser: Lrc<dyn Parser>) -> Self {++ Self { inner: parser }++ }++++ pub fn new_markdown(markdown_options: MarkdownOptions) -> Self {++ Self {++ inner: Lrc::new(Markdown::new(markdown_options)),++ }+ }+}impl Parser for Unit {- fn parse(&mut self, source: &[char]) -> Vec<Token> {+ fn parse(&self, source: &[char]) -> Vec<Token> {let mut tokens = Vec::new();let mut chars_traversed = 0;@@@ -37,7 -28,7 +45,7 @@@continue;}- let mut new_tokens = parse_line(line, &mut self.markdown_parser);- let mut new_tokens = parse_line(line);++ let mut new_tokens = parse_line(line, self.inner.clone());if chars_traversed + line.len() < source.len() {new_tokens.push(Token::new(@@@ -58,7 -49,7 +66,7 @@@}}- fn parse_line(source: &[char], markdown_parser: &mut Markdown) -> Vec<Token> {-fn parse_line(source: &[char]) -> Vec<Token> {++fn parse_line(source: &[char], parser: Lrc<dyn Parser>) -> Vec<Token> {let actual = without_initiators(source);if actual.is_empty() {@@@ -66,8 -57,7 +74,8 @@@}let source = actual.get_content(source);- let mut new_tokens = Markdown.parse(source);+- let mut new_tokens = markdown_parser.parse(source);++ let mut new_tokens = parser.parse(source);new_tokens.iter_mut()diff --cc harper-comments/tests/language_support.rsindex df04a5ea4,ca9df0a3e..49463be7a--- a/harper-comments/tests/language_support.rs+++ b/harper-comments/tests/language_support.rs@@@ -2,7 -2,6 +2,7 @@@ use std::path::Pathuse harper_comments::CommentParser;use harper_core::linting::{LintGroup, LintGroupConfig, Linter};- use harper_core::parsers::Markdown;++use harper_core::parsers::MarkdownOptions;use harper_core::{Document, FstDictionary};/// Creates a unit test checking that the linting of a source file in@@@ -21,7 -20,7 +21,7 @@@ macro_rules! create_test));- let mut parser = CommentParser::new_from_filename(Path::new(filename), Markdown::default()).unwrap();- let mut parser = CommentParser::new_from_filename(Path::new(filename)).unwrap();++ let mut parser = CommentParser::new_from_filename(Path::new(filename), MarkdownOptions::default()).unwrap();let dict = FstDictionary::curated();let document = Document::new(&source, &mut parser, &dict);diff --cc harper-core/benches/parse_demo.rsindex 092215867,d76265321..cbd0a8dbf--- a/harper-core/benches/parse_demo.rs+++ b/harper-core/benches/parse_demo.rs@@@ -7,14 -6,14 +6,14 @@@ static DEMO: &str = include_str!("../..fn parse_demo(c: &mut Criterion) {c.bench_function("parse_demo", |b| {- b.iter(|| Document::new_markdown_curated(black_box(DEMO), MarkdownOptions::default()))- b.iter(|| Document::new_markdown_curated(black_box(DEMO)))++ b.iter(|| Document::new_markdown_default_curated(black_box(DEMO)));});}fn lint_demo(c: &mut Criterion) {let dictionary = FstDictionary::curated();let mut lint_set = LintGroup::new(Default::default(), dictionary);- let document = Document::new_markdown_curated(black_box(DEMO), MarkdownOptions::default());- let document = Document::new_markdown_curated(black_box(DEMO));++ let document = Document::new_markdown_default_curated(black_box(DEMO));c.bench_function("lint_demo", |b| {b.iter(|| lint_set.lint(&document));@@@ -26,8 -25,7 +25,7 @@@ fn lint_demo_uncached(c: &mut Criterionb.iter(|| {let dictionary = FstDictionary::curated();let mut lint_set = LintGroup::new(LintGroupConfig::default(), dictionary.clone());- let document =- Document::new_markdown(black_box(DEMO), MarkdownOptions::default(), &dictionary);- let document = Document::new_markdown(black_box(DEMO), &dictionary);++ let document = Document::new_markdown_default(black_box(DEMO), &dictionary);lint_set.lint(&document)})});diff --cc harper-core/src/document.rsindex 34dab4d77,1741ee63c..7145d0a4b--- a/harper-core/src/document.rs+++ b/harper-core/src/document.rs@@@ -4,13 -4,12 +4,12 @@@ use std::fmt::Displayuse paste::paste;-use crate::parsers::{Markdown, Parser, PlainEnglish};+use crate::parsers::{Markdown, MarkdownOptions, Parser, PlainEnglish};use crate::patterns::{PatternExt, RepeatingPattern, SequencePattern};use crate::punctuation::Punctuation;- use crate::token::NumberSuffix;use crate::vec_ext::VecExt;- use crate::Span;use crate::{Dictionary, FatToken, FstDictionary, Lrc, Token, TokenKind, TokenStringExt};+ use crate::{NumberSuffix, Span};/// A document containing some amount of lexed and parsed English text.#[derive(Debug, Clone)]@@@ -71,22 -70,14 +70,34 @@@ impl Document/// Parse text to produce a document using the built-in [`Markdown`] parser/// and curated dictionary.- pub fn new_markdown_curated(text: &str) -> Self {- Self::new(text, &Markdown, &FstDictionary::curated())+ pub fn new_markdown_curated(text: &str, markdown_options: MarkdownOptions) -> Self {+ Self::new(+ text,- &mut Markdown::new(markdown_options),++ &Markdown::new(markdown_options),+ &FstDictionary::curated(),+ )+ }+++ /// Parse text to produce a document using the built-in [`Markdown`] parser++ /// and curated dictionary with the default markdown configuration.++ pub fn new_markdown_default_curated(text: &str) -> Self {++ Self::new_markdown_curated(text, MarkdownOptions::default())+ }+/// Parse text to produce a document using the built-in [`PlainEnglish`]/// parser and the curated dictionary.- pub fn new_markdown(text: &str, dictionary: &impl Dictionary) -> Self {- Self::new(text, &Markdown, dictionary)+ pub fn new_markdown(+ text: &str,+ markdown_options: MarkdownOptions,+ dictionary: &impl Dictionary,+ ) -> Self {- Self::new(text, &mut Markdown::new(markdown_options), dictionary)++ Self::new(text, &Markdown::new(markdown_options), dictionary)++ }++++ /// Parse text to produce a document using the built-in [`PlainEnglish`]++ /// parser and the curated dictionary with the default markdown configuration.++ pub fn new_markdown_default(text: &str, dictionary: &impl Dictionary) -> Self {++ Self::new_markdown(text, MarkdownOptions::default(), dictionary)}/// Re-parse important language constructs.diff --cc harper-core/src/linting/lint_group.rsindex 728679e9b,b39ff81b8..f9ef86a0b--- a/harper-core/src/linting/lint_group.rs+++ b/harper-core/src/linting/lint_group.rs@@@ -141,3 -192,33 +192,33 @@@ impl<T: Dictionary + Default> Default fSelf::new(LintGroupConfig::default(), T::default())}}++ #[cfg(test)]+ mod tests {+ use crate::{linting::Linter, Document, FstDictionary, FullDictionary};++ use super::{LintGroup, LintGroupConfig};++ #[test]+ fn can_get_all_descriptions() {+ let group = LintGroup::<FullDictionary>::default();+ group.all_descriptions();+ }++ #[test]+ fn lint_descriptions_are_clean() {+ let mut group = LintGroup::new(LintGroupConfig::default(), FstDictionary::curated());+ let pairs: Vec<_> = group+ .all_descriptions()+ .to_vec_pairs()+ .into_iter()+ .map(|(a, b)| (a.to_string(), b.to_string()))+ .collect();++ for (key, value) in pairs {- let doc = Document::new_markdown_curated(&value);++ let doc = Document::new_markdown_default_curated(&value);+ eprintln!("{key}: {value}");+ assert!(group.lint(&doc).is_empty())+ }+ }+ }diff --cc harper-core/src/linting/mod.rsindex 6fa03a07d,8f64ff574..9b8cc5825--- a/harper-core/src/linting/mod.rs+++ b/harper-core/src/linting/mod.rs@@@ -64,19 -78,30 +78,30 @@@ pub trait Linter: Send + Sync#[cfg(test)]mod tests {use super::Linter;- use crate::{parsers::MarkdownOptions, Document};+ use crate::Document;pub fn assert_lint_count(text: &str, mut linter: impl Linter, count: usize) {- let test = Document::new_markdown_curated(text, MarkdownOptions::default());- let test = Document::new_markdown_curated(text);++ let test = Document::new_markdown_default_curated(text);let lints = linter.lint(&test);dbg!(&lints);assert_eq!(lints.len(), count);}+ /// Assert the total number of suggestions produced by a [`Linter`], spread across all produced+ /// [`Lint`]s.+ pub fn assert_suggestion_count(text: &str, mut linter: impl Linter, count: usize) {- let test = Document::new_markdown_curated(text);++ let test = Document::new_markdown_default_curated(text);+ let lints = linter.lint(&test);+ assert_eq!(+ lints.iter().map(|l| l.suggestions.len()).sum::<usize>(),+ count+ );+ }+/// Runs a provided linter on text, applies the first suggestion from each- /// lint and asserts that the result is equal to a given value.+ /// lint and asserts whether the result is equal to a given value.pub fn assert_suggestion_result(text: &str, mut linter: impl Linter, expected_result: &str) {- let test = Document::new_markdown_curated(text, MarkdownOptions::default());- let test = Document::new_markdown_curated(text);++ let test = Document::new_markdown_default_curated(text);let lints = linter.lint(&test);let mut text: Vec<char> = text.chars().collect();diff --cc harper-core/src/parsers/markdown.rsindex 6c51033e5,ed5f965c6..771e5f41f--- a/harper-core/src/parsers/markdown.rs+++ b/harper-core/src/parsers/markdown.rs@@@ -9,29 -7,9 +9,31 @@@ use crate::{Span, Token, TokenKind, Tok/// CommonMark files.////// Will ignore code blocks and tables.- #[derive(Default, Clone, Debug)]- pub struct Markdown(MarkdownOptions);-pub struct Markdown;++#[derive(Default, Clone, Debug, Copy)]++pub struct Markdown {++ options: MarkdownOptions,++}++#[derive(Copy, Clone, Debug, Serialize, Deserialize)]+pub struct MarkdownOptions {+ pub ignore_link_title: bool,+}++// Clippy rule excepted because this can easily be expanded later+#[allow(clippy::derivable_impls)]+impl Default for MarkdownOptions {+ fn default() -> Self {+ Self {+ ignore_link_title: false,+ }+ }+}impl Markdown {+ pub fn new(options: MarkdownOptions) -> Self {- Self(options)++ Self { options }+ }+/// Remove hidden Wikilink target text.////// As in, the stuff to the left of the pipe operator:@@@ -224,15 -202,9 +226,15 @@@ impl Parser for Markdown});continue;}- if matches!(tag, Tag::Link { .. }) && self.0.ignore_link_title {-++ if matches!(tag, Tag::Link { .. }) && self.options.ignore_link_title {+ tokens.push(Token {+ span: Span::new_with_len(traversed_chars, text.chars().count()),+ kind: TokenKind::Unlintable,+ });+ continue;+ }if !(matches!(tag, Tag::Paragraph)- || matches!(tag, Tag::Link { .. }) && !self.0.ignore_link_title- || matches!(tag, Tag::Link { .. })++ || matches!(tag, Tag::Link { .. }) && !self.options.ignore_link_title|| matches!(tag, Tag::Heading { .. })|| matches!(tag, Tag::Item)|| matches!(tag, Tag::TableCell)@@@ -292,9 -264,9 +294,9 @@@ mod tests#[test]fn survives_emojis() {- let source = r#"🤷."#;+ let source = r"🤷.";- Markdown.parse_str(source);+ Markdown::default().parse_str(source);}/// Check whether the Markdown parser will emit a breaking newline@@@ -312,9 -284,9 +314,9 @@@#[test]fn math_becomes_unlintable() {- let source = r#"$\Katex$ $\text{is}$ $\text{great}$."#;+ let source = r"$\Katex$ $\text{is}$ $\text{great}$.";- let tokens = Markdown.parse_str(source);+ let tokens = Markdown::default().parse_str(source);assert_eq!(tokens.iter().map(|t| t.kind).collect::<Vec<_>>(),vec![@@@ -330,9 -302,9 +332,9 @@@#[test]fn hidden_wikilink_text() {- let source = r#"[[this is hidden|this is not]]"#;+ let source = r"[[this is hidden|this is not]]";- let tokens = Markdown.parse_str(source);+ let tokens = Markdown::default().parse_str(source);let token_kinds = tokens.iter().map(|t| t.kind).collect::<Vec<_>>();@@@ -350,9 -322,9 +352,9 @@@#[test]fn improper_wikilink_text() {- let source = r#"this is shown|this is also shown]]"#;+ let source = r"this is shown|this is also shown]]";- let tokens = Markdown.parse_str(source);+ let tokens = Markdown::default().parse_str(source);let token_kinds = tokens.iter().map(|t| t.kind).collect::<Vec<_>>();@@@ -382,8 -354,8 +384,8 @@@#[test]fn normal_wikilink() {- let source = r#"[[Wikilink]]"#;+ let source = r"[[Wikilink]]";- let tokens = Markdown.parse_str(source);+ let tokens = Markdown::default().parse_str(source);let token_kinds = tokens.iter().map(|t| t.kind).collect::<Vec<_>>();dbg!(&token_kinds);@@@ -393,62 -365,8 +395,62 @@@#[test]fn html_is_unlintable() {- let source = r#"The range of inputs from <ctrl-g> to ctrl-z"#;+ let source = r"The range of inputs from <ctrl-g> to ctrl-z";- let tokens = Markdown.parse_str(source);+ let tokens = Markdown::default().parse_str(source);assert_eq!(tokens.iter_unlintables().count(), 1);}++ #[test]+ fn link_title_unlintable() {- let mut parser = Markdown(MarkdownOptions {++ let parser = Markdown::new(MarkdownOptions {+ ignore_link_title: true,+ ..MarkdownOptions::default()+ });+ let source = r#"[elijah-potter/harper](https://github.com/elijah-potter/harper)"#;+ let tokens = parser.parse_str(source);+ let token_kinds = tokens.iter().map(|t| t.kind).collect::<Vec<_>>();++ dbg!(&token_kinds);++ assert!(matches!(token_kinds.as_slice(), &[TokenKind::Unlintable]))+ }++ #[test]+ fn respects_config() {+ let source = r#"[elijah-potter/harper](https://github.com/elijah-potter/harper)"#;- let mut parser = Markdown(MarkdownOptions {++ let parser = Markdown::new(MarkdownOptions {+ ignore_link_title: true,+ ..MarkdownOptions::default()+ });+ let token_kinds = parser+ .parse_str(source)+ .iter()+ .map(|t| t.kind)+ .collect::<Vec<_>>();++ assert!(matches!(token_kinds.as_slice(), &[TokenKind::Unlintable]));+- let mut parser = Markdown(MarkdownOptions {++ let parser = Markdown::new(MarkdownOptions {+ ignore_link_title: false,+ ..MarkdownOptions::default()+ });+ let token_kinds = parser+ .parse_str(source)+ .iter()+ .map(|t| t.kind)+ .collect::<Vec<_>>();++ dbg!(&token_kinds);++ assert!(matches!(+ token_kinds.as_slice(),+ &[+ TokenKind::Word(_),+ TokenKind::Punctuation(Punctuation::Hyphen),+ TokenKind::Word(_),+ TokenKind::Punctuation(Punctuation::ForwardSlash),+ TokenKind::Word(_)+ ]+ ));+ }}diff --cc harper-core/src/parsers/mod.rsindex 4429e054f,91d81980f..5069e3309--- a/harper-core/src/parsers/mod.rs+++ b/harper-core/src/parsers/mod.rs@@@ -11,18 -11,18 +11,18 @@@ pub use markdown::{Markdown, MarkdownOppub use mask::Mask;pub use plain_english::PlainEnglish;- pub use crate::token::{Token, TokenKind, TokenStringExt};+ use crate::{Token, TokenStringExt};#[cfg(not(feature = "concurrent"))]--#[blanket(derive(Box))]++#[blanket(derive(Box, Rc))]pub trait Parser {- fn parse(&mut self, source: &[char]) -> Vec<Token>;+ fn parse(&self, source: &[char]) -> Vec<Token>;}#[cfg(feature = "concurrent")]--#[blanket(derive(Box))]++#[blanket(derive(Box, Arc))]pub trait Parser: Send + Sync {- fn parse(&mut self, source: &[char]) -> Vec<Token>;+ fn parse(&self, source: &[char]) -> Vec<Token>;}pub trait StrParser {diff --cc harper-core/src/patterns/noun_phrase.rsindex 000000000,66d25e4f2..dedf54303mode 000000,100644..100644--- a/harper-core/src/patterns/noun_phrase.rs+++ b/harper-core/src/patterns/noun_phrase.rs@@@ -1,0 -1,99 +1,99 @@@+ use crate::Token;++ use super::Pattern;++ /// A pattern that returns the value of the first non-zero match in a list.+ #[derive(Default)]+ pub struct NounPhrase;++ impl Pattern for NounPhrase {+ fn matches(&self, tokens: &[Token], _source: &[char]) -> usize {+ let mut cursor = 0;++ loop {+ let Some(tok) = tokens.get(cursor) else {+ return 0;+ };++ if tok.kind.is_adjective() || tok.kind.is_article() {+ let Some(next) = tokens.get(cursor + 1) else {+ return 0;+ };++ if !next.kind.is_whitespace() {+ return 0;+ }++ cursor += 2;+ continue;+ }++ if tok.kind.is_noun() {+ return cursor + 1;+ }++ return 0;+ }+ }+ }++ #[cfg(test)]+ mod tests {+ use super::super::DocPattern;+ use super::NounPhrase;+ use crate::{patterns::Pattern, Document, Span};++ #[test]+ fn simple_apple() {- let doc = Document::new_markdown_curated("A red apple");++ let doc = Document::new_markdown_default_curated("A red apple");+ let matches = NounPhrase.find_all_matches_in_doc(&doc);++ assert_eq!(matches, vec![Span::new(0, 5)])+ }++ #[test]+ fn complex_apple() {- let doc = Document::new_markdown_curated("A red apple with a long stem");++ let doc = Document::new_markdown_default_curated("A red apple with a long stem");+ let matches = NounPhrase.find_all_matches_in_doc(&doc);++ assert_eq!(matches, vec![Span::new(0, 5), Span::new(8, 13)])+ }++ #[test]+ fn list_fruit() {- let doc = Document::new_markdown_curated("An apple, a banana and a pear");++ let doc = Document::new_markdown_default_curated("An apple, a banana and a pear");+ let matches = NounPhrase.find_all_matches_in_doc(&doc);++ assert_eq!(+ matches,+ vec![Span::new(0, 3), Span::new(5, 8), Span::new(11, 14)]+ )+ }++ #[test]+ fn simplest_banana() {- let doc = Document::new_markdown_curated("a banana");++ let doc = Document::new_markdown_default_curated("a banana");+ assert!(NounPhrase.matches(doc.get_tokens(), doc.get_source()) != 0);+ }++ #[test]+ fn food() {- let doc = Document::new_markdown_curated(++ let doc = Document::new_markdown_default_curated(+ "My favorite foods are pizza, sushi, tacos and burgers.",+ );+ let matches = NounPhrase.find_all_matches_in_doc(&doc);++ dbg!(&matches);++ assert_eq!(+ matches,+ vec![+ Span::new(2, 5),+ Span::new(8, 9),+ Span::new(11, 12),+ Span::new(14, 15),+ Span::new(18, 19)+ ]+ )+ }+ }diff --cc harper-core/src/patterns/word_set.rsindex 000000000,e8a6eab9c..4372f8f7dmode 000000,100644..100644--- a/harper-core/src/patterns/word_set.rs+++ b/harper-core/src/patterns/word_set.rs@@@ -1,0 -1,93 +1,93 @@@+ use super::Pattern;+ use smallvec::SmallVec;++ use crate::{CharString, Token};++ // A [`Pattern`] that matches against any of a set of provided words.+ // For small sets of short words, it doesn't allocate.+ //+ // Note that any capitalization of the contained words will result in a match.+ #[derive(Debug, Default, Clone)]+ pub struct WordSet {+ words: SmallVec<[CharString; 4]>,+ }++ impl WordSet {+ pub fn add(&mut self, word: &str) {+ let chars = word.chars().collect();++ if !self.words.contains(&chars) {+ self.words.push(chars);+ }+ }++ pub fn all(words: &[&'static str]) -> Self {+ let mut set = Self::default();++ for str in words {+ set.add(str);+ }++ set+ }+ }++ impl Pattern for WordSet {+ fn matches(&self, tokens: &[Token], source: &[char]) -> usize {+ let Some(tok) = tokens.first() else {+ return 0;+ };++ if !tok.kind.is_word() {+ return 0;+ }++ let tok_chars = tok.span.get_content(source);++ for word in &self.words {+ if tok_chars.len() != word.len() {+ continue;+ }++ let partial_match = tok_chars+ .iter()+ .zip(word)+ .all(|(a, b)| a.eq_ignore_ascii_case(b));++ if partial_match {+ return 1;+ }+ }++ 0+ }+ }++ #[cfg(test)]+ mod tests {+ use crate::{patterns::DocPattern, Document, Span};++ use super::WordSet;++ #[test]+ fn fruit() {+ let set = WordSet::all(&["banana", "apple", "orange"]);+- let doc = Document::new_markdown_curated("I ate a banana and an apple today.");++ let doc = Document::new_markdown_default_curated("I ate a banana and an apple today.");++ let matches = set.find_all_matches_in_doc(&doc);++ assert_eq!(matches, vec![Span::new(6, 7), Span::new(12, 13)]);+ }++ #[test]+ fn fruit_whack_capitalization() {+ let set = WordSet::all(&["banana", "apple", "orange"]);+- let doc = Document::new_markdown_curated("I Ate A bAnaNa And aN apPlE today.");++ let doc = Document::new_markdown_default_curated("I Ate A bAnaNa And aN apPlE today.");++ let matches = set.find_all_matches_in_doc(&doc);++ assert_eq!(matches, vec![Span::new(6, 7), Span::new(12, 13)]);+ }+ }diff --cc harper-core/tests/run_tests.rsindex 499cf7149,50f5f3d6e..b6ba18028--- a/harper-core/tests/run_tests.rs+++ b/harper-core/tests/run_tests.rs@@@ -9,29 -8,29 +8,29 @@@ macro_rules! create_testpaste::paste! {#[test]fn [<lints_ $filename _correctly>](){-- let source = include_str!(++ let source = include_str!(concat!("./test_sources/",concat!(stringify!($filename), ".md"))-- );++ );-- let dict = FstDictionary::curated();- let document = Document::new_markdown(&source, MarkdownOptions::default(), &dict);- let document = Document::new_markdown(&source, &dict);++ let dict = FstDictionary::curated();++ let document = Document::new_markdown_default(&source, &dict);-- let mut linter = LintGroup::new(++ let mut linter = LintGroup::new(LintGroupConfig::default(),dict-- );-- let lints = linter.lint(&document);++ );++ let lints = linter.lint(&document);-- dbg!(&lints);-- assert_eq!(lints.len(), $correct_expected);++ dbg!(&lints);++ assert_eq!(lints.len(), $correct_expected);-- // Make sure that all generated tokens span real characters-- for token in document.tokens(){++ // Make sure that all generated tokens span real characters++ for token in document.tokens(){assert!(token.span.try_get_content(document.get_source()).is_some());-- }++ }}}};diff --cc harper-html/tests/run_tests.rsindex 7f39d644f,320ac7c89..dcea70978--- a/harper-html/tests/run_tests.rs+++ b/harper-html/tests/run_tests.rs@@@ -17,7 -16,7 +16,7 @@@ macro_rules! create_test);let dict = FstDictionary::curated();- let document = Document::new_markdown(&source, MarkdownOptions::default(), &dict);- let document = Document::new_markdown(&source, &dict);++ let document = Document::new_markdown_default(&source, &dict);let mut linter = LintGroup::new(LintGroupConfig::default(),diff --cc harper-literate-haskell/src/lib.rsindex 000000000,903f9c5b3..20dd89148mode 000000,100644..100644--- a/harper-literate-haskell/src/lib.rs+++ b/harper-literate-haskell/src/lib.rs@@@ -1,0 -1,31 +1,47 @@@+ use harper_comments::CommentParser;+ use harper_core::{- parsers::{Markdown, Mask, Parser},- FullDictionary, Masker, Token,++ parsers::{Markdown, MarkdownOptions, Mask, Parser},++ FullDictionary, Lrc, Masker, Token,+ };++ mod masker;+ use itertools::Itertools;+ use masker::LiterateHaskellMasker;++ /// Parses a Literate Haskell document by masking out the code and considering text as Markdown.-pub struct LiterateHaskellParser;++pub struct LiterateHaskellParser {++ inner: Lrc<dyn Parser>,++}++ impl LiterateHaskellParser {- pub fn create_ident_dict(&self, source: &[char]) -> Option<FullDictionary> {- let parser = CommentParser::new_from_language_id("haskell").unwrap();++ pub fn new(inner: Lrc<dyn Parser>) -> Self {++ Self { inner }++ }++++ pub fn new_markdown(markdown_options: MarkdownOptions) -> Self {++ Self {++ inner: Lrc::new(Markdown::new(markdown_options)),++ }++ }++++ pub fn create_ident_dict(++ &self,++ source: &[char],++ markdown_options: MarkdownOptions,++ ) -> Option<FullDictionary> {++ let parser = CommentParser::new_from_language_id("haskell", markdown_options).unwrap();+ let mask = LiterateHaskellMasker::code_only().create_mask(source);++ let code = mask+ .iter_allowed(source)+ .flat_map(|(_, src)| src.to_owned())+ .collect_vec();+ parser.create_ident_dict(&code)+ }+ }++ impl Parser for LiterateHaskellParser {+ fn parse(&self, source: &[char]) -> Vec<Token> {- Mask::new(LiterateHaskellMasker::text_only(), Markdown).parse(source)++ Mask::new(LiterateHaskellMasker::text_only(), self.inner.clone()).parse(source)+ }+ }diff --cc harper-literate-haskell/tests/run_tests.rsindex 000000000,f367f66b3..7c8e1791amode 000000,100644..100644--- a/harper-literate-haskell/tests/run_tests.rs+++ b/harper-literate-haskell/tests/run_tests.rs@@@ -1,0 -1,42 +1,43 @@@+ use harper_core::linting::{LintGroup, LintGroupConfig, Linter};++use harper_core::parsers::MarkdownOptions;+ use harper_core::{Document, FstDictionary};+ use harper_literate_haskell::LiterateHaskellParser;++ /// Creates a unit test checking that the linting of a Markdown document (in+ /// `tests_sources`) produces the expected number of lints.+ macro_rules! create_test {+ ($filename:ident.lhs, $correct_expected:expr) => {+ paste::paste! {+ #[test]+ fn [<lints_ $filename _correctly>](){+ let source = include_str!(+ concat!(+ "./test_sources/",+ concat!(stringify!($filename), ".lhs")+ )+ );++ let dict = FstDictionary::curated();- let document = Document::new_curated(&source, &mut LiterateHaskellParser);++ let document = Document::new_curated(&source, &LiterateHaskellParser::new_markdown(MarkdownOptions::default()));++ let mut linter = LintGroup::new(+ LintGroupConfig::default(),+ dict+ );+ let lints = linter.lint(&document);++ dbg!(&lints);+ assert_eq!(lints.len(), $correct_expected);++ // Make sure that all generated tokens span real characters+ for token in document.tokens(){+ assert!(token.span.try_get_content(document.get_source()).is_some());+ }+ }+ }+ };+ }++ create_test!(bird_format.lhs, 2);+ create_test!(latex_format.lhs, 2);+ create_test!(mixed_format.lhs, 4);diff --cc harper-ls/src/backend.rsindex 6f9156e19,91acf5ae2..2e2037224--- a/harper-ls/src/backend.rs+++ b/harper-ls/src/backend.rs@@@ -180,47 -180,78 +180,83 @@@ impl Backendreturn Ok(());};- let parser: Option<Box<dyn Parser>> = if let Some(ts_parser) =- CommentParser::new_from_language_id(- language_id,- Markdown::new(self.config.read().await.markdown_options),- ) {- let source: Vec<char> = text.chars().collect();- let source = Arc::new(source);-- if let Some(new_dict) = ts_parser.create_ident_dict(source.as_slice()) {- let new_dict = Arc::new(new_dict);-- if doc_state.ident_dict != new_dict {- doc_state.ident_dict = new_dict.clone();- let mut merged = self.generate_file_dictionary(url).await?;- merged.add_dictionary(new_dict);- let merged = Arc::new(merged);-- doc_state.linter = LintGroup::new(config_lock.lint_config, merged.clone());- doc_state.dict = merged.clone();- }- Some(Box::new(CollapseIdentifiers::new(- Box::new(ts_parser),- Box::new(doc_state.dict.clone()),- )))- } else {- Some(Box::new(ts_parser))+ async fn use_ident_dict<'a>(+ backend: &'a Backend,+ new_dict: Arc<FullDictionary>,+ parser: impl Parser + 'static,+ url: &'a Url,+ doc_state: &'a mut DocumentState,+ config_lock: tokio::sync::RwLockReadGuard<'a, Config>,+ ) -> Result<Box<dyn Parser>> {+ if doc_state.ident_dict != new_dict {+ doc_state.ident_dict = new_dict.clone();++ let mut merged = backend.generate_file_dictionary(url).await?;+ merged.add_dictionary(new_dict);+ let merged = Arc::new(merged);++ doc_state.linter = LintGroup::new(config_lock.lint_config, merged.clone());+ doc_state.dict = merged.clone();}- } else if language_id == "markdown" {- Some(Box::new(Markdown::new(- self.config.read().await.markdown_options,++ Ok(Box::new(CollapseIdentifiers::new(+ Box::new(parser),+ Box::new(doc_state.dict.clone()),)))- } else if language_id == "git-commit" {- Some(Box::new(GitCommitParser::new(Markdown::new(- self.config.read().await.markdown_options,- ))))- } else if language_id == "html" {- Some(Box::new(HtmlParser::default()))- } else if language_id == "mail" || language_id == "plaintext" {- Some(Box::new(PlainEnglish))- } else {- None+ }+++ let markdown_options = self.config.read().await.markdown_options;+ let source: Vec<char> = text.chars().collect();- let ts_parser = CommentParser::new_from_language_id(language_id);++ let ts_parser = CommentParser::new_from_language_id(language_id, markdown_options);+ let parser: Option<Box<dyn Parser>> = match language_id.as_str() {+ _ if ts_parser.is_some() => {+ let ts_parser = ts_parser.unwrap();++ if let Some(new_dict) = ts_parser.create_ident_dict(&Arc::new(source)) {+ Some(+ use_ident_dict(+ self,+ Arc::new(new_dict),+ ts_parser,+ url,+ doc_state,+ config_lock,+ )+ .await?,+ )+ } else {+ Some(Box::new(ts_parser))+ }+ }+ "lhaskell" => {- let parser = LiterateHaskellParser;++ let parser = LiterateHaskellParser::new_markdown(markdown_options);+- if let Some(new_dict) = parser.create_ident_dict(&Arc::new(source)) {++ if let Some(new_dict) =++ parser.create_ident_dict(&Arc::new(source), markdown_options)++ {+ Some(+ use_ident_dict(+ self,+ Arc::new(new_dict),+ parser,+ url,+ doc_state,+ config_lock,+ )+ .await?,+ )+ } else {+ Some(Box::new(parser))+ }+ }- "markdown" => Some(Box::new(Markdown)),- "git-commit" | "gitcommit" => Some(Box::new(GitCommitParser)),++ "markdown" => Some(Box::new(Markdown::new(markdown_options))),++ "git-commit" | "gitcommit" => {++ Some(Box::new(GitCommitParser::new_markdown(markdown_options)))++ }+ "html" => Some(Box::new(HtmlParser::default())),+ "mail" | "plaintext" => Some(Box::new(PlainEnglish)),+ "typst" => Some(Box::new(Typst)),+ _ => None,};match parser {diff --cc harper-ls/src/git_commit_parser.rsindex c39ca3325,0b86a8ebe..0c93ea986--- a/harper-ls/src/git_commit_parser.rs+++ b/harper-ls/src/git_commit_parser.rs@@@ -1,16 -1,7 +1,24 @@@--use harper_core::parsers::{Markdown, Parser};++use harper_core::parsers::{Markdown, MarkdownOptions, Parser};++use harper_core::Lrc;/// A Harper parser for Git commit files-pub struct GitCommitParser;+#[derive(Clone)]+pub struct GitCommitParser {- markdown_parser: Markdown,++ inner: Lrc<dyn Parser>,+}++impl GitCommitParser {- pub fn new(markdown_parser: Markdown) -> Self {- Self { markdown_parser }++ #[allow(unused)]++ pub fn new(parser: Lrc<dyn Parser>) -> Self {++ Self { inner: parser }++ }++++ pub fn new_markdown(markdown_options: MarkdownOptions) -> Self {++ Self {++ inner: Lrc::new(Markdown::new(markdown_options)),++ }+ }+}impl Parser for GitCommitParser {/// Admittedly a somewhat naive implementation.@@@ -22,6 -13,6 +30,6 @@@.position(|c| *c == '#').unwrap_or(source.len());- self.markdown_parser.parse(&source[0..end])- Markdown.parse(&source[0..end])++ self.inner.parse(&source[0..end])}}diff --cc harper-wasm/src/lib.rsindex 50cf054da,20a0dbfae..923bab940--- a/harper-wasm/src/lib.rs+++ b/harper-wasm/src/lib.rs@@@ -106,12 -116,8 +116,12 @@@ impl Linterlet source: Vec<_> = text.chars().collect();let source = Lrc::new(source);- let document =- Document::new_from_vec(source.clone(), &Markdown, &FullDictionary::curated());+ // TODO: Have a way to configure the markdown parser+ let document = Document::new_from_vec(+ source.clone(),- &mut Markdown::default(),++ &Markdown::default(),+ &FullDictionary::curated(),+ );let mut lints = self.lint_group.lint(&document);