Skip to content

Offline, privacy-first grammar checker. Fast, open-source, Rust-powered

Merge branch 'master' into ignore-link-title-cfg

Grant Lemons809cacdda36b
diff --cc harper-cli/src/main.rs
index 63ee32e12,0044ecde3..b1508df3a
--- a/harper-cli/src/main.rs
+++ b/harper-cli/src/main.rs
@@@ -7,8 -7,9 +7,9 @@@ use ariadne::{Color, Label, Report, Rep
use clap::Parser;
use harper_comments::CommentParser;
use harper_core::linting::{LintGroup, LintGroupConfig, Linter};
-use harper_core::parsers::Markdown;
+use harper_core::parsers::{Markdown, MarkdownOptions};
- use harper_core::{remove_overlaps, Dictionary, Document, FstDictionary};
+ use harper_core::{remove_overlaps, Dictionary, Document, FstDictionary, TokenKind};
+ use harper_literate_haskell::LiterateHaskellParser;
#[derive(Debug, Parser)]
enum Args {
@@@ -91,8 -95,11 +97,11 @@@ fn main() -> anyhow::Result<()>
Ok(())
}
- Args::Spans { file } => {
+ Args::Spans {
+ file,
+ include_newlines,
+ } => {
- let (doc, source) = load_file(&file)?;
+ let (doc, source) = load_file(&file, markdown_options)?;
let primary_color = Color::Blue;
let secondary_color = Color::Magenta;
@@@ -141,19 -166,20 +168,22 @@@
}
}
-fn load_file(file: &Path) -> anyhow::Result<(Document, String)> {
+fn load_file(file: &Path, markdown_options: MarkdownOptions) -> anyhow::Result<(Document, String)> {
let source = std::fs::read_to_string(file)?;
- let mut parser: Box<dyn harper_core::parsers::Parser> =
- if let Some("md") = file.extension().map(|v| v.to_str().unwrap()) {
- Box::new(Markdown::new(markdown_options))
- } else {
- Box::new(
- CommentParser::new_from_filename(file, Markdown::new(markdown_options))
+ let parser: Box<dyn harper_core::parsers::Parser> =
+ match file.extension().map(|v| v.to_str().unwrap()) {
- Some("md") => Box::new(Markdown),
- Some("lhs") => Box::new(LiterateHaskellParser),
++ Some("md") => Box::new(Markdown::default()),
++ Some("lhs") => Box::new(LiterateHaskellParser::new_markdown(
++ MarkdownOptions::default(),
++ )),
+ Some("typ") => Box::new(harper_typst::Typst),
+ _ => Box::new(
- CommentParser::new_from_filename(file)
++ CommentParser::new_from_filename(file, markdown_options)
.map(Box::new)
.ok_or(format_err!("Could not detect language ID."))?,
- )
+ ),
};
- Ok((Document::new_curated(&source, &mut parser), source))
+ Ok((Document::new_curated(&source, &parser), source))
}
diff --cc harper-comments/src/comment_parser.rs
index 5f93706ba,7c15942c1..bf631c7b1
--- a/harper-comments/src/comment_parser.rs
+++ b/harper-comments/src/comment_parser.rs
@@@ -1,7 -1,7 +1,7 @@@
use std::path::Path;
use comment_parsers::{Go, JavaDoc, JsDoc, Unit};
- use harper_core::parsers::{self, Markdown, Parser};
-use harper_core::parsers::{self, Parser};
++use harper_core::parsers::{self, MarkdownOptions, Parser};
use harper_core::{FullDictionary, Token};
use harper_tree_sitter::TreeSitterMasker;
use tree_sitter::Node;
@@@ -17,7 -17,7 +17,10 @@@ impl CommentParser
self.inner.masker.create_ident_dict(source)
}
- pub fn new_from_language_id(language_id: &str, markdown_parser: Markdown) -> Option<Self> {
- pub fn new_from_language_id(language_id: &str) -> Option<Self> {
++ pub fn new_from_language_id(
++ language_id: &str,
++ markdown_options: MarkdownOptions,
++ ) -> Option<Self> {
let language = match language_id {
"rust" => tree_sitter_rust::language(),
"typescriptreact" => tree_sitter_typescript::language_tsx(),
@@@ -40,12 -42,10 +45,12 @@@
};
let comment_parser: Box<dyn Parser> = match language_id {
- "javascriptreact" | "typescript" | "typescriptreact" | "javascript" => Box::new(JsDoc),
+ "javascriptreact" | "typescript" | "typescriptreact" | "javascript" => {
- Box::new(JsDoc::new(markdown_parser))
++ Box::new(JsDoc::new_markdown(markdown_options))
+ }
"java" => Box::new(JavaDoc::default()),
- "go" => Box::new(Go::new(markdown_parser)),
- _ => Box::new(Unit::new(markdown_parser)),
- "go" => Box::new(Go),
- _ => Box::new(Unit),
++ "go" => Box::new(Go::new_markdown(markdown_options)),
++ _ => Box::new(Unit::new_markdown(markdown_options)),
};
Some(Self {
@@@ -57,8 -57,8 +62,8 @@@
}
/// Infer the programming language from a provided filename.
- pub fn new_from_filename(filename: &Path, markdown_parser: Markdown) -> Option<Self> {
- Self::new_from_language_id(Self::filename_to_filetype(filename)?, markdown_parser)
- pub fn new_from_filename(filename: &Path) -> Option<Self> {
- Self::new_from_language_id(Self::filename_to_filetype(filename)?)
++ pub fn new_from_filename(filename: &Path, markdown_options: MarkdownOptions) -> Option<Self> {
++ Self::new_from_language_id(Self::filename_to_filetype(filename)?, markdown_options)
}
/// Convert a provided path to a corresponding Language Server Protocol file
diff --cc harper-comments/src/comment_parsers/go.rs
index 020df3d4c,f7f824dbd..71d499e9d
--- a/harper-comments/src/comment_parsers/go.rs
+++ b/harper-comments/src/comment_parsers/go.rs
@@@ -1,21 -1,13 +1,29 @@@
--use harper_core::parsers::{Markdown, Parser};
++use harper_core::parsers::{Markdown, MarkdownOptions, Parser};
++use harper_core::Lrc;
use harper_core::Token;
use super::without_initiators;
-#[derive(Debug, Clone, Copy)]
-pub struct Go;
+#[derive(Clone)]
+pub struct Go {
- markdown_parser: Markdown,
++ inner: Lrc<dyn Parser>,
+}
+
+impl Go {
- pub fn new(markdown_parser: Markdown) -> Self {
- Self { markdown_parser }
++ #[allow(unused)]
++ pub fn new(parser: Lrc<dyn Parser>) -> Self {
++ Self { inner: parser }
++ }
++
++ pub fn new_markdown(markdown_options: MarkdownOptions) -> Self {
++ Self {
++ inner: Lrc::new(Markdown::new(markdown_options)),
++ }
+ }
+}
impl Parser for Go {
- fn parse(&mut self, source: &[char]) -> Vec<Token> {
+ fn parse(&self, source: &[char]) -> Vec<Token> {
let mut actual = without_initiators(source);
let mut actual_source = actual.get_content(source);
@@@ -33,7 -25,7 +41,7 @@@
actual_source = new_source
}
- let mut new_tokens = self.markdown_parser.parse(actual_source);
- let mut new_tokens = Markdown.parse(actual_source);
++ let mut new_tokens = self.inner.parse(actual_source);
new_tokens
.iter_mut()
diff --cc harper-comments/src/comment_parsers/jsdoc.rs
index 197661145,dcc56a0a8..e8a27820b
--- a/harper-comments/src/comment_parsers/jsdoc.rs
+++ b/harper-comments/src/comment_parsers/jsdoc.rs
@@@ -1,28 -1,19 +1,36 @@@
--use harper_core::parsers::{Markdown, Parser};
++use harper_core::parsers::{Markdown, MarkdownOptions, Parser};
++use harper_core::Lrc;
use harper_core::{Punctuation, Span, Token, TokenKind};
use itertools::Itertools;
use super::without_initiators;
-pub struct JsDoc;
+#[derive(Clone)]
+pub struct JsDoc {
- markdown_parser: Markdown,
++ inner: Lrc<dyn Parser>,
+}
+
+impl JsDoc {
- pub fn new(markdown_parser: Markdown) -> Self {
- Self { markdown_parser }
++ #[allow(unused)]
++ pub fn new(parser: Lrc<dyn Parser>) -> Self {
++ Self { inner: parser }
++ }
++
++ pub fn new_markdown(markdown_options: MarkdownOptions) -> Self {
++ Self {
++ inner: Lrc::new(Markdown::new(markdown_options)),
++ }
+ }
+}
impl Parser for JsDoc {
- fn parse(&mut self, source: &[char]) -> Vec<Token> {
+ fn parse(&self, source: &[char]) -> Vec<Token> {
let mut tokens = Vec::new();
let mut chars_traversed = 0;
for line in source.split(|c| *c == '\n') {
- let mut new_tokens = parse_line(line, &mut self.markdown_parser);
- let mut new_tokens = parse_line(line);
++ let mut new_tokens = parse_line(line, self.inner.clone());
if chars_traversed + line.len() < source.len() {
new_tokens.push(Token::new(
@@@ -43,7 -34,7 +51,7 @@@
}
}
- fn parse_line(source: &[char], markdown_parser: &mut Markdown) -> Vec<Token> {
-fn parse_line(source: &[char]) -> Vec<Token> {
++fn parse_line(source: &[char], parser: Lrc<dyn Parser>) -> Vec<Token> {
let actual_line = without_initiators(source);
if actual_line.is_empty() {
@@@ -52,7 -43,7 +60,7 @@@
let source_line = actual_line.get_content(source);
- let mut new_tokens = markdown_parser.parse(source_line);
- let mut new_tokens = Markdown.parse(source_line);
++ let mut new_tokens = parser.parse(source_line);
// Handle inline tags
mark_inline_tags(&mut new_tokens);
@@@ -157,23 -148,21 +165,23 @@@ fn parse_inline_tag(tokens: &[Token]) -
#[cfg(test)]
mod tests {
- use harper_core::{parsers::Markdown, Document, Punctuation, TokenKind};
- use harper_core::{Document, Punctuation, TokenKind};
++ use harper_core::{parsers::MarkdownOptions, Document, Punctuation, TokenKind};
use crate::CommentParser;
#[test]
fn escapes_loop() {
let source = "/** This should _not_cause an infinite loop: {@ */";
- let mut parser = CommentParser::new_from_language_id("javascript").unwrap();
+ let mut parser =
- CommentParser::new_from_language_id("javascript", Markdown::default()).unwrap();
++ CommentParser::new_from_language_id("javascript", MarkdownOptions::default()).unwrap();
Document::new_curated(source, &mut parser);
}
#[test]
fn handles_inline_link() {
let source = "/** See {@link MyClass} and [MyClass's foo property]{@link MyClass#foo}. */";
- let mut parser = CommentParser::new_from_language_id("javascript").unwrap();
+ let mut parser =
- CommentParser::new_from_language_id("javascript", Markdown::default()).unwrap();
++ CommentParser::new_from_language_id("javascript", MarkdownOptions::default()).unwrap();
let document = Document::new_curated(source, &mut parser);
assert!(matches!(
@@@ -217,8 -206,7 +225,8 @@@
#[test]
fn handles_class() {
let source = "/** @class Circle representing a circle. */";
- let mut parser = CommentParser::new_from_language_id("javascript").unwrap();
+ let mut parser =
- CommentParser::new_from_language_id("javascript", Markdown::default()).unwrap();
++ CommentParser::new_from_language_id("javascript", MarkdownOptions::default()).unwrap();
let document = Document::new_curated(source, &mut parser);
assert!(document
diff --cc harper-comments/src/comment_parsers/unit.rs
index 97af2869a,3185a1725..905a2a4b1
--- a/harper-comments/src/comment_parsers/unit.rs
+++ b/harper-comments/src/comment_parsers/unit.rs
@@@ -1,4 -1,4 +1,5 @@@
--use harper_core::parsers::{Markdown, Parser};
++use harper_core::parsers::{Markdown, MarkdownOptions, Parser};
++use harper_core::Lrc;
use harper_core::{Span, Token};
use super::without_initiators;
@@@ -9,19 -9,10 +10,26 @@@
///
/// It assumes it is being provided a single line of comment at a time,
/// including the comment initiation characters.
-pub struct Unit;
+#[derive(Clone)]
+pub struct Unit {
- markdown_parser: Markdown,
++ inner: Lrc<dyn Parser>,
+}
+
+impl Unit {
- pub fn new(markdown_parser: Markdown) -> Self {
- Self { markdown_parser }
++ #[allow(unused)]
++ pub fn new(parser: Lrc<dyn Parser>) -> Self {
++ Self { inner: parser }
++ }
++
++ pub fn new_markdown(markdown_options: MarkdownOptions) -> Self {
++ Self {
++ inner: Lrc::new(Markdown::new(markdown_options)),
++ }
+ }
+}
impl Parser for Unit {
- fn parse(&mut self, source: &[char]) -> Vec<Token> {
+ fn parse(&self, source: &[char]) -> Vec<Token> {
let mut tokens = Vec::new();
let mut chars_traversed = 0;
@@@ -37,7 -28,7 +45,7 @@@
continue;
}
- let mut new_tokens = parse_line(line, &mut self.markdown_parser);
- let mut new_tokens = parse_line(line);
++ let mut new_tokens = parse_line(line, self.inner.clone());
if chars_traversed + line.len() < source.len() {
new_tokens.push(Token::new(
@@@ -58,7 -49,7 +66,7 @@@
}
}
- fn parse_line(source: &[char], markdown_parser: &mut Markdown) -> Vec<Token> {
-fn parse_line(source: &[char]) -> Vec<Token> {
++fn parse_line(source: &[char], parser: Lrc<dyn Parser>) -> Vec<Token> {
let actual = without_initiators(source);
if actual.is_empty() {
@@@ -66,8 -57,7 +74,8 @@@
}
let source = actual.get_content(source);
- let mut new_tokens = Markdown.parse(source);
+
- let mut new_tokens = markdown_parser.parse(source);
++ let mut new_tokens = parser.parse(source);
new_tokens
.iter_mut()
diff --cc harper-comments/tests/language_support.rs
index df04a5ea4,ca9df0a3e..49463be7a
--- a/harper-comments/tests/language_support.rs
+++ b/harper-comments/tests/language_support.rs
@@@ -2,7 -2,6 +2,7 @@@ use std::path::Path
use harper_comments::CommentParser;
use harper_core::linting::{LintGroup, LintGroupConfig, Linter};
- use harper_core::parsers::Markdown;
++use harper_core::parsers::MarkdownOptions;
use harper_core::{Document, FstDictionary};
/// Creates a unit test checking that the linting of a source file in
@@@ -21,7 -20,7 +21,7 @@@ macro_rules! create_test
)
);
- let mut parser = CommentParser::new_from_filename(Path::new(filename), Markdown::default()).unwrap();
- let mut parser = CommentParser::new_from_filename(Path::new(filename)).unwrap();
++ let mut parser = CommentParser::new_from_filename(Path::new(filename), MarkdownOptions::default()).unwrap();
let dict = FstDictionary::curated();
let document = Document::new(&source, &mut parser, &dict);
diff --cc harper-core/benches/parse_demo.rs
index 092215867,d76265321..cbd0a8dbf
--- a/harper-core/benches/parse_demo.rs
+++ b/harper-core/benches/parse_demo.rs
@@@ -7,14 -6,14 +6,14 @@@ static DEMO: &str = include_str!("../..
fn parse_demo(c: &mut Criterion) {
c.bench_function("parse_demo", |b| {
- b.iter(|| Document::new_markdown_curated(black_box(DEMO), MarkdownOptions::default()))
- b.iter(|| Document::new_markdown_curated(black_box(DEMO)))
++ b.iter(|| Document::new_markdown_default_curated(black_box(DEMO)));
});
}
fn lint_demo(c: &mut Criterion) {
let dictionary = FstDictionary::curated();
let mut lint_set = LintGroup::new(Default::default(), dictionary);
- let document = Document::new_markdown_curated(black_box(DEMO), MarkdownOptions::default());
- let document = Document::new_markdown_curated(black_box(DEMO));
++ let document = Document::new_markdown_default_curated(black_box(DEMO));
c.bench_function("lint_demo", |b| {
b.iter(|| lint_set.lint(&document));
@@@ -26,8 -25,7 +25,7 @@@ fn lint_demo_uncached(c: &mut Criterion
b.iter(|| {
let dictionary = FstDictionary::curated();
let mut lint_set = LintGroup::new(LintGroupConfig::default(), dictionary.clone());
- let document =
- Document::new_markdown(black_box(DEMO), MarkdownOptions::default(), &dictionary);
- let document = Document::new_markdown(black_box(DEMO), &dictionary);
++ let document = Document::new_markdown_default(black_box(DEMO), &dictionary);
lint_set.lint(&document)
})
});
diff --cc harper-core/src/document.rs
index 34dab4d77,1741ee63c..7145d0a4b
--- a/harper-core/src/document.rs
+++ b/harper-core/src/document.rs
@@@ -4,13 -4,12 +4,12 @@@ use std::fmt::Display
use paste::paste;
-use crate::parsers::{Markdown, Parser, PlainEnglish};
+use crate::parsers::{Markdown, MarkdownOptions, Parser, PlainEnglish};
use crate::patterns::{PatternExt, RepeatingPattern, SequencePattern};
use crate::punctuation::Punctuation;
- use crate::token::NumberSuffix;
use crate::vec_ext::VecExt;
- use crate::Span;
use crate::{Dictionary, FatToken, FstDictionary, Lrc, Token, TokenKind, TokenStringExt};
+ use crate::{NumberSuffix, Span};
/// A document containing some amount of lexed and parsed English text.
#[derive(Debug, Clone)]
@@@ -71,22 -70,14 +70,34 @@@ impl Document
/// Parse text to produce a document using the built-in [`Markdown`] parser
/// and curated dictionary.
- pub fn new_markdown_curated(text: &str) -> Self {
- Self::new(text, &Markdown, &FstDictionary::curated())
+ pub fn new_markdown_curated(text: &str, markdown_options: MarkdownOptions) -> Self {
+ Self::new(
+ text,
- &mut Markdown::new(markdown_options),
++ &Markdown::new(markdown_options),
+ &FstDictionary::curated(),
+ )
+ }
+
++ /// Parse text to produce a document using the built-in [`Markdown`] parser
++ /// and curated dictionary with the default markdown configuration.
++ pub fn new_markdown_default_curated(text: &str) -> Self {
++ Self::new_markdown_curated(text, MarkdownOptions::default())
+ }
+
/// Parse text to produce a document using the built-in [`PlainEnglish`]
/// parser and the curated dictionary.
- pub fn new_markdown(text: &str, dictionary: &impl Dictionary) -> Self {
- Self::new(text, &Markdown, dictionary)
+ pub fn new_markdown(
+ text: &str,
+ markdown_options: MarkdownOptions,
+ dictionary: &impl Dictionary,
+ ) -> Self {
- Self::new(text, &mut Markdown::new(markdown_options), dictionary)
++ Self::new(text, &Markdown::new(markdown_options), dictionary)
++ }
++
++ /// Parse text to produce a document using the built-in [`PlainEnglish`]
++ /// parser and the curated dictionary with the default markdown configuration.
++ pub fn new_markdown_default(text: &str, dictionary: &impl Dictionary) -> Self {
++ Self::new_markdown(text, MarkdownOptions::default(), dictionary)
}
/// Re-parse important language constructs.
diff --cc harper-core/src/linting/lint_group.rs
index 728679e9b,b39ff81b8..f9ef86a0b
--- a/harper-core/src/linting/lint_group.rs
+++ b/harper-core/src/linting/lint_group.rs
@@@ -141,3 -192,33 +192,33 @@@ impl<T: Dictionary + Default> Default f
Self::new(LintGroupConfig::default(), T::default())
}
}
+
+ #[cfg(test)]
+ mod tests {
+ use crate::{linting::Linter, Document, FstDictionary, FullDictionary};
+
+ use super::{LintGroup, LintGroupConfig};
+
+ #[test]
+ fn can_get_all_descriptions() {
+ let group = LintGroup::<FullDictionary>::default();
+ group.all_descriptions();
+ }
+
+ #[test]
+ fn lint_descriptions_are_clean() {
+ let mut group = LintGroup::new(LintGroupConfig::default(), FstDictionary::curated());
+ let pairs: Vec<_> = group
+ .all_descriptions()
+ .to_vec_pairs()
+ .into_iter()
+ .map(|(a, b)| (a.to_string(), b.to_string()))
+ .collect();
+
+ for (key, value) in pairs {
- let doc = Document::new_markdown_curated(&value);
++ let doc = Document::new_markdown_default_curated(&value);
+ eprintln!("{key}: {value}");
+ assert!(group.lint(&doc).is_empty())
+ }
+ }
+ }
diff --cc harper-core/src/linting/mod.rs
index 6fa03a07d,8f64ff574..9b8cc5825
--- a/harper-core/src/linting/mod.rs
+++ b/harper-core/src/linting/mod.rs
@@@ -64,19 -78,30 +78,30 @@@ pub trait Linter: Send + Sync
#[cfg(test)]
mod tests {
use super::Linter;
- use crate::{parsers::MarkdownOptions, Document};
+ use crate::Document;
pub fn assert_lint_count(text: &str, mut linter: impl Linter, count: usize) {
- let test = Document::new_markdown_curated(text, MarkdownOptions::default());
- let test = Document::new_markdown_curated(text);
++ let test = Document::new_markdown_default_curated(text);
let lints = linter.lint(&test);
dbg!(&lints);
assert_eq!(lints.len(), count);
}
+ /// Assert the total number of suggestions produced by a [`Linter`], spread across all produced
+ /// [`Lint`]s.
+ pub fn assert_suggestion_count(text: &str, mut linter: impl Linter, count: usize) {
- let test = Document::new_markdown_curated(text);
++ let test = Document::new_markdown_default_curated(text);
+ let lints = linter.lint(&test);
+ assert_eq!(
+ lints.iter().map(|l| l.suggestions.len()).sum::<usize>(),
+ count
+ );
+ }
+
/// Runs a provided linter on text, applies the first suggestion from each
- /// lint and asserts that the result is equal to a given value.
+ /// lint and asserts whether the result is equal to a given value.
pub fn assert_suggestion_result(text: &str, mut linter: impl Linter, expected_result: &str) {
- let test = Document::new_markdown_curated(text, MarkdownOptions::default());
- let test = Document::new_markdown_curated(text);
++ let test = Document::new_markdown_default_curated(text);
let lints = linter.lint(&test);
let mut text: Vec<char> = text.chars().collect();
diff --cc harper-core/src/parsers/markdown.rs
index 6c51033e5,ed5f965c6..771e5f41f
--- a/harper-core/src/parsers/markdown.rs
+++ b/harper-core/src/parsers/markdown.rs
@@@ -9,29 -7,9 +9,31 @@@ use crate::{Span, Token, TokenKind, Tok
/// CommonMark files.
///
/// Will ignore code blocks and tables.
- #[derive(Default, Clone, Debug)]
- pub struct Markdown(MarkdownOptions);
-pub struct Markdown;
++#[derive(Default, Clone, Debug, Copy)]
++pub struct Markdown {
++ options: MarkdownOptions,
++}
+
+#[derive(Copy, Clone, Debug, Serialize, Deserialize)]
+pub struct MarkdownOptions {
+ pub ignore_link_title: bool,
+}
+
+// Clippy rule excepted because this can easily be expanded later
+#[allow(clippy::derivable_impls)]
+impl Default for MarkdownOptions {
+ fn default() -> Self {
+ Self {
+ ignore_link_title: false,
+ }
+ }
+}
impl Markdown {
+ pub fn new(options: MarkdownOptions) -> Self {
- Self(options)
++ Self { options }
+ }
+
/// Remove hidden Wikilink target text.
///
/// As in, the stuff to the left of the pipe operator:
@@@ -224,15 -202,9 +226,15 @@@ impl Parser for Markdown
});
continue;
}
- if matches!(tag, Tag::Link { .. }) && self.0.ignore_link_title {
-
++ if matches!(tag, Tag::Link { .. }) && self.options.ignore_link_title {
+ tokens.push(Token {
+ span: Span::new_with_len(traversed_chars, text.chars().count()),
+ kind: TokenKind::Unlintable,
+ });
+ continue;
+ }
if !(matches!(tag, Tag::Paragraph)
- || matches!(tag, Tag::Link { .. }) && !self.0.ignore_link_title
- || matches!(tag, Tag::Link { .. })
++ || matches!(tag, Tag::Link { .. }) && !self.options.ignore_link_title
|| matches!(tag, Tag::Heading { .. })
|| matches!(tag, Tag::Item)
|| matches!(tag, Tag::TableCell)
@@@ -292,9 -264,9 +294,9 @@@ mod tests
#[test]
fn survives_emojis() {
- let source = r#"🤷."#;
+ let source = r"🤷.";
- Markdown.parse_str(source);
+ Markdown::default().parse_str(source);
}
/// Check whether the Markdown parser will emit a breaking newline
@@@ -312,9 -284,9 +314,9 @@@
#[test]
fn math_becomes_unlintable() {
- let source = r#"$\Katex$ $\text{is}$ $\text{great}$."#;
+ let source = r"$\Katex$ $\text{is}$ $\text{great}$.";
- let tokens = Markdown.parse_str(source);
+ let tokens = Markdown::default().parse_str(source);
assert_eq!(
tokens.iter().map(|t| t.kind).collect::<Vec<_>>(),
vec![
@@@ -330,9 -302,9 +332,9 @@@
#[test]
fn hidden_wikilink_text() {
- let source = r#"[[this is hidden|this is not]]"#;
+ let source = r"[[this is hidden|this is not]]";
- let tokens = Markdown.parse_str(source);
+ let tokens = Markdown::default().parse_str(source);
let token_kinds = tokens.iter().map(|t| t.kind).collect::<Vec<_>>();
@@@ -350,9 -322,9 +352,9 @@@
#[test]
fn improper_wikilink_text() {
- let source = r#"this is shown|this is also shown]]"#;
+ let source = r"this is shown|this is also shown]]";
- let tokens = Markdown.parse_str(source);
+ let tokens = Markdown::default().parse_str(source);
let token_kinds = tokens.iter().map(|t| t.kind).collect::<Vec<_>>();
@@@ -382,8 -354,8 +384,8 @@@
#[test]
fn normal_wikilink() {
- let source = r#"[[Wikilink]]"#;
+ let source = r"[[Wikilink]]";
- let tokens = Markdown.parse_str(source);
+ let tokens = Markdown::default().parse_str(source);
let token_kinds = tokens.iter().map(|t| t.kind).collect::<Vec<_>>();
dbg!(&token_kinds);
@@@ -393,62 -365,8 +395,62 @@@
#[test]
fn html_is_unlintable() {
- let source = r#"The range of inputs from <ctrl-g> to ctrl-z"#;
+ let source = r"The range of inputs from <ctrl-g> to ctrl-z";
- let tokens = Markdown.parse_str(source);
+ let tokens = Markdown::default().parse_str(source);
assert_eq!(tokens.iter_unlintables().count(), 1);
}
+
+ #[test]
+ fn link_title_unlintable() {
- let mut parser = Markdown(MarkdownOptions {
++ let parser = Markdown::new(MarkdownOptions {
+ ignore_link_title: true,
+ ..MarkdownOptions::default()
+ });
+ let source = r#"[elijah-potter/harper](https://github.com/elijah-potter/harper)"#;
+ let tokens = parser.parse_str(source);
+ let token_kinds = tokens.iter().map(|t| t.kind).collect::<Vec<_>>();
+
+ dbg!(&token_kinds);
+
+ assert!(matches!(token_kinds.as_slice(), &[TokenKind::Unlintable]))
+ }
+
+ #[test]
+ fn respects_config() {
+ let source = r#"[elijah-potter/harper](https://github.com/elijah-potter/harper)"#;
- let mut parser = Markdown(MarkdownOptions {
++ let parser = Markdown::new(MarkdownOptions {
+ ignore_link_title: true,
+ ..MarkdownOptions::default()
+ });
+ let token_kinds = parser
+ .parse_str(source)
+ .iter()
+ .map(|t| t.kind)
+ .collect::<Vec<_>>();
+
+ assert!(matches!(token_kinds.as_slice(), &[TokenKind::Unlintable]));
+
- let mut parser = Markdown(MarkdownOptions {
++ let parser = Markdown::new(MarkdownOptions {
+ ignore_link_title: false,
+ ..MarkdownOptions::default()
+ });
+ let token_kinds = parser
+ .parse_str(source)
+ .iter()
+ .map(|t| t.kind)
+ .collect::<Vec<_>>();
+
+ dbg!(&token_kinds);
+
+ assert!(matches!(
+ token_kinds.as_slice(),
+ &[
+ TokenKind::Word(_),
+ TokenKind::Punctuation(Punctuation::Hyphen),
+ TokenKind::Word(_),
+ TokenKind::Punctuation(Punctuation::ForwardSlash),
+ TokenKind::Word(_)
+ ]
+ ));
+ }
}
diff --cc harper-core/src/parsers/mod.rs
index 4429e054f,91d81980f..5069e3309
--- a/harper-core/src/parsers/mod.rs
+++ b/harper-core/src/parsers/mod.rs
@@@ -11,18 -11,18 +11,18 @@@ pub use markdown::{Markdown, MarkdownOp
pub use mask::Mask;
pub use plain_english::PlainEnglish;
- pub use crate::token::{Token, TokenKind, TokenStringExt};
+ use crate::{Token, TokenStringExt};
#[cfg(not(feature = "concurrent"))]
--#[blanket(derive(Box))]
++#[blanket(derive(Box, Rc))]
pub trait Parser {
- fn parse(&mut self, source: &[char]) -> Vec<Token>;
+ fn parse(&self, source: &[char]) -> Vec<Token>;
}
#[cfg(feature = "concurrent")]
--#[blanket(derive(Box))]
++#[blanket(derive(Box, Arc))]
pub trait Parser: Send + Sync {
- fn parse(&mut self, source: &[char]) -> Vec<Token>;
+ fn parse(&self, source: &[char]) -> Vec<Token>;
}
pub trait StrParser {
diff --cc harper-core/src/patterns/noun_phrase.rs
index 000000000,66d25e4f2..dedf54303
mode 000000,100644..100644
--- a/harper-core/src/patterns/noun_phrase.rs
+++ b/harper-core/src/patterns/noun_phrase.rs
@@@ -1,0 -1,99 +1,99 @@@
+ use crate::Token;
+
+ use super::Pattern;
+
+ /// A pattern that returns the value of the first non-zero match in a list.
+ #[derive(Default)]
+ pub struct NounPhrase;
+
+ impl Pattern for NounPhrase {
+ fn matches(&self, tokens: &[Token], _source: &[char]) -> usize {
+ let mut cursor = 0;
+
+ loop {
+ let Some(tok) = tokens.get(cursor) else {
+ return 0;
+ };
+
+ if tok.kind.is_adjective() || tok.kind.is_article() {
+ let Some(next) = tokens.get(cursor + 1) else {
+ return 0;
+ };
+
+ if !next.kind.is_whitespace() {
+ return 0;
+ }
+
+ cursor += 2;
+ continue;
+ }
+
+ if tok.kind.is_noun() {
+ return cursor + 1;
+ }
+
+ return 0;
+ }
+ }
+ }
+
+ #[cfg(test)]
+ mod tests {
+ use super::super::DocPattern;
+ use super::NounPhrase;
+ use crate::{patterns::Pattern, Document, Span};
+
+ #[test]
+ fn simple_apple() {
- let doc = Document::new_markdown_curated("A red apple");
++ let doc = Document::new_markdown_default_curated("A red apple");
+ let matches = NounPhrase.find_all_matches_in_doc(&doc);
+
+ assert_eq!(matches, vec![Span::new(0, 5)])
+ }
+
+ #[test]
+ fn complex_apple() {
- let doc = Document::new_markdown_curated("A red apple with a long stem");
++ let doc = Document::new_markdown_default_curated("A red apple with a long stem");
+ let matches = NounPhrase.find_all_matches_in_doc(&doc);
+
+ assert_eq!(matches, vec![Span::new(0, 5), Span::new(8, 13)])
+ }
+
+ #[test]
+ fn list_fruit() {
- let doc = Document::new_markdown_curated("An apple, a banana and a pear");
++ let doc = Document::new_markdown_default_curated("An apple, a banana and a pear");
+ let matches = NounPhrase.find_all_matches_in_doc(&doc);
+
+ assert_eq!(
+ matches,
+ vec![Span::new(0, 3), Span::new(5, 8), Span::new(11, 14)]
+ )
+ }
+
+ #[test]
+ fn simplest_banana() {
- let doc = Document::new_markdown_curated("a banana");
++ let doc = Document::new_markdown_default_curated("a banana");
+ assert!(NounPhrase.matches(doc.get_tokens(), doc.get_source()) != 0);
+ }
+
+ #[test]
+ fn food() {
- let doc = Document::new_markdown_curated(
++ let doc = Document::new_markdown_default_curated(
+ "My favorite foods are pizza, sushi, tacos and burgers.",
+ );
+ let matches = NounPhrase.find_all_matches_in_doc(&doc);
+
+ dbg!(&matches);
+
+ assert_eq!(
+ matches,
+ vec![
+ Span::new(2, 5),
+ Span::new(8, 9),
+ Span::new(11, 12),
+ Span::new(14, 15),
+ Span::new(18, 19)
+ ]
+ )
+ }
+ }
diff --cc harper-core/src/patterns/word_set.rs
index 000000000,e8a6eab9c..4372f8f7d
mode 000000,100644..100644
--- a/harper-core/src/patterns/word_set.rs
+++ b/harper-core/src/patterns/word_set.rs
@@@ -1,0 -1,93 +1,93 @@@
+ use super::Pattern;
+ use smallvec::SmallVec;
+
+ use crate::{CharString, Token};
+
+ // A [`Pattern`] that matches against any of a set of provided words.
+ // For small sets of short words, it doesn't allocate.
+ //
+ // Note that any capitalization of the contained words will result in a match.
+ #[derive(Debug, Default, Clone)]
+ pub struct WordSet {
+ words: SmallVec<[CharString; 4]>,
+ }
+
+ impl WordSet {
+ pub fn add(&mut self, word: &str) {
+ let chars = word.chars().collect();
+
+ if !self.words.contains(&chars) {
+ self.words.push(chars);
+ }
+ }
+
+ pub fn all(words: &[&'static str]) -> Self {
+ let mut set = Self::default();
+
+ for str in words {
+ set.add(str);
+ }
+
+ set
+ }
+ }
+
+ impl Pattern for WordSet {
+ fn matches(&self, tokens: &[Token], source: &[char]) -> usize {
+ let Some(tok) = tokens.first() else {
+ return 0;
+ };
+
+ if !tok.kind.is_word() {
+ return 0;
+ }
+
+ let tok_chars = tok.span.get_content(source);
+
+ for word in &self.words {
+ if tok_chars.len() != word.len() {
+ continue;
+ }
+
+ let partial_match = tok_chars
+ .iter()
+ .zip(word)
+ .all(|(a, b)| a.eq_ignore_ascii_case(b));
+
+ if partial_match {
+ return 1;
+ }
+ }
+
+ 0
+ }
+ }
+
+ #[cfg(test)]
+ mod tests {
+ use crate::{patterns::DocPattern, Document, Span};
+
+ use super::WordSet;
+
+ #[test]
+ fn fruit() {
+ let set = WordSet::all(&["banana", "apple", "orange"]);
+
- let doc = Document::new_markdown_curated("I ate a banana and an apple today.");
++ let doc = Document::new_markdown_default_curated("I ate a banana and an apple today.");
+
+ let matches = set.find_all_matches_in_doc(&doc);
+
+ assert_eq!(matches, vec![Span::new(6, 7), Span::new(12, 13)]);
+ }
+
+ #[test]
+ fn fruit_whack_capitalization() {
+ let set = WordSet::all(&["banana", "apple", "orange"]);
+
- let doc = Document::new_markdown_curated("I Ate A bAnaNa And aN apPlE today.");
++ let doc = Document::new_markdown_default_curated("I Ate A bAnaNa And aN apPlE today.");
+
+ let matches = set.find_all_matches_in_doc(&doc);
+
+ assert_eq!(matches, vec![Span::new(6, 7), Span::new(12, 13)]);
+ }
+ }
diff --cc harper-core/tests/run_tests.rs
index 499cf7149,50f5f3d6e..b6ba18028
--- a/harper-core/tests/run_tests.rs
+++ b/harper-core/tests/run_tests.rs
@@@ -9,29 -8,29 +8,29 @@@ macro_rules! create_test
paste::paste! {
#[test]
fn [<lints_ $filename _correctly>](){
-- let source = include_str!(
++ let source = include_str!(
concat!(
"./test_sources/",
concat!(stringify!($filename), ".md")
)
-- );
++ );
-- let dict = FstDictionary::curated();
- let document = Document::new_markdown(&source, MarkdownOptions::default(), &dict);
- let document = Document::new_markdown(&source, &dict);
++ let dict = FstDictionary::curated();
++ let document = Document::new_markdown_default(&source, &dict);
-- let mut linter = LintGroup::new(
++ let mut linter = LintGroup::new(
LintGroupConfig::default(),
dict
-- );
-- let lints = linter.lint(&document);
++ );
++ let lints = linter.lint(&document);
-- dbg!(&lints);
-- assert_eq!(lints.len(), $correct_expected);
++ dbg!(&lints);
++ assert_eq!(lints.len(), $correct_expected);
-- // Make sure that all generated tokens span real characters
-- for token in document.tokens(){
++ // Make sure that all generated tokens span real characters
++ for token in document.tokens(){
assert!(token.span.try_get_content(document.get_source()).is_some());
-- }
++ }
}
}
};
diff --cc harper-html/tests/run_tests.rs
index 7f39d644f,320ac7c89..dcea70978
--- a/harper-html/tests/run_tests.rs
+++ b/harper-html/tests/run_tests.rs
@@@ -17,7 -16,7 +16,7 @@@ macro_rules! create_test
);
let dict = FstDictionary::curated();
- let document = Document::new_markdown(&source, MarkdownOptions::default(), &dict);
- let document = Document::new_markdown(&source, &dict);
++ let document = Document::new_markdown_default(&source, &dict);
let mut linter = LintGroup::new(
LintGroupConfig::default(),
diff --cc harper-literate-haskell/src/lib.rs
index 000000000,903f9c5b3..20dd89148
mode 000000,100644..100644
--- a/harper-literate-haskell/src/lib.rs
+++ b/harper-literate-haskell/src/lib.rs
@@@ -1,0 -1,31 +1,47 @@@
+ use harper_comments::CommentParser;
+ use harper_core::{
- parsers::{Markdown, Mask, Parser},
- FullDictionary, Masker, Token,
++ parsers::{Markdown, MarkdownOptions, Mask, Parser},
++ FullDictionary, Lrc, Masker, Token,
+ };
+
+ mod masker;
+ use itertools::Itertools;
+ use masker::LiterateHaskellMasker;
+
+ /// Parses a Literate Haskell document by masking out the code and considering text as Markdown.
-pub struct LiterateHaskellParser;
++pub struct LiterateHaskellParser {
++ inner: Lrc<dyn Parser>,
++}
+
+ impl LiterateHaskellParser {
- pub fn create_ident_dict(&self, source: &[char]) -> Option<FullDictionary> {
- let parser = CommentParser::new_from_language_id("haskell").unwrap();
++ pub fn new(inner: Lrc<dyn Parser>) -> Self {
++ Self { inner }
++ }
++
++ pub fn new_markdown(markdown_options: MarkdownOptions) -> Self {
++ Self {
++ inner: Lrc::new(Markdown::new(markdown_options)),
++ }
++ }
++
++ pub fn create_ident_dict(
++ &self,
++ source: &[char],
++ markdown_options: MarkdownOptions,
++ ) -> Option<FullDictionary> {
++ let parser = CommentParser::new_from_language_id("haskell", markdown_options).unwrap();
+ let mask = LiterateHaskellMasker::code_only().create_mask(source);
+
+ let code = mask
+ .iter_allowed(source)
+ .flat_map(|(_, src)| src.to_owned())
+ .collect_vec();
+ parser.create_ident_dict(&code)
+ }
+ }
+
+ impl Parser for LiterateHaskellParser {
+ fn parse(&self, source: &[char]) -> Vec<Token> {
- Mask::new(LiterateHaskellMasker::text_only(), Markdown).parse(source)
++ Mask::new(LiterateHaskellMasker::text_only(), self.inner.clone()).parse(source)
+ }
+ }
diff --cc harper-literate-haskell/tests/run_tests.rs
index 000000000,f367f66b3..7c8e1791a
mode 000000,100644..100644
--- a/harper-literate-haskell/tests/run_tests.rs
+++ b/harper-literate-haskell/tests/run_tests.rs
@@@ -1,0 -1,42 +1,43 @@@
+ use harper_core::linting::{LintGroup, LintGroupConfig, Linter};
++use harper_core::parsers::MarkdownOptions;
+ use harper_core::{Document, FstDictionary};
+ use harper_literate_haskell::LiterateHaskellParser;
+
+ /// Creates a unit test checking that the linting of a Markdown document (in
+ /// `tests_sources`) produces the expected number of lints.
+ macro_rules! create_test {
+ ($filename:ident.lhs, $correct_expected:expr) => {
+ paste::paste! {
+ #[test]
+ fn [<lints_ $filename _correctly>](){
+ let source = include_str!(
+ concat!(
+ "./test_sources/",
+ concat!(stringify!($filename), ".lhs")
+ )
+ );
+
+ let dict = FstDictionary::curated();
- let document = Document::new_curated(&source, &mut LiterateHaskellParser);
++ let document = Document::new_curated(&source, &LiterateHaskellParser::new_markdown(MarkdownOptions::default()));
+
+ let mut linter = LintGroup::new(
+ LintGroupConfig::default(),
+ dict
+ );
+ let lints = linter.lint(&document);
+
+ dbg!(&lints);
+ assert_eq!(lints.len(), $correct_expected);
+
+ // Make sure that all generated tokens span real characters
+ for token in document.tokens(){
+ assert!(token.span.try_get_content(document.get_source()).is_some());
+ }
+ }
+ }
+ };
+ }
+
+ create_test!(bird_format.lhs, 2);
+ create_test!(latex_format.lhs, 2);
+ create_test!(mixed_format.lhs, 4);
diff --cc harper-ls/src/backend.rs
index 6f9156e19,91acf5ae2..2e2037224
--- a/harper-ls/src/backend.rs
+++ b/harper-ls/src/backend.rs
@@@ -180,47 -180,78 +180,83 @@@ impl Backend
return Ok(());
};
- let parser: Option<Box<dyn Parser>> = if let Some(ts_parser) =
- CommentParser::new_from_language_id(
- language_id,
- Markdown::new(self.config.read().await.markdown_options),
- ) {
- let source: Vec<char> = text.chars().collect();
- let source = Arc::new(source);
-
- if let Some(new_dict) = ts_parser.create_ident_dict(source.as_slice()) {
- let new_dict = Arc::new(new_dict);
-
- if doc_state.ident_dict != new_dict {
- doc_state.ident_dict = new_dict.clone();
- let mut merged = self.generate_file_dictionary(url).await?;
- merged.add_dictionary(new_dict);
- let merged = Arc::new(merged);
-
- doc_state.linter = LintGroup::new(config_lock.lint_config, merged.clone());
- doc_state.dict = merged.clone();
- }
- Some(Box::new(CollapseIdentifiers::new(
- Box::new(ts_parser),
- Box::new(doc_state.dict.clone()),
- )))
- } else {
- Some(Box::new(ts_parser))
+ async fn use_ident_dict<'a>(
+ backend: &'a Backend,
+ new_dict: Arc<FullDictionary>,
+ parser: impl Parser + 'static,
+ url: &'a Url,
+ doc_state: &'a mut DocumentState,
+ config_lock: tokio::sync::RwLockReadGuard<'a, Config>,
+ ) -> Result<Box<dyn Parser>> {
+ if doc_state.ident_dict != new_dict {
+ doc_state.ident_dict = new_dict.clone();
+
+ let mut merged = backend.generate_file_dictionary(url).await?;
+ merged.add_dictionary(new_dict);
+ let merged = Arc::new(merged);
+
+ doc_state.linter = LintGroup::new(config_lock.lint_config, merged.clone());
+ doc_state.dict = merged.clone();
}
- } else if language_id == "markdown" {
- Some(Box::new(Markdown::new(
- self.config.read().await.markdown_options,
+
+ Ok(Box::new(CollapseIdentifiers::new(
+ Box::new(parser),
+ Box::new(doc_state.dict.clone()),
)))
- } else if language_id == "git-commit" {
- Some(Box::new(GitCommitParser::new(Markdown::new(
- self.config.read().await.markdown_options,
- ))))
- } else if language_id == "html" {
- Some(Box::new(HtmlParser::default()))
- } else if language_id == "mail" || language_id == "plaintext" {
- Some(Box::new(PlainEnglish))
- } else {
- None
+ }
+
++ let markdown_options = self.config.read().await.markdown_options;
+ let source: Vec<char> = text.chars().collect();
- let ts_parser = CommentParser::new_from_language_id(language_id);
++ let ts_parser = CommentParser::new_from_language_id(language_id, markdown_options);
+ let parser: Option<Box<dyn Parser>> = match language_id.as_str() {
+ _ if ts_parser.is_some() => {
+ let ts_parser = ts_parser.unwrap();
+
+ if let Some(new_dict) = ts_parser.create_ident_dict(&Arc::new(source)) {
+ Some(
+ use_ident_dict(
+ self,
+ Arc::new(new_dict),
+ ts_parser,
+ url,
+ doc_state,
+ config_lock,
+ )
+ .await?,
+ )
+ } else {
+ Some(Box::new(ts_parser))
+ }
+ }
+ "lhaskell" => {
- let parser = LiterateHaskellParser;
++ let parser = LiterateHaskellParser::new_markdown(markdown_options);
+
- if let Some(new_dict) = parser.create_ident_dict(&Arc::new(source)) {
++ if let Some(new_dict) =
++ parser.create_ident_dict(&Arc::new(source), markdown_options)
++ {
+ Some(
+ use_ident_dict(
+ self,
+ Arc::new(new_dict),
+ parser,
+ url,
+ doc_state,
+ config_lock,
+ )
+ .await?,
+ )
+ } else {
+ Some(Box::new(parser))
+ }
+ }
- "markdown" => Some(Box::new(Markdown)),
- "git-commit" | "gitcommit" => Some(Box::new(GitCommitParser)),
++ "markdown" => Some(Box::new(Markdown::new(markdown_options))),
++ "git-commit" | "gitcommit" => {
++ Some(Box::new(GitCommitParser::new_markdown(markdown_options)))
++ }
+ "html" => Some(Box::new(HtmlParser::default())),
+ "mail" | "plaintext" => Some(Box::new(PlainEnglish)),
+ "typst" => Some(Box::new(Typst)),
+ _ => None,
};
match parser {
diff --cc harper-ls/src/git_commit_parser.rs
index c39ca3325,0b86a8ebe..0c93ea986
--- a/harper-ls/src/git_commit_parser.rs
+++ b/harper-ls/src/git_commit_parser.rs
@@@ -1,16 -1,7 +1,24 @@@
--use harper_core::parsers::{Markdown, Parser};
++use harper_core::parsers::{Markdown, MarkdownOptions, Parser};
++use harper_core::Lrc;
/// A Harper parser for Git commit files
-pub struct GitCommitParser;
+#[derive(Clone)]
+pub struct GitCommitParser {
- markdown_parser: Markdown,
++ inner: Lrc<dyn Parser>,
+}
+
+impl GitCommitParser {
- pub fn new(markdown_parser: Markdown) -> Self {
- Self { markdown_parser }
++ #[allow(unused)]
++ pub fn new(parser: Lrc<dyn Parser>) -> Self {
++ Self { inner: parser }
++ }
++
++ pub fn new_markdown(markdown_options: MarkdownOptions) -> Self {
++ Self {
++ inner: Lrc::new(Markdown::new(markdown_options)),
++ }
+ }
+}
impl Parser for GitCommitParser {
/// Admittedly a somewhat naive implementation.
@@@ -22,6 -13,6 +30,6 @@@
.position(|c| *c == '#')
.unwrap_or(source.len());
- self.markdown_parser.parse(&source[0..end])
- Markdown.parse(&source[0..end])
++ self.inner.parse(&source[0..end])
}
}
diff --cc harper-wasm/src/lib.rs
index 50cf054da,20a0dbfae..923bab940
--- a/harper-wasm/src/lib.rs
+++ b/harper-wasm/src/lib.rs
@@@ -106,12 -116,8 +116,12 @@@ impl Linter
let source: Vec<_> = text.chars().collect();
let source = Lrc::new(source);
- let document =
- Document::new_from_vec(source.clone(), &Markdown, &FullDictionary::curated());
+ // TODO: Have a way to configure the markdown parser
+ let document = Document::new_from_vec(
+ source.clone(),
- &mut Markdown::default(),
++ &Markdown::default(),
+ &FullDictionary::curated(),
+ );
let mut lints = self.lint_group.lint(&document);