Offline, privacy-first grammar checker. Fast, open-source, Rust-powered
feat(#331): create literate haskell masker and parser
Grant Lemonsfca1a9aadb1a
diff --git a/Cargo.lock b/Cargo.lockindex 224e6ff69..5e790c5e3 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -560,6 +560,7 @@ dependencies = ["clap","harper-comments","harper-core",+ "harper-literate-haskell","serde_json",]@@ -626,6 +627,17 @@ dependencies = ["tree-sitter-html",]+[[package]]+name = "harper-literate-haskell"+version = "0.13.0"+dependencies = [+ "harper-comments",+ "harper-core",+ "harper-tree-sitter",+ "itertools 0.13.0",+ "paste",+]+[[package]]name = "harper-ls"version = "0.13.0"@@ -637,6 +649,7 @@ dependencies = ["harper-comments","harper-core","harper-html",+ "harper-literate-haskell","itertools 0.13.0","once_cell","open",diff --git a/Cargo.toml b/Cargo.tomlindex 9a2546b5f..a25b424c0 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -1,5 +1,5 @@[workspace]-members = [ "harper-cli", "harper-core", "harper-ls", "harper-comments", "harper-wasm", "harper-tree-sitter", "harper-html"]+members = [ "harper-cli", "harper-core", "harper-ls", "harper-comments", "harper-wasm", "harper-tree-sitter", "harper-html", "harper-literate-haskell"]resolver = "2"[profile.release]diff --git a/harper-cli/Cargo.toml b/harper-cli/Cargo.tomlindex 7c69db720..34bfde3dc 100644--- a/harper-cli/Cargo.toml+++ b/harper-cli/Cargo.toml@@ -11,5 +11,6 @@ anyhow = "1.0.95"ariadne = "0.4.1"clap = { version = "4.5.23", features = ["derive"] }harper-core = { path = "../harper-core", version = "0.13.0" }+harper-literate-haskell = { path = "../harper-literate-haskell", version = "0.13.0" }harper-comments = { path = "../harper-comments", version = "0.13.0" }serde_json = "1.0.133"diff --git a/harper-cli/src/main.rs b/harper-cli/src/main.rsindex ef57427b6..48ba99a8e 100644--- a/harper-cli/src/main.rs+++ b/harper-cli/src/main.rs@@ -9,6 +9,7 @@ use harper_comments::CommentParser;use harper_core::linting::{LintGroup, LintGroupConfig, Linter};use harper_core::parsers::Markdown;use harper_core::{remove_overlaps, Dictionary, Document, FstDictionary};+use harper_literate_haskell::LiterateHaskellParser;#[derive(Debug, Parser)]enum Args {@@ -143,14 +144,14 @@ fn load_file(file: &Path) -> anyhow::Result<(Document, String)> {let source = std::fs::read_to_string(file)?;let mut parser: Box<dyn harper_core::parsers::Parser> =- if let Some("md") = file.extension().map(|v| v.to_str().unwrap()) {- Box::new(Markdown)- } else {- Box::new(+ match file.extension().map(|v| v.to_str().unwrap()) {+ Some("md") => Box::new(Markdown),+ Some("lhs") => Box::new(LiterateHaskellParser),+ _ => Box::new(CommentParser::new_from_filename(file).map(Box::new).ok_or(format_err!("Could not detect language ID."))?,- )+ ),};Ok((Document::new_curated(&source, &mut parser), source))diff --git a/harper-literate-haskell/Cargo.toml b/harper-literate-haskell/Cargo.tomlnew file mode 100644index 000000000..bdee13887--- /dev/null+++ b/harper-literate-haskell/Cargo.toml@@ -0,0 +1,11 @@+[package]+name = "harper-literate-haskell"+version = "0.13.0"+edition = "2021"++[dependencies]+harper-core = { path = "../harper-core", version = "0.13.0" }+harper-tree-sitter = { path = "../harper-tree-sitter", version = "0.13.0" }+harper-comments = { path = "../harper-comments", version = "0.13.0" }+itertools = "0.13.0"+paste = "1.0.14"diff --git a/harper-literate-haskell/src/lib.rs b/harper-literate-haskell/src/lib.rsnew file mode 100644index 000000000..4317c97ff--- /dev/null+++ b/harper-literate-haskell/src/lib.rs@@ -0,0 +1,41 @@+use harper_comments::CommentParser;+use harper_core::{+ parsers::{Markdown, Mask, Parser},+ FullDictionary, Masker, Token,+};++mod masker;+use itertools::Itertools;+use masker::LiterateHaskellMasker;++pub struct LiterateHaskellParser;++impl LiterateHaskellParser {+ pub fn create_ident_dict(&self, source: &[char]) -> Option<FullDictionary> {+ let parser = CommentParser::new_from_language_id("haskell").unwrap();+ let mask = LiterateHaskellMasker {+ text: false,+ code: true,+ }+ .create_mask(source);++ let code = mask+ .iter_allowed(source)+ .flat_map(|(_, src)| src.to_owned())+ .collect_vec();+ parser.create_ident_dict(&code)+ }+}++impl Parser for LiterateHaskellParser {+ fn parse(&mut self, source: &[char]) -> Vec<Token> {+ Mask::new(+ LiterateHaskellMasker {+ text: true,+ code: false,+ },+ Markdown,+ )+ .parse(source)+ }+}diff --git a/harper-literate-haskell/src/masker.rs b/harper-literate-haskell/src/masker.rsnew file mode 100644index 000000000..2627576a6--- /dev/null+++ b/harper-literate-haskell/src/masker.rs@@ -0,0 +1,39 @@+use harper_core::{CharStringExt, Mask, Masker, Span};++pub struct LiterateHaskellMasker {+ pub text: bool,+ pub code: bool,+}++impl Masker for LiterateHaskellMasker {+ fn create_mask(&mut self, source: &[char]) -> harper_core::Mask {+ let mut mask = Mask::new_blank();++ let mut location = 0;+ let mut in_code_env = false;+ let mut last_line_blank = false;++ for line in source.split(|c| *c == '\n') {+ let string_form = line.to_string();+ let trimmed = string_form.trim();+ let line_is_bird = line.first().map_or(false, |c| *c == '>');++ // Code fencing+ let code_start = trimmed == r"\begin{code}" || (last_line_blank && line_is_bird);+ let code_end = trimmed == r"\end{code}" || trimmed.is_empty();+ if (!in_code_env && code_start) || (in_code_env && code_end) {+ in_code_env = !in_code_env;+ }++ let end_loc = location + line.len();+ if (!in_code_env && self.text) || (in_code_env && self.code) {+ mask.push_allowed(Span::new(location, end_loc));+ }++ location = end_loc + 1; // +1 for the newline split on+ last_line_blank = trimmed.is_empty();+ }++ mask+ }+}diff --git a/harper-literate-haskell/tests/run_tests.rs b/harper-literate-haskell/tests/run_tests.rsnew file mode 100644index 000000000..0bbe25c54--- /dev/null+++ b/harper-literate-haskell/tests/run_tests.rs@@ -0,0 +1,42 @@+use harper_core::linting::{LintGroup, LintGroupConfig, Linter};+use harper_core::{Document, FstDictionary};+use harper_literate_haskell::LiterateHaskellParser;++/// Creates a unit test checking that the linting of a Markdown document (in+/// `tests_sources`) produces the expected number of lints.+macro_rules! create_test {+ ($filename:ident.lhs, $correct_expected:expr) => {+ paste::paste! {+ #[test]+ fn [<lints_ $filename _correctly>](){+ let source = include_str!(+ concat!(+ "./test_sources/",+ concat!(stringify!($filename), ".lhs")+ )+ );++ let dict = FstDictionary::curated();+ let document = Document::new_curated(&source, &mut LiterateHaskellParser);++ let mut linter = LintGroup::new(+ LintGroupConfig::default(),+ dict+ );+ let lints = linter.lint(&document);++ dbg!(&lints);+ assert_eq!(lints.len(), $correct_expected);++ // Make sure that all generated tokens span real characters+ for token in document.tokens(){+ assert!(token.span.try_get_content(document.get_source()).is_some());+ }+ }+ }+ };+}++create_test!(bird_format.lhs, 0);+create_test!(latex_format.lhs, 0);+create_test!(mixed_format.lhs, 0);diff --git a/harper-literate-haskell/tests/test_sources/bird_format.lhs b/harper-literate-haskell/tests/test_sources/bird_format.lhsnew file mode 100644index 000000000..641535318--- /dev/null+++ b/harper-literate-haskell/tests/test_sources/bird_format.lhs@@ -0,0 +1,9 @@+Sourced from https://wiki.haskell.org/Literate_programming.++In Bird-style you have to leave a blank before the code.++> fact :: Integer -> Integer+> fact 0 = 1+> fact n = n * fact (n-1)++And you have to leave a blank line after the code as well.diff --git a/harper-literate-haskell/tests/test_sources/latex_format.lhs b/harper-literate-haskell/tests/test_sources/latex_format.lhsnew file mode 100644index 000000000..44fd441d1--- /dev/null+++ b/harper-literate-haskell/tests/test_sources/latex_format.lhs@@ -0,0 +1,12 @@+Sourced from https://wiki.haskell.org/Literate_programming.++And the definition of the following function+would totally screw up my program, so I'm not+definining it:++\begin{code}+main :: IO ()+main = print "just an example"+\end{code}++See?diff --git a/harper-literate-haskell/tests/test_sources/mixed_format.lhs b/harper-literate-haskell/tests/test_sources/mixed_format.lhsnew file mode 100644index 000000000..b89ef1268--- /dev/null+++ b/harper-literate-haskell/tests/test_sources/mixed_format.lhs@@ -0,0 +1,20 @@+Sourced from https://wiki.haskell.org/Literate_programming.++In Bird-style you have to leave a blank before the code.++> fact :: Integer -> Integer+> fact 0 = 1+> fact n = n * fact (n-1)++And you have to leave a blank line after the code as well.++And the definition of the following function+would totally screw up my program, so I'm not+definining it:++\begin{code}+main :: IO ()+main = print "just an example"+\end{code}++See?diff --git a/harper-ls/Cargo.toml b/harper-ls/Cargo.tomlindex 998ac77d5..3dc16c65b 100644--- a/harper-ls/Cargo.toml+++ b/harper-ls/Cargo.toml@@ -10,6 +10,7 @@ repository = "https://github.com/automattic/harper"[dependencies]harper-core = { path = "../harper-core", version = "0.13.0", features = ["concurrent"] }harper-comments = { path = "../harper-comments", version = "0.13.0" }+harper-literate-haskell = { path = "../harper-literate-haskell", version = "0.13.0" }harper-html = { path = "../harper-html", version = "0.13.0" }tower-lsp = "0.20.0"tokio = { version = "1.42.0", features = ["fs", "rt", "rt-multi-thread", "macros", "io-std", "io-util", "net"] }diff --git a/harper-ls/src/backend.rs b/harper-ls/src/backend.rsindex 73ceedeb1..7c4b5b8e5 100644--- a/harper-ls/src/backend.rs+++ b/harper-ls/src/backend.rs@@ -11,6 +11,7 @@ use harper_core::{WordMetadata,};use harper_html::HtmlParser;+use harper_literate_haskell::LiterateHaskellParser;use serde_json::Value;use tokio::sync::{Mutex, RwLock};use tower_lsp::jsonrpc::Result;@@ -204,6 +205,30 @@ impl Backend {} else {Some(Box::new(ts_parser))}+ } else if language_id == "lhaskell" {+ let source: Vec<char> = text.chars().collect();+ let source = Arc::new(source);++ let parser = LiterateHaskellParser;+ if let Some(new_dict) = parser.create_ident_dict(source.as_slice()) {+ let new_dict = Arc::new(new_dict);++ if doc_state.ident_dict != new_dict {+ doc_state.ident_dict = new_dict.clone();+ let mut merged = self.generate_file_dictionary(url).await?;+ merged.add_dictionary(new_dict);+ let merged = Arc::new(merged);++ doc_state.linter = LintGroup::new(config_lock.lint_config, merged.clone());+ doc_state.dict = merged.clone();+ }+ Some(Box::new(CollapseIdentifiers::new(+ Box::new(parser),+ Box::new(doc_state.dict.clone()),+ )))+ } else {+ Some(Box::new(parser))+ }} else if language_id == "markdown" {Some(Box::new(Markdown))} else if language_id == "git-commit" {