Skip to content

Offline, privacy-first grammar checker. Fast, open-source, Rust-powered

feat(#331): create literate haskell masker and parser

Grant Lemonsfca1a9aadb1a
diff --git a/Cargo.lock b/Cargo.lock
index 224e6ff69..5e790c5e3 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -560,6 +560,7 @@ dependencies = [
"clap",
"harper-comments",
"harper-core",
+ "harper-literate-haskell",
"serde_json",
]
@@ -626,6 +627,17 @@ dependencies = [
"tree-sitter-html",
]
+[[package]]
+name = "harper-literate-haskell"
+version = "0.13.0"
+dependencies = [
+ "harper-comments",
+ "harper-core",
+ "harper-tree-sitter",
+ "itertools 0.13.0",
+ "paste",
+]
+
[[package]]
name = "harper-ls"
version = "0.13.0"
@@ -637,6 +649,7 @@ dependencies = [
"harper-comments",
"harper-core",
"harper-html",
+ "harper-literate-haskell",
"itertools 0.13.0",
"once_cell",
"open",
diff --git a/Cargo.toml b/Cargo.toml
index 9a2546b5f..a25b424c0 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -1,5 +1,5 @@
[workspace]
-members = [ "harper-cli", "harper-core", "harper-ls", "harper-comments", "harper-wasm", "harper-tree-sitter", "harper-html"]
+members = [ "harper-cli", "harper-core", "harper-ls", "harper-comments", "harper-wasm", "harper-tree-sitter", "harper-html", "harper-literate-haskell"]
resolver = "2"
[profile.release]
diff --git a/harper-cli/Cargo.toml b/harper-cli/Cargo.toml
index 7c69db720..34bfde3dc 100644
--- a/harper-cli/Cargo.toml
+++ b/harper-cli/Cargo.toml
@@ -11,5 +11,6 @@ anyhow = "1.0.95"
ariadne = "0.4.1"
clap = { version = "4.5.23", features = ["derive"] }
harper-core = { path = "../harper-core", version = "0.13.0" }
+harper-literate-haskell = { path = "../harper-literate-haskell", version = "0.13.0" }
harper-comments = { path = "../harper-comments", version = "0.13.0" }
serde_json = "1.0.133"
diff --git a/harper-cli/src/main.rs b/harper-cli/src/main.rs
index ef57427b6..48ba99a8e 100644
--- a/harper-cli/src/main.rs
+++ b/harper-cli/src/main.rs
@@ -9,6 +9,7 @@ use harper_comments::CommentParser;
use harper_core::linting::{LintGroup, LintGroupConfig, Linter};
use harper_core::parsers::Markdown;
use harper_core::{remove_overlaps, Dictionary, Document, FstDictionary};
+use harper_literate_haskell::LiterateHaskellParser;
#[derive(Debug, Parser)]
enum Args {
@@ -143,14 +144,14 @@ fn load_file(file: &Path) -> anyhow::Result<(Document, String)> {
let source = std::fs::read_to_string(file)?;
let mut parser: Box<dyn harper_core::parsers::Parser> =
- if let Some("md") = file.extension().map(|v| v.to_str().unwrap()) {
- Box::new(Markdown)
- } else {
- Box::new(
+ match file.extension().map(|v| v.to_str().unwrap()) {
+ Some("md") => Box::new(Markdown),
+ Some("lhs") => Box::new(LiterateHaskellParser),
+ _ => Box::new(
CommentParser::new_from_filename(file)
.map(Box::new)
.ok_or(format_err!("Could not detect language ID."))?,
- )
+ ),
};
Ok((Document::new_curated(&source, &mut parser), source))
diff --git a/harper-literate-haskell/Cargo.toml b/harper-literate-haskell/Cargo.toml
new file mode 100644
index 000000000..bdee13887
--- /dev/null
+++ b/harper-literate-haskell/Cargo.toml
@@ -0,0 +1,11 @@
+[package]
+name = "harper-literate-haskell"
+version = "0.13.0"
+edition = "2021"
+
+[dependencies]
+harper-core = { path = "../harper-core", version = "0.13.0" }
+harper-tree-sitter = { path = "../harper-tree-sitter", version = "0.13.0" }
+harper-comments = { path = "../harper-comments", version = "0.13.0" }
+itertools = "0.13.0"
+paste = "1.0.14"
diff --git a/harper-literate-haskell/src/lib.rs b/harper-literate-haskell/src/lib.rs
new file mode 100644
index 000000000..4317c97ff
--- /dev/null
+++ b/harper-literate-haskell/src/lib.rs
@@ -0,0 +1,41 @@
+use harper_comments::CommentParser;
+use harper_core::{
+ parsers::{Markdown, Mask, Parser},
+ FullDictionary, Masker, Token,
+};
+
+mod masker;
+use itertools::Itertools;
+use masker::LiterateHaskellMasker;
+
+pub struct LiterateHaskellParser;
+
+impl LiterateHaskellParser {
+ pub fn create_ident_dict(&self, source: &[char]) -> Option<FullDictionary> {
+ let parser = CommentParser::new_from_language_id("haskell").unwrap();
+ let mask = LiterateHaskellMasker {
+ text: false,
+ code: true,
+ }
+ .create_mask(source);
+
+ let code = mask
+ .iter_allowed(source)
+ .flat_map(|(_, src)| src.to_owned())
+ .collect_vec();
+ parser.create_ident_dict(&code)
+ }
+}
+
+impl Parser for LiterateHaskellParser {
+ fn parse(&mut self, source: &[char]) -> Vec<Token> {
+ Mask::new(
+ LiterateHaskellMasker {
+ text: true,
+ code: false,
+ },
+ Markdown,
+ )
+ .parse(source)
+ }
+}
diff --git a/harper-literate-haskell/src/masker.rs b/harper-literate-haskell/src/masker.rs
new file mode 100644
index 000000000..2627576a6
--- /dev/null
+++ b/harper-literate-haskell/src/masker.rs
@@ -0,0 +1,39 @@
+use harper_core::{CharStringExt, Mask, Masker, Span};
+
+pub struct LiterateHaskellMasker {
+ pub text: bool,
+ pub code: bool,
+}
+
+impl Masker for LiterateHaskellMasker {
+ fn create_mask(&mut self, source: &[char]) -> harper_core::Mask {
+ let mut mask = Mask::new_blank();
+
+ let mut location = 0;
+ let mut in_code_env = false;
+ let mut last_line_blank = false;
+
+ for line in source.split(|c| *c == '\n') {
+ let string_form = line.to_string();
+ let trimmed = string_form.trim();
+ let line_is_bird = line.first().map_or(false, |c| *c == '>');
+
+ // Code fencing
+ let code_start = trimmed == r"\begin{code}" || (last_line_blank && line_is_bird);
+ let code_end = trimmed == r"\end{code}" || trimmed.is_empty();
+ if (!in_code_env && code_start) || (in_code_env && code_end) {
+ in_code_env = !in_code_env;
+ }
+
+ let end_loc = location + line.len();
+ if (!in_code_env && self.text) || (in_code_env && self.code) {
+ mask.push_allowed(Span::new(location, end_loc));
+ }
+
+ location = end_loc + 1; // +1 for the newline split on
+ last_line_blank = trimmed.is_empty();
+ }
+
+ mask
+ }
+}
diff --git a/harper-literate-haskell/tests/run_tests.rs b/harper-literate-haskell/tests/run_tests.rs
new file mode 100644
index 000000000..0bbe25c54
--- /dev/null
+++ b/harper-literate-haskell/tests/run_tests.rs
@@ -0,0 +1,42 @@
+use harper_core::linting::{LintGroup, LintGroupConfig, Linter};
+use harper_core::{Document, FstDictionary};
+use harper_literate_haskell::LiterateHaskellParser;
+
+/// Creates a unit test checking that the linting of a Markdown document (in
+/// `tests_sources`) produces the expected number of lints.
+macro_rules! create_test {
+ ($filename:ident.lhs, $correct_expected:expr) => {
+ paste::paste! {
+ #[test]
+ fn [<lints_ $filename _correctly>](){
+ let source = include_str!(
+ concat!(
+ "./test_sources/",
+ concat!(stringify!($filename), ".lhs")
+ )
+ );
+
+ let dict = FstDictionary::curated();
+ let document = Document::new_curated(&source, &mut LiterateHaskellParser);
+
+ let mut linter = LintGroup::new(
+ LintGroupConfig::default(),
+ dict
+ );
+ let lints = linter.lint(&document);
+
+ dbg!(&lints);
+ assert_eq!(lints.len(), $correct_expected);
+
+ // Make sure that all generated tokens span real characters
+ for token in document.tokens(){
+ assert!(token.span.try_get_content(document.get_source()).is_some());
+ }
+ }
+ }
+ };
+}
+
+create_test!(bird_format.lhs, 0);
+create_test!(latex_format.lhs, 0);
+create_test!(mixed_format.lhs, 0);
diff --git a/harper-literate-haskell/tests/test_sources/bird_format.lhs b/harper-literate-haskell/tests/test_sources/bird_format.lhs
new file mode 100644
index 000000000..641535318
--- /dev/null
+++ b/harper-literate-haskell/tests/test_sources/bird_format.lhs
@@ -0,0 +1,9 @@
+Sourced from https://wiki.haskell.org/Literate_programming.
+
+In Bird-style you have to leave a blank before the code.
+
+> fact :: Integer -> Integer
+> fact 0 = 1
+> fact n = n * fact (n-1)
+
+And you have to leave a blank line after the code as well.
diff --git a/harper-literate-haskell/tests/test_sources/latex_format.lhs b/harper-literate-haskell/tests/test_sources/latex_format.lhs
new file mode 100644
index 000000000..44fd441d1
--- /dev/null
+++ b/harper-literate-haskell/tests/test_sources/latex_format.lhs
@@ -0,0 +1,12 @@
+Sourced from https://wiki.haskell.org/Literate_programming.
+
+And the definition of the following function
+would totally screw up my program, so I'm not
+definining it:
+
+\begin{code}
+main :: IO ()
+main = print "just an example"
+\end{code}
+
+See?
diff --git a/harper-literate-haskell/tests/test_sources/mixed_format.lhs b/harper-literate-haskell/tests/test_sources/mixed_format.lhs
new file mode 100644
index 000000000..b89ef1268
--- /dev/null
+++ b/harper-literate-haskell/tests/test_sources/mixed_format.lhs
@@ -0,0 +1,20 @@
+Sourced from https://wiki.haskell.org/Literate_programming.
+
+In Bird-style you have to leave a blank before the code.
+
+> fact :: Integer -> Integer
+> fact 0 = 1
+> fact n = n * fact (n-1)
+
+And you have to leave a blank line after the code as well.
+
+And the definition of the following function
+would totally screw up my program, so I'm not
+definining it:
+
+\begin{code}
+main :: IO ()
+main = print "just an example"
+\end{code}
+
+See?
diff --git a/harper-ls/Cargo.toml b/harper-ls/Cargo.toml
index 998ac77d5..3dc16c65b 100644
--- a/harper-ls/Cargo.toml
+++ b/harper-ls/Cargo.toml
@@ -10,6 +10,7 @@ repository = "https://github.com/automattic/harper"
[dependencies]
harper-core = { path = "../harper-core", version = "0.13.0", features = ["concurrent"] }
harper-comments = { path = "../harper-comments", version = "0.13.0" }
+harper-literate-haskell = { path = "../harper-literate-haskell", version = "0.13.0" }
harper-html = { path = "../harper-html", version = "0.13.0" }
tower-lsp = "0.20.0"
tokio = { version = "1.42.0", features = ["fs", "rt", "rt-multi-thread", "macros", "io-std", "io-util", "net"] }
diff --git a/harper-ls/src/backend.rs b/harper-ls/src/backend.rs
index 73ceedeb1..7c4b5b8e5 100644
--- a/harper-ls/src/backend.rs
+++ b/harper-ls/src/backend.rs
@@ -11,6 +11,7 @@ use harper_core::{
WordMetadata,
};
use harper_html::HtmlParser;
+use harper_literate_haskell::LiterateHaskellParser;
use serde_json::Value;
use tokio::sync::{Mutex, RwLock};
use tower_lsp::jsonrpc::Result;
@@ -204,6 +205,30 @@ impl Backend {
} else {
Some(Box::new(ts_parser))
}
+ } else if language_id == "lhaskell" {
+ let source: Vec<char> = text.chars().collect();
+ let source = Arc::new(source);
+
+ let parser = LiterateHaskellParser;
+ if let Some(new_dict) = parser.create_ident_dict(source.as_slice()) {
+ let new_dict = Arc::new(new_dict);
+
+ if doc_state.ident_dict != new_dict {
+ doc_state.ident_dict = new_dict.clone();
+ let mut merged = self.generate_file_dictionary(url).await?;
+ merged.add_dictionary(new_dict);
+ let merged = Arc::new(merged);
+
+ doc_state.linter = LintGroup::new(config_lock.lint_config, merged.clone());
+ doc_state.dict = merged.clone();
+ }
+ Some(Box::new(CollapseIdentifiers::new(
+ Box::new(parser),
+ Box::new(doc_state.dict.clone()),
+ )))
+ } else {
+ Some(Box::new(parser))
+ }
} else if language_id == "markdown" {
Some(Box::new(Markdown))
} else if language_id == "git-commit" {