Skip to content

Offline, privacy-first grammar checker. Fast, open-source, Rust-powered

refactor: move typst parsing to a new crate

Grant Lemons1672de3f592a
diff --git a/Cargo.lock b/Cargo.lock
index 7168ca09d..8d823e6e9 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -594,6 +594,7 @@ dependencies = [
"clap",
"harper-comments",
"harper-core",
+ "harper-typst",
"serde_json",
]
@@ -645,7 +646,6 @@ dependencies = [
"serde_json",
"smallvec",
"thiserror 2.0.9",
- "typst-syntax",
"unicode-blocks",
"unicode-width 0.2.0",
]
@@ -672,6 +672,7 @@ dependencies = [
"harper-comments",
"harper-core",
"harper-html",
+ "harper-typst",
"itertools 0.13.0",
"once_cell",
"open",
@@ -692,6 +693,17 @@ dependencies = [
"tree-sitter",
]
+[[package]]
+name = "harper-typst"
+version = "0.14.0"
+dependencies = [
+ "harper-core",
+ "itertools 0.13.0",
+ "ordered-float",
+ "paste",
+ "typst-syntax",
+]
+
[[package]]
name = "harper-wasm"
version = "0.1.0"
diff --git a/Cargo.toml b/Cargo.toml
index 9a2546b5f..746de49f4 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -1,5 +1,5 @@
[workspace]
-members = [ "harper-cli", "harper-core", "harper-ls", "harper-comments", "harper-wasm", "harper-tree-sitter", "harper-html"]
+members = [ "harper-cli", "harper-core", "harper-ls", "harper-comments", "harper-wasm", "harper-tree-sitter", "harper-html", "harper-typst"]
resolver = "2"
[profile.release]
diff --git a/harper-cli/Cargo.toml b/harper-cli/Cargo.toml
index b23feef22..6efcdb7d4 100644
--- a/harper-cli/Cargo.toml
+++ b/harper-cli/Cargo.toml
@@ -12,8 +12,9 @@ ariadne = "0.4.1"
clap = { version = "4.5.23", features = ["derive"] }
harper-core = { path = "../harper-core", version = "0.14.0" }
harper-comments = { path = "../harper-comments", version = "0.14.0" }
+harper-typst = { path = "../harper-typst", version = "0.14.0", optional = true }
serde_json = "1.0.133"
[features]
default = ["typst"]
-typst = ["harper-core/typst"]
+typst = ["dep:harper-typst"]
diff --git a/harper-cli/src/main.rs b/harper-cli/src/main.rs
index b9ebc22fa..91517ecc9 100644
--- a/harper-cli/src/main.rs
+++ b/harper-cli/src/main.rs
@@ -172,7 +172,7 @@ fn load_file(file: &Path) -> anyhow::Result<(Document, String)> {
match file.extension().map(|v| v.to_str().unwrap()) {
Some("md") => Box::new(Markdown),
#[cfg(feature = "typst")]
- Some("typ") => Box::new(harper_core::parsers::Typst),
+ Some("typ") => Box::new(harper_typst::Typst),
_ => Box::new(
CommentParser::new_from_filename(file)
.map(Box::new)
diff --git a/harper-comments/Cargo.toml b/harper-comments/Cargo.toml
index 6b393c1d5..15539bfa1 100644
--- a/harper-comments/Cargo.toml
+++ b/harper-comments/Cargo.toml
@@ -32,6 +32,3 @@ tree-sitter-haskell = "0.15.0"
[dev-dependencies]
paste = "1.0.15"
-
-[features]
-typst = ["harper-core/typst"]
diff --git a/harper-core/Cargo.toml b/harper-core/Cargo.toml
index 08d722340..456b9a651 100644
--- a/harper-core/Cargo.toml
+++ b/harper-core/Cargo.toml
@@ -24,7 +24,6 @@ thiserror = "2.0.9"
unicode-blocks = "0.1.9"
unicode-width = "0.2.0"
levenshtein_automata = { version = "0.2.1", features = ["fst_automaton"] }
-typst-syntax = { version = "0.12.0", optional = true }
[dev-dependencies]
criterion = { version = "0.5.1", default-features = false }
@@ -35,5 +34,5 @@ name = "parse_demo"
harness = false
[features]
-typst = ["dep:typst-syntax"]
+default = []
concurrent = []
diff --git a/harper-core/src/parsers/mod.rs b/harper-core/src/parsers/mod.rs
index 66038286f..91d81980f 100644
--- a/harper-core/src/parsers/mod.rs
+++ b/harper-core/src/parsers/mod.rs
@@ -4,11 +4,6 @@ mod markdown;
mod mask;
mod plain_english;
-#[cfg(feature = "typst")]
-mod typst;
-#[cfg(feature = "typst")]
-pub use typst::Typst;
-
use blanket::blanket;
pub use collapse_identifiers::CollapseIdentifiers;
pub use isolate_english::IsolateEnglish;
diff --git a/harper-core/src/parsers/typst.rs b/harper-core/src/parsers/typst.rs
deleted file mode 100644
index 4c09cc837..000000000
--- a/harper-core/src/parsers/typst.rs
+++ /dev/null
@@ -1,721 +0,0 @@
-use itertools::Itertools;
-use typst_syntax::{
- ast::{
- Arg, ArrayItem, AstNode, DestructuringItem, DictItem, Expr, Ident, LetBindingKind, Markup,
- Param, Pattern, Spread,
- },
- Source,
-};
-
-use super::{Parser, PlainEnglish};
-use crate::{
- parsers::StrParser,
- patterns::{PatternExt, SequencePattern},
- ConjunctionData, Lrc, NounData, Punctuation, Token, TokenKind, VecExt, WordMetadata,
-};
-
-/// A parser that wraps the [`PlainEnglish`] parser allowing one to parse Typst files.
-pub struct Typst;
-
-/// Encapsulation of the translation between byte-based spans and char-based spans
-#[derive(Debug, Clone, Copy)]
-struct OffsetCursor<'a> {
- doc: &'a Source,
- pub char: usize,
- pub byte: usize,
-}
-
-impl<'a> OffsetCursor<'a> {
- pub fn new(doc: &'a Source) -> Self {
- Self {
- doc,
- char: 0,
- byte: 0,
- }
- }
-
- /// Returns a new [`OffsetCursor`] at the given byte based on the current cursor.
- pub fn push_to(self, new_byte: usize) -> Self {
- assert!(new_byte >= self.byte);
-
- if new_byte == self.byte {
- return self;
- }
-
- Self {
- char: self.char + self.doc.get(self.byte..new_byte).unwrap().chars().count(),
- byte: new_byte,
- ..self
- }
- }
-
- /// Returns a new [`OffsetCursor`] at the beginning of the given [`typst_syntax::Span`] based
- /// on the current cursor.
- pub fn push_to_span(self, span: typst_syntax::Span) -> Self {
- let new_byte = self.doc.range(span).unwrap().start;
-
- self.push_to(new_byte)
- }
-}
-
-macro_rules! def_token {
- ($doc:expr, $a:expr, $kind:expr, $offset:ident) => {{
- let range = $doc.range($a.span()).unwrap();
- let start = $offset.push_to(range.start);
- let end_char_loc = start.push_to(range.end).char;
-
- Some(vec![Token {
- span: crate::Span {
- start: start.char,
- end: end_char_loc,
- },
- kind: $kind,
- }])
- }};
-}
-
-macro_rules! merge {
- ($($inner:expr),*) => {
- Some(
- [$($inner),*]
- .into_iter()
- .flatten()
- .flatten()
- .collect_vec(),
- )
- };
-}
-
-/// Contains values used in parsing so they don't have to be passed around so much
-#[derive(Clone, Copy)]
-struct ParseHelper<'a> {
- parser: PlainEnglish,
- doc: &'a Source,
-}
-
-impl<'a> ParseHelper<'a> {
- pub fn new(doc: &'a Source) -> Self {
- Self {
- parser: PlainEnglish,
- doc,
- }
- }
-
- fn parse_english(self, str: impl Into<String>, offset: OffsetCursor) -> Option<Vec<Token>> {
- Some(
- self.parser
- .parse_str(str.into())
- .into_iter()
- .map(|mut t| {
- t.span.push_by(offset.char);
- t
- })
- .collect_vec(),
- )
- }
-
- fn parse_pattern(self, pat: Pattern, offset: OffsetCursor) -> Option<Vec<Token>> {
- macro_rules! token {
- ($a:expr, $kind:expr) => {
- def_token!(self.doc, $a, $kind, offset)
- };
- }
-
- match pat {
- Pattern::Normal(expr) => self.parse_expr(expr, offset),
- Pattern::Placeholder(underscore) => token!(underscore, TokenKind::Unlintable),
- Pattern::Parenthesized(parenthesized) => merge!(
- self.parse_expr(parenthesized.expr(), offset),
- self.parse_pattern(parenthesized.pattern(), offset)
- ),
- Pattern::Destructuring(destructuring) => Some(
- destructuring
- .items()
- .filter_map(|item| match item {
- DestructuringItem::Pattern(pattern) => self.parse_pattern(pattern, offset),
- DestructuringItem::Named(named) => merge!(
- token!(named.name(), TokenKind::Word(WordMetadata::default())),
- self.parse_pattern(named.pattern(), offset)
- ),
- DestructuringItem::Spread(spread) => merge!(
- spread
- .sink_ident()
- .and_then(|ident| self.parse_ident(ident, offset)),
- spread
- .sink_expr()
- .and_then(|expr| self.parse_expr(expr, offset))
- ),
- })
- .flatten()
- .collect(),
- ),
- }
- }
-
- fn parse_ident(self, ident: Ident, offset: OffsetCursor) -> Option<Vec<Token>> {
- self.parse_expr(Expr::Ident(ident), offset)
- }
-
- /// Do not use for spreads contained in DestructuringItem
- fn parse_spread(self, spread: Spread, offset: OffsetCursor) -> Option<Vec<Token>> {
- merge!(
- self.parse_expr(spread.expr(), offset),
- spread
- .sink_ident()
- .and_then(|ident| self.parse_ident(ident, offset))
- )
- }
-
- pub fn parse_expr(self, ex: Expr, offset: OffsetCursor) -> Option<Vec<Token>> {
- let offset = offset.push_to_span(ex.span());
-
- macro_rules! token {
- ($a:expr, $kind:expr) => {
- def_token!(self.doc, $a, $kind, offset)
- };
- }
- macro_rules! recurse {
- ($inner:expr) => {
- self.parse_expr($inner, offset)
- };
- ($($inner:expr),*) => {
- merge!(
- $(recurse!($inner)),*
- )
- };
- }
-
- let iter_recurse = |exprs: &mut dyn Iterator<Item = Expr>| {
- Some(exprs.filter_map(|e| recurse!(e)).flatten().collect_vec())
- };
- let parse_dict = |dict: &mut dyn Iterator<Item = DictItem>| {
- Some(
- dict.filter_map(|di| match di {
- DictItem::Named(named) => {
- merge!(
- self.parse_ident(named.name(), offset),
- recurse!(named.expr())
- )
- }
- DictItem::Keyed(keyed) => recurse!(keyed.key(), keyed.expr()),
- DictItem::Spread(spread) => self.parse_spread(spread, offset),
- })
- .flatten()
- .collect_vec(),
- )
- };
- let parse_params = |params: &mut dyn Iterator<Item = Param>| {
- Some(
- params
- .filter_map(|p| match p {
- Param::Pos(pattern) => self.parse_pattern(pattern, offset),
- Param::Named(named) => merge!(
- self.parse_ident(named.name(), offset),
- recurse!(named.expr())
- ),
- Param::Spread(spread) => self.parse_spread(spread, offset),
- })
- .flatten()
- .collect_vec(),
- )
- };
- let parse_args = |params: &mut dyn Iterator<Item = Arg>| {
- Some(
- params
- .filter_map(|a| match a {
- Arg::Pos(expr) => recurse!(expr),
- Arg::Named(named) => merge!(
- self.parse_ident(named.name(), offset),
- recurse!(named.expr())
- ),
- Arg::Spread(spread) => self.parse_spread(spread, offset),
- })
- .flatten()
- .collect_vec(),
- )
- };
-
- match ex {
- Expr::Text(text) => self.parse_english(text.get(), offset.push_to_span(text.span())),
- Expr::Space(a) => {
- let mut chars = self
- .doc
- .get(self.doc.range(a.span()).unwrap())
- .unwrap()
- .chars();
- let first_char = chars.next().unwrap();
- let length = chars.count() + 1;
-
- if first_char == '\n' {
- token!(a, TokenKind::Newline(1))
- } else {
- token!(a, TokenKind::Space(length))
- }
- }
- Expr::Linebreak(a) => token!(a, TokenKind::Newline(1)),
- Expr::Parbreak(a) => token!(a, TokenKind::ParagraphBreak),
- Expr::SmartQuote(quote) => {
- if quote.double() {
- token!(
- quote,
- TokenKind::Punctuation(Punctuation::Quote(crate::Quote { twin_loc: None }))
- )
- } else {
- token!(quote, TokenKind::Punctuation(Punctuation::Apostrophe))
- }
- }
- Expr::Strong(strong) => iter_recurse(&mut strong.body().exprs()),
- Expr::Emph(emph) => iter_recurse(&mut emph.body().exprs()),
- Expr::Link(a) => token!(a, TokenKind::Url),
- Expr::Ref(a) => {
- token!(a, TokenKind::Word(WordMetadata::default()))
- }
- Expr::Heading(heading) => iter_recurse(&mut heading.body().exprs()),
- Expr::List(list_item) => iter_recurse(&mut list_item.body().exprs()),
- Expr::Enum(enum_item) => iter_recurse(&mut enum_item.body().exprs()),
- Expr::Term(term_item) => iter_recurse(
- &mut term_item
- .term()
- .exprs()
- .chain(term_item.description().exprs()),
- ),
- Expr::Str(text) => {
- let offset = offset.push_to_span(text.span()).char + 1;
- let string = text.to_untyped().text();
-
- Some(
- self.parser
- .parse_str(&string[1..string.len() - 1])
- .into_iter()
- .map(|mut t| {
- t.span.push_by(offset);
- t
- })
- .collect_vec(),
- )
- }
- Expr::Content(content_block) => iter_recurse(&mut content_block.body().exprs()),
- Expr::Parenthesized(parenthesized) => recurse!(parenthesized.expr()),
- Expr::Array(array) => Some(
- array
- .items()
- .filter_map(|i| {
- if let ArrayItem::Pos(e) = i {
- recurse!(e)
- } else {
- None
- }
- })
- .flatten()
- .collect_vec(),
- ),
- Expr::Dict(a) => parse_dict(&mut a.items()),
- Expr::FieldAccess(field_access) => merge!(
- recurse!(field_access.target()),
- token!(
- field_access.field(),
- TokenKind::Word(WordMetadata::default())
- )
- ),
- Expr::Let(let_binding) => merge!(
- match let_binding.kind() {
- LetBindingKind::Normal(pattern) => self.parse_pattern(pattern, offset),
- LetBindingKind::Closure(ident) => self.parse_ident(ident, offset),
- },
- let_binding.init().and_then(|e| recurse!(e))
- ),
- Expr::DestructAssign(destruct_assignment) => {
- recurse!(destruct_assignment.value())
- }
- Expr::Set(set_rule) => merge!(
- recurse!(set_rule.target()),
- set_rule.condition().and_then(|expr| recurse!(expr)),
- parse_args(&mut set_rule.args().items())
- ),
- Expr::Show(show_rule) => merge!(
- recurse!(show_rule.transform()),
- show_rule.selector().and_then(|expr| recurse!(expr))
- ),
- Expr::Contextual(contextual) => recurse!(contextual.body()),
- Expr::Conditional(conditional) => merge!(
- recurse!(conditional.condition(), conditional.if_body()),
- conditional.else_body().and_then(|expr| recurse!(expr))
- ),
- Expr::While(while_loop) => recurse!(while_loop.condition(), while_loop.body()),
- Expr::For(for_loop) => recurse!(for_loop.iterable(), for_loop.body()),
- Expr::Code(code) => iter_recurse(&mut code.body().exprs()),
- Expr::Closure(closure) => merge!(
- closure
- .name()
- .and_then(|ident| self.parse_ident(ident, offset)),
- parse_params(&mut closure.params().children()),
- recurse!(closure.body())
- ),
- Expr::FuncCall(func) => merge!(
- token!(func.callee(), TokenKind::Unlintable),
- parse_args(&mut func.args().items())
- ),
- a => token!(a, TokenKind::Unlintable),
- }
- }
-}
-
-thread_local! {
- static WORD_APOSTROPHE_WORD: Lrc<SequencePattern> = Lrc::new(SequencePattern::default()
- .then_any_word()
- .then_apostrophe()
- .then_any_word());
-}
-
-impl Parser for Typst {
- fn parse(&self, source: &[char]) -> Vec<Token> {
- let source_str: String = source.iter().collect();
- let typst_document = Source::detached(source_str);
- let typst_tree = Markup::from_untyped(typst_document.root())
- .expect("Unable to create typst document from parsed tree!");
- let parse_helper = ParseHelper::new(&typst_document);
-
- let mut tokens = typst_tree
- .exprs()
- .filter_map(|ex| parse_helper.parse_expr(ex, OffsetCursor::new(&typst_document)))
- .flatten()
- .collect_vec();
-
- // Consolidate conjunctions
- let mut to_remove = std::collections::VecDeque::default();
- for tok_span in WORD_APOSTROPHE_WORD
- .with(|v| v.clone())
- .find_all_matches(&tokens, source)
- {
- let start_tok = &tokens[tok_span.start];
- let end_tok = &tokens[tok_span.end - 1];
- let char_span = crate::Span::new(start_tok.span.start, end_tok.span.end);
-
- if let TokenKind::Word(metadata) = start_tok.kind {
- tokens[tok_span.start].kind =
- TokenKind::Word(if end_tok.span.get_content(source) == ['s'] {
- WordMetadata {
- noun: Some(NounData {
- is_possessive: Some(true),
- ..metadata.noun.unwrap_or_default()
- }),
- conjunction: None,
- ..metadata
- }
- } else {
- WordMetadata {
- noun: metadata.noun.map(|noun| NounData {
- is_possessive: Some(false),
- ..noun
- }),
- conjunction: Some(ConjunctionData {}),
- ..metadata
- }
- });
-
- tokens[tok_span.start].span = char_span;
- to_remove.extend(tok_span.start + 1..tok_span.end);
- } else {
- panic!("Apostrophe consolidation does not start with Word Token!")
- }
- }
- tokens.remove_indices(to_remove.into_iter().sorted().unique().collect());
-
- tokens
- }
-}
-
-#[cfg(test)]
-mod tests {
- use itertools::Itertools;
- use ordered_float::OrderedFloat;
-
- use super::Typst;
- use crate::{parsers::StrParser, NounData, Punctuation, TokenKind, WordMetadata};
-
- #[test]
- fn conjunction() {
- let source = "doesn't";
-
- let tokens = Typst.parse_str(source);
- let token_kinds = tokens.iter().map(|t| t.kind).collect_vec();
- dbg!(&token_kinds);
-
- assert_eq!(token_kinds.len(), 1);
- assert!(token_kinds.into_iter().all(|t| t.is_conjunction()))
- }
-
- #[test]
- fn possessive() {
- let source = "person's";
-
- let tokens = Typst.parse_str(source);
- let token_kinds = tokens.iter().map(|t| t.kind).collect_vec();
- dbg!(&token_kinds);
-
- assert_eq!(token_kinds.len(), 1);
- assert!(token_kinds.into_iter().all(|t| {
- matches!(
- t,
- TokenKind::Word(WordMetadata {
- noun: Some(NounData {
- is_possessive: Some(true),
- ..
- }),
- ..
- })
- )
- }))
- }
-
- #[test]
- fn number() {
- let source = "12 is larger than 11, but much less than 11!";
-
- let tokens = Typst.parse_str(source);
- let token_kinds = tokens.iter().map(|t| t.kind).collect_vec();
- dbg!(&token_kinds);
-
- assert!(matches!(
- token_kinds.as_slice(),
- &[
- TokenKind::Number(OrderedFloat(12.0), None),
- TokenKind::Space(1),
- TokenKind::Word(_),
- TokenKind::Space(1),
- TokenKind::Word(_),
- TokenKind::Space(1),
- TokenKind::Word(_),
- TokenKind::Space(1),
- TokenKind::Number(OrderedFloat(11.0), None),
- TokenKind::Punctuation(Punctuation::Comma),
- TokenKind::Space(1),
- TokenKind::Word(_),
- TokenKind::Space(1),
- TokenKind::Word(_),
- TokenKind::Space(1),
- TokenKind::Word(_),
- TokenKind::Space(1),
- TokenKind::Word(_),
- TokenKind::Space(1),
- TokenKind::Number(OrderedFloat(11.0), None),
- TokenKind::Punctuation(Punctuation::Bang),
- ]
- ))
- }
-
- #[test]
- fn math_unlintable() {
- let source = "$12 > 11$, $12 << 11!$";
-
- let tokens = Typst.parse_str(source);
- let token_kinds = tokens.iter().map(|t| t.kind).collect_vec();
- dbg!(&token_kinds);
-
- assert!(matches!(
- token_kinds.as_slice(),
- &[
- TokenKind::Unlintable,
- TokenKind::Punctuation(Punctuation::Comma),
- TokenKind::Space(1),
- TokenKind::Unlintable,
- ]
- ))
- }
-
- #[test]
- fn dict_parsing() {
- let source = r#"#let dict = (
- name: "Typst",
- born: 2019,
- )"#;
-
- let tokens = Typst.parse_str(source);
- let token_kinds = tokens.iter().map(|t| t.kind).collect_vec();
- dbg!(&token_kinds);
-
- let charslice = source.chars().collect_vec();
- assert_eq!(tokens[2].span.get_content_string(&charslice), "Typst");
-
- assert!(matches!(
- token_kinds.as_slice(),
- &[
- TokenKind::Unlintable, // Ident
- TokenKind::Unlintable, // Key 1
- TokenKind::Word(_), // Value 1
- TokenKind::Unlintable, // Key 2
- TokenKind::Unlintable, // Value 2
- ]
- ))
- }
-
- #[test]
- fn str_parsing() {
- let source = r#"#let ident = "This is a string""#;
-
- let token_kinds = Typst.parse_str(source).iter().map(|t| t.kind).collect_vec();
- dbg!(&token_kinds);
-
- assert!(matches!(
- &token_kinds.as_slice(),
- &[
- TokenKind::Unlintable,
- TokenKind::Word(_), // This
- TokenKind::Space(1),
- TokenKind::Word(_), // Is
- TokenKind::Space(1),
- TokenKind::Word(_), // A
- TokenKind::Space(1),
- TokenKind::Word(_), // String
- ]
- ))
- }
-
- #[test]
- fn non_adjacent_spaces_not_condensed() {
- let source = r#"#authors_slice.join(", ", last: ", and ") bob"#;
-
- let token_kinds = Typst.parse_str(source).iter().map(|t| t.kind).collect_vec();
- dbg!(&token_kinds);
-
- assert!(matches!(
- &token_kinds.as_slice(),
- &[
- TokenKind::Unlintable, // authors_slice.join
- TokenKind::Punctuation(Punctuation::Comma),
- TokenKind::Space(1),
- TokenKind::Unlintable, // Ident
- TokenKind::Punctuation(Punctuation::Comma),
- TokenKind::Space(1),
- TokenKind::Word(_), // and
- TokenKind::Space(1),
- TokenKind::Space(2),
- TokenKind::Word(_),
- ]
- ))
- }
-
- #[test]
- fn header_parsing() {
- let source = r"= Header
- Paragraph";
-
- let tokens = Typst.parse_str(source);
- let token_kinds = tokens.iter().map(|t| t.kind).collect_vec();
- dbg!(&token_kinds);
-
- let charslice = source.chars().collect_vec();
- assert_eq!(tokens[0].span.get_content_string(&charslice), "Header");
- assert_eq!(tokens[2].span.get_content_string(&charslice), "Paragraph");
-
- assert!(matches!(
- &token_kinds.as_slice(),
- &[
- TokenKind::Word(_),
- TokenKind::Newline(1),
- TokenKind::Word(_)
- ]
- ))
- }
-
- #[test]
- fn parbreak() {
- let source = r"Paragraph
-
- Paragraph";
-
- let token_kinds = Typst.parse_str(source).iter().map(|t| t.kind).collect_vec();
- dbg!(&token_kinds);
-
- assert!(matches!(
- &token_kinds.as_slice(),
- &[
- TokenKind::Word(_),
- TokenKind::ParagraphBreak,
- TokenKind::Word(_),
- ]
- ))
- }
-
- #[test]
- fn label_unlintable() {
- let source = r"= Header
- <label>
- Paragraph";
-
- let tokens = Typst.parse_str(source);
- let token_kinds = tokens.iter().map(|t| t.kind).collect_vec();
- dbg!(&token_kinds);
-
- assert!(matches!(
- &token_kinds.as_slice(),
- &[
- TokenKind::Word(_),
- TokenKind::Newline(1),
- TokenKind::Unlintable,
- TokenKind::Newline(1),
- TokenKind::Word(_),
- ]
- ))
- }
-
- #[test]
- fn sentence() {
- let source = "This is a sentence, it is not interesting.";
-
- let tokens = Typst.parse_str(source);
- let token_kinds = tokens.iter().map(|t| t.kind).collect_vec();
- dbg!(&token_kinds);
-
- assert!(matches!(
- token_kinds.as_slice(),
- &[
- TokenKind::Word(_),
- TokenKind::Space(1),
- TokenKind::Word(_),
- TokenKind::Space(1),
- TokenKind::Word(_),
- TokenKind::Space(1),
- TokenKind::Word(_),
- TokenKind::Punctuation(Punctuation::Comma),
- TokenKind::Space(1),
- TokenKind::Word(_),
- TokenKind::Space(1),
- TokenKind::Word(_),
- TokenKind::Space(1),
- TokenKind::Word(_),
- TokenKind::Space(1),
- TokenKind::Word(_),
- TokenKind::Punctuation(Punctuation::Period),
- ]
- ))
- }
-
- #[test]
- fn smart_apostrophe_newline() {
- let source = r#"group’s
-writing"#;
-
- let tokens = Typst.parse_str(source);
- let token_kinds = tokens.iter().map(|t| t.kind).collect_vec();
- dbg!(&token_kinds);
-
- let charslice = source.chars().collect_vec();
- assert_eq!(tokens[2].span.get_content_string(&charslice), "writing");
-
- assert!(matches!(
- token_kinds.as_slice(),
- &[
- TokenKind::Word(WordMetadata {
- noun: Some(NounData {
- is_possessive: Some(true),
- ..
- }),
- ..
- }),
- TokenKind::Newline(1),
- TokenKind::Word(_),
- ]
- ));
- }
-}
diff --git a/harper-core/tests/run_tests.rs b/harper-core/tests/run_tests.rs
index 45b18efcc..c94d4149d 100644
--- a/harper-core/tests/run_tests.rs
+++ b/harper-core/tests/run_tests.rs
@@ -1,8 +1,5 @@
use harper_core::linting::{LintGroup, LintGroupConfig, Linter};
-use harper_core::{
- parsers::{Markdown, Typst},
- Document, FstDictionary,
-};
+use harper_core::{parsers::Markdown, Document, FstDictionary};
/// Creates a unit test checking that the linting of a document in
/// `tests_sources` produces the expected number of lints.
@@ -49,8 +46,3 @@ create_test!(issue_267.md, &Markdown, 0);
create_test!(proper_noun_capitalization.md, &Markdown, 2);
create_test!(amazon_hostname.md, &Markdown, 0);
create_test!(issue_159.md, &Markdown, 1);
-
-#[cfg(feature = "typst")]
-create_test!(complex_typst.typ, &Typst, 0);
-#[cfg(feature = "typst")]
-create_test!(typst_spelling_mistakes.typ, &Typst, 4);
diff --git a/harper-html/Cargo.toml b/harper-html/Cargo.toml
index 5be988b2f..bcfa535ec 100644
--- a/harper-html/Cargo.toml
+++ b/harper-html/Cargo.toml
@@ -14,6 +14,3 @@ tree-sitter = "0.20.10"
[dev-dependencies]
paste = "1.0.15"
-
-[features]
-typst = ["harper-core/typst"]
diff --git a/harper-ls/Cargo.toml b/harper-ls/Cargo.toml
index 419b40180..ced89be0a 100644
--- a/harper-ls/Cargo.toml
+++ b/harper-ls/Cargo.toml
@@ -10,6 +10,7 @@ repository = "https://github.com/automattic/harper"
[dependencies]
harper-core = { path = "../harper-core", version = "0.14.0", features = ["concurrent"] }
harper-comments = { path = "../harper-comments", version = "0.14.0" }
+harper-typst = { path = "../harper-typst", version = "0.14.0", optional = true }
harper-html = { path = "../harper-html", version = "0.14.0" }
tower-lsp = "0.20.0"
tokio = { version = "1.42.0", features = ["fs", "rt", "rt-multi-thread", "macros", "io-std", "io-util", "net"] }
@@ -28,4 +29,4 @@ serde = { version = "1.0.214", features = ["derive"] }
[features]
default = ["typst"]
-typst = ["harper-core/typst"]
+typst = ["dep:harper-typst"]
diff --git a/harper-ls/src/backend.rs b/harper-ls/src/backend.rs
index 2c3765499..851fb60ff 100644
--- a/harper-ls/src/backend.rs
+++ b/harper-ls/src/backend.rs
@@ -5,9 +5,7 @@ use std::sync::Arc;
use anyhow::anyhow;
use harper_comments::CommentParser;
use harper_core::linting::{LintGroup, Linter};
-use harper_core::parsers::{
- CollapseIdentifiers, IsolateEnglish, Markdown, Parser, PlainEnglish, Typst,
-};
+use harper_core::parsers::{CollapseIdentifiers, IsolateEnglish, Markdown, Parser, PlainEnglish};
use harper_core::{
Dictionary, Document, FstDictionary, FullDictionary, MergedDictionary, Token, TokenKind,
WordMetadata,
@@ -29,6 +27,9 @@ use tower_lsp::lsp_types::{
use tower_lsp::{Client, LanguageServer};
use tracing::{error, info};
+#[cfg(feature = "typst")]
+use harper_typst::Typst;
+
use crate::config::Config;
use crate::diagnostics::{lint_to_code_actions, lints_to_diagnostics};
use crate::dictionary_io::{load_dict, save_dict};
diff --git a/harper-tree-sitter/Cargo.toml b/harper-tree-sitter/Cargo.toml
index 9540cdabe..530f4ba49 100644
--- a/harper-tree-sitter/Cargo.toml
+++ b/harper-tree-sitter/Cargo.toml
@@ -9,7 +9,3 @@ repository = "https://github.com/automattic/harper"
[dependencies]
harper-core = { path = "../harper-core", version = "0.14.0" }
tree-sitter = "0.20.10"
-
-[features]
-default = ["typst"]
-typst = ["harper-core/typst"]
diff --git a/harper-typst/Cargo.toml b/harper-typst/Cargo.toml
new file mode 100644
index 000000000..70e107a49
--- /dev/null
+++ b/harper-typst/Cargo.toml
@@ -0,0 +1,11 @@
+[package]
+name = "harper-typst"
+version = "0.14.0"
+edition = "2021"
+
+[dependencies]
+harper-core = { path = "../harper-core", version = "0.14.0" }
+typst-syntax = { version = "0.12.0" }
+ordered-float = { version = "4.6.0", features = ["serde"] }
+itertools = "0.13.0"
+paste = "1.0.14"
diff --git a/harper-typst/src/lib.rs b/harper-typst/src/lib.rs
new file mode 100644
index 000000000..c8813d084
--- /dev/null
+++ b/harper-typst/src/lib.rs
@@ -0,0 +1,379 @@
+mod offset_cursor;
+mod typst_translator;
+
+use offset_cursor::OffsetCursor;
+use typst_translator::TypstTranslator;
+
+use harper_core::{
+ parsers::Parser,
+ patterns::{PatternExt, SequencePattern},
+ ConjunctionData, Lrc, NounData, Token, TokenKind, VecExt, WordMetadata,
+};
+use itertools::Itertools;
+use typst_syntax::{
+ ast::{AstNode, Markup},
+ Source,
+};
+
+/// A parser that wraps the [`PlainEnglish`] parser allowing one to parse Typst files.
+pub struct Typst;
+
+thread_local! {
+ static WORD_APOSTROPHE_WORD: Lrc<SequencePattern> = Lrc::new(SequencePattern::default()
+ .then_any_word()
+ .then_apostrophe()
+ .then_any_word());
+}
+
+impl Parser for Typst {
+ fn parse(&self, source: &[char]) -> Vec<Token> {
+ let source_str: String = source.iter().collect();
+ let typst_document = Source::detached(source_str);
+ let typst_tree = Markup::from_untyped(typst_document.root())
+ .expect("Unable to create typst document from parsed tree!");
+ let parse_helper = TypstTranslator::new(&typst_document);
+
+ let mut tokens = typst_tree
+ .exprs()
+ .filter_map(|ex| parse_helper.parse_expr(ex, OffsetCursor::new(&typst_document)))
+ .flatten()
+ .collect_vec();
+
+ // Consolidate conjunctions
+ let mut to_remove = std::collections::VecDeque::default();
+ for tok_span in WORD_APOSTROPHE_WORD
+ .with(|v| v.clone())
+ .find_all_matches(&tokens, source)
+ {
+ let start_tok = &tokens[tok_span.start];
+ let end_tok = &tokens[tok_span.end - 1];
+ let char_span = harper_core::Span::new(start_tok.span.start, end_tok.span.end);
+
+ if let TokenKind::Word(metadata) = start_tok.kind {
+ tokens[tok_span.start].kind =
+ TokenKind::Word(if end_tok.span.get_content(source) == ['s'] {
+ WordMetadata {
+ noun: Some(NounData {
+ is_possessive: Some(true),
+ ..metadata.noun.unwrap_or_default()
+ }),
+ conjunction: None,
+ ..metadata
+ }
+ } else {
+ WordMetadata {
+ noun: metadata.noun.map(|noun| NounData {
+ is_possessive: Some(false),
+ ..noun
+ }),
+ conjunction: Some(ConjunctionData {}),
+ ..metadata
+ }
+ });
+
+ tokens[tok_span.start].span = char_span;
+ to_remove.extend(tok_span.start + 1..tok_span.end);
+ } else {
+ panic!("Apostrophe consolidation does not start with Word Token!")
+ }
+ }
+ tokens.remove_indices(to_remove.into_iter().sorted().unique().collect());
+
+ tokens
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use itertools::Itertools;
+ use ordered_float::OrderedFloat;
+
+ use super::Typst;
+ use harper_core::{parsers::StrParser, NounData, Punctuation, TokenKind, WordMetadata};
+
+ #[test]
+ fn conjunction() {
+ let source = "doesn't";
+
+ let tokens = Typst.parse_str(source);
+ let token_kinds = tokens.iter().map(|t| t.kind).collect_vec();
+ dbg!(&token_kinds);
+
+ assert_eq!(token_kinds.len(), 1);
+ assert!(token_kinds.into_iter().all(|t| t.is_conjunction()))
+ }
+
+ #[test]
+ fn possessive() {
+ let source = "person's";
+
+ let tokens = Typst.parse_str(source);
+ let token_kinds = tokens.iter().map(|t| t.kind).collect_vec();
+ dbg!(&token_kinds);
+
+ assert_eq!(token_kinds.len(), 1);
+ assert!(token_kinds.into_iter().all(|t| {
+ matches!(
+ t,
+ TokenKind::Word(WordMetadata {
+ noun: Some(NounData {
+ is_possessive: Some(true),
+ ..
+ }),
+ ..
+ })
+ )
+ }))
+ }
+
+ #[test]
+ fn number() {
+ let source = "12 is larger than 11, but much less than 11!";
+
+ let tokens = Typst.parse_str(source);
+ let token_kinds = tokens.iter().map(|t| t.kind).collect_vec();
+ dbg!(&token_kinds);
+
+ assert!(matches!(
+ token_kinds.as_slice(),
+ &[
+ TokenKind::Number(OrderedFloat(12.0), None),
+ TokenKind::Space(1),
+ TokenKind::Word(_),
+ TokenKind::Space(1),
+ TokenKind::Word(_),
+ TokenKind::Space(1),
+ TokenKind::Word(_),
+ TokenKind::Space(1),
+ TokenKind::Number(OrderedFloat(11.0), None),
+ TokenKind::Punctuation(Punctuation::Comma),
+ TokenKind::Space(1),
+ TokenKind::Word(_),
+ TokenKind::Space(1),
+ TokenKind::Word(_),
+ TokenKind::Space(1),
+ TokenKind::Word(_),
+ TokenKind::Space(1),
+ TokenKind::Word(_),
+ TokenKind::Space(1),
+ TokenKind::Number(OrderedFloat(11.0), None),
+ TokenKind::Punctuation(Punctuation::Bang),
+ ]
+ ))
+ }
+
+ #[test]
+ fn math_unlintable() {
+ let source = "$12 > 11$, $12 << 11!$";
+
+ let tokens = Typst.parse_str(source);
+ let token_kinds = tokens.iter().map(|t| t.kind).collect_vec();
+ dbg!(&token_kinds);
+
+ assert!(matches!(
+ token_kinds.as_slice(),
+ &[
+ TokenKind::Unlintable,
+ TokenKind::Punctuation(Punctuation::Comma),
+ TokenKind::Space(1),
+ TokenKind::Unlintable,
+ ]
+ ))
+ }
+
+ #[test]
+ fn dict_parsing() {
+ let source = r#"#let dict = (
+ name: "Typst",
+ born: 2019,
+ )"#;
+
+ let tokens = Typst.parse_str(source);
+ let token_kinds = tokens.iter().map(|t| t.kind).collect_vec();
+ dbg!(&token_kinds);
+
+ let charslice = source.chars().collect_vec();
+ assert_eq!(tokens[2].span.get_content_string(&charslice), "Typst");
+
+ assert!(matches!(
+ token_kinds.as_slice(),
+ &[
+ TokenKind::Unlintable, // Ident
+ TokenKind::Unlintable, // Key 1
+ TokenKind::Word(_), // Value 1
+ TokenKind::Unlintable, // Key 2
+ TokenKind::Unlintable, // Value 2
+ ]
+ ))
+ }
+
+ #[test]
+ fn str_parsing() {
+ let source = r#"#let ident = "This is a string""#;
+
+ let token_kinds = Typst.parse_str(source).iter().map(|t| t.kind).collect_vec();
+ dbg!(&token_kinds);
+
+ assert!(matches!(
+ &token_kinds.as_slice(),
+ &[
+ TokenKind::Unlintable,
+ TokenKind::Word(_), // This
+ TokenKind::Space(1),
+ TokenKind::Word(_), // Is
+ TokenKind::Space(1),
+ TokenKind::Word(_), // A
+ TokenKind::Space(1),
+ TokenKind::Word(_), // String
+ ]
+ ))
+ }
+
+ #[test]
+ fn non_adjacent_spaces_not_condensed() {
+ let source = r#"#authors_slice.join(", ", last: ", and ") bob"#;
+
+ let token_kinds = Typst.parse_str(source).iter().map(|t| t.kind).collect_vec();
+ dbg!(&token_kinds);
+
+ assert!(matches!(
+ &token_kinds.as_slice(),
+ &[
+ TokenKind::Unlintable, // authors_slice.join
+ TokenKind::Punctuation(Punctuation::Comma),
+ TokenKind::Space(1),
+ TokenKind::Unlintable, // Ident
+ TokenKind::Punctuation(Punctuation::Comma),
+ TokenKind::Space(1),
+ TokenKind::Word(_), // and
+ TokenKind::Space(1),
+ TokenKind::Space(2),
+ TokenKind::Word(_),
+ ]
+ ))
+ }
+
+ #[test]
+ fn header_parsing() {
+ let source = r"= Header
+ Paragraph";
+
+ let tokens = Typst.parse_str(source);
+ let token_kinds = tokens.iter().map(|t| t.kind).collect_vec();
+ dbg!(&token_kinds);
+
+ let charslice = source.chars().collect_vec();
+ assert_eq!(tokens[0].span.get_content_string(&charslice), "Header");
+ assert_eq!(tokens[2].span.get_content_string(&charslice), "Paragraph");
+
+ assert!(matches!(
+ &token_kinds.as_slice(),
+ &[
+ TokenKind::Word(_),
+ TokenKind::Newline(1),
+ TokenKind::Word(_)
+ ]
+ ))
+ }
+
+ #[test]
+ fn parbreak() {
+ let source = r"Paragraph
+
+ Paragraph";
+
+ let token_kinds = Typst.parse_str(source).iter().map(|t| t.kind).collect_vec();
+ dbg!(&token_kinds);
+
+ assert!(matches!(
+ &token_kinds.as_slice(),
+ &[
+ TokenKind::Word(_),
+ TokenKind::ParagraphBreak,
+ TokenKind::Word(_),
+ ]
+ ))
+ }
+
+ #[test]
+ fn label_unlintable() {
+ let source = r"= Header
+ <label>
+ Paragraph";
+
+ let tokens = Typst.parse_str(source);
+ let token_kinds = tokens.iter().map(|t| t.kind).collect_vec();
+ dbg!(&token_kinds);
+
+ assert!(matches!(
+ &token_kinds.as_slice(),
+ &[
+ TokenKind::Word(_),
+ TokenKind::Newline(1),
+ TokenKind::Unlintable,
+ TokenKind::Newline(1),
+ TokenKind::Word(_),
+ ]
+ ))
+ }
+
+ #[test]
+ fn sentence() {
+ let source = "This is a sentence, it is not interesting.";
+
+ let tokens = Typst.parse_str(source);
+ let token_kinds = tokens.iter().map(|t| t.kind).collect_vec();
+ dbg!(&token_kinds);
+
+ assert!(matches!(
+ token_kinds.as_slice(),
+ &[
+ TokenKind::Word(_),
+ TokenKind::Space(1),
+ TokenKind::Word(_),
+ TokenKind::Space(1),
+ TokenKind::Word(_),
+ TokenKind::Space(1),
+ TokenKind::Word(_),
+ TokenKind::Punctuation(Punctuation::Comma),
+ TokenKind::Space(1),
+ TokenKind::Word(_),
+ TokenKind::Space(1),
+ TokenKind::Word(_),
+ TokenKind::Space(1),
+ TokenKind::Word(_),
+ TokenKind::Space(1),
+ TokenKind::Word(_),
+ TokenKind::Punctuation(Punctuation::Period),
+ ]
+ ))
+ }
+
+ #[test]
+ fn smart_apostrophe_newline() {
+ let source = r#"group’s
+writing"#;
+
+ let tokens = Typst.parse_str(source);
+ let token_kinds = tokens.iter().map(|t| t.kind).collect_vec();
+ dbg!(&token_kinds);
+
+ let charslice = source.chars().collect_vec();
+ assert_eq!(tokens[2].span.get_content_string(&charslice), "writing");
+
+ assert!(matches!(
+ token_kinds.as_slice(),
+ &[
+ TokenKind::Word(WordMetadata {
+ noun: Some(NounData {
+ is_possessive: Some(true),
+ ..
+ }),
+ ..
+ }),
+ TokenKind::Newline(1),
+ TokenKind::Word(_),
+ ]
+ ));
+ }
+}
diff --git a/harper-typst/src/offset_cursor.rs b/harper-typst/src/offset_cursor.rs
new file mode 100644
index 000000000..769024aaf
--- /dev/null
+++ b/harper-typst/src/offset_cursor.rs
@@ -0,0 +1,42 @@
+use typst_syntax::Source;
+
+/// Encapsulation of the translation between byte-based spans and char-based spans
+#[derive(Debug, Clone, Copy)]
+pub struct OffsetCursor<'a> {
+ doc: &'a Source,
+ pub char: usize,
+ pub byte: usize,
+}
+
+impl<'a> OffsetCursor<'a> {
+ pub fn new(doc: &'a Source) -> Self {
+ Self {
+ doc,
+ char: 0,
+ byte: 0,
+ }
+ }
+
+ /// Returns a new [`OffsetCursor`] at the given byte based on the current cursor.
+ pub fn push_to(self, new_byte: usize) -> Self {
+ assert!(new_byte >= self.byte);
+
+ if new_byte == self.byte {
+ return self;
+ }
+
+ Self {
+ char: self.char + self.doc.get(self.byte..new_byte).unwrap().chars().count(),
+ byte: new_byte,
+ ..self
+ }
+ }
+
+ /// Returns a new [`OffsetCursor`] at the beginning of the given [`typst_syntax::Span`] based
+ /// on the current cursor.
+ pub fn push_to_span(self, span: typst_syntax::Span) -> Self {
+ let new_byte = self.doc.range(span).unwrap().start;
+
+ self.push_to(new_byte)
+ }
+}
diff --git a/harper-typst/src/typst_translator.rs b/harper-typst/src/typst_translator.rs
new file mode 100644
index 000000000..2c9413f22
--- /dev/null
+++ b/harper-typst/src/typst_translator.rs
@@ -0,0 +1,317 @@
+use crate::OffsetCursor;
+use harper_core::{
+ parsers::{PlainEnglish, StrParser},
+ Punctuation, Token, TokenKind, WordMetadata,
+};
+use itertools::Itertools;
+use typst_syntax::{
+ ast::{
+ Arg, ArrayItem, AstNode, DestructuringItem, DictItem, Expr, Ident, LetBindingKind, Param,
+ Pattern, Spread,
+ },
+ Source,
+};
+
+macro_rules! def_token {
+ ($doc:expr, $a:expr, $kind:expr, $offset:ident) => {{
+ let range = $doc.range($a.span()).unwrap();
+ let start = $offset.push_to(range.start);
+ let end_char_loc = start.push_to(range.end).char;
+
+ Some(vec![Token {
+ span: harper_core::Span {
+ start: start.char,
+ end: end_char_loc,
+ },
+ kind: $kind,
+ }])
+ }};
+}
+
+macro_rules! merge {
+ ($($inner:expr),*) => {
+ Some(
+ [$($inner),*]
+ .into_iter()
+ .flatten()
+ .flatten()
+ .collect_vec(),
+ )
+ };
+}
+
+/// Contains values used in parsing so they don't have to be passed around so much
+#[derive(Clone, Copy)]
+pub struct TypstTranslator<'a> {
+ parser: PlainEnglish,
+ doc: &'a Source,
+}
+
+impl<'a> TypstTranslator<'a> {
+ pub fn new(doc: &'a Source) -> Self {
+ Self {
+ parser: PlainEnglish,
+ doc,
+ }
+ }
+
+ fn parse_english(self, str: impl Into<String>, offset: OffsetCursor) -> Option<Vec<Token>> {
+ Some(
+ self.parser
+ .parse_str(str.into())
+ .into_iter()
+ .map(|mut t| {
+ t.span.push_by(offset.char);
+ t
+ })
+ .collect_vec(),
+ )
+ }
+
+ fn parse_pattern(self, pat: Pattern, offset: OffsetCursor) -> Option<Vec<Token>> {
+ macro_rules! token {
+ ($a:expr, $kind:expr) => {
+ def_token!(self.doc, $a, $kind, offset)
+ };
+ }
+
+ match pat {
+ Pattern::Normal(expr) => self.parse_expr(expr, offset),
+ Pattern::Placeholder(underscore) => token!(underscore, TokenKind::Unlintable),
+ Pattern::Parenthesized(parenthesized) => merge!(
+ self.parse_expr(parenthesized.expr(), offset),
+ self.parse_pattern(parenthesized.pattern(), offset)
+ ),
+ Pattern::Destructuring(destructuring) => Some(
+ destructuring
+ .items()
+ .filter_map(|item| match item {
+ DestructuringItem::Pattern(pattern) => self.parse_pattern(pattern, offset),
+ DestructuringItem::Named(named) => merge!(
+ token!(named.name(), TokenKind::Word(WordMetadata::default())),
+ self.parse_pattern(named.pattern(), offset)
+ ),
+ DestructuringItem::Spread(spread) => merge!(
+ spread
+ .sink_ident()
+ .and_then(|ident| self.parse_ident(ident, offset)),
+ spread
+ .sink_expr()
+ .and_then(|expr| self.parse_expr(expr, offset))
+ ),
+ })
+ .flatten()
+ .collect(),
+ ),
+ }
+ }
+
+ fn parse_ident(self, ident: Ident, offset: OffsetCursor) -> Option<Vec<Token>> {
+ self.parse_expr(Expr::Ident(ident), offset)
+ }
+
+ /// Do not use for spreads contained in DestructuringItem
+ fn parse_spread(self, spread: Spread, offset: OffsetCursor) -> Option<Vec<Token>> {
+ merge!(
+ self.parse_expr(spread.expr(), offset),
+ spread
+ .sink_ident()
+ .and_then(|ident| self.parse_ident(ident, offset))
+ )
+ }
+
+ pub fn parse_expr(self, ex: Expr, offset: OffsetCursor) -> Option<Vec<Token>> {
+ let offset = offset.push_to_span(ex.span());
+
+ macro_rules! token {
+ ($a:expr, $kind:expr) => {
+ def_token!(self.doc, $a, $kind, offset)
+ };
+ }
+ macro_rules! recurse {
+ ($inner:expr) => {
+ self.parse_expr($inner, offset)
+ };
+ ($($inner:expr),*) => {
+ merge!(
+ $(recurse!($inner)),*
+ )
+ };
+ }
+
+ let iter_recurse = |exprs: &mut dyn Iterator<Item = Expr>| {
+ Some(exprs.filter_map(|e| recurse!(e)).flatten().collect_vec())
+ };
+ let parse_dict = |dict: &mut dyn Iterator<Item = DictItem>| {
+ Some(
+ dict.filter_map(|di| match di {
+ DictItem::Named(named) => {
+ merge!(
+ self.parse_ident(named.name(), offset),
+ recurse!(named.expr())
+ )
+ }
+ DictItem::Keyed(keyed) => recurse!(keyed.key(), keyed.expr()),
+ DictItem::Spread(spread) => self.parse_spread(spread, offset),
+ })
+ .flatten()
+ .collect_vec(),
+ )
+ };
+ let parse_params = |params: &mut dyn Iterator<Item = Param>| {
+ Some(
+ params
+ .filter_map(|p| match p {
+ Param::Pos(pattern) => self.parse_pattern(pattern, offset),
+ Param::Named(named) => merge!(
+ self.parse_ident(named.name(), offset),
+ recurse!(named.expr())
+ ),
+ Param::Spread(spread) => self.parse_spread(spread, offset),
+ })
+ .flatten()
+ .collect_vec(),
+ )
+ };
+ let parse_args = |params: &mut dyn Iterator<Item = Arg>| {
+ Some(
+ params
+ .filter_map(|a| match a {
+ Arg::Pos(expr) => recurse!(expr),
+ Arg::Named(named) => merge!(
+ self.parse_ident(named.name(), offset),
+ recurse!(named.expr())
+ ),
+ Arg::Spread(spread) => self.parse_spread(spread, offset),
+ })
+ .flatten()
+ .collect_vec(),
+ )
+ };
+
+ match ex {
+ Expr::Text(text) => self.parse_english(text.get(), offset.push_to_span(text.span())),
+ Expr::Space(a) => {
+ let mut chars = self
+ .doc
+ .get(self.doc.range(a.span()).unwrap())
+ .unwrap()
+ .chars();
+ let first_char = chars.next().unwrap();
+ let length = chars.count() + 1;
+
+ if first_char == '\n' {
+ token!(a, TokenKind::Newline(1))
+ } else {
+ token!(a, TokenKind::Space(length))
+ }
+ }
+ Expr::Linebreak(a) => token!(a, TokenKind::Newline(1)),
+ Expr::Parbreak(a) => token!(a, TokenKind::ParagraphBreak),
+ Expr::SmartQuote(quote) => {
+ if quote.double() {
+ token!(
+ quote,
+ TokenKind::Punctuation(Punctuation::Quote(harper_core::Quote {
+ twin_loc: None
+ }))
+ )
+ } else {
+ token!(quote, TokenKind::Punctuation(Punctuation::Apostrophe))
+ }
+ }
+ Expr::Strong(strong) => iter_recurse(&mut strong.body().exprs()),
+ Expr::Emph(emph) => iter_recurse(&mut emph.body().exprs()),
+ Expr::Link(a) => token!(a, TokenKind::Url),
+ Expr::Ref(a) => {
+ token!(a, TokenKind::Word(WordMetadata::default()))
+ }
+ Expr::Heading(heading) => iter_recurse(&mut heading.body().exprs()),
+ Expr::List(list_item) => iter_recurse(&mut list_item.body().exprs()),
+ Expr::Enum(enum_item) => iter_recurse(&mut enum_item.body().exprs()),
+ Expr::Term(term_item) => iter_recurse(
+ &mut term_item
+ .term()
+ .exprs()
+ .chain(term_item.description().exprs()),
+ ),
+ Expr::Str(text) => {
+ let offset = offset.push_to_span(text.span()).char + 1;
+ let string = text.to_untyped().text();
+
+ Some(
+ self.parser
+ .parse_str(&string[1..string.len() - 1])
+ .into_iter()
+ .map(|mut t| {
+ t.span.push_by(offset);
+ t
+ })
+ .collect_vec(),
+ )
+ }
+ Expr::Content(content_block) => iter_recurse(&mut content_block.body().exprs()),
+ Expr::Parenthesized(parenthesized) => recurse!(parenthesized.expr()),
+ Expr::Array(array) => Some(
+ array
+ .items()
+ .filter_map(|i| {
+ if let ArrayItem::Pos(e) = i {
+ recurse!(e)
+ } else {
+ None
+ }
+ })
+ .flatten()
+ .collect_vec(),
+ ),
+ Expr::Dict(a) => parse_dict(&mut a.items()),
+ Expr::FieldAccess(field_access) => merge!(
+ recurse!(field_access.target()),
+ token!(
+ field_access.field(),
+ TokenKind::Word(WordMetadata::default())
+ )
+ ),
+ Expr::Let(let_binding) => merge!(
+ match let_binding.kind() {
+ LetBindingKind::Normal(pattern) => self.parse_pattern(pattern, offset),
+ LetBindingKind::Closure(ident) => self.parse_ident(ident, offset),
+ },
+ let_binding.init().and_then(|e| recurse!(e))
+ ),
+ Expr::DestructAssign(destruct_assignment) => {
+ recurse!(destruct_assignment.value())
+ }
+ Expr::Set(set_rule) => merge!(
+ recurse!(set_rule.target()),
+ set_rule.condition().and_then(|expr| recurse!(expr)),
+ parse_args(&mut set_rule.args().items())
+ ),
+ Expr::Show(show_rule) => merge!(
+ recurse!(show_rule.transform()),
+ show_rule.selector().and_then(|expr| recurse!(expr))
+ ),
+ Expr::Contextual(contextual) => recurse!(contextual.body()),
+ Expr::Conditional(conditional) => merge!(
+ recurse!(conditional.condition(), conditional.if_body()),
+ conditional.else_body().and_then(|expr| recurse!(expr))
+ ),
+ Expr::While(while_loop) => recurse!(while_loop.condition(), while_loop.body()),
+ Expr::For(for_loop) => recurse!(for_loop.iterable(), for_loop.body()),
+ Expr::Code(code) => iter_recurse(&mut code.body().exprs()),
+ Expr::Closure(closure) => merge!(
+ closure
+ .name()
+ .and_then(|ident| self.parse_ident(ident, offset)),
+ parse_params(&mut closure.params().children()),
+ recurse!(closure.body())
+ ),
+ Expr::FuncCall(func) => merge!(
+ token!(func.callee(), TokenKind::Unlintable),
+ parse_args(&mut func.args().items())
+ ),
+ a => token!(a, TokenKind::Unlintable),
+ }
+ }
+}
diff --git a/harper-typst/tests/run_tests.rs b/harper-typst/tests/run_tests.rs
new file mode 100644
index 000000000..c9c15f73e
--- /dev/null
+++ b/harper-typst/tests/run_tests.rs
@@ -0,0 +1,41 @@
+use harper_core::linting::{LintGroup, LintGroupConfig, Linter};
+use harper_core::{Document, FstDictionary};
+use harper_typst::Typst;
+
+/// Creates a unit test checking that the linting of a document in
+/// `tests_sources` produces the expected number of lints.
+macro_rules! create_test {
+ ($filename:ident.$ext:ident, $correct_expected:expr) => {
+ paste::paste! {
+ #[test]
+ fn [<lints_ $filename _correctly>](){
+ let source = include_str!(
+ concat!(
+ "./test_sources/",
+ concat!(stringify!($filename), ".", stringify!($ext))
+ )
+ );
+
+ let dict = FstDictionary::curated();
+ let document = Document::new(&source, &Typst, &dict);
+
+ let mut linter = LintGroup::new(
+ LintGroupConfig::default(),
+ dict
+ );
+ let lints = linter.lint(&document);
+
+ dbg!(&lints);
+ assert_eq!(lints.len(), $correct_expected);
+
+ // Make sure that all generated tokens span real characters
+ for token in document.tokens(){
+ assert!(token.span.try_get_content(document.get_source()).is_some());
+ }
+ }
+ }
+ };
+}
+
+create_test!(complex_typst.typ, 0);
+create_test!(typst_spelling_mistakes.typ, 4);