Skip to content

Offline, privacy-first grammar checker. Fast, open-source, Rust-powered

feat(#230): map basic typst expressions to tokens

Grant Lemonsbed51f7c81fe
diff --git a/Cargo.lock b/Cargo.lock
index 1a2a5767e..c263f4b96 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -322,6 +322,31 @@ dependencies = [
"itertools 0.10.5",
]
+[[package]]
+name = "crossbeam-deque"
+version = "0.8.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "613f8cc01fe9cf1a3eb3d7f488fd2fa8388403e97039e2f73692932e291a770d"
+dependencies = [
+ "crossbeam-epoch",
+ "crossbeam-utils",
+]
+
+[[package]]
+name = "crossbeam-epoch"
+version = "0.9.18"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e"
+dependencies = [
+ "crossbeam-utils",
+]
+
+[[package]]
+name = "crossbeam-utils"
+version = "0.8.20"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80"
+
[[package]]
name = "crunchy"
version = "0.2.2"
@@ -393,6 +418,15 @@ dependencies = [
"syn",
]
+[[package]]
+name = "ecow"
+version = "0.2.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e42fc0a93992b20c58b99e59d61eaf1635a25bfbe49e4275c34ba0aee98119ba"
+dependencies = [
+ "serde",
+]
+
[[package]]
name = "either"
version = "1.13.0"
@@ -610,6 +644,7 @@ dependencies = [
"serde_json",
"smallvec",
"thiserror 2.0.4",
+ "typst-syntax",
"unicode-blocks",
"unicode-width 0.2.0",
]
@@ -850,6 +885,16 @@ dependencies = [
"icu_properties",
]
+[[package]]
+name = "indexmap"
+version = "2.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "707907fe3c25f5424cce2cb7e1cbcafee6bdbe735ca90ef77c29e84591e5b9da"
+dependencies = [
+ "equivalent",
+ "hashbrown 0.15.1",
+]
+
[[package]]
name = "is-docker"
version = "0.2.0"
@@ -1161,6 +1206,12 @@ version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184"
+[[package]]
+name = "portable-atomic"
+version = "1.10.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "280dc24453071f1b63954171985a0b0d30058d287960968b9b2aca264c8d4ee6"
+
[[package]]
name = "ppv-lite86"
version = "0.2.20"
@@ -1239,6 +1290,26 @@ dependencies = [
"serde",
]
+[[package]]
+name = "rayon"
+version = "1.10.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa"
+dependencies = [
+ "either",
+ "rayon-core",
+]
+
+[[package]]
+name = "rayon-core"
+version = "1.12.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2"
+dependencies = [
+ "crossbeam-deque",
+ "crossbeam-utils",
+]
+
[[package]]
name = "redox_syscall"
version = "0.5.7"
@@ -1378,6 +1449,15 @@ dependencies = [
"syn",
]
+[[package]]
+name = "serde_spanned"
+version = "0.6.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "87607cb1398ed59d48732e575a4c28a7a8ebf2454b964fe3f224f2afc07909e1"
+dependencies = [
+ "serde",
+]
+
[[package]]
name = "sharded-slab"
version = "0.1.7"
@@ -1387,6 +1467,12 @@ dependencies = [
"lazy_static",
]
+[[package]]
+name = "siphasher"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "56199f7ddabf13fe5074ce809e7d3f42b42ae711800501b5b16ea82ad029c39d"
+
[[package]]
name = "slab"
version = "0.4.9"
@@ -1449,6 +1535,12 @@ dependencies = [
"syn",
]
+[[package]]
+name = "thin-vec"
+version = "0.2.13"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a38c90d48152c236a3ab59271da4f4ae63d678c5d7ad6b7714d7cb9760be5e4b"
+
[[package]]
name = "thiserror"
version = "1.0.69"
@@ -1559,6 +1651,40 @@ dependencies = [
"tokio",
]
+[[package]]
+name = "toml"
+version = "0.8.19"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a1ed1f98e3fdc28d6d910e6737ae6ab1a93bf1985935a1193e68f93eeb68d24e"
+dependencies = [
+ "serde",
+ "serde_spanned",
+ "toml_datetime",
+ "toml_edit",
+]
+
+[[package]]
+name = "toml_datetime"
+version = "0.6.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0dd7358ecb8fc2f8d014bf86f6f638ce72ba252a2c3a2572f2a795f1d23efb41"
+dependencies = [
+ "serde",
+]
+
+[[package]]
+name = "toml_edit"
+version = "0.22.22"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4ae48d6208a266e853d946088ed816055e556cc6028c5e8e2b84d9fa5dd7c7f5"
+dependencies = [
+ "indexmap",
+ "serde",
+ "serde_spanned",
+ "toml_datetime",
+ "winnow",
+]
+
[[package]]
name = "tower"
version = "0.4.13"
@@ -1857,6 +1983,37 @@ dependencies = [
"tree-sitter",
]
+[[package]]
+name = "typst-syntax"
+version = "0.12.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "05b7be8b6ed6b2cb39ca495947d548a28d7db0ba244008e44c5a759120327693"
+dependencies = [
+ "ecow",
+ "once_cell",
+ "serde",
+ "toml",
+ "typst-utils",
+ "unicode-ident",
+ "unicode-math-class",
+ "unicode-script",
+ "unicode-segmentation",
+ "unscanny",
+]
+
+[[package]]
+name = "typst-utils"
+version = "0.12.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8f0305443ed97f0b658471487228f86bf835705e7525fbdcc671cebd864f7a40"
+dependencies = [
+ "once_cell",
+ "portable-atomic",
+ "rayon",
+ "siphasher",
+ "thin-vec",
+]
+
[[package]]
name = "unicase"
version = "2.8.0"
@@ -1875,6 +2032,24 @@ version = "1.0.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e91b56cd4cadaeb79bbf1a5645f6b4f8dc5bde8834ad5894a8db35fda9efa1fe"
+[[package]]
+name = "unicode-math-class"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7d246cf599d5fae3c8d56e04b20eb519adb89a8af8d0b0fbcded369aa3647d65"
+
+[[package]]
+name = "unicode-script"
+version = "0.5.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9fb421b350c9aff471779e262955939f565ec18b86c15364e6bdf0d662ca7c1f"
+
+[[package]]
+name = "unicode-segmentation"
+version = "1.12.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493"
+
[[package]]
name = "unicode-width"
version = "0.1.14"
@@ -1887,6 +2062,12 @@ version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1fc81956842c57dac11422a97c3b8195a1ff727f06e85c84ed2e8aa277c9a0fd"
+[[package]]
+name = "unscanny"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e9df2af067a7953e9c3831320f35c1cc0600c30d44d9f7a12b01db1cd88d6b47"
+
[[package]]
name = "url"
version = "2.5.3"
@@ -2173,6 +2354,15 @@ version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"
+[[package]]
+name = "winnow"
+version = "0.6.20"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "36c1fec1a2bb5866f07c25f68c26e565c4c200aebb96d7e55710c19d3e8ac49b"
+dependencies = [
+ "memchr",
+]
+
[[package]]
name = "write16"
version = "1.0.0"
diff --git a/harper-core/Cargo.toml b/harper-core/Cargo.toml
index 1d5bdb0c9..69f41a4ca 100644
--- a/harper-core/Cargo.toml
+++ b/harper-core/Cargo.toml
@@ -24,6 +24,7 @@ thiserror = "2.0.4"
unicode-blocks = "0.1.9"
unicode-width = "0.2.0"
levenshtein_automata = { version = "0.2.1", features = ["fst_automaton"] }
+typst-syntax = "0.12.0"
[dev-dependencies]
criterion = { version = "0.5.1", default-features = false }
diff --git a/harper-core/src/parsers/mod.rs b/harper-core/src/parsers/mod.rs
index f35f209bb..08f53bbf2 100644
--- a/harper-core/src/parsers/mod.rs
+++ b/harper-core/src/parsers/mod.rs
@@ -3,6 +3,7 @@ mod isolate_english;
mod markdown;
mod mask;
mod plain_english;
+mod typst;
use blanket::blanket;
pub use collapse_identifiers::CollapseIdentifiers;
diff --git a/harper-core/src/parsers/typst.rs b/harper-core/src/parsers/typst.rs
new file mode 100644
index 000000000..9f5c6f856
--- /dev/null
+++ b/harper-core/src/parsers/typst.rs
@@ -0,0 +1,170 @@
+use itertools::Itertools;
+
+use typst_syntax::ast::{AstNode, Expr};
+
+use super::{Parser, PlainEnglish};
+use crate::{parsers::StrParser, Token, TokenKind, WordMetadata};
+
+/// A parser that wraps the [`PlainEnglish`] parser that allows one to parse
+/// Typst files.
+pub struct Typst;
+
+macro_rules! constant_token {
+ ($offset:ident, $doc:ident, $a:ident, $to:expr) => {{
+ let range = $doc.range($a.span()).unwrap();
+ *$offset += range.len();
+ Some(vec![Token {
+ span: range.into(),
+ kind: $to,
+ }])
+ }};
+}
+macro_rules! recursive_env {
+ ($offset:ident, $expr:ident, $doc:ident, $parser:ident) => {
+ Some(
+ $expr
+ .body()
+ .exprs()
+ .filter_map(|e| map_token(e, $doc, $parser, $offset))
+ .flatten()
+ .collect_vec(),
+ )
+ };
+}
+
+fn map_token(
+ ex: typst_syntax::ast::Expr,
+ doc: &typst_syntax::Source,
+ parser: &mut PlainEnglish,
+ offset: &mut usize,
+) -> Option<Vec<Token>> {
+ match ex {
+ Expr::Text(text) => Some(
+ parser
+ .parse_str(text.get())
+ .into_iter()
+ .map(|mut t| {
+ t.span.push_by(*offset);
+ t
+ })
+ .collect_vec(),
+ ),
+ Expr::Space(a) => constant_token!(offset, doc, a, TokenKind::Space(1)),
+ Expr::Linebreak(a) => constant_token!(offset, doc, a, TokenKind::Newline(1)),
+ Expr::Parbreak(a) => constant_token!(offset, doc, a, TokenKind::Newline(2)),
+ Expr::Escape(_) => None,
+ Expr::Shorthand(_) => None,
+ Expr::SmartQuote(_) => None,
+ Expr::Strong(strong) => recursive_env!(offset, strong, doc, parser),
+ Expr::Emph(emph) => recursive_env!(offset, emph, doc, parser),
+ Expr::Raw(_) => None,
+ Expr::Link(a) => constant_token!(offset, doc, a, TokenKind::Url),
+ Expr::Label(label) => Some(
+ parser
+ .parse_str(label.get())
+ .into_iter()
+ .map(|mut t| {
+ t.span.push_by(*offset);
+ t
+ })
+ .collect_vec(),
+ ),
+ Expr::Ref(a) => {
+ constant_token!(offset, doc, a, TokenKind::Word(WordMetadata::default()))
+ }
+ Expr::Heading(heading) => recursive_env!(offset, heading, doc, parser),
+ Expr::List(list_item) => recursive_env!(offset, list_item, doc, parser),
+ Expr::Enum(enum_item) => recursive_env!(offset, enum_item, doc, parser),
+ Expr::Term(term_item) => Some(
+ term_item
+ .term()
+ .exprs()
+ .chain(term_item.description().exprs())
+ .filter_map(|e| map_token(e, doc, parser, offset))
+ .flatten()
+ .collect_vec(),
+ ),
+ Expr::Equation(a) => constant_token!(offset, doc, a, TokenKind::Unlintable),
+ Expr::Math(_) => None,
+ Expr::MathIdent(_) => None,
+ Expr::MathShorthand(_) => None,
+ Expr::MathAlignPoint(_) => None,
+ Expr::MathDelimited(_) => None,
+ Expr::MathAttach(_) => None,
+ Expr::MathPrimes(_) => None,
+ Expr::MathFrac(_) => None,
+ Expr::MathRoot(_) => None,
+ Expr::Ident(a) => constant_token!(offset, doc, a, TokenKind::Word(WordMetadata::default())),
+ Expr::None(a) => constant_token!(offset, doc, a, TokenKind::Word(WordMetadata::default())),
+ Expr::Auto(a) => constant_token!(offset, doc, a, TokenKind::Word(WordMetadata::default())),
+ Expr::Bool(a) => constant_token!(offset, doc, a, TokenKind::Word(WordMetadata::default())),
+ Expr::Int(int) => todo!(),
+ Expr::Float(float) => todo!(),
+ Expr::Numeric(a) => constant_token!(offset, doc, a, TokenKind::Unlintable),
+ Expr::Str(text) => Some(
+ parser
+ .parse_str(text.get())
+ .into_iter()
+ .map(|mut t| {
+ t.span.push_by(*offset);
+ t
+ })
+ .collect_vec(),
+ ),
+ Expr::Code(a) => constant_token!(offset, doc, a, TokenKind::Unlintable),
+ Expr::Content(content_block) => recursive_env!(offset, content_block, doc, parser),
+ Expr::Parenthesized(parenthesized) => map_token(parenthesized.expr(), doc, parser, offset),
+ Expr::Array(array) => Some(
+ array
+ .items()
+ .filter_map(|i| {
+ if let typst_syntax::ast::ArrayItem::Pos(e) = i {
+ map_token(e, doc, parser, offset)
+ } else {
+ None
+ }
+ })
+ .flatten()
+ .collect_vec(),
+ ),
+ Expr::Dict(dict) => todo!(),
+ Expr::Unary(unary) => todo!(),
+ Expr::Binary(binary) => todo!(),
+ Expr::FieldAccess(field_access) => todo!(),
+ Expr::FuncCall(func_call) => todo!(),
+ Expr::Closure(closure) => todo!(),
+ Expr::Let(let_binding) => todo!(),
+ Expr::DestructAssign(destruct_assignment) => todo!(),
+ Expr::Set(set_rule) => todo!(),
+ Expr::Show(show_rule) => todo!(),
+ Expr::Contextual(contextual) => todo!(),
+ Expr::Conditional(conditional) => todo!(),
+ Expr::While(while_loop) => todo!(),
+ Expr::For(for_loop) => todo!(),
+ Expr::Import(module_import) => todo!(),
+ Expr::Include(module_include) => todo!(),
+ Expr::Break(loop_break) => todo!(),
+ Expr::Continue(loop_continue) => todo!(),
+ Expr::Return(func_return) => todo!(),
+ }
+}
+
+impl Parser for Typst {
+ fn parse(&mut self, source: &[char]) -> Vec<Token> {
+ let mut english_parser = PlainEnglish;
+
+ let source_str: String = source.iter().collect();
+ let typst_document = typst_syntax::Source::detached(source_str);
+ let typst_tree = typst_syntax::ast::Markup::from_untyped(typst_document.root())
+ .expect("Unable to create typst document from parsed tree!");
+ let mut offset = 0;
+
+ // NOTE: the range spits out __byte__ indices, not char indices.
+ // This is why we keep track above.
+ typst_tree
+ .exprs()
+ .filter_map(|ex| map_token(ex, &typst_document, &mut english_parser, &mut offset))
+ .flatten()
+ .collect_vec()
+ }
+}