Offline, privacy-first grammar checker. Fast, open-source, Rust-powered
feat(#230): map basic typst expressions to tokens
Grant Lemonsbed51f7c81fe
diff --git a/Cargo.lock b/Cargo.lockindex 1a2a5767e..c263f4b96 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -322,6 +322,31 @@ dependencies = ["itertools 0.10.5",]+[[package]]+name = "crossbeam-deque"+version = "0.8.5"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "613f8cc01fe9cf1a3eb3d7f488fd2fa8388403e97039e2f73692932e291a770d"+dependencies = [+ "crossbeam-epoch",+ "crossbeam-utils",+]++[[package]]+name = "crossbeam-epoch"+version = "0.9.18"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e"+dependencies = [+ "crossbeam-utils",+]++[[package]]+name = "crossbeam-utils"+version = "0.8.20"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80"+[[package]]name = "crunchy"version = "0.2.2"@@ -393,6 +418,15 @@ dependencies = ["syn",]+[[package]]+name = "ecow"+version = "0.2.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "e42fc0a93992b20c58b99e59d61eaf1635a25bfbe49e4275c34ba0aee98119ba"+dependencies = [+ "serde",+]+[[package]]name = "either"version = "1.13.0"@@ -610,6 +644,7 @@ dependencies = ["serde_json","smallvec","thiserror 2.0.4",+ "typst-syntax","unicode-blocks","unicode-width 0.2.0",]@@ -850,6 +885,16 @@ dependencies = ["icu_properties",]+[[package]]+name = "indexmap"+version = "2.6.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "707907fe3c25f5424cce2cb7e1cbcafee6bdbe735ca90ef77c29e84591e5b9da"+dependencies = [+ "equivalent",+ "hashbrown 0.15.1",+]+[[package]]name = "is-docker"version = "0.2.0"@@ -1161,6 +1206,12 @@ version = "0.1.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184"+[[package]]+name = "portable-atomic"+version = "1.10.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "280dc24453071f1b63954171985a0b0d30058d287960968b9b2aca264c8d4ee6"+[[package]]name = "ppv-lite86"version = "0.2.20"@@ -1239,6 +1290,26 @@ dependencies = ["serde",]+[[package]]+name = "rayon"+version = "1.10.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa"+dependencies = [+ "either",+ "rayon-core",+]++[[package]]+name = "rayon-core"+version = "1.12.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2"+dependencies = [+ "crossbeam-deque",+ "crossbeam-utils",+]+[[package]]name = "redox_syscall"version = "0.5.7"@@ -1378,6 +1449,15 @@ dependencies = ["syn",]+[[package]]+name = "serde_spanned"+version = "0.6.8"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "87607cb1398ed59d48732e575a4c28a7a8ebf2454b964fe3f224f2afc07909e1"+dependencies = [+ "serde",+]+[[package]]name = "sharded-slab"version = "0.1.7"@@ -1387,6 +1467,12 @@ dependencies = ["lazy_static",]+[[package]]+name = "siphasher"+version = "1.0.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "56199f7ddabf13fe5074ce809e7d3f42b42ae711800501b5b16ea82ad029c39d"+[[package]]name = "slab"version = "0.4.9"@@ -1449,6 +1535,12 @@ dependencies = ["syn",]+[[package]]+name = "thin-vec"+version = "0.2.13"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "a38c90d48152c236a3ab59271da4f4ae63d678c5d7ad6b7714d7cb9760be5e4b"+[[package]]name = "thiserror"version = "1.0.69"@@ -1559,6 +1651,40 @@ dependencies = ["tokio",]+[[package]]+name = "toml"+version = "0.8.19"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "a1ed1f98e3fdc28d6d910e6737ae6ab1a93bf1985935a1193e68f93eeb68d24e"+dependencies = [+ "serde",+ "serde_spanned",+ "toml_datetime",+ "toml_edit",+]++[[package]]+name = "toml_datetime"+version = "0.6.8"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "0dd7358ecb8fc2f8d014bf86f6f638ce72ba252a2c3a2572f2a795f1d23efb41"+dependencies = [+ "serde",+]++[[package]]+name = "toml_edit"+version = "0.22.22"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "4ae48d6208a266e853d946088ed816055e556cc6028c5e8e2b84d9fa5dd7c7f5"+dependencies = [+ "indexmap",+ "serde",+ "serde_spanned",+ "toml_datetime",+ "winnow",+]+[[package]]name = "tower"version = "0.4.13"@@ -1857,6 +1983,37 @@ dependencies = ["tree-sitter",]+[[package]]+name = "typst-syntax"+version = "0.12.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "05b7be8b6ed6b2cb39ca495947d548a28d7db0ba244008e44c5a759120327693"+dependencies = [+ "ecow",+ "once_cell",+ "serde",+ "toml",+ "typst-utils",+ "unicode-ident",+ "unicode-math-class",+ "unicode-script",+ "unicode-segmentation",+ "unscanny",+]++[[package]]+name = "typst-utils"+version = "0.12.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "8f0305443ed97f0b658471487228f86bf835705e7525fbdcc671cebd864f7a40"+dependencies = [+ "once_cell",+ "portable-atomic",+ "rayon",+ "siphasher",+ "thin-vec",+]+[[package]]name = "unicase"version = "2.8.0"@@ -1875,6 +2032,24 @@ version = "1.0.13"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "e91b56cd4cadaeb79bbf1a5645f6b4f8dc5bde8834ad5894a8db35fda9efa1fe"+[[package]]+name = "unicode-math-class"+version = "0.1.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "7d246cf599d5fae3c8d56e04b20eb519adb89a8af8d0b0fbcded369aa3647d65"++[[package]]+name = "unicode-script"+version = "0.5.7"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "9fb421b350c9aff471779e262955939f565ec18b86c15364e6bdf0d662ca7c1f"++[[package]]+name = "unicode-segmentation"+version = "1.12.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493"+[[package]]name = "unicode-width"version = "0.1.14"@@ -1887,6 +2062,12 @@ version = "0.2.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "1fc81956842c57dac11422a97c3b8195a1ff727f06e85c84ed2e8aa277c9a0fd"+[[package]]+name = "unscanny"+version = "0.1.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "e9df2af067a7953e9c3831320f35c1cc0600c30d44d9f7a12b01db1cd88d6b47"+[[package]]name = "url"version = "2.5.3"@@ -2173,6 +2354,15 @@ version = "0.52.6"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"+[[package]]+name = "winnow"+version = "0.6.20"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "36c1fec1a2bb5866f07c25f68c26e565c4c200aebb96d7e55710c19d3e8ac49b"+dependencies = [+ "memchr",+]+[[package]]name = "write16"version = "1.0.0"diff --git a/harper-core/Cargo.toml b/harper-core/Cargo.tomlindex 1d5bdb0c9..69f41a4ca 100644--- a/harper-core/Cargo.toml+++ b/harper-core/Cargo.toml@@ -24,6 +24,7 @@ thiserror = "2.0.4"unicode-blocks = "0.1.9"unicode-width = "0.2.0"levenshtein_automata = { version = "0.2.1", features = ["fst_automaton"] }+typst-syntax = "0.12.0"[dev-dependencies]criterion = { version = "0.5.1", default-features = false }diff --git a/harper-core/src/parsers/mod.rs b/harper-core/src/parsers/mod.rsindex f35f209bb..08f53bbf2 100644--- a/harper-core/src/parsers/mod.rs+++ b/harper-core/src/parsers/mod.rs@@ -3,6 +3,7 @@ mod isolate_english;mod markdown;mod mask;mod plain_english;+mod typst;use blanket::blanket;pub use collapse_identifiers::CollapseIdentifiers;diff --git a/harper-core/src/parsers/typst.rs b/harper-core/src/parsers/typst.rsnew file mode 100644index 000000000..9f5c6f856--- /dev/null+++ b/harper-core/src/parsers/typst.rs@@ -0,0 +1,170 @@+use itertools::Itertools;++use typst_syntax::ast::{AstNode, Expr};++use super::{Parser, PlainEnglish};+use crate::{parsers::StrParser, Token, TokenKind, WordMetadata};++/// A parser that wraps the [`PlainEnglish`] parser that allows one to parse+/// Typst files.+pub struct Typst;++macro_rules! constant_token {+ ($offset:ident, $doc:ident, $a:ident, $to:expr) => {{+ let range = $doc.range($a.span()).unwrap();+ *$offset += range.len();+ Some(vec![Token {+ span: range.into(),+ kind: $to,+ }])+ }};+}+macro_rules! recursive_env {+ ($offset:ident, $expr:ident, $doc:ident, $parser:ident) => {+ Some(+ $expr+ .body()+ .exprs()+ .filter_map(|e| map_token(e, $doc, $parser, $offset))+ .flatten()+ .collect_vec(),+ )+ };+}++fn map_token(+ ex: typst_syntax::ast::Expr,+ doc: &typst_syntax::Source,+ parser: &mut PlainEnglish,+ offset: &mut usize,+) -> Option<Vec<Token>> {+ match ex {+ Expr::Text(text) => Some(+ parser+ .parse_str(text.get())+ .into_iter()+ .map(|mut t| {+ t.span.push_by(*offset);+ t+ })+ .collect_vec(),+ ),+ Expr::Space(a) => constant_token!(offset, doc, a, TokenKind::Space(1)),+ Expr::Linebreak(a) => constant_token!(offset, doc, a, TokenKind::Newline(1)),+ Expr::Parbreak(a) => constant_token!(offset, doc, a, TokenKind::Newline(2)),+ Expr::Escape(_) => None,+ Expr::Shorthand(_) => None,+ Expr::SmartQuote(_) => None,+ Expr::Strong(strong) => recursive_env!(offset, strong, doc, parser),+ Expr::Emph(emph) => recursive_env!(offset, emph, doc, parser),+ Expr::Raw(_) => None,+ Expr::Link(a) => constant_token!(offset, doc, a, TokenKind::Url),+ Expr::Label(label) => Some(+ parser+ .parse_str(label.get())+ .into_iter()+ .map(|mut t| {+ t.span.push_by(*offset);+ t+ })+ .collect_vec(),+ ),+ Expr::Ref(a) => {+ constant_token!(offset, doc, a, TokenKind::Word(WordMetadata::default()))+ }+ Expr::Heading(heading) => recursive_env!(offset, heading, doc, parser),+ Expr::List(list_item) => recursive_env!(offset, list_item, doc, parser),+ Expr::Enum(enum_item) => recursive_env!(offset, enum_item, doc, parser),+ Expr::Term(term_item) => Some(+ term_item+ .term()+ .exprs()+ .chain(term_item.description().exprs())+ .filter_map(|e| map_token(e, doc, parser, offset))+ .flatten()+ .collect_vec(),+ ),+ Expr::Equation(a) => constant_token!(offset, doc, a, TokenKind::Unlintable),+ Expr::Math(_) => None,+ Expr::MathIdent(_) => None,+ Expr::MathShorthand(_) => None,+ Expr::MathAlignPoint(_) => None,+ Expr::MathDelimited(_) => None,+ Expr::MathAttach(_) => None,+ Expr::MathPrimes(_) => None,+ Expr::MathFrac(_) => None,+ Expr::MathRoot(_) => None,+ Expr::Ident(a) => constant_token!(offset, doc, a, TokenKind::Word(WordMetadata::default())),+ Expr::None(a) => constant_token!(offset, doc, a, TokenKind::Word(WordMetadata::default())),+ Expr::Auto(a) => constant_token!(offset, doc, a, TokenKind::Word(WordMetadata::default())),+ Expr::Bool(a) => constant_token!(offset, doc, a, TokenKind::Word(WordMetadata::default())),+ Expr::Int(int) => todo!(),+ Expr::Float(float) => todo!(),+ Expr::Numeric(a) => constant_token!(offset, doc, a, TokenKind::Unlintable),+ Expr::Str(text) => Some(+ parser+ .parse_str(text.get())+ .into_iter()+ .map(|mut t| {+ t.span.push_by(*offset);+ t+ })+ .collect_vec(),+ ),+ Expr::Code(a) => constant_token!(offset, doc, a, TokenKind::Unlintable),+ Expr::Content(content_block) => recursive_env!(offset, content_block, doc, parser),+ Expr::Parenthesized(parenthesized) => map_token(parenthesized.expr(), doc, parser, offset),+ Expr::Array(array) => Some(+ array+ .items()+ .filter_map(|i| {+ if let typst_syntax::ast::ArrayItem::Pos(e) = i {+ map_token(e, doc, parser, offset)+ } else {+ None+ }+ })+ .flatten()+ .collect_vec(),+ ),+ Expr::Dict(dict) => todo!(),+ Expr::Unary(unary) => todo!(),+ Expr::Binary(binary) => todo!(),+ Expr::FieldAccess(field_access) => todo!(),+ Expr::FuncCall(func_call) => todo!(),+ Expr::Closure(closure) => todo!(),+ Expr::Let(let_binding) => todo!(),+ Expr::DestructAssign(destruct_assignment) => todo!(),+ Expr::Set(set_rule) => todo!(),+ Expr::Show(show_rule) => todo!(),+ Expr::Contextual(contextual) => todo!(),+ Expr::Conditional(conditional) => todo!(),+ Expr::While(while_loop) => todo!(),+ Expr::For(for_loop) => todo!(),+ Expr::Import(module_import) => todo!(),+ Expr::Include(module_include) => todo!(),+ Expr::Break(loop_break) => todo!(),+ Expr::Continue(loop_continue) => todo!(),+ Expr::Return(func_return) => todo!(),+ }+}++impl Parser for Typst {+ fn parse(&mut self, source: &[char]) -> Vec<Token> {+ let mut english_parser = PlainEnglish;++ let source_str: String = source.iter().collect();+ let typst_document = typst_syntax::Source::detached(source_str);+ let typst_tree = typst_syntax::ast::Markup::from_untyped(typst_document.root())+ .expect("Unable to create typst document from parsed tree!");+ let mut offset = 0;++ // NOTE: the range spits out __byte__ indices, not char indices.+ // This is why we keep track above.+ typst_tree+ .exprs()+ .filter_map(|ex| map_token(ex, &typst_document, &mut english_parser, &mut offset))+ .flatten()+ .collect_vec()+ }+}