Skip to content

Offline, privacy-first grammar checker. Fast, open-source, Rust-powered

feat(#230): add some tests for dictionary parsing, and improve dict parsing to fit better

Grant Lemons24e055182a7f
diff --git a/harper-core/src/parsers/typst.rs b/harper-core/src/parsers/typst.rs
index 5d2b235e2..2beac0afa 100644
--- a/harper-core/src/parsers/typst.rs
+++ b/harper-core/src/parsers/typst.rs
@@ -53,15 +53,17 @@ fn parse_english(
parser: &mut PlainEnglish,
span: &typst_syntax::Span,
) -> Option<Vec<Token>> {
- let res = parser
- .parse_str(str.into())
- .into_iter()
- .map(|mut t| {
- t.span.push_by(doc.range(*span).unwrap().start);
- t
- })
- .collect_vec();
- Some(res)
+ let offset = doc.range(*span).unwrap().start;
+ Some(
+ parser
+ .parse_str(str.into())
+ .into_iter()
+ .map(|mut t| {
+ t.span.push_by(offset);
+ t
+ })
+ .collect_vec(),
+ )
}
fn parse_dict(
@@ -73,8 +75,7 @@ fn parse_dict(
dict.filter_map(|di| match di {
typst_syntax::ast::DictItem::Named(named) => merge_expr!(
constant_token!(doc, named.name(), TokenKind::Word(WordMetadata::default())),
- map_token(named.expr(), doc, parser),
- parse_pattern(named.pattern(), doc, parser)
+ map_token(named.expr(), doc, parser)
),
typst_syntax::ast::DictItem::Keyed(keyed) => merge_expr!(
map_token(keyed.key(), doc, parser),
@@ -241,7 +242,15 @@ fn map_token(
),
Expr::FuncCall(a) => constant_token!(doc, a, TokenKind::Unlintable),
Expr::Closure(a) => constant_token!(doc, a, TokenKind::Unlintable),
- Expr::Let(let_binding) => let_binding.init().and_then(|e| map_token(e, doc, parser)),
+ Expr::Let(let_binding) => merge_expr!(
+ match let_binding.kind() {
+ typst_syntax::ast::LetBindingKind::Normal(pattern) =>
+ parse_pattern(pattern, doc, parser),
+ typst_syntax::ast::LetBindingKind::Closure(ident) =>
+ constant_token!(doc, ident, TokenKind::Word(WordMetadata::default())),
+ },
+ let_binding.init().and_then(|e| map_token(e, doc, parser))
+ ),
Expr::DestructAssign(destruct_assignment) => {
map_token(destruct_assignment.value(), doc, parser)
}
@@ -354,6 +363,7 @@ impl Parser for Typst {
#[cfg(test)]
mod tests {
+ use itertools::Itertools;
use ordered_float::OrderedFloat;
use super::Typst;
@@ -361,12 +371,10 @@ mod tests {
#[test]
fn conjunction() {
- let source = r"doesn't";
+ let source = "doesn't";
let tokens = Typst.parse_str(source);
-
let token_kinds = tokens.iter().map(|t| t.kind).collect::<Vec<_>>();
-
dbg!(&token_kinds);
assert_eq!(token_kinds.len(), 1);
@@ -375,12 +383,10 @@ mod tests {
#[test]
fn possessive() {
- let source = r"person's";
+ let source = "person's";
let tokens = Typst.parse_str(source);
-
let token_kinds = tokens.iter().map(|t| t.kind).collect::<Vec<_>>();
-
dbg!(&token_kinds);
assert_eq!(token_kinds.len(), 1);
@@ -400,12 +406,10 @@ mod tests {
#[test]
fn number() {
- let source = r"12 is larger than 11, but much less than 11!";
+ let source = "12 is larger than 11, but much less than 11!";
let tokens = Typst.parse_str(source);
-
let token_kinds = tokens.iter().map(|t| t.kind).collect::<Vec<_>>();
-
dbg!(&token_kinds);
assert!(matches!(
@@ -438,12 +442,10 @@ mod tests {
#[test]
fn math_unlintable() {
- let source = r"$12 > 11$, $12 << 11!$";
+ let source = "$12 > 11$, $12 << 11!$";
let tokens = Typst.parse_str(source);
-
let token_kinds = tokens.iter().map(|t| t.kind).collect::<Vec<_>>();
-
dbg!(&token_kinds);
assert!(matches!(
@@ -458,13 +460,45 @@ mod tests {
}
#[test]
- fn sentence() {
- let source = r"This is a sentence, it does not have any particularly interesting elements of the typst syntax.";
+ fn dict_parsing() {
+ let source = r#"#let dict = (
+ name: "Typst",
+ born: 2019,
+ )"#;
let tokens = Typst.parse_str(source);
-
let token_kinds = tokens.iter().map(|t| t.kind).collect::<Vec<_>>();
+ dbg!(&token_kinds);
+
+ let typst_document = typst_syntax::Source::detached(source);
+ let typst_tree = <typst_syntax::ast::Markup as typst_syntax::ast::AstNode>::from_untyped(
+ typst_document.root(),
+ )
+ .expect("Unable to create typst document from parsed tree!");
+ dbg!(typst_tree.exprs().collect_vec());
+ let charslice = source.chars().collect_vec();
+ assert_eq!(tokens[2].span.get_content_string(&charslice), "Typst");
+ assert!(matches!(
+ token_kinds.as_slice(),
+ &[
+ TokenKind::Word(_),
+ TokenKind::Word(_),
+ TokenKind::Punctuation(Punctuation::Quote { .. }),
+ TokenKind::Word(_),
+ TokenKind::Punctuation(Punctuation::Quote { .. }),
+ TokenKind::Word(_),
+ TokenKind::Number(OrderedFloat(2019.0), None),
+ ]
+ ))
+ }
+
+ #[test]
+ fn sentence() {
+ let source = "This is a sentence, it does not have any particularly interesting elements of the typst syntax.";
+
+ let tokens = Typst.parse_str(source);
+ let token_kinds = tokens.iter().map(|t| t.kind).collect::<Vec<_>>();
dbg!(&token_kinds);
assert!(matches!(