Offline, privacy-first grammar checker. Fast, open-source, Rust-powered
test(core): added property testing to hopefully resolve #373
Elijah Potter0ab718b150c7
diff --git a/Cargo.lock b/Cargo.lockindex b8c4bdf6e..dbf72369a 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -111,7 +111,7 @@ checksum = "1b1244b10dcd56c92219da4e14caa97e312079e185f04ba3eea25061561dc0a0"dependencies = ["proc-macro2","quote",- "syn",+ "syn 2.0.95",][[package]]@@ -122,7 +122,7 @@ checksum = "3c87f3f15e7794432337fc718554eaa4dc8f04c9677a950ffe366f20a162ae42"dependencies = ["proc-macro2","quote",- "syn",+ "syn 2.0.95",][[package]]@@ -166,7 +166,7 @@ checksum = "56791e4bd64c99fc361e01008f45c984baa93f12a0957d1b3c51dd2c6baab453"dependencies = ["proc-macro2","quote",- "syn",+ "syn 2.0.95",][[package]]@@ -263,7 +263,7 @@ dependencies = ["heck","proc-macro2","quote",- "syn",+ "syn 2.0.95",][[package]]@@ -415,7 +415,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0"dependencies = ["proc-macro2","quote",- "syn",+ "syn 2.0.95",][[package]]@@ -433,6 +433,16 @@ version = "1.13.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0"+[[package]]+name = "env_logger"+version = "0.8.4"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "a19187fea3ac7e84da7dacf48de0c45d63c6a76f9490dae389aead16c243fce3"+dependencies = [+ "log",+ "regex",+]+[[package]]name = "equivalent"version = "1.0.1"@@ -516,7 +526,7 @@ checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650"dependencies = ["proc-macro2","quote",- "syn",+ "syn 2.0.95",][[package]]@@ -643,6 +653,8 @@ dependencies = ["ordered-float","paste","pulldown-cmark",+ "quickcheck",+ "quickcheck_macros","rand","serde","serde_json",@@ -884,7 +896,7 @@ checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6"dependencies = ["proc-macro2","quote",- "syn",+ "syn 2.0.95",][[package]]@@ -936,7 +948,7 @@ dependencies = ["heck","proc-macro2","quote",- "syn",+ "syn 2.0.95",][[package]]@@ -1223,7 +1235,7 @@ checksum = "d56a66c0c55993aa927429d0f8a0abfd74f084e4d9c192cffed01e418d83eefb"dependencies = ["proc-macro2","quote",- "syn",+ "syn 2.0.95",][[package]]@@ -1281,6 +1293,28 @@ version = "0.11.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "007d8adb5ddab6f8e3f491ac63566a7d5002cc7ed73901f72057943fa71ae1ae"+[[package]]+name = "quickcheck"+version = "1.0.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "588f6378e4dd99458b60ec275b4477add41ce4fa9f64dcba6f15adccb19b50d6"+dependencies = [+ "env_logger",+ "log",+ "rand",+]++[[package]]+name = "quickcheck_macros"+version = "1.0.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "b22a693222d716a9587786f37ac3f6b4faedb5b80c23914e7303ff5a1d8016e9"+dependencies = [+ "proc-macro2",+ "quote",+ "syn 1.0.109",+]+[[package]]name = "quote"version = "1.0.38"@@ -1466,7 +1500,7 @@ checksum = "5a9bf7cf98d04a2b28aead066b7496853d4779c9cc183c440dbac457641e19a0"dependencies = ["proc-macro2","quote",- "syn",+ "syn 2.0.95",][[package]]@@ -1489,7 +1523,7 @@ checksum = "6c64451ba24fc7a6a2d60fc75dd9c83c90903b19028d4eff35e88fc1e86564e9"dependencies = ["proc-macro2","quote",- "syn",+ "syn 2.0.95",][[package]]@@ -1556,6 +1590,17 @@ version = "0.11.1"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f"+[[package]]+name = "syn"+version = "1.0.109"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237"+dependencies = [+ "proc-macro2",+ "quote",+ "unicode-ident",+]+[[package]]name = "syn"version = "2.0.95"@@ -1575,7 +1620,7 @@ checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971"dependencies = ["proc-macro2","quote",- "syn",+ "syn 2.0.95",][[package]]@@ -1610,7 +1655,7 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1"dependencies = ["proc-macro2","quote",- "syn",+ "syn 2.0.95",][[package]]@@ -1621,7 +1666,7 @@ checksum = "26afc1baea8a989337eeb52b6e72a039780ce45c3edfcc9c5b9d112feeb173c2"dependencies = ["proc-macro2","quote",- "syn",+ "syn 2.0.95",][[package]]@@ -1678,7 +1723,7 @@ checksum = "6e06d43f1345a3bcd39f6a56dbb7dcab2ba47e68e8ac134855e7e2bdbaf8cab8"dependencies = ["proc-macro2","quote",- "syn",+ "syn 2.0.95",][[package]]@@ -1779,7 +1824,7 @@ checksum = "84fd902d4e0b9a4b27f2f440108dc034e1758628a9b702f8ec61ad66355422fa"dependencies = ["proc-macro2","quote",- "syn",+ "syn 2.0.95",][[package]]@@ -1807,7 +1852,7 @@ checksum = "395ae124c09f9e6918a2310af6038fba074bcf474ac352496d5910dd59a2226d"dependencies = ["proc-macro2","quote",- "syn",+ "syn 2.0.95",][[package]]@@ -2204,7 +2249,7 @@ dependencies = ["log","proc-macro2","quote",- "syn",+ "syn 2.0.95","wasm-bindgen-shared",]@@ -2226,7 +2271,7 @@ checksum = "30d7a95b763d3c45903ed6c81f156801839e5ee968bb07e534c44df0fcd330c2"dependencies = ["proc-macro2","quote",- "syn",+ "syn 2.0.95","wasm-bindgen-backend","wasm-bindgen-shared",]@@ -2397,7 +2442,7 @@ checksum = "2380878cad4ac9aac1e2435f3eb4020e8374b5f13c296cb75b4620ff8e229154"dependencies = ["proc-macro2","quote",- "syn",+ "syn 2.0.95","synstructure",]@@ -2419,7 +2464,7 @@ checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e"dependencies = ["proc-macro2","quote",- "syn",+ "syn 2.0.95",][[package]]@@ -2439,7 +2484,7 @@ checksum = "595eed982f7d355beb85837f651fa22e90b3c044842dc7f2c2842c086f295808"dependencies = ["proc-macro2","quote",- "syn",+ "syn 2.0.95","synstructure",]@@ -2462,5 +2507,5 @@ checksum = "6eafa6dfb17584ea3e2bd6e76e0cc15ad7af12b09abdd1ca55961bed9b1063c6"dependencies = ["proc-macro2","quote",- "syn",+ "syn 2.0.95",]diff --git a/harper-core/Cargo.toml b/harper-core/Cargo.tomlindex e011929b8..9f2f9310e 100644--- a/harper-core/Cargo.toml+++ b/harper-core/Cargo.toml@@ -28,6 +28,8 @@ levenshtein_automata = { version = "0.2.1", features = ["fst_automaton"] }[dev-dependencies]criterion = { version = "0.5.1", default-features = false }rand = "0.8.5"+quickcheck = "1.0.3"+quickcheck_macros = "1.0.0"[[bench]]name = "parse_demo"diff --git a/harper-core/src/document.rs b/harper-core/src/document.rsindex 1741ee63c..a3f1da1bd 100644--- a/harper-core/src/document.rs+++ b/harper-core/src/document.rs@@ -496,6 +496,7 @@ macro_rules! create_fns_on_doc {impl TokenStringExt for Document {create_fns_on_doc!(word);+ create_fns_on_doc!(word_like);create_fns_on_doc!(conjunction);create_fns_on_doc!(space);create_fns_on_doc!(apostrophe);diff --git a/harper-core/src/lexing/mod.rs b/harper-core/src/lexing/mod.rsindex 04a7184e9..d5f0f74a0 100644--- a/harper-core/src/lexing/mod.rs+++ b/harper-core/src/lexing/mod.rs@@ -72,18 +72,21 @@ pub fn lex_number(source: &[char]) -> Option<FoundToken> {.rev().find_map(|(i, v)| v.is_numeric().then_some(i))?;- {- let s: String = source[0..end + 1].iter().collect();+ let mut s: String = source[0..end + 1].iter().collect();+ // Find the longest possible valid number+ while !s.is_empty() {if let Ok(n) = s.parse::<f64>() {return Some(FoundToken {token: TokenKind::Number(n.into(), None),- next_index: end + 1,+ next_index: s.len(),});}++ s.pop();}- lex_number(&source[0..end])+ None}fn lex_newlines(source: &[char]) -> Option<FoundToken> {diff --git a/harper-core/src/title_case.rs b/harper-core/src/title_case.rsindex 0c7ccd095..beda4e99c 100644--- a/harper-core/src/title_case.rs+++ b/harper-core/src/title_case.rs@@ -34,7 +34,7 @@ pub fn make_title_case(toks: &[Token], source: &[char], dict: &impl Dictionary)let start_index = toks.first().unwrap().span.start;- let mut words = toks.iter_words().enumerate().peekable();+ let mut words = toks.iter_word_likes().enumerate().peekable();let mut output = toks.span().unwrap().get_content(source).to_vec();// Only specific conjunctions are not capitalized.@@ -46,6 +46,10 @@ pub fn make_title_case(toks: &[Token], source: &[char], dict: &impl Dictionary)}while let Some((index, word)) = words.next() {+ if !word.kind.is_word() {+ continue;+ }+let chars = word.span.get_content(source);let chars_lower = chars.to_lower();@@ -82,8 +86,15 @@ pub fn make_title_case(toks: &[Token], source: &[char], dict: &impl Dictionary)#[cfg(test)]mod tests {++ use quickcheck::{Arbitrary, TestResult};+ use quickcheck_macros::quickcheck;+use super::make_title_case_str;- use crate::{parsers::PlainEnglish, FstDictionary};+ use crate::{+ parsers::{Markdown, PlainEnglish},+ FstDictionary, FullDictionary,+ };#[test]fn normal() {@@ -120,4 +131,113 @@ mod tests {"This Is a Test")}++ #[derive(Debug, Clone)]+ struct Word(String);++ impl Arbitrary for Word {+ fn arbitrary(g: &mut quickcheck::Gen) -> Self {+ let mut s = String::new();++ for _ in 0..g.size() {+ let c = loop {+ let gen = char::arbitrary(g);++ if gen.is_ascii_alphanumeric() {+ break gen;+ }+ };++ s.push(c);+ }++ Self(s)+ }+ }++ #[derive(Debug, Clone)]+ struct Sentence(String);++ /// Builds a sentence out of words from the curated [`FullDictionary`].+ impl Arbitrary for Sentence {+ fn arbitrary(g: &mut quickcheck::Gen) -> Self {+ let mut s = String::new();++ let Word(first_word) = Word::arbitrary(g);+ s.push_str(&first_word);++ for _ in 0..g.size() {+ let Word(word) = Word::arbitrary(g);++ s.push(' ');+ s.push_str(&word);+ }++ Self(s)+ }+ }++ #[quickcheck]+ fn a_stays_lowercase(prefix: String, postfix: String) -> TestResult {+ // There must be words other than the `a`.+ if prefix.chars().any(|c| !c.is_alphanumeric())+ || prefix.is_empty()+ || postfix.chars().any(|c| !c.is_alphanumeric())+ || postfix.is_empty()+ {+ return TestResult::discard();+ }++ let title_case: Vec<_> = make_title_case_str(+ &format!("{prefix} a {postfix}"),+ &mut Markdown,+ &FstDictionary::curated(),+ )+ .chars()+ .collect();++ dbg!(&title_case);++ TestResult::from_bool(title_case[prefix.chars().count() + 1] == 'a')+ }++ #[quickcheck]+ fn about_becomes_uppercase(prefix: String, postfix: String) -> TestResult {+ // There must be words other than the `a`.+ if prefix.chars().any(|c| !c.is_alphanumeric())+ || prefix.is_empty()+ || postfix.chars().any(|c| !c.is_alphanumeric())+ || postfix.is_empty()+ {+ return TestResult::discard();+ }++ let title_case: Vec<_> = make_title_case_str(+ &format!("{prefix} about {postfix}"),+ &mut Markdown,+ &FstDictionary::curated(),+ )+ .chars()+ .collect();++ TestResult::from_bool(title_case[prefix.chars().count() + 1] == 'A')+ }++ #[quickcheck]+ fn first_word_is_upcase(sentence: Sentence) -> TestResult {+ let title_case: Vec<_> =+ make_title_case_str(&sentence.0, &mut Markdown, &FstDictionary::curated())+ .chars()+ .collect();++ if let Some(first) = title_case.first() {+ if first.is_alphabetic() {+ TestResult::from_bool(first.is_uppercase())+ } else {+ TestResult::discard()+ }+ } else {+ TestResult::discard()+ }+ }}diff --git a/harper-core/src/token.rs b/harper-core/src/token.rsindex 4435173c8..f5e66c0ec 100644--- a/harper-core/src/token.rs+++ b/harper-core/src/token.rs@@ -79,6 +79,7 @@ pub trait TokenStringExt {fn span(&self) -> Option<Span>;create_decl_for!(word);+ create_decl_for!(word_like);create_decl_for!(conjunction);create_decl_for!(space);create_decl_for!(apostrophe);@@ -118,6 +119,7 @@ pub trait TokenStringExt {impl TokenStringExt for [Token] {create_fns_for!(word);+ create_fns_for!(word_like);create_fns_for!(conjunction);create_fns_for!(space);create_fns_for!(apostrophe);diff --git a/harper-core/src/token_kind.rs b/harper-core/src/token_kind.rsindex 015d82db1..c3145849d 100644--- a/harper-core/src/token_kind.rs+++ b/harper-core/src/token_kind.rs@@ -39,6 +39,18 @@ impl TokenKind {matches!(self, TokenKind::Punctuation(Punctuation::Pipe))}+ /// Checks whether a token is word-like--meaning it is more complex than punctuation and can+ /// hold semantic meaning in the way a word does.+ pub fn is_word_like(&self) -> bool {+ matches!(+ self,+ TokenKind::Word(..)+ | TokenKind::EmailAddress+ | TokenKind::Hostname+ | TokenKind::Number(..)+ )+ }+pub fn is_pronoun(&self) -> bool {matches!(self,