Skip to content

Offline, privacy-first grammar checker. Fast, open-source, Rust-powered

test(core): added property testing to hopefully resolve #373

Elijah Potter0ab718b150c7
diff --git a/Cargo.lock b/Cargo.lock
index b8c4bdf6e..dbf72369a 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -111,7 +111,7 @@ checksum = "1b1244b10dcd56c92219da4e14caa97e312079e185f04ba3eea25061561dc0a0"
dependencies = [
"proc-macro2",
"quote",
- "syn",
+ "syn 2.0.95",
]
[[package]]
@@ -122,7 +122,7 @@ checksum = "3c87f3f15e7794432337fc718554eaa4dc8f04c9677a950ffe366f20a162ae42"
dependencies = [
"proc-macro2",
"quote",
- "syn",
+ "syn 2.0.95",
]
[[package]]
@@ -166,7 +166,7 @@ checksum = "56791e4bd64c99fc361e01008f45c984baa93f12a0957d1b3c51dd2c6baab453"
dependencies = [
"proc-macro2",
"quote",
- "syn",
+ "syn 2.0.95",
]
[[package]]
@@ -263,7 +263,7 @@ dependencies = [
"heck",
"proc-macro2",
"quote",
- "syn",
+ "syn 2.0.95",
]
[[package]]
@@ -415,7 +415,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0"
dependencies = [
"proc-macro2",
"quote",
- "syn",
+ "syn 2.0.95",
]
[[package]]
@@ -433,6 +433,16 @@ version = "1.13.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0"
+[[package]]
+name = "env_logger"
+version = "0.8.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a19187fea3ac7e84da7dacf48de0c45d63c6a76f9490dae389aead16c243fce3"
+dependencies = [
+ "log",
+ "regex",
+]
+
[[package]]
name = "equivalent"
version = "1.0.1"
@@ -516,7 +526,7 @@ checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650"
dependencies = [
"proc-macro2",
"quote",
- "syn",
+ "syn 2.0.95",
]
[[package]]
@@ -643,6 +653,8 @@ dependencies = [
"ordered-float",
"paste",
"pulldown-cmark",
+ "quickcheck",
+ "quickcheck_macros",
"rand",
"serde",
"serde_json",
@@ -884,7 +896,7 @@ checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6"
dependencies = [
"proc-macro2",
"quote",
- "syn",
+ "syn 2.0.95",
]
[[package]]
@@ -936,7 +948,7 @@ dependencies = [
"heck",
"proc-macro2",
"quote",
- "syn",
+ "syn 2.0.95",
]
[[package]]
@@ -1223,7 +1235,7 @@ checksum = "d56a66c0c55993aa927429d0f8a0abfd74f084e4d9c192cffed01e418d83eefb"
dependencies = [
"proc-macro2",
"quote",
- "syn",
+ "syn 2.0.95",
]
[[package]]
@@ -1281,6 +1293,28 @@ version = "0.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "007d8adb5ddab6f8e3f491ac63566a7d5002cc7ed73901f72057943fa71ae1ae"
+[[package]]
+name = "quickcheck"
+version = "1.0.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "588f6378e4dd99458b60ec275b4477add41ce4fa9f64dcba6f15adccb19b50d6"
+dependencies = [
+ "env_logger",
+ "log",
+ "rand",
+]
+
+[[package]]
+name = "quickcheck_macros"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b22a693222d716a9587786f37ac3f6b4faedb5b80c23914e7303ff5a1d8016e9"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 1.0.109",
+]
+
[[package]]
name = "quote"
version = "1.0.38"
@@ -1466,7 +1500,7 @@ checksum = "5a9bf7cf98d04a2b28aead066b7496853d4779c9cc183c440dbac457641e19a0"
dependencies = [
"proc-macro2",
"quote",
- "syn",
+ "syn 2.0.95",
]
[[package]]
@@ -1489,7 +1523,7 @@ checksum = "6c64451ba24fc7a6a2d60fc75dd9c83c90903b19028d4eff35e88fc1e86564e9"
dependencies = [
"proc-macro2",
"quote",
- "syn",
+ "syn 2.0.95",
]
[[package]]
@@ -1556,6 +1590,17 @@ version = "0.11.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f"
+[[package]]
+name = "syn"
+version = "1.0.109"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "unicode-ident",
+]
+
[[package]]
name = "syn"
version = "2.0.95"
@@ -1575,7 +1620,7 @@ checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971"
dependencies = [
"proc-macro2",
"quote",
- "syn",
+ "syn 2.0.95",
]
[[package]]
@@ -1610,7 +1655,7 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1"
dependencies = [
"proc-macro2",
"quote",
- "syn",
+ "syn 2.0.95",
]
[[package]]
@@ -1621,7 +1666,7 @@ checksum = "26afc1baea8a989337eeb52b6e72a039780ce45c3edfcc9c5b9d112feeb173c2"
dependencies = [
"proc-macro2",
"quote",
- "syn",
+ "syn 2.0.95",
]
[[package]]
@@ -1678,7 +1723,7 @@ checksum = "6e06d43f1345a3bcd39f6a56dbb7dcab2ba47e68e8ac134855e7e2bdbaf8cab8"
dependencies = [
"proc-macro2",
"quote",
- "syn",
+ "syn 2.0.95",
]
[[package]]
@@ -1779,7 +1824,7 @@ checksum = "84fd902d4e0b9a4b27f2f440108dc034e1758628a9b702f8ec61ad66355422fa"
dependencies = [
"proc-macro2",
"quote",
- "syn",
+ "syn 2.0.95",
]
[[package]]
@@ -1807,7 +1852,7 @@ checksum = "395ae124c09f9e6918a2310af6038fba074bcf474ac352496d5910dd59a2226d"
dependencies = [
"proc-macro2",
"quote",
- "syn",
+ "syn 2.0.95",
]
[[package]]
@@ -2204,7 +2249,7 @@ dependencies = [
"log",
"proc-macro2",
"quote",
- "syn",
+ "syn 2.0.95",
"wasm-bindgen-shared",
]
@@ -2226,7 +2271,7 @@ checksum = "30d7a95b763d3c45903ed6c81f156801839e5ee968bb07e534c44df0fcd330c2"
dependencies = [
"proc-macro2",
"quote",
- "syn",
+ "syn 2.0.95",
"wasm-bindgen-backend",
"wasm-bindgen-shared",
]
@@ -2397,7 +2442,7 @@ checksum = "2380878cad4ac9aac1e2435f3eb4020e8374b5f13c296cb75b4620ff8e229154"
dependencies = [
"proc-macro2",
"quote",
- "syn",
+ "syn 2.0.95",
"synstructure",
]
@@ -2419,7 +2464,7 @@ checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e"
dependencies = [
"proc-macro2",
"quote",
- "syn",
+ "syn 2.0.95",
]
[[package]]
@@ -2439,7 +2484,7 @@ checksum = "595eed982f7d355beb85837f651fa22e90b3c044842dc7f2c2842c086f295808"
dependencies = [
"proc-macro2",
"quote",
- "syn",
+ "syn 2.0.95",
"synstructure",
]
@@ -2462,5 +2507,5 @@ checksum = "6eafa6dfb17584ea3e2bd6e76e0cc15ad7af12b09abdd1ca55961bed9b1063c6"
dependencies = [
"proc-macro2",
"quote",
- "syn",
+ "syn 2.0.95",
]
diff --git a/harper-core/Cargo.toml b/harper-core/Cargo.toml
index e011929b8..9f2f9310e 100644
--- a/harper-core/Cargo.toml
+++ b/harper-core/Cargo.toml
@@ -28,6 +28,8 @@ levenshtein_automata = { version = "0.2.1", features = ["fst_automaton"] }
[dev-dependencies]
criterion = { version = "0.5.1", default-features = false }
rand = "0.8.5"
+quickcheck = "1.0.3"
+quickcheck_macros = "1.0.0"
[[bench]]
name = "parse_demo"
diff --git a/harper-core/src/document.rs b/harper-core/src/document.rs
index 1741ee63c..a3f1da1bd 100644
--- a/harper-core/src/document.rs
+++ b/harper-core/src/document.rs
@@ -496,6 +496,7 @@ macro_rules! create_fns_on_doc {
impl TokenStringExt for Document {
create_fns_on_doc!(word);
+ create_fns_on_doc!(word_like);
create_fns_on_doc!(conjunction);
create_fns_on_doc!(space);
create_fns_on_doc!(apostrophe);
diff --git a/harper-core/src/lexing/mod.rs b/harper-core/src/lexing/mod.rs
index 04a7184e9..d5f0f74a0 100644
--- a/harper-core/src/lexing/mod.rs
+++ b/harper-core/src/lexing/mod.rs
@@ -72,18 +72,21 @@ pub fn lex_number(source: &[char]) -> Option<FoundToken> {
.rev()
.find_map(|(i, v)| v.is_numeric().then_some(i))?;
- {
- let s: String = source[0..end + 1].iter().collect();
+ let mut s: String = source[0..end + 1].iter().collect();
+ // Find the longest possible valid number
+ while !s.is_empty() {
if let Ok(n) = s.parse::<f64>() {
return Some(FoundToken {
token: TokenKind::Number(n.into(), None),
- next_index: end + 1,
+ next_index: s.len(),
});
}
+
+ s.pop();
}
- lex_number(&source[0..end])
+ None
}
fn lex_newlines(source: &[char]) -> Option<FoundToken> {
diff --git a/harper-core/src/title_case.rs b/harper-core/src/title_case.rs
index 0c7ccd095..beda4e99c 100644
--- a/harper-core/src/title_case.rs
+++ b/harper-core/src/title_case.rs
@@ -34,7 +34,7 @@ pub fn make_title_case(toks: &[Token], source: &[char], dict: &impl Dictionary)
let start_index = toks.first().unwrap().span.start;
- let mut words = toks.iter_words().enumerate().peekable();
+ let mut words = toks.iter_word_likes().enumerate().peekable();
let mut output = toks.span().unwrap().get_content(source).to_vec();
// Only specific conjunctions are not capitalized.
@@ -46,6 +46,10 @@ pub fn make_title_case(toks: &[Token], source: &[char], dict: &impl Dictionary)
}
while let Some((index, word)) = words.next() {
+ if !word.kind.is_word() {
+ continue;
+ }
+
let chars = word.span.get_content(source);
let chars_lower = chars.to_lower();
@@ -82,8 +86,15 @@ pub fn make_title_case(toks: &[Token], source: &[char], dict: &impl Dictionary)
#[cfg(test)]
mod tests {
+
+ use quickcheck::{Arbitrary, TestResult};
+ use quickcheck_macros::quickcheck;
+
use super::make_title_case_str;
- use crate::{parsers::PlainEnglish, FstDictionary};
+ use crate::{
+ parsers::{Markdown, PlainEnglish},
+ FstDictionary, FullDictionary,
+ };
#[test]
fn normal() {
@@ -120,4 +131,113 @@ mod tests {
"This Is a Test"
)
}
+
+ #[derive(Debug, Clone)]
+ struct Word(String);
+
+ impl Arbitrary for Word {
+ fn arbitrary(g: &mut quickcheck::Gen) -> Self {
+ let mut s = String::new();
+
+ for _ in 0..g.size() {
+ let c = loop {
+ let gen = char::arbitrary(g);
+
+ if gen.is_ascii_alphanumeric() {
+ break gen;
+ }
+ };
+
+ s.push(c);
+ }
+
+ Self(s)
+ }
+ }
+
+ #[derive(Debug, Clone)]
+ struct Sentence(String);
+
+ /// Builds a sentence out of words from the curated [`FullDictionary`].
+ impl Arbitrary for Sentence {
+ fn arbitrary(g: &mut quickcheck::Gen) -> Self {
+ let mut s = String::new();
+
+ let Word(first_word) = Word::arbitrary(g);
+ s.push_str(&first_word);
+
+ for _ in 0..g.size() {
+ let Word(word) = Word::arbitrary(g);
+
+ s.push(' ');
+ s.push_str(&word);
+ }
+
+ Self(s)
+ }
+ }
+
+ #[quickcheck]
+ fn a_stays_lowercase(prefix: String, postfix: String) -> TestResult {
+ // There must be words other than the `a`.
+ if prefix.chars().any(|c| !c.is_alphanumeric())
+ || prefix.is_empty()
+ || postfix.chars().any(|c| !c.is_alphanumeric())
+ || postfix.is_empty()
+ {
+ return TestResult::discard();
+ }
+
+ let title_case: Vec<_> = make_title_case_str(
+ &format!("{prefix} a {postfix}"),
+ &mut Markdown,
+ &FstDictionary::curated(),
+ )
+ .chars()
+ .collect();
+
+ dbg!(&title_case);
+
+ TestResult::from_bool(title_case[prefix.chars().count() + 1] == 'a')
+ }
+
+ #[quickcheck]
+ fn about_becomes_uppercase(prefix: String, postfix: String) -> TestResult {
+ // There must be words other than the `a`.
+ if prefix.chars().any(|c| !c.is_alphanumeric())
+ || prefix.is_empty()
+ || postfix.chars().any(|c| !c.is_alphanumeric())
+ || postfix.is_empty()
+ {
+ return TestResult::discard();
+ }
+
+ let title_case: Vec<_> = make_title_case_str(
+ &format!("{prefix} about {postfix}"),
+ &mut Markdown,
+ &FstDictionary::curated(),
+ )
+ .chars()
+ .collect();
+
+ TestResult::from_bool(title_case[prefix.chars().count() + 1] == 'A')
+ }
+
+ #[quickcheck]
+ fn first_word_is_upcase(sentence: Sentence) -> TestResult {
+ let title_case: Vec<_> =
+ make_title_case_str(&sentence.0, &mut Markdown, &FstDictionary::curated())
+ .chars()
+ .collect();
+
+ if let Some(first) = title_case.first() {
+ if first.is_alphabetic() {
+ TestResult::from_bool(first.is_uppercase())
+ } else {
+ TestResult::discard()
+ }
+ } else {
+ TestResult::discard()
+ }
+ }
}
diff --git a/harper-core/src/token.rs b/harper-core/src/token.rs
index 4435173c8..f5e66c0ec 100644
--- a/harper-core/src/token.rs
+++ b/harper-core/src/token.rs
@@ -79,6 +79,7 @@ pub trait TokenStringExt {
fn span(&self) -> Option<Span>;
create_decl_for!(word);
+ create_decl_for!(word_like);
create_decl_for!(conjunction);
create_decl_for!(space);
create_decl_for!(apostrophe);
@@ -118,6 +119,7 @@ pub trait TokenStringExt {
impl TokenStringExt for [Token] {
create_fns_for!(word);
+ create_fns_for!(word_like);
create_fns_for!(conjunction);
create_fns_for!(space);
create_fns_for!(apostrophe);
diff --git a/harper-core/src/token_kind.rs b/harper-core/src/token_kind.rs
index 015d82db1..c3145849d 100644
--- a/harper-core/src/token_kind.rs
+++ b/harper-core/src/token_kind.rs
@@ -39,6 +39,18 @@ impl TokenKind {
matches!(self, TokenKind::Punctuation(Punctuation::Pipe))
}
+ /// Checks whether a token is word-like--meaning it is more complex than punctuation and can
+ /// hold semantic meaning in the way a word does.
+ pub fn is_word_like(&self) -> bool {
+ matches!(
+ self,
+ TokenKind::Word(..)
+ | TokenKind::EmailAddress
+ | TokenKind::Hostname
+ | TokenKind::Number(..)
+ )
+ }
+
pub fn is_pronoun(&self) -> bool {
matches!(
self,