Skip to content

Offline, privacy-first grammar checker. Fast, open-source, Rust-powered

perf(test): improve test performance (#2473)

Steve Kurchd66c45932197
diff --git a/Cargo.lock b/Cargo.lock
index 3984ee8e2..212f02ab4 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1282,7 +1282,7 @@ dependencies = [
"half",
"log",
"serde",
- "sysinfo",
+ "sysinfo 0.36.1",
"tracel-llvm",
"tracel-llvm-bundler",
]
@@ -2964,6 +2964,7 @@ dependencies = [
"serde",
"serde-wasm-bindgen",
"serde_json",
+ "sysinfo 0.37.2",
"tracing",
"tracing-wasm",
"wasm-bindgen",
@@ -5219,6 +5220,20 @@ dependencies = [
"windows 0.61.3",
]
+[[package]]
+name = "sysinfo"
+version = "0.37.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "16607d5caffd1c07ce073528f9ed972d88db15dd44023fa57142963be3feb11f"
+dependencies = [
+ "libc",
+ "memchr",
+ "ntapi",
+ "objc2-core-foundation",
+ "objc2-io-kit",
+ "windows 0.61.3",
+]
+
[[package]]
name = "tar"
version = "0.4.44"
diff --git a/harper-core/src/document.rs b/harper-core/src/document.rs
index 0ee714afe..174366c7a 100644
--- a/harper-core/src/document.rs
+++ b/harper-core/src/document.rs
@@ -88,6 +88,19 @@ impl Document {
Self::new(text, &PlainEnglish, &FstDictionary::curated())
}
+ /// Create a new document simply by tokenizing the provided input and applying fix-ups. The
+ /// contained words will not contain any metadata.
+ ///
+ /// This avoids running potentially expensive metadata generation code, so this is more
+ /// efficient if you don't need that information.
+ pub(crate) fn new_basic_tokenize(text: &str, parser: &impl Parser) -> Self {
+ let source = Lrc::new(text.chars().collect_vec());
+ let tokens = parser.parse(&source);
+ let mut document = Self { source, tokens };
+ document.apply_fixups();
+ document
+ }
+
/// Parse text to produce a document using the built-in [`PlainEnglish`]
/// parser and a provided dictionary.
pub fn new_plain_english(text: &str, dictionary: &impl Dictionary) -> Self {
@@ -126,10 +139,7 @@ impl Document {
Self::new_markdown(text, MarkdownOptions::default(), dictionary)
}
- /// Re-parse important language constructs.
- ///
- /// Should be run after every change to the underlying [`Self::source`].
- fn parse(&mut self, dictionary: &impl Dictionary) {
+ fn apply_fixups(&mut self) {
self.condense_spaces();
self.condense_newlines();
self.newlines_to_breaks();
@@ -142,6 +152,13 @@ impl Document {
self.condense_ampersand_pairs();
self.condense_slash_pairs();
self.match_quotes();
+ }
+
+ /// Re-parse important language constructs.
+ ///
+ /// Should be run after every change to the underlying [`Self::source`].
+ fn parse(&mut self, dictionary: &impl Dictionary) {
+ self.apply_fixups();
let chunker = burn_chunker();
let tagger = brill_tagger();
diff --git a/harper-core/src/expr/fixed_phrase.rs b/harper-core/src/expr/fixed_phrase.rs
index 6a3844544..2a3fc6452 100644
--- a/harper-core/src/expr/fixed_phrase.rs
+++ b/harper-core/src/expr/fixed_phrase.rs
@@ -1,3 +1,4 @@
+use crate::parsers::PlainEnglish;
use crate::patterns::Word;
use crate::{Document, Span, Token, TokenKind};
@@ -24,7 +25,7 @@ impl FixedPhrase {
/// Creates a [`FixedPhrase`] from a plaintext string.
/// Uses plain English tokenization rules.
pub fn from_phrase(text: &str) -> Self {
- let document = Document::new_plain_english_curated(text);
+ let document = Document::new_basic_tokenize(text, &PlainEnglish);
Self::from_document(&document)
}
diff --git a/harper-core/src/linting/proper_noun_capitalization_linters.rs b/harper-core/src/linting/proper_noun_capitalization_linters.rs
index 665403e7f..4ef6f9a83 100644
--- a/harper-core/src/linting/proper_noun_capitalization_linters.rs
+++ b/harper-core/src/linting/proper_noun_capitalization_linters.rs
@@ -22,37 +22,19 @@ pub struct ProperNounCapitalizationLinter<D: Dictionary + 'static> {
}
impl<D: Dictionary + 'static> ProperNounCapitalizationLinter<D> {
- /// Wrapper function around [`Self::new`] that allows construction with Strings.
+ /// Create a linter that corrects the capitalization of phrases provided.
pub fn new_strs(
canonical_versions: impl IntoIterator<Item = impl AsRef<str>>,
description: impl ToString,
dictionary: D,
- ) -> Self {
- Self::new(
- canonical_versions
- .into_iter()
- .map(|s| s.as_ref().chars().collect::<Vec<_>>()),
- description,
- dictionary,
- )
- }
-
- /// Create a linter that corrects the capitalization of phrases provided.
- pub fn new(
- canonical_versions: impl IntoIterator<Item = impl AsRef<[char]>>,
- description: impl ToString,
- dictionary: D,
) -> Self {
let dictionary = Arc::new(dictionary);
let mut expr_map = ExprMap::default();
for can_vers in canonical_versions {
- let doc = Document::new_from_vec(
- can_vers.as_ref().to_vec().into(),
- &PlainEnglish,
- &dictionary,
- );
+ let doc = Document::new_basic_tokenize(can_vers.as_ref(), &PlainEnglish);
+
let expr = FixedPhrase::from_document(&doc);
expr_map.insert(expr, doc);
diff --git a/harper-wasm/Cargo.toml b/harper-wasm/Cargo.toml
index 71abd017f..7817c6c54 100644
--- a/harper-wasm/Cargo.toml
+++ b/harper-wasm/Cargo.toml
@@ -20,3 +20,6 @@ serde_json = "1.0.149"
serde = { version = "1.0.228", features = ["derive"] }
getrandom = { version = "0.3.4", default-features = false, features = ["wasm_js"] }
harper-stats = { path = "../harper-stats", version = "1.0.0", features = ["js"] }
+
+[dev-dependencies]
+sysinfo = { version = "0.37.2", default-features = false, features = [ "system" ] }
diff --git a/harper-wasm/src/lib.rs b/harper-wasm/src/lib.rs
index a2466a66b..a4ac9e5b6 100644
--- a/harper-wasm/src/lib.rs
+++ b/harper-wasm/src/lib.rs
@@ -645,6 +645,20 @@ pub struct OrganizedGroup {
#[cfg(test)]
mod tests {
use super::*;
+ use sysinfo::{Pid, ProcessRefreshKind, ProcessesToUpdate, System};
+
+ /// Get memory usage for the process with the given PID, in bytes.
+ ///
+ /// This will fail if the requested process does not exist.
+ #[must_use]
+ fn get_mem_usage_of_process(sys: &mut System, pid: Pid) -> Option<u64> {
+ sys.refresh_processes_specifics(
+ ProcessesToUpdate::Some(&[pid]),
+ false,
+ ProcessRefreshKind::nothing().with_memory(),
+ );
+ sys.process(pid).map(|process| process.memory())
+ }
/// If a word from another dialect is added to the user dictionary, it should be considered
/// part of the user's dialect as well.
@@ -661,17 +675,32 @@ mod tests {
}
#[test]
- fn does_not_oom_with_repeated_lints() {
- for _ in 0..3000 {
- let mut linter = Linter::new(Dialect::American);
+ fn no_memory_leak_with_repeated_lints() {
+ let pid = Pid::from_u32(std::process::id());
+ let mut sys = System::new();
- let results = linter.lint(
- "This is a grammatically correct sentence.".to_string(),
- Language::Plain,
- false,
- );
+ let mut prev_memory_usage = get_mem_usage_of_process(&mut sys, pid).unwrap();
- assert!(results.is_empty())
+ if (0..10).all(|_| {
+ // Run a few times.
+ for _ in 0..10 {
+ let mut linter = Linter::new(Dialect::American);
+
+ let results = linter.lint(
+ "This is a grammatically correct sentence.".to_string(),
+ Language::Plain,
+ false,
+ );
+
+ assert!(results.is_empty())
+ }
+ // Check if our process' memory usage increased.
+ let curr_memory_usage = get_mem_usage_of_process(&mut sys, pid).unwrap();
+ let memory_usage_increased = curr_memory_usage > prev_memory_usage;
+ prev_memory_usage = curr_memory_usage;
+ memory_usage_increased
+ }) {
+ panic!("Memory leak!");
}
}
}