Skip to content

Offline, privacy-first grammar checker. Fast, open-source, Rust-powered

fix(harper.js): create disposal function and use it (#2445)

Elijah Pottereab8bba001ce
diff --git a/Cargo.lock b/Cargo.lock
index 943c45451..2b3139d0a 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -3372,9 +3372,9 @@ dependencies = [
[[package]]
name = "js-sys"
-version = "0.3.77"
+version = "0.3.83"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1cfaf33c695fc6e08064efbc1f72ec937429614f25eef83af942d0e227c3a28f"
+checksum = "464a3709c7f55f1f721e5389aa6ea4e3bc6aba669353300af094b29ffbdde1d8"
dependencies = [
"once_cell",
"wasm-bindgen",
@@ -6336,35 +6336,22 @@ dependencies = [
[[package]]
name = "wasm-bindgen"
-version = "0.2.100"
+version = "0.2.106"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1edc8929d7499fc4e8f0be2262a241556cfc54a0bea223790e71446f2aab1ef5"
+checksum = "0d759f433fa64a2d763d1340820e46e111a7a5ab75f993d1852d70b03dbb80fd"
dependencies = [
"cfg-if",
"once_cell",
"rustversion",
"wasm-bindgen-macro",
-]
-
-[[package]]
-name = "wasm-bindgen-backend"
-version = "0.2.100"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2f0a0651a5c2bc21487bde11ee802ccaf4c51935d0d3d42a6101f98161700bc6"
-dependencies = [
- "bumpalo",
- "log",
- "proc-macro2",
- "quote",
- "syn",
"wasm-bindgen-shared",
]
[[package]]
name = "wasm-bindgen-futures"
-version = "0.4.50"
+version = "0.4.56"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "555d470ec0bc3bb57890405e5d4322cc9ea83cebb085523ced7be4144dac1e61"
+checksum = "836d9622d604feee9e5de25ac10e3ea5f2d65b41eac0d9ce72eb5deae707ce7c"
dependencies = [
"cfg-if",
"js-sys",
@@ -6375,9 +6362,9 @@ dependencies = [
[[package]]
name = "wasm-bindgen-macro"
-version = "0.2.100"
+version = "0.2.106"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7fe63fc6d09ed3792bd0897b314f53de8e16568c2b3f7982f468c0bf9bd0b407"
+checksum = "48cb0d2638f8baedbc542ed444afc0644a29166f1595371af4fecf8ce1e7eeb3"
dependencies = [
"quote",
"wasm-bindgen-macro-support",
@@ -6385,31 +6372,31 @@ dependencies = [
[[package]]
name = "wasm-bindgen-macro-support"
-version = "0.2.100"
+version = "0.2.106"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de"
+checksum = "cefb59d5cd5f92d9dcf80e4683949f15ca4b511f4ac0a6e14d4e1ac60c6ecd40"
dependencies = [
+ "bumpalo",
"proc-macro2",
"quote",
"syn",
- "wasm-bindgen-backend",
"wasm-bindgen-shared",
]
[[package]]
name = "wasm-bindgen-shared"
-version = "0.2.100"
+version = "0.2.106"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1a05d73b933a847d6cccdda8f838a22ff101ad9bf93e33684f39c1f5f0eece3d"
+checksum = "cbc538057e648b67f72a982e708d485b2efa771e1ac05fec311f9f63e5800db4"
dependencies = [
"unicode-ident",
]
[[package]]
name = "web-sys"
-version = "0.3.77"
+version = "0.3.83"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "33b6dd2ef9186f1f2072e409e99cd22a975331a6b3591b12c764e0e55c60d5d2"
+checksum = "9b32828d774c412041098d182a8b38b16ea816958e07cf40eec2bc080ae137ac"
dependencies = [
"js-sys",
"wasm-bindgen",
diff --git a/harper-wasm/src/lib.rs b/harper-wasm/src/lib.rs
index e71eb9382..a2466a66b 100644
--- a/harper-wasm/src/lib.rs
+++ b/harper-wasm/src/lib.rs
@@ -659,4 +659,19 @@ mod tests {
let lints = linter.lint(text, Language::Plain, false);
assert!(lints.is_empty());
}
+
+ #[test]
+ fn does_not_oom_with_repeated_lints() {
+ for _ in 0..3000 {
+ let mut linter = Linter::new(Dialect::American);
+
+ let results = linter.lint(
+ "This is a grammatically correct sentence.".to_string(),
+ Language::Plain,
+ false,
+ );
+
+ assert!(results.is_empty())
+ }
+ }
}
diff --git a/packages/chrome-plugin/src/background/index.ts b/packages/chrome-plugin/src/background/index.ts
index df10e5f76..dcc068129 100644
--- a/packages/chrome-plugin/src/background/index.ts
+++ b/packages/chrome-plugin/src/background/index.ts
@@ -394,6 +394,10 @@ async function setActivationKey(key: ActivationKey) {
}
function initializeLinter(dialect: Dialect) {
+ if (linter != null) {
+ linter.dispose();
+ }
+
linter = new LocalLinter({
binary: BinaryModule.create(chrome.runtime.getURL('./wasm/harper_wasm_bg.wasm')),
dialect,
diff --git a/packages/harper.js/src/Linter.test.ts b/packages/harper.js/src/Linter.test.ts
index 7835b3694..6bfd3fd5f 100644
--- a/packages/harper.js/src/Linter.test.ts
+++ b/packages/harper.js/src/Linter.test.ts
@@ -3,6 +3,15 @@ import { binary } from './binary';
import LocalLinter from './LocalLinter';
import WorkerLinter from './WorkerLinter';
+function randomString(length: number): string {
+ const chars = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz';
+ let result = '';
+ for (let i = 0; i < length; i++) {
+ result += chars.charAt(Math.floor(Math.random() * chars.length));
+ }
+ return result;
+}
+
const linters = {
WorkerLinter: WorkerLinter,
LocalLinter: LocalLinter,
@@ -15,6 +24,8 @@ for (const [linterName, Linter] of Object.entries(linters)) {
const lints = await linter.lint('The the problem is...');
expect(lints.length).toBe(1);
+
+ await linter.dispose();
});
test(`${linterName} emits organized lints the same as it emits normal lints`, async () => {
@@ -37,6 +48,8 @@ for (const [linterName, Linter] of Object.entries(linters)) {
const item = flattened[0];
expect(item.message().length).not.toBe(0);
+
+ await linter.dispose();
});
test(`${linterName} detects repeated words with multiple synchronous requests`, async () => {
@@ -54,6 +67,8 @@ for (const [linterName, Linter] of Object.entries(linters)) {
expect(results[0][0].suggestions().length).toBe(1);
expect(results[1].length).toBe(0);
expect(results[2].length).toBe(1);
+
+ await linter.dispose();
});
test(`${linterName} detects repeated words with concurrent requests`, async () => {
@@ -71,6 +86,8 @@ for (const [linterName, Linter] of Object.entries(linters)) {
expect(results[0][0].suggestions().length).toBe(1);
expect(results[1].length).toBe(0);
expect(results[2].length).toBe(1);
+
+ await linter.dispose();
});
test(`${linterName} detects lorem ipsum paragraph as not english`, async () => {
@@ -82,6 +99,8 @@ for (const [linterName, Linter] of Object.entries(linters)) {
expect(result).toBeTypeOf('boolean');
expect(result).toBe(false);
+
+ await linter.dispose();
});
test(`${linterName} can run setup without issues`, async () => {
@@ -95,6 +114,8 @@ for (const [linterName, Linter] of Object.entries(linters)) {
const lintConfig = await linter.getLintConfig();
expect(lintConfig).toHaveProperty('RepeatedWords');
+
+ await linter.dispose();
});
test(`${linterName} can set its configuration away and to default`, async () => {
@@ -119,6 +140,8 @@ for (const [linterName, Linter] of Object.entries(linters)) {
for (const key of Object.keys(lintConfig)) {
expect(lintConfig[key]).toBe(null);
}
+
+ await linter.dispose();
});
test(`${linterName} can both get and set its configuration`, async () => {
@@ -136,6 +159,8 @@ for (const [linterName, Linter] of Object.entries(linters)) {
for (const key of Object.keys(lintConfig)) {
expect(lintConfig[key]).toBe(true);
}
+
+ await linter.dispose();
});
test(`${linterName} can make things title case`, async () => {
@@ -144,6 +169,8 @@ for (const [linterName, Linter] of Object.entries(linters)) {
const titleCase = await linter.toTitleCase('this is a test for making titles');
expect(titleCase).toBe('This Is a Test for Making Titles');
+
+ await linter.dispose();
});
test(`${linterName} can get rule descriptions`, async () => {
@@ -152,6 +179,8 @@ for (const [linterName, Linter] of Object.entries(linters)) {
const descriptions = await linter.getLintDescriptions();
expect(descriptions).toBeTypeOf('object');
+
+ await linter.dispose();
});
test(`${linterName} can get rule descriptions in HTML.`, async () => {
@@ -160,6 +189,8 @@ for (const [linterName, Linter] of Object.entries(linters)) {
const descriptions = await linter.getLintDescriptionsHTML();
expect(descriptions).toBeTypeOf('object');
+
+ await linter.dispose();
});
test(`${linterName} rule descriptions are not empty`, async () => {
@@ -171,6 +202,8 @@ for (const [linterName, Linter] of Object.entries(linters)) {
expect(value).toBeTypeOf('string');
expect(value).not.toHaveLength(0);
}
+
+ await linter.dispose();
});
test(`${linterName} default lint config has no null values`, async () => {
@@ -181,6 +214,8 @@ for (const [linterName, Linter] of Object.entries(linters)) {
for (const value of Object.values(lintConfig)) {
expect(value).not.toBeNull();
}
+
+ await linter.dispose();
});
test(`${linterName} can generate lint context hashes`, async () => {
@@ -192,6 +227,8 @@ for (const [linterName, Linter] of Object.entries(linters)) {
expect(lints.length).toBeGreaterThanOrEqual(1);
await linter.contextHash(source, lints[0]);
+
+ await linter.dispose();
});
test(`${linterName} can ignore lints`, async () => {
@@ -207,6 +244,7 @@ for (const [linterName, Linter] of Object.entries(linters)) {
const secondRound = await linter.lint(source);
expect(secondRound.length).toBeLessThan(firstRound.length);
+ await linter.dispose();
});
test(`${linterName} can ignore lints with hashes`, async () => {
@@ -223,6 +261,7 @@ for (const [linterName, Linter] of Object.entries(linters)) {
const secondRound = await linter.lint(source);
expect(secondRound.length).toBeLessThan(firstRound.length);
+ await linter.dispose();
});
test(`${linterName} can ignore larger lints to reveal smaller ones`, async () => {
@@ -238,6 +277,8 @@ for (const [linterName, Linter] of Object.entries(linters)) {
const secondRound = await linter.lint(source);
expect(secondRound.length).toBe(4);
+
+ await linter.dispose();
});
test(`${linterName} can reimport ignored lints.`, async () => {
@@ -261,6 +302,9 @@ for (const [linterName, Linter] of Object.entries(linters)) {
expect(firstLints.length).toBeGreaterThan(secondLints.length);
expect(secondLints.length).toBe(0);
+
+ await firstLinter.dispose();
+ await secondLinter.dispose();
});
test(`${linterName} can add words to the dictionary`, async () => {
@@ -275,6 +319,8 @@ for (const [linterName, Linter] of Object.entries(linters)) {
lints = await linter.lint(source);
expect(lints).toHaveLength(0);
+
+ await linter.dispose();
});
test(`${linterName} allows correct capitalization of "United States"`, async () => {
@@ -282,6 +328,8 @@ for (const [linterName, Linter] of Object.entries(linters)) {
const lints = await linter.lint('The United States is a big country.');
expect(lints).toHaveLength(0);
+
+ await linter.dispose();
});
test(`${linterName} can summarize simple stat records`, async () => {
@@ -306,6 +354,8 @@ for (const [linterName, Linter] of Object.entries(linters)) {
const summary = await linter.summarizeStats();
expect(summary).toBeTypeOf('object');
+
+ await linter.dispose();
});
test(`${linterName} can save and restore stat records`, async () => {
@@ -332,6 +382,8 @@ for (const [linterName, Linter] of Object.entries(linters)) {
const newLinter = new Linter({ binary });
await newLinter.importStatsFile(stats);
+
+ await linter.dispose();
});
test(`${linterName} emits the correct span indices`, async () => {
@@ -346,6 +398,8 @@ for (const [linterName, Linter] of Object.entries(linters)) {
expect(span.end).toBe(54);
expect(text.slice(span.start, span.end)).toBe('sdssda');
+
+ await linter.dispose();
});
test(`${linterName} lints headings when forced to mark them as such`, async () => {
@@ -359,6 +413,8 @@ for (const [linterName, Linter] of Object.entries(linters)) {
const lint = lints[0];
expect(lint.lint_kind()).toBe('Capitalization');
expect(lint.get_problem_text()).toBe(text);
+
+ await linter.dispose();
});
test(`${linterName} lints headings when forced to mark them as such with organized mode`, async () => {
@@ -374,9 +430,36 @@ for (const [linterName, Linter] of Object.entries(linters)) {
const lint = titleCaseLints[0];
expect(lint.lint_kind()).toBe('Capitalization');
expect(lint.get_problem_text()).toBe(text);
+
+ await linter.dispose();
});
+
+ test(`${linterName} will lint many random strings with a single instance`, async () => {
+ const linter = new Linter({ binary });
+
+ for (let i = 0; i < 250; i++) {
+ const text = randomString(10);
+ const lints = await linter.organizedLints(text);
+
+ expect(lints).not.toBeNull();
+ }
+
+ await linter.dispose();
+ }, 120000);
}
+test('LocalLinters will lint many times with fresh instances', async () => {
+ for (let i = 0; i < 1000; i++) {
+ const linter = new LocalLinter({ binary });
+
+ const text = 'This is a grammatically correct sentence.';
+ const lints = await linter.organizedLints(text);
+ expect(lints).not.toBeNull();
+
+ await linter.dispose();
+ }
+}, 120000);
+
test('Linters have the same config format', async () => {
const configs = [];
@@ -384,6 +467,8 @@ test('Linters have the same config format', async () => {
const linter = new Linter({ binary });
configs.push(await linter.getLintConfig());
+
+ await linter.dispose();
}
for (const config of configs) {
@@ -399,6 +484,7 @@ test('Linters have the same JSON config format', async () => {
const linter = new Linter({ binary });
configs.push(await linter.getLintConfigAsJSON());
+ await linter.dispose();
}
for (const config of configs) {
diff --git a/packages/harper.js/src/Linter.ts b/packages/harper.js/src/Linter.ts
index 3d657f6fe..7796505a8 100644
--- a/packages/harper.js/src/Linter.ts
+++ b/packages/harper.js/src/Linter.ts
@@ -31,11 +31,11 @@ export default interface Linter {
getLintConfig(): Promise<LintConfig>;
/** Get the default (unset) linter configuration as JSON.
- * This method does not effect the caller's lint configuration, nor does it return the current one. */
+ * This method does not affect the caller's lint configuration, nor does it return the current one. */
getDefaultLintConfigAsJSON(): Promise<string>;
/** Get the default (unset) linter configuration.
- * This method does not effect the caller's lint configuration, nor does it return the current one. */
+ * This method does not affect the caller's lint configuration, nor does it return the current one. */
getDefaultLintConfig(): Promise<LintConfig>;
/** Set the linter's current configuration. */
@@ -65,6 +65,9 @@ export default interface Linter {
Wraps the function on the BinaryModule by the same name. */
toTitleCase(text: string): Promise<string>;
+ /** Release resources held by this linter instance. */
+ dispose(): Promise<void>;
+
/** Ignore future instances of a lint from a previous linting run in future invocations. */
ignoreLint(source: string, lint: Lint): Promise<void>;
diff --git a/packages/harper.js/src/LocalLinter.ts b/packages/harper.js/src/LocalLinter.ts
index c2c274c89..31d2e4385 100644
--- a/packages/harper.js/src/LocalLinter.ts
+++ b/packages/harper.js/src/LocalLinter.ts
@@ -11,6 +11,7 @@ import type { LintConfig, LintOptions } from './main';
export default class LocalLinter implements Linter {
binary: SuperBinaryModule;
private inner: Promise<WasmLinter>;
+ private disposed = false;
constructor(init: LinterInit) {
this.binary = init.binary as SuperBinaryModule;
@@ -47,8 +48,9 @@ export default class LocalLinter implements Linter {
const output: Record<string, Lint[]> = {};
- for (const { group, lints } of lintGroups) {
- output[group] = lints;
+ for (const group of lintGroups) {
+ output[group.group] = group.lints;
+ group.free();
}
return output;
@@ -179,6 +181,7 @@ export default class LocalLinter implements Linter {
const inner = await this.inner;
if (inner.get_dialect() !== dialect) {
+ inner.free();
this.inner = this.createInner(dialect);
}
@@ -199,4 +202,14 @@ export default class LocalLinter implements Linter {
const inner = await this.inner;
return inner.import_stats_file(statsFile);
}
+
+ async dispose(): Promise<void> {
+ if (this.disposed) {
+ return;
+ }
+
+ this.disposed = true;
+ const inner = await this.inner;
+ inner.free();
+ }
}
diff --git a/packages/harper.js/src/Serializer.test.ts b/packages/harper.js/src/Serializer.test.ts
index 8dd64357d..64a881390 100644
--- a/packages/harper.js/src/Serializer.test.ts
+++ b/packages/harper.js/src/Serializer.test.ts
@@ -80,5 +80,7 @@ describe('Serializer', () => {
expect(end.message()).toBe(start.message());
expect(end.lint_kind()).toBe(start.lint_kind());
+
+ await linter.dispose();
});
});
diff --git a/packages/harper.js/src/WorkerLinter/index.ts b/packages/harper.js/src/WorkerLinter/index.ts
index 885c06b0e..c79c991b5 100644
--- a/packages/harper.js/src/WorkerLinter/index.ts
+++ b/packages/harper.js/src/WorkerLinter/index.ts
@@ -25,6 +25,7 @@ export default class WorkerLinter implements Linter {
private worker: Worker;
private requestQueue: RequestItem[];
private working = true;
+ private disposed = false;
constructor(init: LinterInit) {
this.binary = init.binary;
@@ -133,6 +134,18 @@ export default class WorkerLinter implements Linter {
return JSON.parse(await this.getDefaultLintConfigAsJSON()) as LintConfig;
}
+ async dispose(): Promise<void> {
+ if (this.disposed) {
+ return;
+ }
+
+ await this.rpc('dispose', []);
+
+ this.disposed = true;
+ this.requestQueue = [];
+ this.worker.terminate();
+ }
+
ignoreLint(source: string, lint: Lint): Promise<void> {
return this.rpc('ignoreLint', [source, lint]);
}
@@ -191,6 +204,10 @@ export default class WorkerLinter implements Linter {
/** Run a procedure on the remote worker. */
private async rpc(procName: string, args: unknown[]): Promise<any> {
+ if (this.disposed) {
+ throw new Error('WorkerLinter has been disposed.');
+ }
+
const promise = new Promise((resolve, reject) => {
this.requestQueue.push({
resolve,
diff --git a/packages/harper.js/vite.config.ts b/packages/harper.js/vite.config.ts
index b455a524d..34d486fea 100644
--- a/packages/harper.js/vite.config.ts
+++ b/packages/harper.js/vite.config.ts
@@ -69,7 +69,7 @@ export default defineConfig({
enabled: true,
headless: true,
screenshotFailures: false,
- instances: [{ browser: 'chromium' }],
+ instances: [{ browser: 'chromium' }, { browser: 'firefox' }],
},
},
assetsInclude: ['**/*.wasm'],
diff --git a/packages/obsidian-plugin/src/State.ts b/packages/obsidian-plugin/src/State.ts
index aa2b0346b..556144900 100644
--- a/packages/obsidian-plugin/src/State.ts
+++ b/packages/obsidian-plugin/src/State.ts
@@ -73,8 +73,10 @@ export default class State {
settings.dialect !== oldSettings.dialect
) {
if (settings.useWebWorker) {
+ this.harper.dispose();
this.harper = new WorkerLinter({ binary: binaryInlined, dialect: settings.dialect });
} else {
+ this.harper.dispose();
this.harper = new LocalLinter({ binary: binaryInlined, dialect: settings.dialect });
}
} else {