Offline, privacy-first grammar checker. Fast, open-source, Rust-powered
fix(harper.js): create disposal function and use it (#2445)
Elijah Pottereab8bba001ce
diff --git a/Cargo.lock b/Cargo.lockindex 943c45451..2b3139d0a 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -3372,9 +3372,9 @@ dependencies = [[[package]]name = "js-sys"-version = "0.3.77"+version = "0.3.83"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "1cfaf33c695fc6e08064efbc1f72ec937429614f25eef83af942d0e227c3a28f"+checksum = "464a3709c7f55f1f721e5389aa6ea4e3bc6aba669353300af094b29ffbdde1d8"dependencies = ["once_cell","wasm-bindgen",@@ -6336,35 +6336,22 @@ dependencies = [[[package]]name = "wasm-bindgen"-version = "0.2.100"+version = "0.2.106"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "1edc8929d7499fc4e8f0be2262a241556cfc54a0bea223790e71446f2aab1ef5"+checksum = "0d759f433fa64a2d763d1340820e46e111a7a5ab75f993d1852d70b03dbb80fd"dependencies = ["cfg-if","once_cell","rustversion","wasm-bindgen-macro",-]--[[package]]-name = "wasm-bindgen-backend"-version = "0.2.100"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "2f0a0651a5c2bc21487bde11ee802ccaf4c51935d0d3d42a6101f98161700bc6"-dependencies = [- "bumpalo",- "log",- "proc-macro2",- "quote",- "syn","wasm-bindgen-shared",][[package]]name = "wasm-bindgen-futures"-version = "0.4.50"+version = "0.4.56"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "555d470ec0bc3bb57890405e5d4322cc9ea83cebb085523ced7be4144dac1e61"+checksum = "836d9622d604feee9e5de25ac10e3ea5f2d65b41eac0d9ce72eb5deae707ce7c"dependencies = ["cfg-if","js-sys",@@ -6375,9 +6362,9 @@ dependencies = [[[package]]name = "wasm-bindgen-macro"-version = "0.2.100"+version = "0.2.106"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "7fe63fc6d09ed3792bd0897b314f53de8e16568c2b3f7982f468c0bf9bd0b407"+checksum = "48cb0d2638f8baedbc542ed444afc0644a29166f1595371af4fecf8ce1e7eeb3"dependencies = ["quote","wasm-bindgen-macro-support",@@ -6385,31 +6372,31 @@ dependencies = [[[package]]name = "wasm-bindgen-macro-support"-version = "0.2.100"+version = "0.2.106"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de"+checksum = "cefb59d5cd5f92d9dcf80e4683949f15ca4b511f4ac0a6e14d4e1ac60c6ecd40"dependencies = [+ "bumpalo","proc-macro2","quote","syn",- "wasm-bindgen-backend","wasm-bindgen-shared",][[package]]name = "wasm-bindgen-shared"-version = "0.2.100"+version = "0.2.106"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "1a05d73b933a847d6cccdda8f838a22ff101ad9bf93e33684f39c1f5f0eece3d"+checksum = "cbc538057e648b67f72a982e708d485b2efa771e1ac05fec311f9f63e5800db4"dependencies = ["unicode-ident",][[package]]name = "web-sys"-version = "0.3.77"+version = "0.3.83"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "33b6dd2ef9186f1f2072e409e99cd22a975331a6b3591b12c764e0e55c60d5d2"+checksum = "9b32828d774c412041098d182a8b38b16ea816958e07cf40eec2bc080ae137ac"dependencies = ["js-sys","wasm-bindgen",diff --git a/harper-wasm/src/lib.rs b/harper-wasm/src/lib.rsindex e71eb9382..a2466a66b 100644--- a/harper-wasm/src/lib.rs+++ b/harper-wasm/src/lib.rs@@ -659,4 +659,19 @@ mod tests {let lints = linter.lint(text, Language::Plain, false);assert!(lints.is_empty());}++ #[test]+ fn does_not_oom_with_repeated_lints() {+ for _ in 0..3000 {+ let mut linter = Linter::new(Dialect::American);++ let results = linter.lint(+ "This is a grammatically correct sentence.".to_string(),+ Language::Plain,+ false,+ );++ assert!(results.is_empty())+ }+ }}diff --git a/packages/chrome-plugin/src/background/index.ts b/packages/chrome-plugin/src/background/index.tsindex df10e5f76..dcc068129 100644--- a/packages/chrome-plugin/src/background/index.ts+++ b/packages/chrome-plugin/src/background/index.ts@@ -394,6 +394,10 @@ async function setActivationKey(key: ActivationKey) {}function initializeLinter(dialect: Dialect) {+ if (linter != null) {+ linter.dispose();+ }+linter = new LocalLinter({binary: BinaryModule.create(chrome.runtime.getURL('./wasm/harper_wasm_bg.wasm')),dialect,diff --git a/packages/harper.js/src/Linter.test.ts b/packages/harper.js/src/Linter.test.tsindex 7835b3694..6bfd3fd5f 100644--- a/packages/harper.js/src/Linter.test.ts+++ b/packages/harper.js/src/Linter.test.ts@@ -3,6 +3,15 @@ import { binary } from './binary';import LocalLinter from './LocalLinter';import WorkerLinter from './WorkerLinter';+function randomString(length: number): string {+ const chars = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz';+ let result = '';+ for (let i = 0; i < length; i++) {+ result += chars.charAt(Math.floor(Math.random() * chars.length));+ }+ return result;+}+const linters = {WorkerLinter: WorkerLinter,LocalLinter: LocalLinter,@@ -15,6 +24,8 @@ for (const [linterName, Linter] of Object.entries(linters)) {const lints = await linter.lint('The the problem is...');expect(lints.length).toBe(1);++ await linter.dispose();});test(`${linterName} emits organized lints the same as it emits normal lints`, async () => {@@ -37,6 +48,8 @@ for (const [linterName, Linter] of Object.entries(linters)) {const item = flattened[0];expect(item.message().length).not.toBe(0);++ await linter.dispose();});test(`${linterName} detects repeated words with multiple synchronous requests`, async () => {@@ -54,6 +67,8 @@ for (const [linterName, Linter] of Object.entries(linters)) {expect(results[0][0].suggestions().length).toBe(1);expect(results[1].length).toBe(0);expect(results[2].length).toBe(1);++ await linter.dispose();});test(`${linterName} detects repeated words with concurrent requests`, async () => {@@ -71,6 +86,8 @@ for (const [linterName, Linter] of Object.entries(linters)) {expect(results[0][0].suggestions().length).toBe(1);expect(results[1].length).toBe(0);expect(results[2].length).toBe(1);++ await linter.dispose();});test(`${linterName} detects lorem ipsum paragraph as not english`, async () => {@@ -82,6 +99,8 @@ for (const [linterName, Linter] of Object.entries(linters)) {expect(result).toBeTypeOf('boolean');expect(result).toBe(false);++ await linter.dispose();});test(`${linterName} can run setup without issues`, async () => {@@ -95,6 +114,8 @@ for (const [linterName, Linter] of Object.entries(linters)) {const lintConfig = await linter.getLintConfig();expect(lintConfig).toHaveProperty('RepeatedWords');++ await linter.dispose();});test(`${linterName} can set its configuration away and to default`, async () => {@@ -119,6 +140,8 @@ for (const [linterName, Linter] of Object.entries(linters)) {for (const key of Object.keys(lintConfig)) {expect(lintConfig[key]).toBe(null);}++ await linter.dispose();});test(`${linterName} can both get and set its configuration`, async () => {@@ -136,6 +159,8 @@ for (const [linterName, Linter] of Object.entries(linters)) {for (const key of Object.keys(lintConfig)) {expect(lintConfig[key]).toBe(true);}++ await linter.dispose();});test(`${linterName} can make things title case`, async () => {@@ -144,6 +169,8 @@ for (const [linterName, Linter] of Object.entries(linters)) {const titleCase = await linter.toTitleCase('this is a test for making titles');expect(titleCase).toBe('This Is a Test for Making Titles');++ await linter.dispose();});test(`${linterName} can get rule descriptions`, async () => {@@ -152,6 +179,8 @@ for (const [linterName, Linter] of Object.entries(linters)) {const descriptions = await linter.getLintDescriptions();expect(descriptions).toBeTypeOf('object');++ await linter.dispose();});test(`${linterName} can get rule descriptions in HTML.`, async () => {@@ -160,6 +189,8 @@ for (const [linterName, Linter] of Object.entries(linters)) {const descriptions = await linter.getLintDescriptionsHTML();expect(descriptions).toBeTypeOf('object');++ await linter.dispose();});test(`${linterName} rule descriptions are not empty`, async () => {@@ -171,6 +202,8 @@ for (const [linterName, Linter] of Object.entries(linters)) {expect(value).toBeTypeOf('string');expect(value).not.toHaveLength(0);}++ await linter.dispose();});test(`${linterName} default lint config has no null values`, async () => {@@ -181,6 +214,8 @@ for (const [linterName, Linter] of Object.entries(linters)) {for (const value of Object.values(lintConfig)) {expect(value).not.toBeNull();}++ await linter.dispose();});test(`${linterName} can generate lint context hashes`, async () => {@@ -192,6 +227,8 @@ for (const [linterName, Linter] of Object.entries(linters)) {expect(lints.length).toBeGreaterThanOrEqual(1);await linter.contextHash(source, lints[0]);++ await linter.dispose();});test(`${linterName} can ignore lints`, async () => {@@ -207,6 +244,7 @@ for (const [linterName, Linter] of Object.entries(linters)) {const secondRound = await linter.lint(source);expect(secondRound.length).toBeLessThan(firstRound.length);+ await linter.dispose();});test(`${linterName} can ignore lints with hashes`, async () => {@@ -223,6 +261,7 @@ for (const [linterName, Linter] of Object.entries(linters)) {const secondRound = await linter.lint(source);expect(secondRound.length).toBeLessThan(firstRound.length);+ await linter.dispose();});test(`${linterName} can ignore larger lints to reveal smaller ones`, async () => {@@ -238,6 +277,8 @@ for (const [linterName, Linter] of Object.entries(linters)) {const secondRound = await linter.lint(source);expect(secondRound.length).toBe(4);++ await linter.dispose();});test(`${linterName} can reimport ignored lints.`, async () => {@@ -261,6 +302,9 @@ for (const [linterName, Linter] of Object.entries(linters)) {expect(firstLints.length).toBeGreaterThan(secondLints.length);expect(secondLints.length).toBe(0);++ await firstLinter.dispose();+ await secondLinter.dispose();});test(`${linterName} can add words to the dictionary`, async () => {@@ -275,6 +319,8 @@ for (const [linterName, Linter] of Object.entries(linters)) {lints = await linter.lint(source);expect(lints).toHaveLength(0);++ await linter.dispose();});test(`${linterName} allows correct capitalization of "United States"`, async () => {@@ -282,6 +328,8 @@ for (const [linterName, Linter] of Object.entries(linters)) {const lints = await linter.lint('The United States is a big country.');expect(lints).toHaveLength(0);++ await linter.dispose();});test(`${linterName} can summarize simple stat records`, async () => {@@ -306,6 +354,8 @@ for (const [linterName, Linter] of Object.entries(linters)) {const summary = await linter.summarizeStats();expect(summary).toBeTypeOf('object');++ await linter.dispose();});test(`${linterName} can save and restore stat records`, async () => {@@ -332,6 +382,8 @@ for (const [linterName, Linter] of Object.entries(linters)) {const newLinter = new Linter({ binary });await newLinter.importStatsFile(stats);++ await linter.dispose();});test(`${linterName} emits the correct span indices`, async () => {@@ -346,6 +398,8 @@ for (const [linterName, Linter] of Object.entries(linters)) {expect(span.end).toBe(54);expect(text.slice(span.start, span.end)).toBe('sdssda');++ await linter.dispose();});test(`${linterName} lints headings when forced to mark them as such`, async () => {@@ -359,6 +413,8 @@ for (const [linterName, Linter] of Object.entries(linters)) {const lint = lints[0];expect(lint.lint_kind()).toBe('Capitalization');expect(lint.get_problem_text()).toBe(text);++ await linter.dispose();});test(`${linterName} lints headings when forced to mark them as such with organized mode`, async () => {@@ -374,9 +430,36 @@ for (const [linterName, Linter] of Object.entries(linters)) {const lint = titleCaseLints[0];expect(lint.lint_kind()).toBe('Capitalization');expect(lint.get_problem_text()).toBe(text);++ await linter.dispose();});++ test(`${linterName} will lint many random strings with a single instance`, async () => {+ const linter = new Linter({ binary });++ for (let i = 0; i < 250; i++) {+ const text = randomString(10);+ const lints = await linter.organizedLints(text);++ expect(lints).not.toBeNull();+ }++ await linter.dispose();+ }, 120000);}+test('LocalLinters will lint many times with fresh instances', async () => {+ for (let i = 0; i < 1000; i++) {+ const linter = new LocalLinter({ binary });++ const text = 'This is a grammatically correct sentence.';+ const lints = await linter.organizedLints(text);+ expect(lints).not.toBeNull();++ await linter.dispose();+ }+}, 120000);+test('Linters have the same config format', async () => {const configs = [];@@ -384,6 +467,8 @@ test('Linters have the same config format', async () => {const linter = new Linter({ binary });configs.push(await linter.getLintConfig());++ await linter.dispose();}for (const config of configs) {@@ -399,6 +484,7 @@ test('Linters have the same JSON config format', async () => {const linter = new Linter({ binary });configs.push(await linter.getLintConfigAsJSON());+ await linter.dispose();}for (const config of configs) {diff --git a/packages/harper.js/src/Linter.ts b/packages/harper.js/src/Linter.tsindex 3d657f6fe..7796505a8 100644--- a/packages/harper.js/src/Linter.ts+++ b/packages/harper.js/src/Linter.ts@@ -31,11 +31,11 @@ export default interface Linter {getLintConfig(): Promise<LintConfig>;/** Get the default (unset) linter configuration as JSON.- * This method does not effect the caller's lint configuration, nor does it return the current one. */+ * This method does not affect the caller's lint configuration, nor does it return the current one. */getDefaultLintConfigAsJSON(): Promise<string>;/** Get the default (unset) linter configuration.- * This method does not effect the caller's lint configuration, nor does it return the current one. */+ * This method does not affect the caller's lint configuration, nor does it return the current one. */getDefaultLintConfig(): Promise<LintConfig>;/** Set the linter's current configuration. */@@ -65,6 +65,9 @@ export default interface Linter {Wraps the function on the BinaryModule by the same name. */toTitleCase(text: string): Promise<string>;+ /** Release resources held by this linter instance. */+ dispose(): Promise<void>;+/** Ignore future instances of a lint from a previous linting run in future invocations. */ignoreLint(source: string, lint: Lint): Promise<void>;diff --git a/packages/harper.js/src/LocalLinter.ts b/packages/harper.js/src/LocalLinter.tsindex c2c274c89..31d2e4385 100644--- a/packages/harper.js/src/LocalLinter.ts+++ b/packages/harper.js/src/LocalLinter.ts@@ -11,6 +11,7 @@ import type { LintConfig, LintOptions } from './main';export default class LocalLinter implements Linter {binary: SuperBinaryModule;private inner: Promise<WasmLinter>;+ private disposed = false;constructor(init: LinterInit) {this.binary = init.binary as SuperBinaryModule;@@ -47,8 +48,9 @@ export default class LocalLinter implements Linter {const output: Record<string, Lint[]> = {};- for (const { group, lints } of lintGroups) {- output[group] = lints;+ for (const group of lintGroups) {+ output[group.group] = group.lints;+ group.free();}return output;@@ -179,6 +181,7 @@ export default class LocalLinter implements Linter {const inner = await this.inner;if (inner.get_dialect() !== dialect) {+ inner.free();this.inner = this.createInner(dialect);}@@ -199,4 +202,14 @@ export default class LocalLinter implements Linter {const inner = await this.inner;return inner.import_stats_file(statsFile);}++ async dispose(): Promise<void> {+ if (this.disposed) {+ return;+ }++ this.disposed = true;+ const inner = await this.inner;+ inner.free();+ }}diff --git a/packages/harper.js/src/Serializer.test.ts b/packages/harper.js/src/Serializer.test.tsindex 8dd64357d..64a881390 100644--- a/packages/harper.js/src/Serializer.test.ts+++ b/packages/harper.js/src/Serializer.test.ts@@ -80,5 +80,7 @@ describe('Serializer', () => {expect(end.message()).toBe(start.message());expect(end.lint_kind()).toBe(start.lint_kind());++ await linter.dispose();});});diff --git a/packages/harper.js/src/WorkerLinter/index.ts b/packages/harper.js/src/WorkerLinter/index.tsindex 885c06b0e..c79c991b5 100644--- a/packages/harper.js/src/WorkerLinter/index.ts+++ b/packages/harper.js/src/WorkerLinter/index.ts@@ -25,6 +25,7 @@ export default class WorkerLinter implements Linter {private worker: Worker;private requestQueue: RequestItem[];private working = true;+ private disposed = false;constructor(init: LinterInit) {this.binary = init.binary;@@ -133,6 +134,18 @@ export default class WorkerLinter implements Linter {return JSON.parse(await this.getDefaultLintConfigAsJSON()) as LintConfig;}+ async dispose(): Promise<void> {+ if (this.disposed) {+ return;+ }++ await this.rpc('dispose', []);++ this.disposed = true;+ this.requestQueue = [];+ this.worker.terminate();+ }+ignoreLint(source: string, lint: Lint): Promise<void> {return this.rpc('ignoreLint', [source, lint]);}@@ -191,6 +204,10 @@ export default class WorkerLinter implements Linter {/** Run a procedure on the remote worker. */private async rpc(procName: string, args: unknown[]): Promise<any> {+ if (this.disposed) {+ throw new Error('WorkerLinter has been disposed.');+ }+const promise = new Promise((resolve, reject) => {this.requestQueue.push({resolve,diff --git a/packages/harper.js/vite.config.ts b/packages/harper.js/vite.config.tsindex b455a524d..34d486fea 100644--- a/packages/harper.js/vite.config.ts+++ b/packages/harper.js/vite.config.ts@@ -69,7 +69,7 @@ export default defineConfig({enabled: true,headless: true,screenshotFailures: false,- instances: [{ browser: 'chromium' }],+ instances: [{ browser: 'chromium' }, { browser: 'firefox' }],},},assetsInclude: ['**/*.wasm'],diff --git a/packages/obsidian-plugin/src/State.ts b/packages/obsidian-plugin/src/State.tsindex aa2b0346b..556144900 100644--- a/packages/obsidian-plugin/src/State.ts+++ b/packages/obsidian-plugin/src/State.ts@@ -73,8 +73,10 @@ export default class State {settings.dialect !== oldSettings.dialect) {if (settings.useWebWorker) {+ this.harper.dispose();this.harper = new WorkerLinter({ binary: binaryInlined, dialect: settings.dialect });} else {+ this.harper.dispose();this.harper = new LocalLinter({ binary: binaryInlined, dialect: settings.dialect });}} else {