Offline, privacy-first grammar checker. Fast, open-source, Rust-powered
feat(core): allow users to include dictionaries in Weirpacks (#2922)
Elijah Potter0e49ee1d452f
diff --git a/harper-core/src/weirpack/error.rs b/harper-core/src/weirpack/error.rsindex 79878e8e0..948d11179 100644--- a/harper-core/src/weirpack/error.rs+++ b/harper-core/src/weirpack/error.rs@@ -20,4 +20,6 @@ pub enum Error {DuplicateManifest(&'static str),#[error("Invalid rule filename '{0}'.")]InvalidRuleFileName(String),+ #[error("The Rune dictionary is not valid.")]+ InvalidDictionaryFormat,}diff --git a/harper-core/src/weirpack/mod.rs b/harper-core/src/weirpack/mod.rsindex f034c97e3..dd75b3242 100644--- a/harper-core/src/weirpack/mod.rs+++ b/harper-core/src/weirpack/mod.rs@@ -8,6 +8,7 @@ use zip::write::FileOptions;use zip::{CompressionMethod, ZipArchive, ZipWriter};use crate::linting::LintGroup;+use crate::spell::MutableDictionary;use crate::weir::{TestResult, WeirLinter};mod error;@@ -21,6 +22,10 @@ pub use manifest::WeirpackManifest;#[derive(Debug, Clone, Default)]pub struct Weirpack {pub rules: HashMap<String, String>,+ /// The `dictionary.dict` file, if it exists.+ pub dictionary: Option<String>,+ /// The `annotations.json` file, if it exists.+ pub annotations: Option<String>,pub manifest: WeirpackManifest,}@@ -29,6 +34,8 @@ impl Weirpack {pub fn new(manifest: WeirpackManifest) -> Self {Self {rules: HashMap::new(),+ annotations: None,+ dictionary: None,manifest,}}@@ -86,6 +93,25 @@ impl Weirpack {Ok(())}+ /// Loads the dictionary that may or may not be contained within the Weirpack.+ ///+ /// The dictionary is in the Rune format and thus is composed of two files, `annotations.json`+ /// and `dictionary.dict`.+ ///+ /// Returns `None` if the relevant files are not present in the Weirpack.+ pub fn load_dictionary(&self) -> Result<Option<MutableDictionary>, Error> {+ if let Some(dict) = &self.dictionary+ && let Some(annot) = &self.annotations+ {+ Ok(Some(+ MutableDictionary::from_rune_files(dict, annot)+ .map_err(|_| Error::InvalidDictionaryFormat)?,+ ))+ } else {+ Ok(None)+ }+ }+/// Load a Weirpack from bytes.pub fn from_bytes(bytes: &[u8]) -> Result<Self, Error> {let cursor = std::io::Cursor::new(bytes);@@ -93,6 +119,8 @@ impl Weirpack {let mut manifest = None;let mut rules = HashMap::new();+ let mut dictionary = None;+ let mut annotations = None;for i in 0..archive.len() {let mut file = archive.by_index(i)?;@@ -110,28 +138,39 @@ impl Weirpack {continue;}- if !name.ends_with(".weir") {- continue;+ if name.ends_with(".weir") {+ let path = Path::new(&name);+ let file_name = path+ .file_name()+ .and_then(|segment| segment.to_str())+ .ok_or_else(|| Error::InvalidRuleFileName(name.clone()))?;+ let rule_name = Path::new(file_name)+ .file_stem()+ .and_then(|stem| stem.to_str())+ .ok_or_else(|| Error::InvalidRuleFileName(name.clone()))?;++ let mut contents = String::new();+ file.read_to_string(&mut contents)?;+ rules.insert(rule_name.to_string(), contents);+ } else if name == "dictionary.dict" {+ let mut contents = String::new();+ file.read_to_string(&mut contents)?;+ dictionary = Some(contents);+ } else if name == "annotations.json" {+ let mut contents = String::new();+ file.read_to_string(&mut contents)?;+ annotations = Some(contents);}-- let path = Path::new(&name);- let file_name = path- .file_name()- .and_then(|segment| segment.to_str())- .ok_or_else(|| Error::InvalidRuleFileName(name.clone()))?;- let rule_name = Path::new(file_name)- .file_stem()- .and_then(|stem| stem.to_str())- .ok_or_else(|| Error::InvalidRuleFileName(name.clone()))?;-- let mut contents = String::new();- file.read_to_string(&mut contents)?;- rules.insert(rule_name.to_string(), contents);}let manifest = manifest.ok_or(Error::MissingManifest("manifest.json"))?;- Ok(Self { rules, manifest })+ Ok(Self {+ rules,+ manifest,+ annotations,+ dictionary,+ })}/// Write a Weirpack into bytes.@@ -144,6 +183,16 @@ impl Weirpack {zip.start_file("manifest.json", options)?;zip.write_all(&manifest_bytes)?;+ if let Some(annot) = &self.annotations {+ zip.start_file("annotations.json", options)?;+ zip.write_all(annot.as_bytes())?;+ }++ if let Some(dict) = &self.dictionary {+ zip.start_file("dictionary.dict", options)?;+ zip.write_all(dict.as_bytes())?;+ }+let mut rule_names: Vec<_> = self.rules.keys().collect();rule_names.sort();diff --git a/harper-wasm/src/lib.rs b/harper-wasm/src/lib.rsindex e98a8db99..589dbabf5 100644--- a/harper-wasm/src/lib.rs+++ b/harper-wasm/src/lib.rs@@ -100,6 +100,7 @@ pub struct Linter {/// To make changes affect linting, run [`Self::synchronize_lint_dict`].user_dictionary: MutableDictionary,dictionary: Arc<MergedDictionary>,+ weirpack_dictionaries: Vec<Arc<MutableDictionary>>,ignored_lints: IgnoredLints,dialect: Dialect,stats: Stats,@@ -117,12 +118,13 @@ impl Linter {/// Note that this can mean constructing the curated dictionary, which is the most expensive operation/// in Harper.pub fn new(dialect: Dialect) -> Self {- let dictionary = Self::construct_merged_dict(MutableDictionary::default());+ let dictionary = Self::construct_merged_dict(&[Arc::new(MutableDictionary::default())]);let lint_group = LintGroup::new_curated_empty_config(dictionary.clone(), dialect.into());Self {lint_group,user_dictionary: MutableDictionary::new(),+ weirpack_dictionaries: Vec::new(),dictionary,ignored_lints: IgnoredLints::default(),dialect,@@ -134,19 +136,28 @@ impl Linter {/// This clears any linter caches, so use it sparingly.fn synchronize_lint_dict(&mut self) {let mut lint_config = self.lint_group.config.clone();- self.dictionary = Self::construct_merged_dict(self.user_dictionary.clone());++ let mut constituent_dictionaries = vec![Arc::new(self.user_dictionary.clone())];+ constituent_dictionaries.extend(self.weirpack_dictionaries.iter().cloned());++ self.dictionary = Self::construct_merged_dict(&constituent_dictionaries);+self.lint_group =LintGroup::new_curated_empty_config(self.dictionary.clone(), self.dialect.into());+self.lint_group.config.merge_from(&mut lint_config);}/// Construct the actual dictionary to be used for linting and parsing from the curated dictionary- /// and [`Self::user_dictionary`].- fn construct_merged_dict(user_dictionary: MutableDictionary) -> Arc<MergedDictionary> {+ /// and any other runtime-provided dictionaries.+ fn construct_merged_dict(dicts: &[Arc<impl Dictionary + 'static>]) -> Arc<MergedDictionary> {let mut lint_dict = MergedDictionary::new();lint_dict.add_dictionary(FstDictionary::curated());- lint_dict.add_dictionary(Arc::new(user_dictionary));++ for dict in dicts {+ lint_dict.add_dictionary(Arc::new(dict.clone()));+ }Arc::new(lint_dict)}@@ -502,6 +513,11 @@ impl Linter {return Ok(value);}+ if let Some(dict) = pack.load_dictionary().map_err(|err| err.to_string())? {+ self.weirpack_dictionaries.push(Arc::new(dict));+ self.synchronize_lint_dict();+ }+let mut group = pack.to_lint_group().map_err(|err| err.to_string())?;self.lint_group.merge_from(&mut group);Ok(JsValue::UNDEFINED)diff --git a/packages/chrome-plugin/app.css b/packages/chrome-plugin/app.cssindex 9063eb38c..ff98f77b3 100644--- a/packages/chrome-plugin/app.css+++ b/packages/chrome-plugin/app.css@@ -72,6 +72,10 @@ code {@apply bg-primary-100 rounded p-1 dark:text-black;}+a {+ @apply underline;+}+#app {@apply min-h-screen bg-white text-black dark:bg-black dark:text-white transition-colors duration-150;diff --git a/packages/chrome-plugin/src/options/Options.svelte b/packages/chrome-plugin/src/options/Options.svelteindex 33fca7ca9..a49e09648 100644--- a/packages/chrome-plugin/src/options/Options.svelte+++ b/packages/chrome-plugin/src/options/Options.svelte@@ -358,6 +358,7 @@ async function removeWeirpack(id: string) {<div class="space-y-2 flex flex-row w-full justify-between"><p class="text-xs text-gray-600 dark:text-gray-400">Upload one or more <code>.weirpack</code> files to add custom rule packs.+ <a href="https://writewithharper.com/docs/weir#Weirpacks">What is a Weirpack?</a></p><inputtype="file"diff --git a/packages/chrome-plugin/src/popup/Main.svelte b/packages/chrome-plugin/src/popup/Main.svelteindex 04daf6fcc..48ef26ef9 100644--- a/packages/chrome-plugin/src/popup/Main.svelte+++ b/packages/chrome-plugin/src/popup/Main.svelte@@ -115,7 +115,7 @@ function daysSince(date: Date): number {</h1><p class="text-sm font-medium font-sans dark:text-white text-right">- Harper is {enabled ? 'enabled on ' : 'disabled on '}{domain}+ Harper is {enabled ? 'enabled on ' : 'disabled on '}{domain.length > 10 ? "this page" : domain}.</p></section></section>diff --git a/packages/harper.js/src/Linter.test.ts b/packages/harper.js/src/Linter.test.tsindex c77c0e797..604c67a41 100644--- a/packages/harper.js/src/Linter.test.ts+++ b/packages/harper.js/src/Linter.test.ts@@ -2,6 +2,7 @@ import { expect, test } from 'vitest';import { binary } from './binary';import LocalLinter from './LocalLinter';import WorkerLinter from './WorkerLinter';+import { packWeirpackFiles } from './weirpack';function randomString(length: number): string {const chars = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz';@@ -12,6 +13,35 @@ function randomString(length: number): string {return result;}+function createTestWeirpack(): Map<string, string> {+ const manifest = `{+ "name": "Test Weirpack",+ "author": "Anonymous",+ "version": "0.1.0",+ "description": "",+ "license": "MIT"+ }`;++ const annotations = `+ {+ "affixes": {+ },+ "properties": {+ }+ }+ `;++ const dict = '1000\n\n';++ const pack = new Map();++ pack.set('annotations.json', annotations);+ pack.set('manifest.json', manifest);+ pack.set('dictionary.dict', dict);++ return pack;+}+const WEIRPACK_PASS_BASE64 ='UEsDBBQAAAAIAFR7LFx+V+AhbQAAAIgAAAANAAAAbWFuaWZlc3QuanNvbi2MMQuDMBBG9/yKI3MJdnXrWNBNcE7iaQ9DLlxil+J/l5iO33u876cAtD3Kh0X3oCfMBV5tPqr6omTiWF1nOvNsdMHshVL5m7uakSRZv8PKAjPLjjJQLCjgAjsIbBeKm2kHgTzGjDUe35NW5wVQSwMEFAAAAAgAVHssXLV8hxV2AAAAoQAAABUAAABXZWlycGFja1Rlc3RSdWxlLndlaXJNzrENwzAMBMBeUxDskx0yQMoMQEtvh7BECaKMePwYdorgyz/gH3vrVESNJrEjIWQMKnCXBcQvB4036APtTeJKHS1LRIGNO582wWPXNrQa8eNPbhk0104DPtSWH1/VEvFTPSJnMdTNr2JCrMcu8XXkNuuOxOELUEsBAhQDFAAAAAgAVHssXH5X4CFtAAAAiAAAAA0AAAAAAAAAAAAAAIABAAAAAG1hbmlmZXN0Lmpzb25QSwECFAMUAAAACABUeyxctXyHFXYAAAChAAAAFQAAAAAAAAAAAAAAgAGYAAAAV2VpcnBhY2tUZXN0UnVsZS53ZWlyUEsFBgAAAAACAAIAfgAAAEEBAAAAAA==';const WEIRPACK_FAIL_BASE64 =@@ -591,6 +621,26 @@ for (const [linterName, Linter] of Object.entries(linters)) {await linter.dispose();});++ test(`${linterName} can load dictionaries from Weirpacks`, async () => {+ const linter = new Linter({ binary });++ const source = 'I adore this akljsfhl group!';++ // Show that the word is marked as misspelled when the Weirpack is not included.+ let lints = await linter.lint(source);+ expect(lints).toHaveLength(1);++ // Load the dictionary+ const weirpack = createTestWeirpack();+ weirpack.set('dictionary.dict', '1000\n\nakljsfhl');+ const packed = packWeirpackFiles(weirpack);+ await linter.loadWeirpackFromBytes(packed);++ // It should pass now that we've included the "word" in the dictionary via the Weirpack.+ lints = await linter.lint(source);+ expect(lints).toHaveLength(0);+ }, 30000);}// Disabled because it significantly slows down CI