Skip to content

Offline, privacy-first grammar checker. Fast, open-source, Rust-powered

feat(core): allow users to include dictionaries in Weirpacks (#2922)

Elijah Potter0e49ee1d452f
diff --git a/harper-core/src/weirpack/error.rs b/harper-core/src/weirpack/error.rs
index 79878e8e0..948d11179 100644
--- a/harper-core/src/weirpack/error.rs
+++ b/harper-core/src/weirpack/error.rs
@@ -20,4 +20,6 @@ pub enum Error {
DuplicateManifest(&'static str),
#[error("Invalid rule filename '{0}'.")]
InvalidRuleFileName(String),
+ #[error("The Rune dictionary is not valid.")]
+ InvalidDictionaryFormat,
}
diff --git a/harper-core/src/weirpack/mod.rs b/harper-core/src/weirpack/mod.rs
index f034c97e3..dd75b3242 100644
--- a/harper-core/src/weirpack/mod.rs
+++ b/harper-core/src/weirpack/mod.rs
@@ -8,6 +8,7 @@ use zip::write::FileOptions;
use zip::{CompressionMethod, ZipArchive, ZipWriter};
use crate::linting::LintGroup;
+use crate::spell::MutableDictionary;
use crate::weir::{TestResult, WeirLinter};
mod error;
@@ -21,6 +22,10 @@ pub use manifest::WeirpackManifest;
#[derive(Debug, Clone, Default)]
pub struct Weirpack {
pub rules: HashMap<String, String>,
+ /// The `dictionary.dict` file, if it exists.
+ pub dictionary: Option<String>,
+ /// The `annotations.json` file, if it exists.
+ pub annotations: Option<String>,
pub manifest: WeirpackManifest,
}
@@ -29,6 +34,8 @@ impl Weirpack {
pub fn new(manifest: WeirpackManifest) -> Self {
Self {
rules: HashMap::new(),
+ annotations: None,
+ dictionary: None,
manifest,
}
}
@@ -86,6 +93,25 @@ impl Weirpack {
Ok(())
}
+ /// Loads the dictionary that may or may not be contained within the Weirpack.
+ ///
+ /// The dictionary is in the Rune format and thus is composed of two files, `annotations.json`
+ /// and `dictionary.dict`.
+ ///
+ /// Returns `None` if the relevant files are not present in the Weirpack.
+ pub fn load_dictionary(&self) -> Result<Option<MutableDictionary>, Error> {
+ if let Some(dict) = &self.dictionary
+ && let Some(annot) = &self.annotations
+ {
+ Ok(Some(
+ MutableDictionary::from_rune_files(dict, annot)
+ .map_err(|_| Error::InvalidDictionaryFormat)?,
+ ))
+ } else {
+ Ok(None)
+ }
+ }
+
/// Load a Weirpack from bytes.
pub fn from_bytes(bytes: &[u8]) -> Result<Self, Error> {
let cursor = std::io::Cursor::new(bytes);
@@ -93,6 +119,8 @@ impl Weirpack {
let mut manifest = None;
let mut rules = HashMap::new();
+ let mut dictionary = None;
+ let mut annotations = None;
for i in 0..archive.len() {
let mut file = archive.by_index(i)?;
@@ -110,28 +138,39 @@ impl Weirpack {
continue;
}
- if !name.ends_with(".weir") {
- continue;
+ if name.ends_with(".weir") {
+ let path = Path::new(&name);
+ let file_name = path
+ .file_name()
+ .and_then(|segment| segment.to_str())
+ .ok_or_else(|| Error::InvalidRuleFileName(name.clone()))?;
+ let rule_name = Path::new(file_name)
+ .file_stem()
+ .and_then(|stem| stem.to_str())
+ .ok_or_else(|| Error::InvalidRuleFileName(name.clone()))?;
+
+ let mut contents = String::new();
+ file.read_to_string(&mut contents)?;
+ rules.insert(rule_name.to_string(), contents);
+ } else if name == "dictionary.dict" {
+ let mut contents = String::new();
+ file.read_to_string(&mut contents)?;
+ dictionary = Some(contents);
+ } else if name == "annotations.json" {
+ let mut contents = String::new();
+ file.read_to_string(&mut contents)?;
+ annotations = Some(contents);
}
-
- let path = Path::new(&name);
- let file_name = path
- .file_name()
- .and_then(|segment| segment.to_str())
- .ok_or_else(|| Error::InvalidRuleFileName(name.clone()))?;
- let rule_name = Path::new(file_name)
- .file_stem()
- .and_then(|stem| stem.to_str())
- .ok_or_else(|| Error::InvalidRuleFileName(name.clone()))?;
-
- let mut contents = String::new();
- file.read_to_string(&mut contents)?;
- rules.insert(rule_name.to_string(), contents);
}
let manifest = manifest.ok_or(Error::MissingManifest("manifest.json"))?;
- Ok(Self { rules, manifest })
+ Ok(Self {
+ rules,
+ manifest,
+ annotations,
+ dictionary,
+ })
}
/// Write a Weirpack into bytes.
@@ -144,6 +183,16 @@ impl Weirpack {
zip.start_file("manifest.json", options)?;
zip.write_all(&manifest_bytes)?;
+ if let Some(annot) = &self.annotations {
+ zip.start_file("annotations.json", options)?;
+ zip.write_all(annot.as_bytes())?;
+ }
+
+ if let Some(dict) = &self.dictionary {
+ zip.start_file("dictionary.dict", options)?;
+ zip.write_all(dict.as_bytes())?;
+ }
+
let mut rule_names: Vec<_> = self.rules.keys().collect();
rule_names.sort();
diff --git a/harper-wasm/src/lib.rs b/harper-wasm/src/lib.rs
index e98a8db99..589dbabf5 100644
--- a/harper-wasm/src/lib.rs
+++ b/harper-wasm/src/lib.rs
@@ -100,6 +100,7 @@ pub struct Linter {
/// To make changes affect linting, run [`Self::synchronize_lint_dict`].
user_dictionary: MutableDictionary,
dictionary: Arc<MergedDictionary>,
+ weirpack_dictionaries: Vec<Arc<MutableDictionary>>,
ignored_lints: IgnoredLints,
dialect: Dialect,
stats: Stats,
@@ -117,12 +118,13 @@ impl Linter {
/// Note that this can mean constructing the curated dictionary, which is the most expensive operation
/// in Harper.
pub fn new(dialect: Dialect) -> Self {
- let dictionary = Self::construct_merged_dict(MutableDictionary::default());
+ let dictionary = Self::construct_merged_dict(&[Arc::new(MutableDictionary::default())]);
let lint_group = LintGroup::new_curated_empty_config(dictionary.clone(), dialect.into());
Self {
lint_group,
user_dictionary: MutableDictionary::new(),
+ weirpack_dictionaries: Vec::new(),
dictionary,
ignored_lints: IgnoredLints::default(),
dialect,
@@ -134,19 +136,28 @@ impl Linter {
/// This clears any linter caches, so use it sparingly.
fn synchronize_lint_dict(&mut self) {
let mut lint_config = self.lint_group.config.clone();
- self.dictionary = Self::construct_merged_dict(self.user_dictionary.clone());
+
+ let mut constituent_dictionaries = vec![Arc::new(self.user_dictionary.clone())];
+ constituent_dictionaries.extend(self.weirpack_dictionaries.iter().cloned());
+
+ self.dictionary = Self::construct_merged_dict(&constituent_dictionaries);
+
self.lint_group =
LintGroup::new_curated_empty_config(self.dictionary.clone(), self.dialect.into());
+
self.lint_group.config.merge_from(&mut lint_config);
}
/// Construct the actual dictionary to be used for linting and parsing from the curated dictionary
- /// and [`Self::user_dictionary`].
- fn construct_merged_dict(user_dictionary: MutableDictionary) -> Arc<MergedDictionary> {
+ /// and any other runtime-provided dictionaries.
+ fn construct_merged_dict(dicts: &[Arc<impl Dictionary + 'static>]) -> Arc<MergedDictionary> {
let mut lint_dict = MergedDictionary::new();
lint_dict.add_dictionary(FstDictionary::curated());
- lint_dict.add_dictionary(Arc::new(user_dictionary));
+
+ for dict in dicts {
+ lint_dict.add_dictionary(Arc::new(dict.clone()));
+ }
Arc::new(lint_dict)
}
@@ -502,6 +513,11 @@ impl Linter {
return Ok(value);
}
+ if let Some(dict) = pack.load_dictionary().map_err(|err| err.to_string())? {
+ self.weirpack_dictionaries.push(Arc::new(dict));
+ self.synchronize_lint_dict();
+ }
+
let mut group = pack.to_lint_group().map_err(|err| err.to_string())?;
self.lint_group.merge_from(&mut group);
Ok(JsValue::UNDEFINED)
diff --git a/packages/chrome-plugin/app.css b/packages/chrome-plugin/app.css
index 9063eb38c..ff98f77b3 100644
--- a/packages/chrome-plugin/app.css
+++ b/packages/chrome-plugin/app.css
@@ -72,6 +72,10 @@ code {
@apply bg-primary-100 rounded p-1 dark:text-black;
}
+a {
+ @apply underline;
+}
+
#app {
@apply min-h-screen bg-white text-black dark:bg-black dark:text-white transition-colors duration-150;
diff --git a/packages/chrome-plugin/src/options/Options.svelte b/packages/chrome-plugin/src/options/Options.svelte
index 33fca7ca9..a49e09648 100644
--- a/packages/chrome-plugin/src/options/Options.svelte
+++ b/packages/chrome-plugin/src/options/Options.svelte
@@ -358,6 +358,7 @@ async function removeWeirpack(id: string) {
<div class="space-y-2 flex flex-row w-full justify-between">
<p class="text-xs text-gray-600 dark:text-gray-400">
Upload one or more <code>.weirpack</code> files to add custom rule packs.
+ <a href="https://writewithharper.com/docs/weir#Weirpacks">What is a Weirpack?</a>
</p>
<input
type="file"
diff --git a/packages/chrome-plugin/src/popup/Main.svelte b/packages/chrome-plugin/src/popup/Main.svelte
index 04daf6fcc..48ef26ef9 100644
--- a/packages/chrome-plugin/src/popup/Main.svelte
+++ b/packages/chrome-plugin/src/popup/Main.svelte
@@ -115,7 +115,7 @@ function daysSince(date: Date): number {
</h1>
<p class="text-sm font-medium font-sans dark:text-white text-right">
- Harper is {enabled ? 'enabled on ' : 'disabled on '}{domain}
+ Harper is {enabled ? 'enabled on ' : 'disabled on '}{domain.length > 10 ? "this page" : domain}.
</p>
</section>
</section>
diff --git a/packages/harper.js/src/Linter.test.ts b/packages/harper.js/src/Linter.test.ts
index c77c0e797..604c67a41 100644
--- a/packages/harper.js/src/Linter.test.ts
+++ b/packages/harper.js/src/Linter.test.ts
@@ -2,6 +2,7 @@ import { expect, test } from 'vitest';
import { binary } from './binary';
import LocalLinter from './LocalLinter';
import WorkerLinter from './WorkerLinter';
+import { packWeirpackFiles } from './weirpack';
function randomString(length: number): string {
const chars = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz';
@@ -12,6 +13,35 @@ function randomString(length: number): string {
return result;
}
+function createTestWeirpack(): Map<string, string> {
+ const manifest = `{
+ "name": "Test Weirpack",
+ "author": "Anonymous",
+ "version": "0.1.0",
+ "description": "",
+ "license": "MIT"
+ }`;
+
+ const annotations = `
+ {
+ "affixes": {
+ },
+ "properties": {
+ }
+ }
+ `;
+
+ const dict = '1000\n\n';
+
+ const pack = new Map();
+
+ pack.set('annotations.json', annotations);
+ pack.set('manifest.json', manifest);
+ pack.set('dictionary.dict', dict);
+
+ return pack;
+}
+
const WEIRPACK_PASS_BASE64 =
'UEsDBBQAAAAIAFR7LFx+V+AhbQAAAIgAAAANAAAAbWFuaWZlc3QuanNvbi2MMQuDMBBG9/yKI3MJdnXrWNBNcE7iaQ9DLlxil+J/l5iO33u876cAtD3Kh0X3oCfMBV5tPqr6omTiWF1nOvNsdMHshVL5m7uakSRZv8PKAjPLjjJQLCjgAjsIbBeKm2kHgTzGjDUe35NW5wVQSwMEFAAAAAgAVHssXLV8hxV2AAAAoQAAABUAAABXZWlycGFja1Rlc3RSdWxlLndlaXJNzrENwzAMBMBeUxDskx0yQMoMQEtvh7BECaKMePwYdorgyz/gH3vrVESNJrEjIWQMKnCXBcQvB4036APtTeJKHS1LRIGNO582wWPXNrQa8eNPbhk0104DPtSWH1/VEvFTPSJnMdTNr2JCrMcu8XXkNuuOxOELUEsBAhQDFAAAAAgAVHssXH5X4CFtAAAAiAAAAA0AAAAAAAAAAAAAAIABAAAAAG1hbmlmZXN0Lmpzb25QSwECFAMUAAAACABUeyxctXyHFXYAAAChAAAAFQAAAAAAAAAAAAAAgAGYAAAAV2VpcnBhY2tUZXN0UnVsZS53ZWlyUEsFBgAAAAACAAIAfgAAAEEBAAAAAA==';
const WEIRPACK_FAIL_BASE64 =
@@ -591,6 +621,26 @@ for (const [linterName, Linter] of Object.entries(linters)) {
await linter.dispose();
});
+
+ test(`${linterName} can load dictionaries from Weirpacks`, async () => {
+ const linter = new Linter({ binary });
+
+ const source = 'I adore this akljsfhl group!';
+
+ // Show that the word is marked as misspelled when the Weirpack is not included.
+ let lints = await linter.lint(source);
+ expect(lints).toHaveLength(1);
+
+ // Load the dictionary
+ const weirpack = createTestWeirpack();
+ weirpack.set('dictionary.dict', '1000\n\nakljsfhl');
+ const packed = packWeirpackFiles(weirpack);
+ await linter.loadWeirpackFromBytes(packed);
+
+ // It should pass now that we've included the "word" in the dictionary via the Weirpack.
+ lints = await linter.lint(source);
+ expect(lints).toHaveLength(0);
+ }, 30000);
}
// Disabled because it significantly slows down CI