Offline, privacy-first grammar checker. Fast, open-source, Rust-powered
feat(obsidian): allow users to mask out text using regular expressions (#2684)
Elijah Potter2076d2e9ee25
diff --git a/Cargo.lock b/Cargo.lockindex e60a359d2..de906d75d 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -2785,6 +2785,7 @@ dependencies = ["quickcheck_macros","rand 0.8.5","rayon",+ "regex","serde","serde_json","smallvec",@@ -4554,9 +4555,9 @@ dependencies = [[[package]]name = "regex"-version = "1.12.2"+version = "1.12.3"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "843bc0191f75f3e22651ae5f1e72939ab2f72a4bc30fa80a066bd66edefc24d4"+checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276"dependencies = ["aho-corasick","memchr",diff --git a/harper-core/Cargo.toml b/harper-core/Cargo.tomlindex fe99f5323..ba0f98b29 100644--- a/harper-core/Cargo.toml+++ b/harper-core/Cargo.toml@@ -35,6 +35,7 @@ harper-thesaurus = { path = "../harper-thesaurus", version = "1.4.1", optional =bitflags = { version = "2.10.0", features = ["serde"] }trie-rs = "0.4.2"zip = { version = "2.2.0", default-features = false, features = ["deflate"] }+regex = "1.12.3"[dev-dependencies]criterion = { version = "0.8.2", default-features = false }diff --git a/harper-core/src/lib.rs b/harper-core/src/lib.rsindex 4c37bcce9..5a3fa205e 100644--- a/harper-core/src/lib.rs+++ b/harper-core/src/lib.rs@@ -19,6 +19,7 @@ mod lexing;pub mod linting;mod mask;mod number;+mod offsets;pub mod parsers;pub mod patterns;mod punctuation;@@ -52,7 +53,7 @@ pub use ignored_lints::{IgnoredLints, LintContext};pub use irregular_nouns::IrregularNouns;pub use irregular_verbs::IrregularVerbs;use linting::Lint;-pub use mask::{Mask, Masker};+pub use mask::{Mask, Masker, RegexMasker};pub use number::{Number, OrdinalSuffix};pub use punctuation::{Punctuation, Quote};pub use span::Span;diff --git a/harper-core/src/mask/mod.rs b/harper-core/src/mask/mod.rsindex a9b78ccc4..b34517719 100644--- a/harper-core/src/mask/mod.rs+++ b/harper-core/src/mask/mod.rs@@ -1,3 +1,6 @@+mod regex_masker;+pub use regex_masker::RegexMasker;+use itertools::Itertools;use crate::Span;diff --git a/harper-core/src/mask/regex_masker.rs b/harper-core/src/mask/regex_masker.rsnew file mode 100644index 000000000..b0eecb278--- /dev/null+++ b/harper-core/src/mask/regex_masker.rs@@ -0,0 +1,135 @@+use regex::Regex;++use crate::{Span, offsets::build_byte_to_char_map};++use super::{Mask, Masker};++/// Allows one to mask the sections of a document that match a regular expression (or vice versa).+pub struct RegexMasker {+ regex: Regex,+ exclude_matches: bool,+}++impl RegexMasker {+ /// Parses and compiles the provided Regex expression. Returns None if an invalid expression+ /// was provided.+ ///+ /// If `exclude_matches` is marked `true`, then the areas selected by the regular expression+ /// will be _removed_ from Harper's view. If it is `false`, those areas will be the only ones+ /// _included_.+ pub fn new(regex: &str, exclude_matches: bool) -> Option<Self> {+ Some(Self {+ regex: Regex::new(regex).ok()?,+ exclude_matches,+ })+ }+}++impl Masker for RegexMasker {+ fn create_mask(&self, source: &[char]) -> Mask {+ let source_s: String = source.iter().collect();+ let byte_to_char = build_byte_to_char_map(&source_s);++ let mut mask = Mask::new_blank();++ if self.exclude_matches {+ let mut allowed_start = 0;++ for m in self.regex.find_iter(&source_s) {+ let match_start = byte_to_char[m.start()];+ let match_end = byte_to_char[m.end()];++ if allowed_start < match_start {+ mask.push_allowed(Span::new(allowed_start, match_start));+ }++ allowed_start = match_end;+ }++ if allowed_start < source.len() {+ mask.push_allowed(Span::new(allowed_start, source.len()));+ }+ } else {+ for m in self.regex.find_iter(&source_s) {+ let match_start = byte_to_char[m.start()];+ let match_end = byte_to_char[m.end()];++ if match_start < match_end {+ mask.push_allowed(Span::new(match_start, match_end));+ }+ }+ }++ mask+ }+}++#[cfg(test)]+mod tests {+ use quickcheck::TestResult;+ use quickcheck_macros::quickcheck;++ use super::RegexMasker;+ use crate::{Masker, Span};++ #[test]+ fn include_matches() {+ let source: Vec<_> = "foo [ignore] bar [drop]".chars().collect();+ let masker = RegexMasker::new(r"\[[^\]]+\]", false).unwrap();++ let allowed = masker+ .create_mask(&source)+ .iter_allowed(&source)+ .map(|(_, chars)| chars.iter().collect::<String>())+ .collect::<Vec<_>>();++ assert_eq!(allowed, vec!["[ignore]", "[drop]"]);+ }++ #[test]+ fn exclude_matches() {+ let source: Vec<_> = "foo [ignore] bar [drop]".chars().collect();+ let masker = RegexMasker::new(r"\[[^\]]+\]", true).unwrap();++ let allowed = masker+ .create_mask(&source)+ .iter_allowed(&source)+ .map(|(_, chars)| chars.iter().collect::<String>())+ .collect::<Vec<_>>();++ assert_eq!(allowed, vec!["foo ", " bar "]);+ }++ #[test]+ fn unicode_offsets_are_converted_to_char_spans() {+ let source: Vec<_> = "AπBπC".chars().collect();+ let masker = RegexMasker::new(r"πBπ", false).unwrap();++ let allowed = masker+ .create_mask(&source)+ .iter_allowed(&source)+ .map(|(_, chars)| chars.iter().collect::<String>())+ .collect::<Vec<_>>();++ assert_eq!(allowed, vec!["πBπ"]);+ }++ #[quickcheck]+ fn can_match_everything(source: String) -> TestResult {+ if source.contains(|s: char| !s.is_ascii() || s.is_control()) {+ return TestResult::discard();+ }++ let masker = RegexMasker::new(".*", false).unwrap();++ let chars: Vec<_> = source.chars().collect();+ let mask = masker.create_mask(&chars);++ if !chars.is_empty() {+ assert_eq!(mask.allowed, vec![Span::new_with_len(0, chars.len())]);+ TestResult::passed()+ } else {+ TestResult::discard()+ }+ }+}diff --git a/harper-core/src/offsets.rs b/harper-core/src/offsets.rsnew file mode 100644index 000000000..7ee4d5610--- /dev/null+++ b/harper-core/src/offsets.rs@@ -0,0 +1,36 @@+/// Build a lookup table that maps every byte offset in `source` to its+/// corresponding character offset.+pub(crate) fn build_byte_to_char_map(source: &str) -> Vec<usize> {+ let mut byte_to_char = vec![0; source.len() + 1];+ let mut char_idx = 0;++ for (byte_idx, ch) in source.char_indices() {+ let next_byte_idx = byte_idx + ch.len_utf8();++ for slot in &mut byte_to_char[byte_idx..next_byte_idx] {+ *slot = char_idx;+ }++ char_idx += 1;+ byte_to_char[next_byte_idx] = char_idx;+ }++ byte_to_char+}++#[cfg(test)]+mod tests {+ use super::build_byte_to_char_map;++ #[test]+ fn maps_ascii_offsets() {+ let map = build_byte_to_char_map("abc");+ assert_eq!(map, vec![0, 1, 2, 3]);+ }++ #[test]+ fn maps_unicode_offsets() {+ let map = build_byte_to_char_map("AπB");+ assert_eq!(map, vec![0, 1, 1, 1, 1, 2, 3]);+ }+}diff --git a/harper-core/src/parsers/markdown.rs b/harper-core/src/parsers/markdown.rsindex be64f5fad..3f9d2aa69 100644--- a/harper-core/src/parsers/markdown.rs+++ b/harper-core/src/parsers/markdown.rs@@ -3,7 +3,7 @@ use std::collections::VecDeque;use serde::{Deserialize, Serialize};use super::{Parser, PlainEnglish};-use crate::{Span, Token, TokenKind, TokenStringExt, VecExt};+use crate::{Span, Token, TokenKind, TokenStringExt, VecExt, offsets::build_byte_to_char_map};/// A parser that wraps the [`PlainEnglish`] parser that allows one to parse/// CommonMark files.@@ -162,19 +162,8 @@ impl Parser for Markdown {let mut tokens = Vec::new();// Build a mapping from the inner parser's byte-based indexing to Harper's char-based- // indexing- let mut byte_to_char = vec![0; source_str.len() + 1];- let mut char_index = 0;- let mut byte_idx = 0;- for ch in source_str.chars() {- let char_len = ch.len_utf8();- for _ in 0..char_len {- byte_to_char[byte_idx] = char_index;- byte_idx += 1;- }- char_index += 1;- }- byte_to_char[source_str.len()] = char_index;+ // indexing.+ let byte_to_char = build_byte_to_char_map(&source_str);let mut stack = Vec::new();diff --git a/harper-wasm/src/lib.rs b/harper-wasm/src/lib.rsindex 2d98829a5..e98a8db99 100644--- a/harper-wasm/src/lib.rs+++ b/harper-wasm/src/lib.rs@@ -5,16 +5,16 @@ use std::convert::Into;use std::io::Cursor;use std::sync::Arc;-use harper_core::DialectFlags;use harper_core::language_detection::is_doc_likely_english;use harper_core::linting::{LintGroup, Linter as _};-use harper_core::parsers::{IsolateEnglish, Markdown, OopsAllHeadings, Parser, PlainEnglish};+use harper_core::parsers::{IsolateEnglish, Markdown, Mask, OopsAllHeadings, Parser, PlainEnglish};use harper_core::remove_overlaps_map;use harper_core::weirpack::Weirpack;use harper_core::{CharString, DictWordMetadata, Document, IgnoredLints, LintContext, Lrc, remove_overlaps,spell::{Dictionary, FstDictionary, MergedDictionary, MutableDictionary},};+use harper_core::{DialectFlags, RegexMasker};use harper_stats::{Record, RecordKind, Stats};use harper_typst::Typst;use serde::{Deserialize, Serialize};@@ -271,12 +271,22 @@ impl Linter {text: String,language: Language,all_headings: bool,+ regex_mask: Option<String>,) -> Vec<OrganizedGroup> {let source: Vec<_> = text.chars().collect();let source = Lrc::new(source);let mut parser = language.create_parser();+ if let Some(regex) = regex_mask {+ let masker_maybe = RegexMasker::new(regex.as_str(), true);+ if let Some(masker) = masker_maybe {+ parser = Box::new(Mask::new(masker, parser));+ } else {+ return vec![];+ }+ }+if all_headings {parser = Box::new(OopsAllHeadings::new(parser));}@@ -314,12 +324,29 @@ impl Linter {}/// Perform the configured linting on the provided text.- pub fn lint(&mut self, text: String, language: Language, all_headings: bool) -> Vec<Lint> {+ ///+ /// If the provided regex mask cannot be parsed, this method will return an empty array.+ pub fn lint(+ &mut self,+ text: String,+ language: Language,+ all_headings: bool,+ regex_mask: Option<String>,+ ) -> Vec<Lint> {let source: Vec<_> = text.chars().collect();let source = Lrc::new(source);let mut parser = language.create_parser();+ if let Some(regex) = regex_mask {+ let masker_maybe = RegexMasker::new(regex.as_str(), true);+ if let Some(masker) = masker_maybe {+ parser = Box::new(Mask::new(masker, parser));+ } else {+ return vec![];+ }+ }+if all_headings {parser = Box::new(OopsAllHeadings::new(parser));}@@ -714,7 +741,7 @@ mod tests {linter.import_words(vec![text.clone()]);dbg!(linter.dictionary.get_word_metadata_str(&text));- let lints = linter.lint(text, Language::Plain, false);+ let lints = linter.lint(text, Language::Plain, false, None);assert!(lints.is_empty());}@@ -734,6 +761,7 @@ mod tests {"This is a grammatically correct sentence.".to_string(),Language::Plain,false,+ None,);assert!(results.is_empty())diff --git a/packages/harper.js/src/Linter.test.ts b/packages/harper.js/src/Linter.test.tsindex dbc725889..43fc1981f 100644--- a/packages/harper.js/src/Linter.test.ts+++ b/packages/harper.js/src/Linter.test.ts@@ -527,6 +527,25 @@ for (const [linterName, Linter] of Object.entries(linters)) {await linter.dispose();});+ test(`${linterName} can exclude things with Regex in organized lint function.`, async () => {+ const linter = new Linter({ binary });++ const regex = 'errorz';+ const source = 'This text contains errorz.';++ // Without regex, Harper should detect the error.+ let lints = await linter.organizedLints(source);+ let flattened = Object.values(lints).flat();+ expect(flattened).toHaveLength(1);++ // With regex, Harper should not detect the error.+ lints = await linter.organizedLints(source, { regex_mask: regex });+ flattened = Object.values(lints).flat();+ expect(flattened).toHaveLength(0);++ await linter.dispose();+ });+test(`${linterName} returns correct suggestion for 'ned' with organizedLints.`, async () => {const linter = new Linter({ binary });@@ -543,6 +562,23 @@ for (const [linterName, Linter] of Object.entries(linters)) {await linter.dispose();});+ test(`${linterName} can exclude things with Regex in normal lint function.`, async () => {+ const linter = new Linter({ binary });++ const regex = 'errorz';+ const source = 'This text contains errorz.';++ // Without regex, Harper should detect the error.+ let lints = await linter.lint(source);+ expect(lints).toHaveLength(1);++ // With regex, Harper should not detect the error.+ lints = await linter.lint(source, { regex_mask: regex });+ expect(lints).toHaveLength(0);++ await linter.dispose();+ });+test(`${linterName} returns correct suggestion for 'ned'.`, async () => {const linter = new Linter({ binary });diff --git a/packages/harper.js/src/LocalLinter.ts b/packages/harper.js/src/LocalLinter.tsindex edd221486..517557b43 100644--- a/packages/harper.js/src/LocalLinter.ts+++ b/packages/harper.js/src/LocalLinter.ts@@ -49,7 +49,12 @@ export default class LocalLinter implements Linter {language = Language.Typst;}- const lints = inner.lint(text, language, options?.forceAllHeadings ?? false);+ const lints = inner.lint(+ text,+ language,+ options?.forceAllHeadings ?? false,+ options?.regex_mask,+ );return lints;}@@ -57,7 +62,12 @@ export default class LocalLinter implements Linter {async organizedLints(text: string, options?: LintOptions): Promise<Record<string, Lint[]>> {const inner = await this.inner;const language = options?.language === 'plaintext' ? Language.Plain : Language.Markdown;- const lintGroups = inner.organized_lints(text, language, options?.forceAllHeadings ?? false);+ const lintGroups = inner.organized_lints(+ text,+ language,+ options?.forceAllHeadings ?? false,+ options?.regex_mask,+ );const output: Record<string, Lint[]> = {};diff --git a/packages/harper.js/src/main.ts b/packages/harper.js/src/main.tsindex ca1c82788..3d340c835 100644--- a/packages/harper.js/src/main.ts+++ b/packages/harper.js/src/main.ts@@ -24,6 +24,7 @@ export type LintConfig = Record<string, boolean | null>;export interface LintOptions {/** The markup language that is being passed. Defaults to `markdown`. */language?: 'plaintext' | 'markdown' | 'typst';+ regex_mask?: string;/** Force the entirety of the document to be composed of headings. An undefined value is assumed to be false.*/forceAllHeadings?: boolean;diff --git a/packages/obsidian-plugin/src/HarperSettingTab.ts b/packages/obsidian-plugin/src/HarperSettingTab.tsindex 9cf0c747c..c0dd3ef9d 100644--- a/packages/obsidian-plugin/src/HarperSettingTab.ts+++ b/packages/obsidian-plugin/src/HarperSettingTab.ts@@ -100,6 +100,18 @@ export class HarperSettingTab extends PluginSettingTab {}),);+ new Setting(containerEl)+ .setName('Mask')+ .setDesc(+ "Hide certain text from Harper's pedantic gaze with a regular expression. Follows the standard Rust syntax.",+ )+ .addTextArea((ta) =>+ ta.setValue(this.settings.regexMask ?? '').onChange(async (value) => {+ this.settings.regexMask = value;+ await this.state.initializeFromSettings(this.settings);+ }),+ );+new Setting(containerEl).setName('Personal Dictionary').setDesc(diff --git a/packages/obsidian-plugin/src/State.ts b/packages/obsidian-plugin/src/State.tsindex 556144900..2c1a55a0b 100644--- a/packages/obsidian-plugin/src/State.ts+++ b/packages/obsidian-plugin/src/State.ts@@ -14,6 +14,7 @@ export type Settings = {delay?: number;ignoredGlobs?: string[];lintEnabled?: boolean;+ regexMask?: string;};const DEFAULT_DELAY = -1;@@ -29,6 +30,7 @@ export default class State {private ignoredGlobs?: string[];private editorInfoField?: StateField<MarkdownFileInfo>;private lintEnabled?: boolean;+ private regexMask?: string;/** The CodeMirror extension objects that should be inserted by the host. */private editorExtensions: Extension[];@@ -100,6 +102,7 @@ export default class State {this.delay = settings.delay ?? DEFAULT_DELAY;this.ignoredGlobs = settings.ignoredGlobs;this.lintEnabled = settings.lintEnabled;+ this.regexMask = settings.regexMask;// Reinitialize it.if (this.hasEditorLinter()) {@@ -131,9 +134,7 @@ export default class State {}const text = view.state.doc.sliceString(-1);- const chars = Array.from(text);-- const lints = await this.harper.organizedLints(text);+ const lints = await this.harper.organizedLints(text, { regex_mask: this.regexMask });return Object.entries(lints).flatMap(([linterName, lints]) =>lints.map((lint) => {@@ -251,6 +252,7 @@ export default class State {delay: this.delay,ignoredGlobs: this.ignoredGlobs,lintEnabled: this.lintEnabled,+ regexMask: this.regexMask,};}