Skip to content

Offline, privacy-first grammar checker. Fast, open-source, Rust-powered

feat(obsidian): allow users to mask out text using regular expressions (#2684)

Elijah Potter2076d2e9ee25
diff --git a/Cargo.lock b/Cargo.lock
index e60a359d2..de906d75d 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -2785,6 +2785,7 @@ dependencies = [
"quickcheck_macros",
"rand 0.8.5",
"rayon",
+ "regex",
"serde",
"serde_json",
"smallvec",
@@ -4554,9 +4555,9 @@ dependencies = [
[[package]]
name = "regex"
-version = "1.12.2"
+version = "1.12.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "843bc0191f75f3e22651ae5f1e72939ab2f72a4bc30fa80a066bd66edefc24d4"
+checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276"
dependencies = [
"aho-corasick",
"memchr",
diff --git a/harper-core/Cargo.toml b/harper-core/Cargo.toml
index fe99f5323..ba0f98b29 100644
--- a/harper-core/Cargo.toml
+++ b/harper-core/Cargo.toml
@@ -35,6 +35,7 @@ harper-thesaurus = { path = "../harper-thesaurus", version = "1.4.1", optional =
bitflags = { version = "2.10.0", features = ["serde"] }
trie-rs = "0.4.2"
zip = { version = "2.2.0", default-features = false, features = ["deflate"] }
+regex = "1.12.3"
[dev-dependencies]
criterion = { version = "0.8.2", default-features = false }
diff --git a/harper-core/src/lib.rs b/harper-core/src/lib.rs
index 4c37bcce9..5a3fa205e 100644
--- a/harper-core/src/lib.rs
+++ b/harper-core/src/lib.rs
@@ -19,6 +19,7 @@ mod lexing;
pub mod linting;
mod mask;
mod number;
+mod offsets;
pub mod parsers;
pub mod patterns;
mod punctuation;
@@ -52,7 +53,7 @@ pub use ignored_lints::{IgnoredLints, LintContext};
pub use irregular_nouns::IrregularNouns;
pub use irregular_verbs::IrregularVerbs;
use linting::Lint;
-pub use mask::{Mask, Masker};
+pub use mask::{Mask, Masker, RegexMasker};
pub use number::{Number, OrdinalSuffix};
pub use punctuation::{Punctuation, Quote};
pub use span::Span;
diff --git a/harper-core/src/mask/mod.rs b/harper-core/src/mask/mod.rs
index a9b78ccc4..b34517719 100644
--- a/harper-core/src/mask/mod.rs
+++ b/harper-core/src/mask/mod.rs
@@ -1,3 +1,6 @@
+mod regex_masker;
+pub use regex_masker::RegexMasker;
+
use itertools::Itertools;
use crate::Span;
diff --git a/harper-core/src/mask/regex_masker.rs b/harper-core/src/mask/regex_masker.rs
new file mode 100644
index 000000000..b0eecb278
--- /dev/null
+++ b/harper-core/src/mask/regex_masker.rs
@@ -0,0 +1,135 @@
+use regex::Regex;
+
+use crate::{Span, offsets::build_byte_to_char_map};
+
+use super::{Mask, Masker};
+
+/// Allows one to mask the sections of a document that match a regular expression (or vice versa).
+pub struct RegexMasker {
+ regex: Regex,
+ exclude_matches: bool,
+}
+
+impl RegexMasker {
+ /// Parses and compiles the provided Regex expression. Returns None if an invalid expression
+ /// was provided.
+ ///
+ /// If `exclude_matches` is marked `true`, then the areas selected by the regular expression
+ /// will be _removed_ from Harper's view. If it is `false`, those areas will be the only ones
+ /// _included_.
+ pub fn new(regex: &str, exclude_matches: bool) -> Option<Self> {
+ Some(Self {
+ regex: Regex::new(regex).ok()?,
+ exclude_matches,
+ })
+ }
+}
+
+impl Masker for RegexMasker {
+ fn create_mask(&self, source: &[char]) -> Mask {
+ let source_s: String = source.iter().collect();
+ let byte_to_char = build_byte_to_char_map(&source_s);
+
+ let mut mask = Mask::new_blank();
+
+ if self.exclude_matches {
+ let mut allowed_start = 0;
+
+ for m in self.regex.find_iter(&source_s) {
+ let match_start = byte_to_char[m.start()];
+ let match_end = byte_to_char[m.end()];
+
+ if allowed_start < match_start {
+ mask.push_allowed(Span::new(allowed_start, match_start));
+ }
+
+ allowed_start = match_end;
+ }
+
+ if allowed_start < source.len() {
+ mask.push_allowed(Span::new(allowed_start, source.len()));
+ }
+ } else {
+ for m in self.regex.find_iter(&source_s) {
+ let match_start = byte_to_char[m.start()];
+ let match_end = byte_to_char[m.end()];
+
+ if match_start < match_end {
+ mask.push_allowed(Span::new(match_start, match_end));
+ }
+ }
+ }
+
+ mask
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use quickcheck::TestResult;
+ use quickcheck_macros::quickcheck;
+
+ use super::RegexMasker;
+ use crate::{Masker, Span};
+
+ #[test]
+ fn include_matches() {
+ let source: Vec<_> = "foo [ignore] bar [drop]".chars().collect();
+ let masker = RegexMasker::new(r"\[[^\]]+\]", false).unwrap();
+
+ let allowed = masker
+ .create_mask(&source)
+ .iter_allowed(&source)
+ .map(|(_, chars)| chars.iter().collect::<String>())
+ .collect::<Vec<_>>();
+
+ assert_eq!(allowed, vec!["[ignore]", "[drop]"]);
+ }
+
+ #[test]
+ fn exclude_matches() {
+ let source: Vec<_> = "foo [ignore] bar [drop]".chars().collect();
+ let masker = RegexMasker::new(r"\[[^\]]+\]", true).unwrap();
+
+ let allowed = masker
+ .create_mask(&source)
+ .iter_allowed(&source)
+ .map(|(_, chars)| chars.iter().collect::<String>())
+ .collect::<Vec<_>>();
+
+ assert_eq!(allowed, vec!["foo ", " bar "]);
+ }
+
+ #[test]
+ fn unicode_offsets_are_converted_to_char_spans() {
+ let source: Vec<_> = "AπŸ™‚BπŸ™‚C".chars().collect();
+ let masker = RegexMasker::new(r"πŸ™‚BπŸ™‚", false).unwrap();
+
+ let allowed = masker
+ .create_mask(&source)
+ .iter_allowed(&source)
+ .map(|(_, chars)| chars.iter().collect::<String>())
+ .collect::<Vec<_>>();
+
+ assert_eq!(allowed, vec!["πŸ™‚BπŸ™‚"]);
+ }
+
+ #[quickcheck]
+ fn can_match_everything(source: String) -> TestResult {
+ if source.contains(|s: char| !s.is_ascii() || s.is_control()) {
+ return TestResult::discard();
+ }
+
+ let masker = RegexMasker::new(".*", false).unwrap();
+
+ let chars: Vec<_> = source.chars().collect();
+ let mask = masker.create_mask(&chars);
+
+ if !chars.is_empty() {
+ assert_eq!(mask.allowed, vec![Span::new_with_len(0, chars.len())]);
+ TestResult::passed()
+ } else {
+ TestResult::discard()
+ }
+ }
+}
diff --git a/harper-core/src/offsets.rs b/harper-core/src/offsets.rs
new file mode 100644
index 000000000..7ee4d5610
--- /dev/null
+++ b/harper-core/src/offsets.rs
@@ -0,0 +1,36 @@
+/// Build a lookup table that maps every byte offset in `source` to its
+/// corresponding character offset.
+pub(crate) fn build_byte_to_char_map(source: &str) -> Vec<usize> {
+ let mut byte_to_char = vec![0; source.len() + 1];
+ let mut char_idx = 0;
+
+ for (byte_idx, ch) in source.char_indices() {
+ let next_byte_idx = byte_idx + ch.len_utf8();
+
+ for slot in &mut byte_to_char[byte_idx..next_byte_idx] {
+ *slot = char_idx;
+ }
+
+ char_idx += 1;
+ byte_to_char[next_byte_idx] = char_idx;
+ }
+
+ byte_to_char
+}
+
+#[cfg(test)]
+mod tests {
+ use super::build_byte_to_char_map;
+
+ #[test]
+ fn maps_ascii_offsets() {
+ let map = build_byte_to_char_map("abc");
+ assert_eq!(map, vec![0, 1, 2, 3]);
+ }
+
+ #[test]
+ fn maps_unicode_offsets() {
+ let map = build_byte_to_char_map("AπŸ™‚B");
+ assert_eq!(map, vec![0, 1, 1, 1, 1, 2, 3]);
+ }
+}
diff --git a/harper-core/src/parsers/markdown.rs b/harper-core/src/parsers/markdown.rs
index be64f5fad..3f9d2aa69 100644
--- a/harper-core/src/parsers/markdown.rs
+++ b/harper-core/src/parsers/markdown.rs
@@ -3,7 +3,7 @@ use std::collections::VecDeque;
use serde::{Deserialize, Serialize};
use super::{Parser, PlainEnglish};
-use crate::{Span, Token, TokenKind, TokenStringExt, VecExt};
+use crate::{Span, Token, TokenKind, TokenStringExt, VecExt, offsets::build_byte_to_char_map};
/// A parser that wraps the [`PlainEnglish`] parser that allows one to parse
/// CommonMark files.
@@ -162,19 +162,8 @@ impl Parser for Markdown {
let mut tokens = Vec::new();
// Build a mapping from the inner parser's byte-based indexing to Harper's char-based
- // indexing
- let mut byte_to_char = vec![0; source_str.len() + 1];
- let mut char_index = 0;
- let mut byte_idx = 0;
- for ch in source_str.chars() {
- let char_len = ch.len_utf8();
- for _ in 0..char_len {
- byte_to_char[byte_idx] = char_index;
- byte_idx += 1;
- }
- char_index += 1;
- }
- byte_to_char[source_str.len()] = char_index;
+ // indexing.
+ let byte_to_char = build_byte_to_char_map(&source_str);
let mut stack = Vec::new();
diff --git a/harper-wasm/src/lib.rs b/harper-wasm/src/lib.rs
index 2d98829a5..e98a8db99 100644
--- a/harper-wasm/src/lib.rs
+++ b/harper-wasm/src/lib.rs
@@ -5,16 +5,16 @@ use std::convert::Into;
use std::io::Cursor;
use std::sync::Arc;
-use harper_core::DialectFlags;
use harper_core::language_detection::is_doc_likely_english;
use harper_core::linting::{LintGroup, Linter as _};
-use harper_core::parsers::{IsolateEnglish, Markdown, OopsAllHeadings, Parser, PlainEnglish};
+use harper_core::parsers::{IsolateEnglish, Markdown, Mask, OopsAllHeadings, Parser, PlainEnglish};
use harper_core::remove_overlaps_map;
use harper_core::weirpack::Weirpack;
use harper_core::{
CharString, DictWordMetadata, Document, IgnoredLints, LintContext, Lrc, remove_overlaps,
spell::{Dictionary, FstDictionary, MergedDictionary, MutableDictionary},
};
+use harper_core::{DialectFlags, RegexMasker};
use harper_stats::{Record, RecordKind, Stats};
use harper_typst::Typst;
use serde::{Deserialize, Serialize};
@@ -271,12 +271,22 @@ impl Linter {
text: String,
language: Language,
all_headings: bool,
+ regex_mask: Option<String>,
) -> Vec<OrganizedGroup> {
let source: Vec<_> = text.chars().collect();
let source = Lrc::new(source);
let mut parser = language.create_parser();
+ if let Some(regex) = regex_mask {
+ let masker_maybe = RegexMasker::new(regex.as_str(), true);
+ if let Some(masker) = masker_maybe {
+ parser = Box::new(Mask::new(masker, parser));
+ } else {
+ return vec![];
+ }
+ }
+
if all_headings {
parser = Box::new(OopsAllHeadings::new(parser));
}
@@ -314,12 +324,29 @@ impl Linter {
}
/// Perform the configured linting on the provided text.
- pub fn lint(&mut self, text: String, language: Language, all_headings: bool) -> Vec<Lint> {
+ ///
+ /// If the provided regex mask cannot be parsed, this method will return an empty array.
+ pub fn lint(
+ &mut self,
+ text: String,
+ language: Language,
+ all_headings: bool,
+ regex_mask: Option<String>,
+ ) -> Vec<Lint> {
let source: Vec<_> = text.chars().collect();
let source = Lrc::new(source);
let mut parser = language.create_parser();
+ if let Some(regex) = regex_mask {
+ let masker_maybe = RegexMasker::new(regex.as_str(), true);
+ if let Some(masker) = masker_maybe {
+ parser = Box::new(Mask::new(masker, parser));
+ } else {
+ return vec![];
+ }
+ }
+
if all_headings {
parser = Box::new(OopsAllHeadings::new(parser));
}
@@ -714,7 +741,7 @@ mod tests {
linter.import_words(vec![text.clone()]);
dbg!(linter.dictionary.get_word_metadata_str(&text));
- let lints = linter.lint(text, Language::Plain, false);
+ let lints = linter.lint(text, Language::Plain, false, None);
assert!(lints.is_empty());
}
@@ -734,6 +761,7 @@ mod tests {
"This is a grammatically correct sentence.".to_string(),
Language::Plain,
false,
+ None,
);
assert!(results.is_empty())
diff --git a/packages/harper.js/src/Linter.test.ts b/packages/harper.js/src/Linter.test.ts
index dbc725889..43fc1981f 100644
--- a/packages/harper.js/src/Linter.test.ts
+++ b/packages/harper.js/src/Linter.test.ts
@@ -527,6 +527,25 @@ for (const [linterName, Linter] of Object.entries(linters)) {
await linter.dispose();
});
+ test(`${linterName} can exclude things with Regex in organized lint function.`, async () => {
+ const linter = new Linter({ binary });
+
+ const regex = 'errorz';
+ const source = 'This text contains errorz.';
+
+ // Without regex, Harper should detect the error.
+ let lints = await linter.organizedLints(source);
+ let flattened = Object.values(lints).flat();
+ expect(flattened).toHaveLength(1);
+
+ // With regex, Harper should not detect the error.
+ lints = await linter.organizedLints(source, { regex_mask: regex });
+ flattened = Object.values(lints).flat();
+ expect(flattened).toHaveLength(0);
+
+ await linter.dispose();
+ });
+
test(`${linterName} returns correct suggestion for 'ned' with organizedLints.`, async () => {
const linter = new Linter({ binary });
@@ -543,6 +562,23 @@ for (const [linterName, Linter] of Object.entries(linters)) {
await linter.dispose();
});
+ test(`${linterName} can exclude things with Regex in normal lint function.`, async () => {
+ const linter = new Linter({ binary });
+
+ const regex = 'errorz';
+ const source = 'This text contains errorz.';
+
+ // Without regex, Harper should detect the error.
+ let lints = await linter.lint(source);
+ expect(lints).toHaveLength(1);
+
+ // With regex, Harper should not detect the error.
+ lints = await linter.lint(source, { regex_mask: regex });
+ expect(lints).toHaveLength(0);
+
+ await linter.dispose();
+ });
+
test(`${linterName} returns correct suggestion for 'ned'.`, async () => {
const linter = new Linter({ binary });
diff --git a/packages/harper.js/src/LocalLinter.ts b/packages/harper.js/src/LocalLinter.ts
index edd221486..517557b43 100644
--- a/packages/harper.js/src/LocalLinter.ts
+++ b/packages/harper.js/src/LocalLinter.ts
@@ -49,7 +49,12 @@ export default class LocalLinter implements Linter {
language = Language.Typst;
}
- const lints = inner.lint(text, language, options?.forceAllHeadings ?? false);
+ const lints = inner.lint(
+ text,
+ language,
+ options?.forceAllHeadings ?? false,
+ options?.regex_mask,
+ );
return lints;
}
@@ -57,7 +62,12 @@ export default class LocalLinter implements Linter {
async organizedLints(text: string, options?: LintOptions): Promise<Record<string, Lint[]>> {
const inner = await this.inner;
const language = options?.language === 'plaintext' ? Language.Plain : Language.Markdown;
- const lintGroups = inner.organized_lints(text, language, options?.forceAllHeadings ?? false);
+ const lintGroups = inner.organized_lints(
+ text,
+ language,
+ options?.forceAllHeadings ?? false,
+ options?.regex_mask,
+ );
const output: Record<string, Lint[]> = {};
diff --git a/packages/harper.js/src/main.ts b/packages/harper.js/src/main.ts
index ca1c82788..3d340c835 100644
--- a/packages/harper.js/src/main.ts
+++ b/packages/harper.js/src/main.ts
@@ -24,6 +24,7 @@ export type LintConfig = Record<string, boolean | null>;
export interface LintOptions {
/** The markup language that is being passed. Defaults to `markdown`. */
language?: 'plaintext' | 'markdown' | 'typst';
+ regex_mask?: string;
/** Force the entirety of the document to be composed of headings. An undefined value is assumed to be false.*/
forceAllHeadings?: boolean;
diff --git a/packages/obsidian-plugin/src/HarperSettingTab.ts b/packages/obsidian-plugin/src/HarperSettingTab.ts
index 9cf0c747c..c0dd3ef9d 100644
--- a/packages/obsidian-plugin/src/HarperSettingTab.ts
+++ b/packages/obsidian-plugin/src/HarperSettingTab.ts
@@ -100,6 +100,18 @@ export class HarperSettingTab extends PluginSettingTab {
}),
);
+ new Setting(containerEl)
+ .setName('Mask')
+ .setDesc(
+ "Hide certain text from Harper's pedantic gaze with a regular expression. Follows the standard Rust syntax.",
+ )
+ .addTextArea((ta) =>
+ ta.setValue(this.settings.regexMask ?? '').onChange(async (value) => {
+ this.settings.regexMask = value;
+ await this.state.initializeFromSettings(this.settings);
+ }),
+ );
+
new Setting(containerEl)
.setName('Personal Dictionary')
.setDesc(
diff --git a/packages/obsidian-plugin/src/State.ts b/packages/obsidian-plugin/src/State.ts
index 556144900..2c1a55a0b 100644
--- a/packages/obsidian-plugin/src/State.ts
+++ b/packages/obsidian-plugin/src/State.ts
@@ -14,6 +14,7 @@ export type Settings = {
delay?: number;
ignoredGlobs?: string[];
lintEnabled?: boolean;
+ regexMask?: string;
};
const DEFAULT_DELAY = -1;
@@ -29,6 +30,7 @@ export default class State {
private ignoredGlobs?: string[];
private editorInfoField?: StateField<MarkdownFileInfo>;
private lintEnabled?: boolean;
+ private regexMask?: string;
/** The CodeMirror extension objects that should be inserted by the host. */
private editorExtensions: Extension[];
@@ -100,6 +102,7 @@ export default class State {
this.delay = settings.delay ?? DEFAULT_DELAY;
this.ignoredGlobs = settings.ignoredGlobs;
this.lintEnabled = settings.lintEnabled;
+ this.regexMask = settings.regexMask;
// Reinitialize it.
if (this.hasEditorLinter()) {
@@ -131,9 +134,7 @@ export default class State {
}
const text = view.state.doc.sliceString(-1);
- const chars = Array.from(text);
-
- const lints = await this.harper.organizedLints(text);
+ const lints = await this.harper.organizedLints(text, { regex_mask: this.regexMask });
return Object.entries(lints).flatMap(([linterName, lints]) =>
lints.map((lint) => {
@@ -251,6 +252,7 @@ export default class State {
delay: this.delay,
ignoredGlobs: this.ignoredGlobs,
lintEnabled: this.lintEnabled,
+ regexMask: this.regexMask,
};
}