Skip to content

Offline, privacy-first grammar checker. Fast, open-source, Rust-powered

feat: created page to demo language detection

Elijah Potter10481244073d
diff --git a/harper-core/src/lib.rs b/harper-core/src/lib.rs
index c7f25c61c..29a7b1015 100644
--- a/harper-core/src/lib.rs
+++ b/harper-core/src/lib.rs
@@ -4,7 +4,7 @@
mod char_ext;
mod char_string;
mod document;
-mod language_detection;
+pub mod language_detection;
mod lexing;
pub mod linting;
mod mask;
diff --git a/harper-wasm/src/lib.rs b/harper-wasm/src/lib.rs
index c4dcd4f9d..ad3f6f343 100644
--- a/harper-wasm/src/lib.rs
+++ b/harper-wasm/src/lib.rs
@@ -2,6 +2,7 @@
use std::sync::Mutex;
+use harper_core::language_detection::is_doc_likely_english;
use harper_core::linting::{LintGroup, LintGroupConfig, Linter};
use harper_core::parsers::Markdown;
use harper_core::{remove_overlaps, Document, FullDictionary, Lrc};
@@ -27,6 +28,13 @@ pub fn setup() {
tracing_wasm::set_as_global_default();
}
+/// Helper method to quickly check if a Markdown string is likely intended to be English
+#[wasm_bindgen]
+pub fn is_likely_english(text: String) -> bool {
+ let document = Document::new_markdown_curated(&text);
+ is_doc_likely_english(&document, &FullDictionary::curated())
+}
+
#[wasm_bindgen]
pub fn get_lint_config_as_object() -> JsValue {
let linter = LINTER.lock().unwrap();
diff --git a/packages/web/src/lib/analysis.ts b/packages/web/src/lib/analysis.ts
index f95b6860c..a5b9f7582 100644
--- a/packages/web/src/lib/analysis.ts
+++ b/packages/web/src/lib/analysis.ts
@@ -25,3 +25,9 @@ export async function applySuggestion(
const applied = wasm.apply_suggestion(text, span, suggestion);
return applied;
}
+
+export async function isLikelyEnglish(text: string): Promise<boolean> {
+ const wasm = await import('wasm');
+
+ return wasm.is_likely_english(text);
+}
diff --git a/packages/web/src/routes/languagedetection/+page.svelte b/packages/web/src/routes/languagedetection/+page.svelte
new file mode 100644
index 000000000..0bd630819
--- /dev/null
+++ b/packages/web/src/routes/languagedetection/+page.svelte
@@ -0,0 +1,52 @@
+<script lang="ts">
+ import { isLikelyEnglish } from '$lib/analysis';
+ import { Textarea, Select } from 'flowbite-svelte';
+ import demoText from '../../../../../demo.md?raw';
+
+ let isEnglish: boolean | null = null;
+ let text = '';
+
+ $: isLikelyEnglish(text).then((v) => (isEnglish = v));
+
+ $: color = isEnglish == null ? '' : isEnglish ? 'bg-green-100' : 'bg-red-100';
+
+ let templates = [
+ {
+ name: 'Java Code',
+ value: `public class Main {
+ public static void main(String[] args) {
+ System.out.println("Hello World");
+ }
+}`
+ },
+ { name: 'Poor Grammar', value: demoText },
+ {
+ name: 'Chinese Lorem Ipsum',
+ value:
+ '食棵支每躲種。奶象打星爪子二細喜才記行在發像原斤!頁固點子衣點豆看身蝴看苗急午公何足,筆娘經色蝶行元香也要。麻了綠尼固世,色北書目登功;因告黑。'
+ }
+ ];
+
+ function selectChange(e: Event) {
+ text = (e.target as HTMLSelectElement).value;
+ }
+</script>
+
+<div class="[&>*]:mt-12 p-4">
+ <h1 class="text-2xl">Language Detection Demo</h1>
+ <p>
+ This is demonstration of Harper's ability to quickly (under 1 ms for large documents) determine
+ whether a provided document is intended to be English. The algorithm is flexible to bad grammar.
+ <br />
+ Since this is used to redact commented-out code, it airs on the side of producing false-positives.
+ </p>
+
+ <Select items={templates} on:change={selectChange} />
+
+ <Textarea
+ rows={8}
+ class={color}
+ bind:value={text}
+ placeholder="Is your text supposed to be English?"
+ />
+</div>