Skip to content

Offline, privacy-first grammar checker. Fast, open-source, Rust-powered

feat: prototyped initial language detection

Elijah Potter078ac694ff14
diff --git a/harper-core/src/document.rs b/harper-core/src/document.rs
index fbaab8716..47df1c006 100644
--- a/harper-core/src/document.rs
+++ b/harper-core/src/document.rs
@@ -181,14 +181,6 @@ impl Document {
first_sentence.into_iter().chain(rest).chain(last)
}
- /** Returns all tokens whose `kind` is [`TokenKind::Word`] */
- pub fn words(&self) -> impl Iterator<Item = Token> + '_ {
- self.tokens
- .iter()
- .filter(|token| token.kind.is_word())
- .cloned()
- }
-
pub fn get_span_content(&self, span: Span) -> &[char] {
span.get_content(&self.source)
}
diff --git a/harper-core/src/language_detection.rs b/harper-core/src/language_detection.rs
new file mode 100644
index 000000000..af64776d3
--- /dev/null
+++ b/harper-core/src/language_detection.rs
@@ -0,0 +1,117 @@
+use crate::{Dictionary, Document, TokenKind};
+
+/// Check if the contents of the document are likely intended to represent
+/// English.
+pub fn is_likely_english(doc: &Document, dict: &impl Dictionary) -> bool {
+ let mut total_words = 0;
+ let mut valid_words = 0;
+ let mut punctuation = 0;
+ let mut spaces = 0;
+
+ for token in doc.tokens() {
+ match token.kind {
+ TokenKind::Word => {
+ total_words += 1;
+
+ let word_content = doc.get_span_content(token.span);
+ if dict.contains_word(word_content) {
+ valid_words += 1;
+ }
+ }
+ TokenKind::Punctuation(_) => punctuation += 1,
+ TokenKind::Space(n) => spaces += n,
+ _ => ()
+ }
+ }
+
+ dbg!(total_words);
+ dbg!(valid_words);
+ dbg!(punctuation);
+ dbg!(spaces);
+
+ if spaces > total_words * 3 {
+ return false;
+ }
+
+ if punctuation > valid_words {
+ return false;
+ }
+
+ if (valid_words as f64 / total_words as f64) < 0.4 {
+ return false;
+ }
+
+ true
+}
+
+#[cfg(test)]
+mod tests {
+ use super::is_likely_english;
+ use crate::{Document, FullDictionary};
+
+ fn assert_not_english(source: &'static str) {
+ let dict = FullDictionary::create_from_curated();
+ let doc = Document::new_plain_english(source);
+ let is_likely_english = is_likely_english(&doc, &dict);
+ dbg!(source);
+ assert!(!is_likely_english);
+ }
+
+ fn assert_english(source: &'static str) {
+ let dict = FullDictionary::create_from_curated();
+ let doc = Document::new_plain_english(source);
+ let is_likely_english = is_likely_english(&doc, &dict);
+ dbg!(source);
+ assert!(is_likely_english);
+ }
+
+ #[test]
+ fn detects_spanish() {
+ assert_not_english("Esto es español. Harper no debería marcarlo como inglés.");
+ }
+
+ #[test]
+ fn detects_french() {
+ assert_not_english(
+ "C'est du français. Il ne devrait pas être marqué comme anglais par Harper."
+ );
+ }
+
+ #[test]
+ fn detects_shebang() {
+ assert_not_english("#! /bin/bash");
+ assert_not_english("#! /usr/bin/fish");
+ }
+
+ #[test]
+ fn detects_short_english() {
+ assert_english("This is English!");
+ }
+
+ #[test]
+ fn detects_english() {
+ assert_english("This is perfectly valid English, evn if it has a cople typos.")
+ }
+
+ #[test]
+ fn detects_expressive_english() {
+ assert_english("Look above! That is real English! So is this: bippity bop!")
+ }
+
+ /// Useful for detecting commented-out code.
+ #[test]
+ fn detects_python_fib() {
+ assert_not_english(
+ r#"
+def fibIter(n):
+ if n < 2:
+ return n
+ fibPrev = 1
+ fib = 1
+ for _ in range(2, n):
+ fibPrev, fib = fib, fib + fibPrev
+ return fib
+ "#
+ );
+ }
+}
diff --git a/harper-core/src/lexing/mod.rs b/harper-core/src/lexing/mod.rs
index ec63ca162..bd6d3c7f0 100644
--- a/harper-core/src/lexing/mod.rs
+++ b/harper-core/src/lexing/mod.rs
@@ -20,6 +20,7 @@ pub struct FoundToken {
pub fn lex_token(source: &[char]) -> Option<FoundToken> {
let lexers = [
lex_punctuation,
+ lex_tabs,
lex_spaces,
lex_newlines,
lex_number,
@@ -96,6 +97,19 @@ fn lex_newlines(source: &[char]) -> Option<FoundToken> {
}
}
+fn lex_tabs(source: &[char]) -> Option<FoundToken> {
+ let count = source.iter().take_while(|c| **c == '\t').count();
+
+ if count > 0 {
+ Some(FoundToken {
+ token: TokenKind::Space(count * 2),
+ next_index: count
+ })
+ } else {
+ None
+ }
+}
+
fn lex_spaces(source: &[char]) -> Option<FoundToken> {
let count = source.iter().take_while(|c| **c == ' ').count();
diff --git a/harper-core/src/lib.rs b/harper-core/src/lib.rs
index d85041fa1..baac1f84d 100644
--- a/harper-core/src/lib.rs
+++ b/harper-core/src/lib.rs
@@ -3,6 +3,7 @@
mod char_ext;
mod char_string;
mod document;
+mod language_detection;
mod lexing;
mod linting;
pub mod parsers;
diff --git a/harper-core/src/linting/spell_check.rs b/harper-core/src/linting/spell_check.rs
index e0578587c..4accfd0bf 100644
--- a/harper-core/src/linting/spell_check.rs
+++ b/harper-core/src/linting/spell_check.rs
@@ -4,7 +4,7 @@ use super::lint::Suggestion;
use super::{Lint, LintKind, Linter};
use crate::document::Document;
use crate::spell::suggest_correct_spelling;
-use crate::Dictionary;
+use crate::{Dictionary, TokenStringExt};
pub struct SpellCheck<T>
where
@@ -53,7 +53,7 @@ impl<T: Dictionary> Linter for SpellCheck<T> {
fn lint(&mut self, document: &Document) -> Vec<Lint> {
let mut lints = Vec::new();
- for word in document.words() {
+ for word in document.iter_words() {
let word_chars = document.get_span_content(word.span);
if self.dictionary.contains_word(word_chars) {
continue;
diff --git a/harper-ls/src/backend.rs b/harper-ls/src/backend.rs
index 509c49233..bc16cf53d 100644
--- a/harper-ls/src/backend.rs
+++ b/harper-ls/src/backend.rs
@@ -349,9 +349,7 @@ impl Backend {
#[tower_lsp::async_trait]
impl LanguageServer for Backend {
- async fn initialize(&self, p: InitializeParams) -> Result<InitializeResult> {
- dbg!(p.capabilities);
-
+ async fn initialize(&self, _: InitializeParams) -> Result<InitializeResult> {
Ok(InitializeResult {
server_info: None,
capabilities: ServerCapabilities {