Offline, privacy-first grammar checker. Fast, open-source, Rust-powered
feat: prototyped initial language detection
Elijah Potter078ac694ff14
diff --git a/harper-core/src/document.rs b/harper-core/src/document.rsindex fbaab8716..47df1c006 100644--- a/harper-core/src/document.rs+++ b/harper-core/src/document.rs@@ -181,14 +181,6 @@ impl Document {first_sentence.into_iter().chain(rest).chain(last)}- /** Returns all tokens whose `kind` is [`TokenKind::Word`] */- pub fn words(&self) -> impl Iterator<Item = Token> + '_ {- self.tokens- .iter()- .filter(|token| token.kind.is_word())- .cloned()- }-pub fn get_span_content(&self, span: Span) -> &[char] {span.get_content(&self.source)}diff --git a/harper-core/src/language_detection.rs b/harper-core/src/language_detection.rsnew file mode 100644index 000000000..af64776d3--- /dev/null+++ b/harper-core/src/language_detection.rs@@ -0,0 +1,117 @@+use crate::{Dictionary, Document, TokenKind};++/// Check if the contents of the document are likely intended to represent+/// English.+pub fn is_likely_english(doc: &Document, dict: &impl Dictionary) -> bool {+ let mut total_words = 0;+ let mut valid_words = 0;+ let mut punctuation = 0;+ let mut spaces = 0;++ for token in doc.tokens() {+ match token.kind {+ TokenKind::Word => {+ total_words += 1;++ let word_content = doc.get_span_content(token.span);+ if dict.contains_word(word_content) {+ valid_words += 1;+ }+ }+ TokenKind::Punctuation(_) => punctuation += 1,+ TokenKind::Space(n) => spaces += n,+ _ => ()+ }+ }++ dbg!(total_words);+ dbg!(valid_words);+ dbg!(punctuation);+ dbg!(spaces);++ if spaces > total_words * 3 {+ return false;+ }++ if punctuation > valid_words {+ return false;+ }++ if (valid_words as f64 / total_words as f64) < 0.4 {+ return false;+ }++ true+}++#[cfg(test)]+mod tests {+ use super::is_likely_english;+ use crate::{Document, FullDictionary};++ fn assert_not_english(source: &'static str) {+ let dict = FullDictionary::create_from_curated();+ let doc = Document::new_plain_english(source);+ let is_likely_english = is_likely_english(&doc, &dict);+ dbg!(source);+ assert!(!is_likely_english);+ }++ fn assert_english(source: &'static str) {+ let dict = FullDictionary::create_from_curated();+ let doc = Document::new_plain_english(source);+ let is_likely_english = is_likely_english(&doc, &dict);+ dbg!(source);+ assert!(is_likely_english);+ }++ #[test]+ fn detects_spanish() {+ assert_not_english("Esto es español. Harper no debería marcarlo como inglés.");+ }++ #[test]+ fn detects_french() {+ assert_not_english(+ "C'est du français. Il ne devrait pas être marqué comme anglais par Harper."+ );+ }++ #[test]+ fn detects_shebang() {+ assert_not_english("#! /bin/bash");+ assert_not_english("#! /usr/bin/fish");+ }++ #[test]+ fn detects_short_english() {+ assert_english("This is English!");+ }++ #[test]+ fn detects_english() {+ assert_english("This is perfectly valid English, evn if it has a cople typos.")+ }++ #[test]+ fn detects_expressive_english() {+ assert_english("Look above! That is real English! So is this: bippity bop!")+ }++ /// Useful for detecting commented-out code.+ #[test]+ fn detects_python_fib() {+ assert_not_english(+ r#"+def fibIter(n):+ if n < 2:+ return n+ fibPrev = 1+ fib = 1+ for _ in range(2, n):+ fibPrev, fib = fib, fib + fibPrev+ return fib+ "#+ );+ }+}diff --git a/harper-core/src/lexing/mod.rs b/harper-core/src/lexing/mod.rsindex ec63ca162..bd6d3c7f0 100644--- a/harper-core/src/lexing/mod.rs+++ b/harper-core/src/lexing/mod.rs@@ -20,6 +20,7 @@ pub struct FoundToken {pub fn lex_token(source: &[char]) -> Option<FoundToken> {let lexers = [lex_punctuation,+ lex_tabs,lex_spaces,lex_newlines,lex_number,@@ -96,6 +97,19 @@ fn lex_newlines(source: &[char]) -> Option<FoundToken> {}}+fn lex_tabs(source: &[char]) -> Option<FoundToken> {+ let count = source.iter().take_while(|c| **c == '\t').count();++ if count > 0 {+ Some(FoundToken {+ token: TokenKind::Space(count * 2),+ next_index: count+ })+ } else {+ None+ }+}+fn lex_spaces(source: &[char]) -> Option<FoundToken> {let count = source.iter().take_while(|c| **c == ' ').count();diff --git a/harper-core/src/lib.rs b/harper-core/src/lib.rsindex d85041fa1..baac1f84d 100644--- a/harper-core/src/lib.rs+++ b/harper-core/src/lib.rs@@ -3,6 +3,7 @@mod char_ext;mod char_string;mod document;+mod language_detection;mod lexing;mod linting;pub mod parsers;diff --git a/harper-core/src/linting/spell_check.rs b/harper-core/src/linting/spell_check.rsindex e0578587c..4accfd0bf 100644--- a/harper-core/src/linting/spell_check.rs+++ b/harper-core/src/linting/spell_check.rs@@ -4,7 +4,7 @@ use super::lint::Suggestion;use super::{Lint, LintKind, Linter};use crate::document::Document;use crate::spell::suggest_correct_spelling;-use crate::Dictionary;+use crate::{Dictionary, TokenStringExt};pub struct SpellCheck<T>where@@ -53,7 +53,7 @@ impl<T: Dictionary> Linter for SpellCheck<T> {fn lint(&mut self, document: &Document) -> Vec<Lint> {let mut lints = Vec::new();- for word in document.words() {+ for word in document.iter_words() {let word_chars = document.get_span_content(word.span);if self.dictionary.contains_word(word_chars) {continue;diff --git a/harper-ls/src/backend.rs b/harper-ls/src/backend.rsindex 509c49233..bc16cf53d 100644--- a/harper-ls/src/backend.rs+++ b/harper-ls/src/backend.rs@@ -349,9 +349,7 @@ impl Backend {#[tower_lsp::async_trait]impl LanguageServer for Backend {- async fn initialize(&self, p: InitializeParams) -> Result<InitializeResult> {- dbg!(p.capabilities);-+ async fn initialize(&self, _: InitializeParams) -> Result<InitializeResult> {Ok(InitializeResult {server_info: None,capabilities: ServerCapabilities {