Offline, privacy-first grammar checker. Fast, open-source, Rust-powered
Fixed issues
Elijah Potter1a019ed06591
diff --git a/alice.txt b/alice.txtindex a9054835a..bbee48392 100644--- a/alice.txt+++ b/alice.txt@@ -17,4 +17,3 @@ the same when I got up this morning? I almost think I can rememberfeeling a little different. But if I'm not the same, the next questionis, who in the world am I? Ah, _that's_ the great puzzle!" And she beganthinking over all the children she knew that were of the same age as-herself, to see if she could have been changed for any of them.ubscribe to our email newsletter to hear about new eBooks.diff --git a/lt-core/src/lexer.rs b/lt-core/src/lexer.rsindex ab3c2973a..3f9cd53ac 100644--- a/lt-core/src/lexer.rs+++ b/lt-core/src/lexer.rs@@ -10,14 +10,20 @@ pub struct FoundToken {pub token: TokenKind,}+pub fn lex_to_end_str(source: impl AsRef<str>) -> Vec<Token> {+ let r = source.as_ref();++ let chars: Vec<_> = r.chars().collect();++ lex_to_end(&chars)+}+/// Lex all tokens, if possible.pub fn lex_to_end(source: &[char]) -> Vec<Token> {let mut cursor = 0;let mut tokens = Vec::new();loop {- cursor += lex_ignorables(&source[cursor..]);-if cursor == source.len() {return tokens;}@@ -29,13 +35,19 @@ pub fn lex_to_end(source: &[char]) -> Vec<Token> {});cursor += next_index;} else {- cursor += 1;+ panic!()}}}fn lex_token(source: &[char]) -> Option<FoundToken> {- let lexers = [lex_punctuation, lex_number, lex_word];+ let lexers = [+ lex_spaces,+ lex_newlines,+ lex_punctuation,+ lex_number,+ lex_word,+ ];for lexer in lexers {if let Some(f) = lexer(source) {@@ -49,7 +61,10 @@ fn lex_word(source: &[char]) -> Option<FoundToken> {let mut end = 0;while end < source.len() {- if lex_punctuation(&source[end + 1..]).is_none() && lex_ignorables(&source[end + 1..]) == 0+ if lex_punctuation(&source[end + 1..]).is_none()+ && lex_spaces(&source[end + 1..]).is_none()+ && lex_newlines(&source[end + 1..]).is_none()+ && end + 1 != source.len(){end += 1;} else {@@ -95,32 +110,30 @@ pub fn lex_number(source: &[char]) -> Option<FoundToken> {lex_number(&source[0..end])}-/// Find the first token _after_ any characters that can be ignored (whitespace, mostly).-fn lex_ignorables(source: &[char]) -> usize {- let mut cursor = 0;-- loop {- let last_cursor = cursor;-- cursor += lex_whitespace(&source[cursor..]);+fn lex_newlines(source: &[char]) -> Option<FoundToken> {+ let count = source.iter().take_while(|c| **c == '\n').count();- if last_cursor == cursor {- break;- }+ if count > 0 {+ Some(FoundToken {+ token: TokenKind::Newline(count),+ next_index: count,+ })+ } else {+ None}-- cursor}-/// Find the first token _after_ whitespace.-fn lex_whitespace(source: &[char]) -> usize {- for (index, c) in source.iter().enumerate() {- if !c.is_whitespace() {- return index;- }- }+fn lex_spaces(source: &[char]) -> Option<FoundToken> {+ let count = source.iter().take_while(|c| **c == ' ').count();- source.len()+ if count > 0 {+ Some(FoundToken {+ token: TokenKind::Space(count),+ next_index: count,+ })+ } else {+ None+ }}fn lex_characters(source: &[char], cs: &str, token: TokenKind) -> Option<FoundToken> {@@ -167,3 +180,32 @@ lex_punctuation! {")" => CloseRound,"#" => Hash}++#[cfg(test)]+mod tests {+ use crate::{+ lex_to_end_str,+ token::Punctuation::{self, *},+ TokenKind::{self, *},+ };++ fn assert_tokens_eq(test_str: impl AsRef<str>, expected: &[TokenKind]) {+ let tokens = lex_to_end_str(test_str);+ let kinds: Vec<_> = tokens.into_iter().map(|v| v.kind).collect();++ assert_eq!(&kinds, expected)+ }++ #[test]+ fn single_letter() {+ assert_tokens_eq("a", &[Word])+ }++ #[test]+ fn sentence() {+ assert_tokens_eq(+ "hello world, my friend",+ &[Word, Word, Punctuation(Comma), Word, Word],+ )+ }+}diff --git a/lt-core/src/lib.rs b/lt-core/src/lib.rsindex 161a6ff14..9f1144b11 100644--- a/lt-core/src/lib.rs+++ b/lt-core/src/lib.rs@@ -3,6 +3,6 @@ mod lint;mod span;mod token;-pub use lexer::lex_to_end;+pub use lexer::{lex_to_end, lex_to_end_str};pub use lint::{Lint, LintKind};pub use token::{Token, TokenKind};diff --git a/lt-core/src/span.rs b/lt-core/src/span.rsindex 1958264b0..a5e8f1a13 100644--- a/lt-core/src/span.rs+++ b/lt-core/src/span.rs@@ -15,4 +15,8 @@ impl Span {pub fn get_content<'a>(&self, source: &'a [char]) -> &'a [char] {&source[self.start..self.end]}++ pub fn get_content_string(&self, source: &[char]) -> String {+ String::from_iter(self.get_content(source))+ }}diff --git a/lt-core/src/token.rs b/lt-core/src/token.rsindex 31616d46c..c77914fe6 100644--- a/lt-core/src/token.rs+++ b/lt-core/src/token.rs@@ -9,14 +9,18 @@ pub struct Token {pub kind: TokenKind,}-#[derive(Debug, Is, Clone, Serialize, Deserialize)]+#[derive(Debug, Is, Clone, Serialize, Deserialize, PartialEq)]pub enum TokenKind {Word,Punctuation(Punctuation),Number(f64),+ /// A sequence of " " spaces.+ Space(usize),+ /// A sequence of "\n" newlines+ Newline(usize),}-#[derive(Debug, Is, Clone, Serialize, Deserialize)]+#[derive(Debug, Is, Clone, Serialize, Deserialize, PartialEq)]pub enum Punctuation {/// .Period,diff --git a/lt-serve/src/main.rs b/lt-serve/src/main.rsindex 2051a165e..2cb1edb0f 100644--- a/lt-serve/src/main.rs+++ b/lt-serve/src/main.rs@@ -34,14 +34,48 @@ async fn root() -> &'static str {"Hello, World!"}-async fn parse_text(Query(payload): Query<ParseRequest>) -> (StatusCode, Json<Vec<Token>>) {- info!("Parse request for: \"{}\"", payload.text);+async fn parse_text(Query(payload): Query<ParseRequest>) -> (StatusCode, String) {+ info!("Parse request for: {:?}", payload.text);- let chars: Vec<_> = payload.text.chars().collect();+ let chars: Vec<_> = payload+ .text+ .replace('\u{a0}', " ")+ .replace("<br>", "\n")+ .chars()+ .collect();let lexed = lex_to_end(&chars);- (StatusCode::ACCEPTED, Json(lexed))+ let mut html = String::new();++ for token in lexed {+ let chunk = match token.kind {+ lt_core::TokenKind::Word => {+ format!(+ r#"<span style="background-color: #F0CEA0; border-radius: 2px;">{}</span>"#,+ token.span.get_content_string(&chars)+ )+ }+ lt_core::TokenKind::Punctuation(_) => {+ format!(+ r#"<span style="background-color: #DB2B39; border-radius: 2px;">{}</span>"#,+ token.span.get_content_string(&chars)+ )+ }+ lt_core::TokenKind::Number(_) => {+ format!(+ r#"<span style="background-color: #F3A712; border-radius: 2px;">{}</span>"#,+ token.span.get_content_string(&chars)+ )+ }+ lt_core::TokenKind::Space(count) => "\u{a0}".repeat(count),+ lt_core::TokenKind::Newline(count) => "<br>".repeat(count),+ };++ html.push_str(&chunk);+ }++ (StatusCode::ACCEPTED, html)}#[derive(Deserialize)]diff --git a/web/src/lib/parse.ts b/web/src/lib/parse.tsnew file mode 100644index 000000000..33bd673bc--- /dev/null+++ b/web/src/lib/parse.ts@@ -0,0 +1,5 @@+export async function highlightText(text: string): Promise<string> {+ const req = await fetch(`/parse?text=${encodeURIComponent(text)}`);++ return req.text();+}diff --git a/web/src/routes/+page.svelte b/web/src/routes/+page.svelteindex 2d8f53e93..a9590a8c4 100644--- a/web/src/routes/+page.svelte+++ b/web/src/routes/+page.svelte@@ -1,15 +1,15 @@<script lang="ts">import Suggestion from '$lib/Suggestion.svelte';- import { onMount } from 'svelte';import alice from '../../../alice.txt?raw';- import { position, offset } from 'caret-pos';+ import { position } from 'caret-pos';+ import { highlightText } from '$lib/parse';let content = alice;let editor: HTMLDivElement;function updateContent() {if (editor !== null) {- content = editor.textContent;+ content = editor.innerText;}}@@ -17,8 +17,14 @@function action(text: string) {if (editor) {const pos = position(editor);- if (text) node.innerHTML = text.replaceAll('Duchess', `<b class="text-bold">Duchess</b>`);- position(editor, pos.pos);+ highlightText(text).then((highlighted) => {+ node.innerHTML = highlighted;+ position(editor, pos.pos);+ });+ } else {+ highlightText(text).then((highlighted) => {+ node.innerHTML = highlighted;+ });}}action(text);@@ -39,9 +45,10 @@contenteditable="true"use:highlight={content}bind:this={editor}+ placeholder="Tell your story..."on:input={updateContent}/>- <div class="flex flex-col">+ <div class="flex flex-col flex-none"><Suggestion category="grammar" title="Word Repeated"><p>The word "the" was repeated twice</p></Suggestion>