Skip to content

Offline, privacy-first grammar checker. Fast, open-source, Rust-powered

Fixed issues

Elijah Potter1a019ed06591
diff --git a/alice.txt b/alice.txt
index a9054835a..bbee48392 100644
--- a/alice.txt
+++ b/alice.txt
@@ -17,4 +17,3 @@ the same when I got up this morning? I almost think I can remember
feeling a little different. But if I'm not the same, the next question
is, who in the world am I? Ah, _that's_ the great puzzle!" And she began
thinking over all the children she knew that were of the same age as
-herself, to see if she could have been changed for any of them.ubscribe to our email newsletter to hear about new eBooks.
diff --git a/lt-core/src/lexer.rs b/lt-core/src/lexer.rs
index ab3c2973a..3f9cd53ac 100644
--- a/lt-core/src/lexer.rs
+++ b/lt-core/src/lexer.rs
@@ -10,14 +10,20 @@ pub struct FoundToken {
pub token: TokenKind,
}
+pub fn lex_to_end_str(source: impl AsRef<str>) -> Vec<Token> {
+ let r = source.as_ref();
+
+ let chars: Vec<_> = r.chars().collect();
+
+ lex_to_end(&chars)
+}
+
/// Lex all tokens, if possible.
pub fn lex_to_end(source: &[char]) -> Vec<Token> {
let mut cursor = 0;
let mut tokens = Vec::new();
loop {
- cursor += lex_ignorables(&source[cursor..]);
-
if cursor == source.len() {
return tokens;
}
@@ -29,13 +35,19 @@ pub fn lex_to_end(source: &[char]) -> Vec<Token> {
});
cursor += next_index;
} else {
- cursor += 1;
+ panic!()
}
}
}
fn lex_token(source: &[char]) -> Option<FoundToken> {
- let lexers = [lex_punctuation, lex_number, lex_word];
+ let lexers = [
+ lex_spaces,
+ lex_newlines,
+ lex_punctuation,
+ lex_number,
+ lex_word,
+ ];
for lexer in lexers {
if let Some(f) = lexer(source) {
@@ -49,7 +61,10 @@ fn lex_word(source: &[char]) -> Option<FoundToken> {
let mut end = 0;
while end < source.len() {
- if lex_punctuation(&source[end + 1..]).is_none() && lex_ignorables(&source[end + 1..]) == 0
+ if lex_punctuation(&source[end + 1..]).is_none()
+ && lex_spaces(&source[end + 1..]).is_none()
+ && lex_newlines(&source[end + 1..]).is_none()
+ && end + 1 != source.len()
{
end += 1;
} else {
@@ -95,32 +110,30 @@ pub fn lex_number(source: &[char]) -> Option<FoundToken> {
lex_number(&source[0..end])
}
-/// Find the first token _after_ any characters that can be ignored (whitespace, mostly).
-fn lex_ignorables(source: &[char]) -> usize {
- let mut cursor = 0;
-
- loop {
- let last_cursor = cursor;
-
- cursor += lex_whitespace(&source[cursor..]);
+fn lex_newlines(source: &[char]) -> Option<FoundToken> {
+ let count = source.iter().take_while(|c| **c == '\n').count();
- if last_cursor == cursor {
- break;
- }
+ if count > 0 {
+ Some(FoundToken {
+ token: TokenKind::Newline(count),
+ next_index: count,
+ })
+ } else {
+ None
}
-
- cursor
}
-/// Find the first token _after_ whitespace.
-fn lex_whitespace(source: &[char]) -> usize {
- for (index, c) in source.iter().enumerate() {
- if !c.is_whitespace() {
- return index;
- }
- }
+fn lex_spaces(source: &[char]) -> Option<FoundToken> {
+ let count = source.iter().take_while(|c| **c == ' ').count();
- source.len()
+ if count > 0 {
+ Some(FoundToken {
+ token: TokenKind::Space(count),
+ next_index: count,
+ })
+ } else {
+ None
+ }
}
fn lex_characters(source: &[char], cs: &str, token: TokenKind) -> Option<FoundToken> {
@@ -167,3 +180,32 @@ lex_punctuation! {
")" => CloseRound,
"#" => Hash
}
+
+#[cfg(test)]
+mod tests {
+ use crate::{
+ lex_to_end_str,
+ token::Punctuation::{self, *},
+ TokenKind::{self, *},
+ };
+
+ fn assert_tokens_eq(test_str: impl AsRef<str>, expected: &[TokenKind]) {
+ let tokens = lex_to_end_str(test_str);
+ let kinds: Vec<_> = tokens.into_iter().map(|v| v.kind).collect();
+
+ assert_eq!(&kinds, expected)
+ }
+
+ #[test]
+ fn single_letter() {
+ assert_tokens_eq("a", &[Word])
+ }
+
+ #[test]
+ fn sentence() {
+ assert_tokens_eq(
+ "hello world, my friend",
+ &[Word, Word, Punctuation(Comma), Word, Word],
+ )
+ }
+}
diff --git a/lt-core/src/lib.rs b/lt-core/src/lib.rs
index 161a6ff14..9f1144b11 100644
--- a/lt-core/src/lib.rs
+++ b/lt-core/src/lib.rs
@@ -3,6 +3,6 @@ mod lint;
mod span;
mod token;
-pub use lexer::lex_to_end;
+pub use lexer::{lex_to_end, lex_to_end_str};
pub use lint::{Lint, LintKind};
pub use token::{Token, TokenKind};
diff --git a/lt-core/src/span.rs b/lt-core/src/span.rs
index 1958264b0..a5e8f1a13 100644
--- a/lt-core/src/span.rs
+++ b/lt-core/src/span.rs
@@ -15,4 +15,8 @@ impl Span {
pub fn get_content<'a>(&self, source: &'a [char]) -> &'a [char] {
&source[self.start..self.end]
}
+
+ pub fn get_content_string(&self, source: &[char]) -> String {
+ String::from_iter(self.get_content(source))
+ }
}
diff --git a/lt-core/src/token.rs b/lt-core/src/token.rs
index 31616d46c..c77914fe6 100644
--- a/lt-core/src/token.rs
+++ b/lt-core/src/token.rs
@@ -9,14 +9,18 @@ pub struct Token {
pub kind: TokenKind,
}
-#[derive(Debug, Is, Clone, Serialize, Deserialize)]
+#[derive(Debug, Is, Clone, Serialize, Deserialize, PartialEq)]
pub enum TokenKind {
Word,
Punctuation(Punctuation),
Number(f64),
+ /// A sequence of " " spaces.
+ Space(usize),
+ /// A sequence of "\n" newlines
+ Newline(usize),
}
-#[derive(Debug, Is, Clone, Serialize, Deserialize)]
+#[derive(Debug, Is, Clone, Serialize, Deserialize, PartialEq)]
pub enum Punctuation {
/// .
Period,
diff --git a/lt-serve/src/main.rs b/lt-serve/src/main.rs
index 2051a165e..2cb1edb0f 100644
--- a/lt-serve/src/main.rs
+++ b/lt-serve/src/main.rs
@@ -34,14 +34,48 @@ async fn root() -> &'static str {
"Hello, World!"
}
-async fn parse_text(Query(payload): Query<ParseRequest>) -> (StatusCode, Json<Vec<Token>>) {
- info!("Parse request for: \"{}\"", payload.text);
+async fn parse_text(Query(payload): Query<ParseRequest>) -> (StatusCode, String) {
+ info!("Parse request for: {:?}", payload.text);
- let chars: Vec<_> = payload.text.chars().collect();
+ let chars: Vec<_> = payload
+ .text
+ .replace('\u{a0}', " ")
+ .replace("<br>", "\n")
+ .chars()
+ .collect();
let lexed = lex_to_end(&chars);
- (StatusCode::ACCEPTED, Json(lexed))
+ let mut html = String::new();
+
+ for token in lexed {
+ let chunk = match token.kind {
+ lt_core::TokenKind::Word => {
+ format!(
+ r#"<span style="background-color: #F0CEA0; border-radius: 2px;">{}</span>"#,
+ token.span.get_content_string(&chars)
+ )
+ }
+ lt_core::TokenKind::Punctuation(_) => {
+ format!(
+ r#"<span style="background-color: #DB2B39; border-radius: 2px;">{}</span>"#,
+ token.span.get_content_string(&chars)
+ )
+ }
+ lt_core::TokenKind::Number(_) => {
+ format!(
+ r#"<span style="background-color: #F3A712; border-radius: 2px;">{}</span>"#,
+ token.span.get_content_string(&chars)
+ )
+ }
+ lt_core::TokenKind::Space(count) => "\u{a0}".repeat(count),
+ lt_core::TokenKind::Newline(count) => "<br>".repeat(count),
+ };
+
+ html.push_str(&chunk);
+ }
+
+ (StatusCode::ACCEPTED, html)
}
#[derive(Deserialize)]
diff --git a/web/src/lib/parse.ts b/web/src/lib/parse.ts
new file mode 100644
index 000000000..33bd673bc
--- /dev/null
+++ b/web/src/lib/parse.ts
@@ -0,0 +1,5 @@
+export async function highlightText(text: string): Promise<string> {
+ const req = await fetch(`/parse?text=${encodeURIComponent(text)}`);
+
+ return req.text();
+}
diff --git a/web/src/routes/+page.svelte b/web/src/routes/+page.svelte
index 2d8f53e93..a9590a8c4 100644
--- a/web/src/routes/+page.svelte
+++ b/web/src/routes/+page.svelte
@@ -1,15 +1,15 @@
<script lang="ts">
import Suggestion from '$lib/Suggestion.svelte';
- import { onMount } from 'svelte';
import alice from '../../../alice.txt?raw';
- import { position, offset } from 'caret-pos';
+ import { position } from 'caret-pos';
+ import { highlightText } from '$lib/parse';
let content = alice;
let editor: HTMLDivElement;
function updateContent() {
if (editor !== null) {
- content = editor.textContent;
+ content = editor.innerText;
}
}
@@ -17,8 +17,14 @@
function action(text: string) {
if (editor) {
const pos = position(editor);
- if (text) node.innerHTML = text.replaceAll('Duchess', `<b class="text-bold">Duchess</b>`);
- position(editor, pos.pos);
+ highlightText(text).then((highlighted) => {
+ node.innerHTML = highlighted;
+ position(editor, pos.pos);
+ });
+ } else {
+ highlightText(text).then((highlighted) => {
+ node.innerHTML = highlighted;
+ });
}
}
action(text);
@@ -39,9 +45,10 @@
contenteditable="true"
use:highlight={content}
bind:this={editor}
+ placeholder="Tell your story..."
on:input={updateContent}
/>
- <div class="flex flex-col">
+ <div class="flex flex-col flex-none">
<Suggestion category="grammar" title="Word Repeated">
<p>The word "the" was repeated twice</p>
</Suggestion>