Offline, privacy-first grammar checker. Fast, open-source, Rust-powered
feat(ls): add support for `TeX` and derivative languages (#2689)
Elijah Potter12292d50f10c
diff --git a/Cargo.lock b/Cargo.lockindex 815f3e8b1..ee87a8c82 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -2862,6 +2862,7 @@ dependencies = ["harper-literate-haskell","harper-python","harper-stats",+ "harper-tex","harper-typst","itertools 0.14.0","open",@@ -2914,6 +2915,14 @@ dependencies = ["uuid",]+[[package]]+name = "harper-tex"+version = "1.6.0"+dependencies = [+ "harper-core",+ "paste",+]+[[package]]name = "harper-thesaurus"version = "1.7.0"diff --git a/Cargo.toml b/Cargo.tomlindex a69753a14..0d98f2261 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -1,5 +1,5 @@[workspace]-members = ["harper-cli", "harper-core", "harper-ls", "harper-comments", "harper-wasm", "harper-tree-sitter", "harper-html", "harper-literate-haskell", "harper-typst", "harper-stats", "harper-pos-utils", "harper-brill", "harper-ink", "harper-python", "harper-jjdescription", "harper-thesaurus", "harper-asciidoc", "fuzz"]+members = ["harper-cli", "harper-core", "harper-ls", "harper-comments", "harper-wasm", "harper-tree-sitter", "harper-html", "harper-literate-haskell", "harper-typst", "harper-stats", "harper-pos-utils", "harper-brill", "harper-ink", "harper-python", "harper-jjdescription", "harper-thesaurus", "harper-asciidoc", "fuzz", "harper-tex"]resolver = "2"[profile.test]diff --git a/harper-core/src/expr/sequence_expr.rs b/harper-core/src/expr/sequence_expr.rsindex 54e2cb9a0..8467f9660 100644--- a/harper-core/src/expr/sequence_expr.rs+++ b/harper-core/src/expr/sequence_expr.rs@@ -613,6 +613,9 @@ impl SequenceExpr {gen_then_from_is!(period);gen_then_from_is!(semicolon);gen_then_from_is!(quote);+ gen_then_from_is!(backslash);+ gen_then_from_is!(slash);+ gen_then_from_is!(percent);// Otherdiff --git a/harper-core/src/token_kind.rs b/harper-core/src/token_kind.rsindex 73c12a309..80616be7b 100644--- a/harper-core/src/token_kind.rs+++ b/harper-core/src/token_kind.rs@@ -268,10 +268,18 @@ impl TokenKind {matches!(self, TokenKind::Punctuation(Punctuation::Ampersand))}+ pub fn is_backslash(&self) -> bool {+ matches!(self, TokenKind::Punctuation(Punctuation::Backslash))+ }+pub fn is_slash(&self) -> bool {matches!(self, TokenKind::Punctuation(Punctuation::ForwardSlash))}+ pub fn is_percent(&self) -> bool {+ matches!(self, TokenKind::Punctuation(Punctuation::Percent))+ }+// Miscellaneous is-methods/// Checks whether a token is word-like--meaning it is more complex than punctuation and candiff --git a/harper-ls/Cargo.toml b/harper-ls/Cargo.tomlindex 98df4069b..6ed7d976d 100644--- a/harper-ls/Cargo.toml+++ b/harper-ls/Cargo.toml@@ -14,6 +14,7 @@ harper-core = { path = "../harper-core", version = "1.0.0", features = ["concurrharper-comments = { path = "../harper-comments", version = "1.0.0" }harper-jjdescription = { path = "../harper-jjdescription", version = "1.0.0" }harper-typst = { path = "../harper-typst", version = "1.0.0" }+harper-tex = { path = "../harper-tex", version = "1.0.0" }harper-html = { path = "../harper-html", version = "1.0.0" }harper-python = { path = "../harper-python", version = "1.0.0" }harper-asciidoc = { path = "../harper-asciidoc", version = "1.0.0" }diff --git a/harper-ls/src/backend.rs b/harper-ls/src/backend.rsindex eff28e758..0ae0ed386 100644--- a/harper-ls/src/backend.rs+++ b/harper-ls/src/backend.rs@@ -26,6 +26,7 @@ use harper_jjdescription::JJDescriptionParser;use harper_literate_haskell::LiterateHaskellParser;use harper_python::PythonParser;use harper_stats::{Record, Stats};+use harper_tex::TeX;use harper_typst::Typst;use serde_json::{Value, json};use tokio::sync::{Mutex, RwLock};@@ -397,6 +398,7 @@ impl Backend {"plaintext" | "text" => Some(Box::new(PlainEnglish)),"python" => Some(Box::new(PythonParser::default())),"typst" => Some(Box::new(Typst)),+ "tex" | "plaintex" | "latex" => Some(Box::new(TeX::default())),_ => None,};diff --git a/harper-tex/Cargo.toml b/harper-tex/Cargo.tomlnew file mode 100644index 000000000..7a6f96600--- /dev/null+++ b/harper-tex/Cargo.toml@@ -0,0 +1,10 @@+[package]+name = "harper-tex"+version = "1.6.0"+edition = "2024"++[dependencies]+harper-core = { path = "../harper-core", version = "1.0.0" }++[dev-dependencies]+paste = "1.0.15"diff --git a/harper-tex/src/lib.rs b/harper-tex/src/lib.rsnew file mode 100644index 000000000..d1be80eb7--- /dev/null+++ b/harper-tex/src/lib.rs@@ -0,0 +1,69 @@+mod masker;++use harper_core::Token;+use harper_core::parsers::{Mask, Parser, PlainEnglish};++use self::masker::Masker;++/// A parser for Harper that wraps the native `PlainEnglish` parser, allowing one use Harper on+/// documents written in TeX, LaTeX, or any other TeX derivative.+///+/// This parser is crude, and could definitely use work if all features of Harper wish to be+/// supported for the language.+pub struct TeX {+ inner: Mask<Masker, PlainEnglish>,+}++impl Default for TeX {+ fn default() -> Self {+ Self {+ inner: Mask::new(Default::default(), PlainEnglish),+ }+ }+}++impl Parser for TeX {+ fn parse(&self, source: &[char]) -> Vec<Token> {+ self.inner.parse(source)+ }+}++#[cfg(test)]+mod tests {+ use harper_core::TokenKind;+ use harper_core::parsers::StrParser;++ use crate::TeX;++ #[test]+ fn ignores_comment_characters() {+ let source = r"%!!%";++ let toks = TeX::default().parse_str(source);+ let tok_kinds: Vec<_> = toks.into_iter().map(|t| t.kind).collect();++ assert_eq!(+ tok_kinds,+ vec![+ TokenKind::Punctuation(harper_core::Punctuation::Bang),+ TokenKind::Punctuation(harper_core::Punctuation::Bang),+ ]+ )+ }++ #[test]+ fn passes_comment_characters_preceded_by_backslash() {+ let source = r"\%!!";++ let toks = TeX::default().parse_str(source);+ let tok_kinds: Vec<_> = toks.into_iter().map(|t| t.kind).collect();++ assert_eq!(+ tok_kinds,+ vec![+ TokenKind::Punctuation(harper_core::Punctuation::Bang),+ TokenKind::Punctuation(harper_core::Punctuation::Bang)+ ]+ )+ }+}diff --git a/harper-tex/src/masker.rs b/harper-tex/src/masker.rsnew file mode 100644index 000000000..55c991707--- /dev/null+++ b/harper-tex/src/masker.rs@@ -0,0 +1,196 @@+use std::collections::VecDeque;++use harper_core::{Mask, Span};++#[derive(Debug, Default)]+pub struct Masker {}++impl harper_core::Masker for Masker {+ fn create_mask(&self, source: &[char]) -> Mask {+ let mut cursor = 0;+ let mut mask = Mask::new_blank();++ let mut cur_mask_start = 0;+ let mut actions = VecDeque::new();++ loop {+ if cursor >= source.len() {+ break;+ }++ let c = source[cursor];++ if matches!(c, '%') {+ actions.push_back(CursorAction::PushMaskAndIncBy(1));+ } else if !command_at_cursor(cursor, source, &mut actions)+ && let Some(s) = math_mode_at_cursor(cursor, source)+ {+ actions.push_back(CursorAction::PushMaskAndIncBy(s));+ } else {+ actions.push_back(CursorAction::IncBy(1));+ }++ while let Some(action) = actions.pop_front() {+ match action {+ CursorAction::IncBy(n) => cursor = (cursor + n).min(source.len()),+ CursorAction::PushMaskAndIncBy(mut n) => {+ if cur_mask_start != cursor {+ mask.push_allowed(Span::new(cur_mask_start, cursor));+ }++ n = (cursor + n).min(source.len());++ cursor = n;+ cur_mask_start = n;+ }+ }+ }+ }++ if cur_mask_start != cursor {+ mask.push_allowed(Span::new(cur_mask_start, cursor));+ }++ mask+ }+}++/// Check whether there is a math mode block at the current cursor. If so, this function will return the+/// index of the next non-math-block index.+fn math_mode_at_cursor(cursor: usize, source: &[char]) -> Option<usize> {+ if *source.get(cursor)? != '$' {+ return None;+ }++ Some(+ source+ .iter()+ .skip(cursor)+ .take_while(|t| **t != '$')+ .count()+ + cursor+ + 1,+ )+}++/// Check whether there is a command at the current cursor. If so, this function will update the action queue to mask out the hidden elements.+/// Returns whether the action queue was modified.+fn command_at_cursor(+ mut cursor: usize,+ source: &[char],+ actions: &mut VecDeque<CursorAction>,+) -> bool {+ let orig_cursor = cursor;++ if source.get(cursor) != Some(&'\\') {+ return false;+ }++ cursor += 1;++ // The name of the command+ let name_len = source+ .iter()+ .skip(cursor + 1)+ .take_while(|t| t.is_alphabetic())+ .count();+ let name = &source[cursor..cursor + 1 + name_len];++ cursor += name_len + 1;++ // The optional square braces+ if source.get(cursor) == Some(&'[') {+ cursor += source+ .iter()+ .skip(cursor)+ .take_while(|t| **t != ']')+ .count()+ + 1;+ }++ let content_commands = [+ "section",+ "title",+ "subsection",+ "subsubsection",+ "textbf",+ "textit",+ "emph",+ "author",+ "part",+ "chapter",+ "caption",+ ];+ let is_content_command = content_commands+ .iter()+ .any(|c| name.iter().copied().eq(c.chars()));++ // The optional curly braces+ if source.get(cursor) == Some(&'{') {+ let brace_len = source+ .iter()+ .skip(cursor)+ .take_while(|t| **t != '}')+ .count();++ if is_content_command {+ actions.push_back(CursorAction::PushMaskAndIncBy(cursor - orig_cursor));+ actions.push_back(CursorAction::IncBy(brace_len));+ actions.push_back(CursorAction::PushMaskAndIncBy(1));+ true+ } else {+ actions.push_back(CursorAction::PushMaskAndIncBy(+ brace_len + cursor + 1 - orig_cursor,+ ));+ true+ }+ } else {+ actions.push_back(CursorAction::PushMaskAndIncBy(cursor - orig_cursor));+ true+ }+}++#[derive(Debug)]+enum CursorAction {+ IncBy(usize),+ PushMaskAndIncBy(usize),+}++#[cfg(test)]+mod tests {+ use harper_core::Masker as _;++ use super::Masker;++ #[test]+ fn ignores_many_comment_signs() {+ let source: Vec<_> = "%%%".chars().collect();+ let mask = Masker::default().create_mask(&source);++ assert_eq!(mask.iter_allowed(&source).next(), None)+ }++ #[test]+ fn ignores_single_comment_sign() {+ let source: Vec<_> = "%".chars().collect();+ let mask = Masker::default().create_mask(&source);++ assert_eq!(mask.iter_allowed(&source).next(), None)+ }++ #[test]+ fn ignores_single_comment_sign_in_phrase() {+ let source: Vec<_> = "this is a comment: % here it is!".chars().collect();+ let mask = Masker::default().create_mask(&source);++ assert_eq!(mask.iter_allowed(&source).count(), 2)+ }++ #[test]+ fn ignores_latex_command() {+ let source: Vec<_> = r"this is a command: \LaTeX there it was!".chars().collect();+ let mask = Masker::default().create_mask(&source);++ assert_eq!(mask.iter_allowed(&source).count(), 2)+ }+}diff --git a/harper-tex/tests/run_tests.rs b/harper-tex/tests/run_tests.rsnew file mode 100644index 000000000..24a498795--- /dev/null+++ b/harper-tex/tests/run_tests.rs@@ -0,0 +1,44 @@+use harper_core::linting::{LintGroup, Linter};+use harper_core::spell::FstDictionary;+use harper_core::{Dialect, Document};+use harper_tex::TeX;++/// Creates a unit test checking that the linting of a document in+/// `tests_sources` produces the expected number of lints.+macro_rules! create_test {+ ($filename:ident.$ext:ident, $correct_expected:expr) => {+ paste::paste! {+ #[test]+ fn [<lints_ $filename _correctly>](){+ let source = include_str!(+ concat!(+ "./test_sources/",+ concat!(stringify!($filename), ".", stringify!($ext))+ )+ );++ let dict = FstDictionary::curated();+ let document = Document::new(&source, &TeX::default(), &dict);++ let mut linter = LintGroup::new_curated(dict, Dialect::American);+ let lints = linter.lint(&document);++ dbg!(&lints);+ assert_eq!(lints.len(), $correct_expected);++ // Make sure that all generated tokens span real characters+ for token in document.tokens(){+ assert!(token.span.try_get_content(document.get_source()).is_some());+ }+ }+ }+ };+}++create_test!(clean.tex, 0);+create_test!(simple.tex, 1);+create_test!(heading.tex, 1);+create_test!(title.tex, 1);+create_test!(city.tex, 0);+create_test!(many_tags.tex, 6);+create_test!(many_more_tags.tex, 11);diff --git a/harper-tex/tests/test_sources/city.tex b/harper-tex/tests/test_sources/city.texnew file mode 100644index 000000000..7ad5a759c--- /dev/null+++ b/harper-tex/tests/test_sources/city.tex@@ -0,0 +1,45 @@+\documentclass{article}++\usepackage[margin=1in]{geometry}++\title{Urban Development Report: The fictional city}+\author{Department of Civic Studies}+\date{\today}++\begin{document}++\maketitle++\section{Executive Summary}++The fictional city is a fictional coastal city known for its advanced infrastructure, cultural diversity, and sustainable urban planning. The city serves as a model for integrated transportation systems and renewable energy adoption.++\section{Geography}++The fictional city is located along a crescent-shaped bay bordered by low mountain ranges to the north and wetlands to the south. The natural harbor enables a strong shipping industry while also supporting marine biodiversity zones protected by city law.++\section{Population}++The population of The fictional city is approximately 2.3 million residents. The city has a balanced demographic distribution with strong representation across age groups and professional sectors. Over 40 percent of residents work in research, technology, or education fields.++\section{Infrastructure}++The transportation network consists of elevated rail, underground metro lines, and autonomous electric bus corridors. Private car ownership is low due to efficient public transit coverage and pedestrian-first city zoning.++\section{Energy and Environment}++The fictional city operates on a hybrid renewable grid powered by offshore wind farms, tidal generators, and urban solar arrays integrated into building materials. The city maintains a carbon-negative energy profile during most months of the year.++\section{Culture}++The city is known for its annual Festival of Lights, public waterfront amphitheaters, and a dense network of public libraries and maker spaces. Local policy requires that new construction projects allocate space for public art installations.++\section{Economy}++Major economic sectors include marine engineering, climate technology research, and digital education services. The city government provides tax incentives for companies developing environmental restoration technologies.++\section{Conclusion}++The fictional city represents a theoretical model of a future-oriented city balancing technological progress, environmental stewardship, and high quality of life for residents.++\end{document}diff --git a/harper-tex/tests/test_sources/clean.tex b/harper-tex/tests/test_sources/clean.texnew file mode 100644index 000000000..4793f3877--- /dev/null+++ b/harper-tex/tests/test_sources/clean.tex@@ -0,0 +1,66 @@+\documentclass[12pt]{article}++\usepackage[utf8]{inputenc}+\usepackage[T1]{fontenc}+\usepackage{geometry}+\usepackage{amsmath}+\usepackage{graphicx}+\usepackage{hyperref}++\geometry{margin=1in}++\title{A Small Representative Document}+\author{Author Name}+\date{\today}++\begin{document}++\maketitle++\begin{abstract}+This document demonstrates a minimal but representative structure of a LaTeX article, including sections, mathematics, figures, and references.+\end{abstract}++\section{Introduction}++LaTeX is widely used for scientific and technical documents because of its precision and typographic quality. This document provides a concise example of common elements.++\section{Mathematics Example}++Inline math example: $E = mc^2$.++Displayed equation:+\begin{equation}+\int_0^\infty e^{-x^2} dx = \frac{\sqrt{\pi}}{2}+\end{equation}++\section{Lists}++\subsection{Itemized List}+\begin{itemize}+\item First item+\item Second item+\item Third item+\end{itemize}++\subsection{Enumerated List}+\begin{enumerate}+\item Step one+\item Step two+\item Step three+\end{enumerate}++\section{Figures}++\begin{figure}[h]+\centering+\includegraphics[width=0.5\textwidth]{example-image}+\caption{Example placeholder image}+\label{fig:example}+\end{figure}++\section{Conclusion}++This sample illustrates a typical structure suitable for academic or technical writing.++\end{document}diff --git a/harper-tex/tests/test_sources/heading.tex b/harper-tex/tests/test_sources/heading.texnew file mode 100644index 000000000..cb73cf095--- /dev/null+++ b/harper-tex/tests/test_sources/heading.tex@@ -0,0 +1,9 @@+\documentclass{article}++\begin{document}++\section{This is an test.}++Hello, world.++\end{document}diff --git a/harper-tex/tests/test_sources/many_more_tags.tex b/harper-tex/tests/test_sources/many_more_tags.texnew file mode 100644index 000000000..5b8374e72--- /dev/null+++ b/harper-tex/tests/test_sources/many_more_tags.tex@@ -0,0 +1,45 @@+\documentclass{report}++\usepackage[margin=1in]{geometry}++\title{Shwcase of Common Readable Content Commands}+\author{Exmple Author}+\date{\today}++\begin{document}++\maketitle++\begin{abstract}+This abstract demonstrates readable narrative text stored inside the abstract environment.+\end{abstract}++\part{Mjor Structural Content}++\chapter{Chapter Level Contet}++\section{ection Level Content}++This is normal readable body text inside the document environment.++Here is inline readable text using \emph{emphasized cntent} and \textbf{stongly highlighted content}.++\subsection{Subsection Level Cntent}++This subsection contains additional explanatory readable text that expands on the section above.++\subsubsection{Sbsubsection Level Content}++This level often contains very specific technical or descriptive readable information.++\paragraph{Pragraph Command Content}++This command typically introduces a short readable heading followed immediately by readable explanatory text.++\begin{figure}[h]+\centering+\rule{6cm}{3cm}+\caption{This cption demonstrates dense readable description explaining the figure meaning and Kontext.}+\end{figure}++\end{document}diff --git a/harper-tex/tests/test_sources/many_tags.tex b/harper-tex/tests/test_sources/many_tags.texnew file mode 100644index 000000000..1990e03e0--- /dev/null+++ b/harper-tex/tests/test_sources/many_tags.tex@@ -0,0 +1,44 @@+\documentclass{article}++\usepackage[margin=1in]{geometry}+\title{Demonstration of Readable Content Structures}+\author{Exmple Author}+\date{\today}++\begin{document}++\maketitle++\begin{abstract}+This document demonstrates common LaTeX structures that typically contain readable content for humans rather than configuration or layout-only logic.+\end{abstract}++\section{Introduction}++This is normal body text, which in LaTeX is already primary readable content.+Here is some \textbf{bold readable text} and some \emph{emhasized readable text}.++\subsection{Structured Environmnts}++\begin{quote}+Science is what we understnd well enough to explain to a computer. Art is everything else we do.+\end{quote}++\begin{quotation}+Long-form quoted text often lives in qotation environments. These typically contain dense narrative or explanatory prose and are high-signal readable regions.+\end{quotation}++\begin{verse}+In structured lines the meaning flows,\\+Between the commands the language grows.+\end{verse}++\section{Figures and aptions}++\begin{figure}[h]+\centering+\rule{6cm}{3cm}+\caption{This caption contains concise human-readable explanatory text describing the figure.}+\end{figure}++\end{document}diff --git a/harper-tex/tests/test_sources/simple.tex b/harper-tex/tests/test_sources/simple.texnew file mode 100644index 000000000..1f088a3b6--- /dev/null+++ b/harper-tex/tests/test_sources/simple.tex@@ -0,0 +1,7 @@+\documentclass{article}++\begin{document}++This is an test.++\end{document}diff --git a/harper-tex/tests/test_sources/title.tex b/harper-tex/tests/test_sources/title.texnew file mode 100644index 000000000..0061d6f07--- /dev/null+++ b/harper-tex/tests/test_sources/title.tex@@ -0,0 +1,9 @@+\documentclass{article}++\title{This is an test.}++\begin{document}++Hello, world.++\end{document}diff --git a/packages/vscode-plugin/package.json b/packages/vscode-plugin/package.jsonindex 3c6148758..1faab96f1 100644--- a/packages/vscode-plugin/package.json+++ b/packages/vscode-plugin/package.json@@ -63,6 +63,8 @@"onLanguage:shellscript","onLanguage:solidity","onLanguage:swift",+ "onLanguage:tex",+ "onLanguage:latex","onLanguage:toml","onLanguage:typescript","onLanguage:typescriptreact",diff --git a/packages/vscode-plugin/src/tests/fixtures/languages/latex.tex b/packages/vscode-plugin/src/tests/fixtures/languages/latex.texnew file mode 100644index 000000000..027769f61--- /dev/null+++ b/packages/vscode-plugin/src/tests/fixtures/languages/latex.tex@@ -0,0 +1,7 @@+\documentclass{article}++\begin{document}++Errorz++\end{document}diff --git a/packages/vscode-plugin/src/tests/suite/languages.test.ts b/packages/vscode-plugin/src/tests/suite/languages.test.tsindex 0b7e3b1a7..614be3ae1 100644--- a/packages/vscode-plugin/src/tests/suite/languages.test.ts+++ b/packages/vscode-plugin/src/tests/suite/languages.test.ts@@ -50,6 +50,7 @@ describe('Languages >', () => {{ type: 'Swift', file: 'swift.swift', row: 9, column: 26 },{ type: 'TypeScript', file: 'typescript.ts', row: 0, column: 32 },{ type: 'TypeScript JSX', file: 'typescriptreact.tsx', row: 3, column: 7 },+ { type: 'LaTeX', file: 'latex.tex', row: 4, column: 0 },].forEach((testCase) => {it(`gives correct diagnostics for ${testCase.type} files`, async () => {const uri = getUri('languages', testCase.file);diff --git a/packages/web/src/routes/docs/integrations/language-server/+page.md b/packages/web/src/routes/docs/integrations/language-server/+page.mdindex 4a8c3e141..afaa6c696 100644--- a/packages/web/src/routes/docs/integrations/language-server/+page.md+++ b/packages/web/src/routes/docs/integrations/language-server/+page.md@@ -308,6 +308,7 @@ These configs are under the `markdown` key:| TypeScript React | `typescriptreact` | ✅ || Typst | `typst` | || Zig | `zig` | ✅ |+| LaTeX/TeX | `latex`/`tex`/`plaintex` | |Want your language added?Let us know by [commenting on this issue](https://github.com/Automattic/harper/issues/79).