Offline, privacy-first grammar checker. Fast, open-source, Rust-powered
feat: Added ORGMODE parser (#1369)
Eloy Coto55f5e8b6d323
diff --git a/harper-cli/src/main.rs b/harper-cli/src/main.rsindex 575248f03..af778453f 100644--- a/harper-cli/src/main.rs+++ b/harper-cli/src/main.rs@@ -12,7 +12,7 @@ use clap::Parser;use dirs::{config_dir, data_local_dir};use harper_comments::CommentParser;use harper_core::linting::{LintGroup, Linter};-use harper_core::parsers::{Markdown, MarkdownOptions, PlainEnglish};+use harper_core::parsers::{Markdown, MarkdownOptions, OrgMode, PlainEnglish};use harper_core::{remove_overlaps, CharStringExt, Dialect, Dictionary, Document, FstDictionary, MergedDictionary,MutableDictionary, TokenKind, TokenStringExt, WordId, WordMetadata,@@ -439,6 +439,7 @@ fn load_file(Some("lhs") => Box::new(LiterateHaskellParser::new_markdown(MarkdownOptions::default(),)),+ Some("org") => Box::new(OrgMode),Some("typ") => Box::new(harper_typst::Typst),_ => {if let Some(comment_parser) = CommentParser::new_from_filename(file, markdown_options) {diff --git a/harper-core/src/parsers/mod.rs b/harper-core/src/parsers/mod.rsindex 9909308d2..11c97f8ba 100644--- a/harper-core/src/parsers/mod.rs+++ b/harper-core/src/parsers/mod.rs@@ -2,6 +2,7 @@ mod collapse_identifiers;mod isolate_english;mod markdown;mod mask;+mod org_mode;mod plain_english;use blanket::blanket;@@ -9,6 +10,7 @@ pub use collapse_identifiers::CollapseIdentifiers;pub use isolate_english::IsolateEnglish;pub use markdown::{Markdown, MarkdownOptions};pub use mask::Mask;+pub use org_mode::OrgMode;pub use plain_english::PlainEnglish;use crate::{LSend, Token, TokenStringExt};@@ -35,7 +37,7 @@ where#[cfg(test)]mod tests {- use super::{Markdown, Parser, PlainEnglish};+ use super::{Markdown, OrgMode, Parser, PlainEnglish};use crate::Punctuation;use crate::TokenKind::{self, *};@@ -55,6 +57,10 @@ mod tests {assert_tokens_eq(test_str, expected, &Markdown::default())}+ fn assert_tokens_eq_org(test_str: impl AsRef<str>, expected: &[TokenKind]) {+ assert_tokens_eq(test_str, expected, &OrgMode)+ }+#[test]fn single_letter() {assert_tokens_eq_plain("a", &[TokenKind::blank_word()])@@ -118,4 +124,12 @@ mod tests {assert_tokens_eq_plain("Løvetann", &[TokenKind::blank_word()]);assert_tokens_eq_plain("Naïve", &[TokenKind::blank_word()]);}++ #[test]+ fn org_mode_basic() {+ assert_tokens_eq_org(+ "hello world",+ &[TokenKind::blank_word(), Space(1), TokenKind::blank_word()],+ );+ }}diff --git a/harper-core/src/parsers/org_mode.rs b/harper-core/src/parsers/org_mode.rsnew file mode 100644index 000000000..54fdf14c5--- /dev/null+++ b/harper-core/src/parsers/org_mode.rs@@ -0,0 +1,296 @@+use super::{Parser, PlainEnglish};+use crate::{Span, Token, TokenKind};++#[derive(Debug, PartialEq, Copy, Clone)]+enum SourceBlockMarker {+ Begin,+ End,+}++// Check if a line starts with a header (starts with one or more '*')+fn is_header_line(chars: &[char], start: usize) -> bool {+ chars.get(start).is_some_and(|c| *c == '*')+}++// Check if a line starts with a source block begin/end+fn is_source_block_marker(chars: &[char], start: usize) -> Option<SourceBlockMarker> {+ let line = get_line_from_start(chars, start);+ let line_str: String = line.iter().collect();+ let line_str = line_str.trim();++ if line_str.starts_with("#+BEGIN_SRC") || line_str.starts_with("#+begin_src") {+ Some(SourceBlockMarker::Begin)+ } else if line_str.starts_with("#+END_SRC") || line_str.starts_with("#+end_src") {+ Some(SourceBlockMarker::End)+ } else {+ None+ }+}++// Check if a line is a directive (starts with #+)+fn is_directive(chars: &[char], start: usize) -> bool {+ if start + 1 >= chars.len() {+ return false;+ }+ chars[start] == '#' && chars[start + 1] == '+'+}++// Get the rest of the line from a starting position+fn get_line_from_start(chars: &[char], start: usize) -> &[char] {+ let mut end = start;+ while end < chars.len() && chars[end] != '\n' {+ end += 1;+ }+ &chars[start..end]+}++// Find the end of the current line starting from position+fn find_line_end(chars: &[char], start: usize) -> usize {+ let mut pos = start;+ while pos < chars.len() && chars[pos] != '\n' {+ pos += 1;+ }+ if pos < chars.len() && chars[pos] == '\n' {+ pos + 1 // Include the newline+ } else {+ pos+ }+}++// Find the start of the line containing the given position+fn find_line_start(chars: &[char], pos: usize) -> usize {+ let mut start = pos;+ while start > 0 && chars[start - 1] != '\n' {+ start -= 1;+ }+ start+}++/// A parser that wraps the [`PlainEnglish`] parser that allows one to parse+/// Org-mode files.+///+/// Will ignore code blocks, source blocks, and other org-mode specific elements+/// that should not be linted for prose.+#[derive(Default, Clone, Debug, Copy)]+pub struct OrgMode;++impl OrgMode {}++impl Parser for OrgMode {+ fn parse(&self, source: &[char]) -> Vec<Token> {+ let english_parser = PlainEnglish;+ let mut tokens = Vec::new();+ let mut cursor = 0;+ let mut in_source_block = false;++ while cursor < source.len() {+ let line_start = find_line_start(source, cursor);++ // Check for source block markers+ let source_marker = is_source_block_marker(source, line_start);+ if let Some(marker) = source_marker {+ in_source_block = marker == SourceBlockMarker::Begin;+ }++ // If we're in a source block or found a source block marker, make the line unlintable+ if in_source_block || source_marker.is_some() {+ let line_end = find_line_end(source, line_start);+ tokens.push(Token {+ span: Span::new(line_start, line_end),+ kind: TokenKind::Unlintable,+ });+ cursor = line_end;+ continue;+ }++ // Check for headers+ if is_header_line(source, line_start) {+ let line_end = find_line_end(source, line_start);++ // Find where the header text starts (after the stars and spaces)+ let mut header_text_start = line_start;+ while header_text_start < line_end+ && (source[header_text_start] == '*' || source[header_text_start] == ' ')+ {+ header_text_start += 1;+ }++ // If there's actual text after the stars, parse it+ if header_text_start < line_end {+ let mut header_tokens =+ english_parser.parse(&source[header_text_start..line_end]);+ header_tokens+ .iter_mut()+ .for_each(|token| token.span.push_by(header_text_start));+ tokens.append(&mut header_tokens);+ }++ // Add paragraph break after header+ tokens.push(Token {+ span: Span::new_with_len(line_end.saturating_sub(1), 0),+ kind: TokenKind::ParagraphBreak,+ });++ cursor = line_end;+ continue;+ }++ // Check for directives (#+SOMETHING)+ if is_directive(source, line_start) {+ let line_end = find_line_end(source, line_start);+ tokens.push(Token {+ span: Span::new(line_start, line_end),+ kind: TokenKind::Unlintable,+ });+ cursor = line_end;+ continue;+ }++ // For normal text, parse with the English parser+ let line_end = find_line_end(source, cursor);+ if cursor < line_end {+ let mut line_tokens = english_parser.parse(&source[cursor..line_end]);+ line_tokens+ .iter_mut()+ .for_each(|token| token.span.push_by(cursor));+ tokens.append(&mut line_tokens);+ }++ cursor = line_end;+ }++ // Remove trailing newline/paragraph break tokens if the source doesn't actually end with a newline.+ if matches!(+ tokens.last(),+ Some(Token {+ kind: TokenKind::Newline(_) | TokenKind::ParagraphBreak,+ ..+ })+ ) && source.last() != Some(&'\n')+ {+ tokens.pop();+ }++ tokens+ }+}++#[cfg(test)]+mod tests {+ use super::super::StrParser;+ use super::OrgMode;+ use crate::TokenKind;++ #[test]+ fn simple_text() {+ let source = "This is simple text.";+ let tokens = OrgMode.parse_str(source);+ assert!(!tokens.is_empty());+ assert!(tokens.iter().any(|t| matches!(t.kind, TokenKind::Word(_))));+ }++ #[test]+ fn header_parsing() {+ let source = "* This is a header\nThis is regular text.";+ let tokens = OrgMode.parse_str(source);+ let token_kinds: Vec<_> = tokens.iter().map(|t| &t.kind).collect();++ // Should have words from header and paragraph break+ assert!(token_kinds.iter().any(|k| matches!(k, TokenKind::Word(_))));+ assert!(+ token_kinds+ .iter()+ .any(|k| matches!(k, TokenKind::ParagraphBreak))+ );+ }++ #[test]+ fn multiple_level_headers() {+ let source = "** Second level header\n*** Third level header";+ let tokens = OrgMode.parse_str(source);+ let token_kinds: Vec<_> = tokens.iter().map(|t| &t.kind).collect();++ // Should parse text from both headers+ let word_count = token_kinds+ .iter()+ .filter(|k| matches!(k, TokenKind::Word(_)))+ .count();+ assert!(word_count >= 4); // "Second", "level", "Third", "header"+ }++ #[test]+ fn source_block_unlintable() {+ let source = r#"Regular text.+#+BEGIN_SRC rust+fn main() {+ println!("Hello, world!");+}+#+END_SRC+More regular text."#;++ let tokens = OrgMode.parse_str(source);+ let unlintable_count = tokens+ .iter()+ .filter(|t| matches!(t.kind, TokenKind::Unlintable))+ .count();++ // Should have unlintable tokens for the source block lines+ assert!(unlintable_count > 0);++ // Should still have regular words from the non-source-block text+ assert!(tokens.iter().any(|t| matches!(t.kind, TokenKind::Word(_))));+ }++ #[test]+ fn directive_unlintable() {+ let source = r#"#+TITLE: My Document+#+AUTHOR: Test Author+This is regular text."#;++ let tokens = OrgMode.parse_str(source);+ let unlintable_count = tokens+ .iter()+ .filter(|t| matches!(t.kind, TokenKind::Unlintable))+ .count();++ // Should have unlintable tokens for directives+ assert_eq!(unlintable_count, 2);++ // Should still have regular words+ assert!(tokens.iter().any(|t| matches!(t.kind, TokenKind::Word(_))));+ }++ #[test]+ fn case_insensitive_source_blocks() {+ let source = r#"#+begin_src python+print("hello")+#+end_src"#;++ let tokens = OrgMode.parse_str(source);+ let unlintable_count = tokens+ .iter()+ .filter(|t| matches!(t.kind, TokenKind::Unlintable))+ .count();++ // All lines should be unlintable+ assert_eq!(unlintable_count, 3);+ }++ #[test]+ fn empty_header() {+ let source = "*\nRegular text.";+ let tokens = OrgMode.parse_str(source);++ // Should handle empty headers gracefully+ assert!(tokens.iter().any(|t| matches!(t.kind, TokenKind::Word(_))));+ }++ #[test]+ fn no_trailing_newline() {+ let source = "Simple text without newline";+ let tokens = OrgMode.parse_str(source);++ // Should not end with newline token if source doesn't+ assert!(!tokens.last().unwrap().kind.is_newline());+ }+}diff --git a/harper-core/tests/run_tests.rs b/harper-core/tests/run_tests.rsindex 5a3729153..536b63220 100644--- a/harper-core/tests/run_tests.rs+++ b/harper-core/tests/run_tests.rs@@ -1,4 +1,5 @@use harper_core::linting::{LintGroup, Linter};+use harper_core::parsers::OrgMode;use harper_core::{Dialect, Document, FstDictionary};/// Creates a unit test checking that the linting of a Markdown document (in@@ -33,6 +34,38 @@ macro_rules! create_test {};}+/// Creates a unit test checking that the linting of an Org mode document (in+/// `tests_sources`) produces the expected number of lints.+macro_rules! create_org_test {+ ($filename:ident.org, $correct_expected:expr, $dialect:expr) => {+ paste::paste! {+ #[test]+ fn [<lints_ $filename _correctly>](){+ let source = include_str!(+ concat!(+ "./test_sources/",+ concat!(stringify!($filename), ".org")+ )+ );++ let dict = FstDictionary::curated();+ let document = Document::new(&source, &OrgMode, &dict);++ let mut linter = LintGroup::new_curated(dict, $dialect);+ let lints = linter.lint(&document);++ dbg!(&lints);+ assert_eq!(lints.len(), $correct_expected);++ // Make sure that all generated tokens span real characters+ for token in document.tokens(){+ assert!(token.span.try_get_content(document.get_source()).is_some());+ }+ }+ }+ };+}+create_test!(whack_bullets.md, 1, Dialect::American);create_test!(preexisting.md, 0, Dialect::American);create_test!(issue_109.md, 0, Dialect::American);@@ -56,3 +89,6 @@ create_test!(yogurt_british_clean.md, 0, Dialect::British);// Make sure it doesn't paniccreate_test!(lukas_homework.md, 3, Dialect::American);++// Org mode tests+create_org_test!(index.org, 33, Dialect::American);diff --git a/harper-core/tests/test_sources/index.org b/harper-core/tests/test_sources/index.orgnew file mode 100644index 000000000..d94c37baf--- /dev/null+++ b/harper-core/tests/test_sources/index.org@@ -0,0 +1,81 @@+#+OPTIONS: H:9 ^:nil+* Nvim Orgmode++Nvim orgmode is a clone of Emacs Orgmode for Neovim 0.10.3+.+It aims to be a feature-complete implementation of Orgmode features in Neovim.++Online version of this documentation is available at [[https://nvim-orgmode.github.io]].++To view this documentation offline in Neovim, run =:Org help=. More info in [[#globals-and-commands][Globals and commands]] section.++** Quick start+:PROPERTIES:+:CUSTOM_ID: quick-start+:END:+- Install with [[https://github.com/folke/lazy.nvim][lazy.nvim]]:+ #+begin_src lua+ {+ 'nvim-orgmode/orgmode',+ event = 'VeryLazy',+ config = function()+ -- Setup orgmode+ require('orgmode').setup({+ org_agenda_files = '~/orgfiles/**/*',+ org_default_notes_file = '~/orgfiles/refile.org',+ })+ end,+ }+ #+end_src+- Capture your first note with =<leader>oc=+- Open up the prompt for agenda with =<leader>oa=++For more details about the installation and usage, check [[./installation.org][Installation page]].++To see all configuration options, check [[file:./configuration.org][Configuration page]].++** Getting started with orgmode+:PROPERTIES:+:CUSTOM_ID: getting-started+:END:+To get a basic idea how Orgmode works, check our hands-on [[file:./tutorial.org][tutorial]].++You can also check this screencast from [[https://github.com/dhruvasagar][@dhruvasagar]]+that demonstrates how the similar Orgmode clone [[https://github.com/dhruvasagar/vim-dotoo][vim-dotoo]] works.++[[https://www.youtube.com/watch?v=nsv33iOnH34]]++** API docs+:PROPERTIES:+:CUSTOM_ID: api-docs+:END:+Nvim-orgmode exposes a Lua API that can be used to interact with the orgmode. To view it, check [[file:../doc/orgmode_api.txt][orgmode_api.txt]]+or do =:h OrgApi= in Neovim.++** Globals and commands+:PROPERTIES:+:CUSTOM_ID: globals-and-commands+:END:+There are 2 additional ways to interact with Orgmode:+1. Through the =:Org= command+2. Through =Org= Lua global variable++List of available actions:+- =:Org help= - Open this documentation in new tab, set working directory to the docs folder for the tab to allow browsing+- =:Org helpgrep= - Open search agenda view that allows searching through the documentation+- =:Org agenda {type?}= - Open agenda view by the shortcut, for example =:Org agenda M= will open =tags_todo= view. When =type= is omitted, it opens up Agenda view.+- =:Org capture {type?}= - Open capture template by the shortcut, for example =:Org capture t=. When =type= is omitted, it opens up Capture prompt.+- =:Org install_treesitter_grammar= - Install the treesitter grammar for Orgmode. If installed, prompt to reinstall. Grammar is installed automatically+ on first run, but this is useful in case when there are issues with the grammar.+- =:Org store_link= - [[file:./configuration.org::#org_store_link][Store link]] to the headline under cursor. Works in both agenda and org files.+- =:Org indent_mode= - Toggle virtual indent mode in the current buffer. See [[file:./configuration.org::#org_startup_indented][osg_startup_indented]] for additional info.++All of the commands above can be executed through the global Lua =Org= variable. Examples:+- =Org.help()=+- =Org.helpgrep()=+- =Org.install_treesitter_grammar()=+- =Org.store_link()=+- =Org.indent_mode()=+- =Org.agenda()= - Opens =agenda= prompt+- =Org.agenda.m()= - Opens =tags= view+- =Org.capture()= - Opens capture prompt+- =Org.capture.t()= - Opens capture template for =t= shortcut\ No newline at end of filediff --git a/harper-ls/src/backend.rs b/harper-ls/src/backend.rsindex 73dd35497..952c25450 100644--- a/harper-ls/src/backend.rs+++ b/harper-ls/src/backend.rs@@ -8,7 +8,9 @@ use anyhow::{Context, Result, anyhow};use futures::future::join;use harper_comments::CommentParser;use harper_core::linting::{LintGroup, LintGroupConfig};-use harper_core::parsers::{CollapseIdentifiers, IsolateEnglish, Markdown, Parser, PlainEnglish};+use harper_core::parsers::{+ CollapseIdentifiers, IsolateEnglish, Markdown, OrgMode, Parser, PlainEnglish,+};use harper_core::{Dialect, Dictionary, Document, FstDictionary, IgnoredLints, MergedDictionary,MutableDictionary, WordMetadata,@@ -337,6 +339,7 @@ impl Backend {"html" => Some(Box::new(HtmlParser::default())),"mail" | "plaintext" | "text" => Some(Box::new(PlainEnglish)),"typst" => Some(Box::new(Typst)),+ "org" => Some(Box::new(OrgMode)),_ => None,};