Skip to content

Offline, privacy-first grammar checker. Fast, open-source, Rust-powered

feat: Added ORGMODE parser (#1369)

Eloy Coto55f5e8b6d323
diff --git a/harper-cli/src/main.rs b/harper-cli/src/main.rs
index 575248f03..af778453f 100644
--- a/harper-cli/src/main.rs
+++ b/harper-cli/src/main.rs
@@ -12,7 +12,7 @@ use clap::Parser;
use dirs::{config_dir, data_local_dir};
use harper_comments::CommentParser;
use harper_core::linting::{LintGroup, Linter};
-use harper_core::parsers::{Markdown, MarkdownOptions, PlainEnglish};
+use harper_core::parsers::{Markdown, MarkdownOptions, OrgMode, PlainEnglish};
use harper_core::{
remove_overlaps, CharStringExt, Dialect, Dictionary, Document, FstDictionary, MergedDictionary,
MutableDictionary, TokenKind, TokenStringExt, WordId, WordMetadata,
@@ -439,6 +439,7 @@ fn load_file(
Some("lhs") => Box::new(LiterateHaskellParser::new_markdown(
MarkdownOptions::default(),
)),
+ Some("org") => Box::new(OrgMode),
Some("typ") => Box::new(harper_typst::Typst),
_ => {
if let Some(comment_parser) = CommentParser::new_from_filename(file, markdown_options) {
diff --git a/harper-core/src/parsers/mod.rs b/harper-core/src/parsers/mod.rs
index 9909308d2..11c97f8ba 100644
--- a/harper-core/src/parsers/mod.rs
+++ b/harper-core/src/parsers/mod.rs
@@ -2,6 +2,7 @@ mod collapse_identifiers;
mod isolate_english;
mod markdown;
mod mask;
+mod org_mode;
mod plain_english;
use blanket::blanket;
@@ -9,6 +10,7 @@ pub use collapse_identifiers::CollapseIdentifiers;
pub use isolate_english::IsolateEnglish;
pub use markdown::{Markdown, MarkdownOptions};
pub use mask::Mask;
+pub use org_mode::OrgMode;
pub use plain_english::PlainEnglish;
use crate::{LSend, Token, TokenStringExt};
@@ -35,7 +37,7 @@ where
#[cfg(test)]
mod tests {
- use super::{Markdown, Parser, PlainEnglish};
+ use super::{Markdown, OrgMode, Parser, PlainEnglish};
use crate::Punctuation;
use crate::TokenKind::{self, *};
@@ -55,6 +57,10 @@ mod tests {
assert_tokens_eq(test_str, expected, &Markdown::default())
}
+ fn assert_tokens_eq_org(test_str: impl AsRef<str>, expected: &[TokenKind]) {
+ assert_tokens_eq(test_str, expected, &OrgMode)
+ }
+
#[test]
fn single_letter() {
assert_tokens_eq_plain("a", &[TokenKind::blank_word()])
@@ -118,4 +124,12 @@ mod tests {
assert_tokens_eq_plain("Løvetann", &[TokenKind::blank_word()]);
assert_tokens_eq_plain("Naïve", &[TokenKind::blank_word()]);
}
+
+ #[test]
+ fn org_mode_basic() {
+ assert_tokens_eq_org(
+ "hello world",
+ &[TokenKind::blank_word(), Space(1), TokenKind::blank_word()],
+ );
+ }
}
diff --git a/harper-core/src/parsers/org_mode.rs b/harper-core/src/parsers/org_mode.rs
new file mode 100644
index 000000000..54fdf14c5
--- /dev/null
+++ b/harper-core/src/parsers/org_mode.rs
@@ -0,0 +1,296 @@
+use super::{Parser, PlainEnglish};
+use crate::{Span, Token, TokenKind};
+
+#[derive(Debug, PartialEq, Copy, Clone)]
+enum SourceBlockMarker {
+ Begin,
+ End,
+}
+
+// Check if a line starts with a header (starts with one or more '*')
+fn is_header_line(chars: &[char], start: usize) -> bool {
+ chars.get(start).is_some_and(|c| *c == '*')
+}
+
+// Check if a line starts with a source block begin/end
+fn is_source_block_marker(chars: &[char], start: usize) -> Option<SourceBlockMarker> {
+ let line = get_line_from_start(chars, start);
+ let line_str: String = line.iter().collect();
+ let line_str = line_str.trim();
+
+ if line_str.starts_with("#+BEGIN_SRC") || line_str.starts_with("#+begin_src") {
+ Some(SourceBlockMarker::Begin)
+ } else if line_str.starts_with("#+END_SRC") || line_str.starts_with("#+end_src") {
+ Some(SourceBlockMarker::End)
+ } else {
+ None
+ }
+}
+
+// Check if a line is a directive (starts with #+)
+fn is_directive(chars: &[char], start: usize) -> bool {
+ if start + 1 >= chars.len() {
+ return false;
+ }
+ chars[start] == '#' && chars[start + 1] == '+'
+}
+
+// Get the rest of the line from a starting position
+fn get_line_from_start(chars: &[char], start: usize) -> &[char] {
+ let mut end = start;
+ while end < chars.len() && chars[end] != '\n' {
+ end += 1;
+ }
+ &chars[start..end]
+}
+
+// Find the end of the current line starting from position
+fn find_line_end(chars: &[char], start: usize) -> usize {
+ let mut pos = start;
+ while pos < chars.len() && chars[pos] != '\n' {
+ pos += 1;
+ }
+ if pos < chars.len() && chars[pos] == '\n' {
+ pos + 1 // Include the newline
+ } else {
+ pos
+ }
+}
+
+// Find the start of the line containing the given position
+fn find_line_start(chars: &[char], pos: usize) -> usize {
+ let mut start = pos;
+ while start > 0 && chars[start - 1] != '\n' {
+ start -= 1;
+ }
+ start
+}
+
+/// A parser that wraps the [`PlainEnglish`] parser that allows one to parse
+/// Org-mode files.
+///
+/// Will ignore code blocks, source blocks, and other org-mode specific elements
+/// that should not be linted for prose.
+#[derive(Default, Clone, Debug, Copy)]
+pub struct OrgMode;
+
+impl OrgMode {}
+
+impl Parser for OrgMode {
+ fn parse(&self, source: &[char]) -> Vec<Token> {
+ let english_parser = PlainEnglish;
+ let mut tokens = Vec::new();
+ let mut cursor = 0;
+ let mut in_source_block = false;
+
+ while cursor < source.len() {
+ let line_start = find_line_start(source, cursor);
+
+ // Check for source block markers
+ let source_marker = is_source_block_marker(source, line_start);
+ if let Some(marker) = source_marker {
+ in_source_block = marker == SourceBlockMarker::Begin;
+ }
+
+ // If we're in a source block or found a source block marker, make the line unlintable
+ if in_source_block || source_marker.is_some() {
+ let line_end = find_line_end(source, line_start);
+ tokens.push(Token {
+ span: Span::new(line_start, line_end),
+ kind: TokenKind::Unlintable,
+ });
+ cursor = line_end;
+ continue;
+ }
+
+ // Check for headers
+ if is_header_line(source, line_start) {
+ let line_end = find_line_end(source, line_start);
+
+ // Find where the header text starts (after the stars and spaces)
+ let mut header_text_start = line_start;
+ while header_text_start < line_end
+ && (source[header_text_start] == '*' || source[header_text_start] == ' ')
+ {
+ header_text_start += 1;
+ }
+
+ // If there's actual text after the stars, parse it
+ if header_text_start < line_end {
+ let mut header_tokens =
+ english_parser.parse(&source[header_text_start..line_end]);
+ header_tokens
+ .iter_mut()
+ .for_each(|token| token.span.push_by(header_text_start));
+ tokens.append(&mut header_tokens);
+ }
+
+ // Add paragraph break after header
+ tokens.push(Token {
+ span: Span::new_with_len(line_end.saturating_sub(1), 0),
+ kind: TokenKind::ParagraphBreak,
+ });
+
+ cursor = line_end;
+ continue;
+ }
+
+ // Check for directives (#+SOMETHING)
+ if is_directive(source, line_start) {
+ let line_end = find_line_end(source, line_start);
+ tokens.push(Token {
+ span: Span::new(line_start, line_end),
+ kind: TokenKind::Unlintable,
+ });
+ cursor = line_end;
+ continue;
+ }
+
+ // For normal text, parse with the English parser
+ let line_end = find_line_end(source, cursor);
+ if cursor < line_end {
+ let mut line_tokens = english_parser.parse(&source[cursor..line_end]);
+ line_tokens
+ .iter_mut()
+ .for_each(|token| token.span.push_by(cursor));
+ tokens.append(&mut line_tokens);
+ }
+
+ cursor = line_end;
+ }
+
+ // Remove trailing newline/paragraph break tokens if the source doesn't actually end with a newline.
+ if matches!(
+ tokens.last(),
+ Some(Token {
+ kind: TokenKind::Newline(_) | TokenKind::ParagraphBreak,
+ ..
+ })
+ ) && source.last() != Some(&'\n')
+ {
+ tokens.pop();
+ }
+
+ tokens
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::super::StrParser;
+ use super::OrgMode;
+ use crate::TokenKind;
+
+ #[test]
+ fn simple_text() {
+ let source = "This is simple text.";
+ let tokens = OrgMode.parse_str(source);
+ assert!(!tokens.is_empty());
+ assert!(tokens.iter().any(|t| matches!(t.kind, TokenKind::Word(_))));
+ }
+
+ #[test]
+ fn header_parsing() {
+ let source = "* This is a header\nThis is regular text.";
+ let tokens = OrgMode.parse_str(source);
+ let token_kinds: Vec<_> = tokens.iter().map(|t| &t.kind).collect();
+
+ // Should have words from header and paragraph break
+ assert!(token_kinds.iter().any(|k| matches!(k, TokenKind::Word(_))));
+ assert!(
+ token_kinds
+ .iter()
+ .any(|k| matches!(k, TokenKind::ParagraphBreak))
+ );
+ }
+
+ #[test]
+ fn multiple_level_headers() {
+ let source = "** Second level header\n*** Third level header";
+ let tokens = OrgMode.parse_str(source);
+ let token_kinds: Vec<_> = tokens.iter().map(|t| &t.kind).collect();
+
+ // Should parse text from both headers
+ let word_count = token_kinds
+ .iter()
+ .filter(|k| matches!(k, TokenKind::Word(_)))
+ .count();
+ assert!(word_count >= 4); // "Second", "level", "Third", "header"
+ }
+
+ #[test]
+ fn source_block_unlintable() {
+ let source = r#"Regular text.
+#+BEGIN_SRC rust
+fn main() {
+ println!("Hello, world!");
+}
+#+END_SRC
+More regular text."#;
+
+ let tokens = OrgMode.parse_str(source);
+ let unlintable_count = tokens
+ .iter()
+ .filter(|t| matches!(t.kind, TokenKind::Unlintable))
+ .count();
+
+ // Should have unlintable tokens for the source block lines
+ assert!(unlintable_count > 0);
+
+ // Should still have regular words from the non-source-block text
+ assert!(tokens.iter().any(|t| matches!(t.kind, TokenKind::Word(_))));
+ }
+
+ #[test]
+ fn directive_unlintable() {
+ let source = r#"#+TITLE: My Document
+#+AUTHOR: Test Author
+This is regular text."#;
+
+ let tokens = OrgMode.parse_str(source);
+ let unlintable_count = tokens
+ .iter()
+ .filter(|t| matches!(t.kind, TokenKind::Unlintable))
+ .count();
+
+ // Should have unlintable tokens for directives
+ assert_eq!(unlintable_count, 2);
+
+ // Should still have regular words
+ assert!(tokens.iter().any(|t| matches!(t.kind, TokenKind::Word(_))));
+ }
+
+ #[test]
+ fn case_insensitive_source_blocks() {
+ let source = r#"#+begin_src python
+print("hello")
+#+end_src"#;
+
+ let tokens = OrgMode.parse_str(source);
+ let unlintable_count = tokens
+ .iter()
+ .filter(|t| matches!(t.kind, TokenKind::Unlintable))
+ .count();
+
+ // All lines should be unlintable
+ assert_eq!(unlintable_count, 3);
+ }
+
+ #[test]
+ fn empty_header() {
+ let source = "*\nRegular text.";
+ let tokens = OrgMode.parse_str(source);
+
+ // Should handle empty headers gracefully
+ assert!(tokens.iter().any(|t| matches!(t.kind, TokenKind::Word(_))));
+ }
+
+ #[test]
+ fn no_trailing_newline() {
+ let source = "Simple text without newline";
+ let tokens = OrgMode.parse_str(source);
+
+ // Should not end with newline token if source doesn't
+ assert!(!tokens.last().unwrap().kind.is_newline());
+ }
+}
diff --git a/harper-core/tests/run_tests.rs b/harper-core/tests/run_tests.rs
index 5a3729153..536b63220 100644
--- a/harper-core/tests/run_tests.rs
+++ b/harper-core/tests/run_tests.rs
@@ -1,4 +1,5 @@
use harper_core::linting::{LintGroup, Linter};
+use harper_core::parsers::OrgMode;
use harper_core::{Dialect, Document, FstDictionary};
/// Creates a unit test checking that the linting of a Markdown document (in
@@ -33,6 +34,38 @@ macro_rules! create_test {
};
}
+/// Creates a unit test checking that the linting of an Org mode document (in
+/// `tests_sources`) produces the expected number of lints.
+macro_rules! create_org_test {
+ ($filename:ident.org, $correct_expected:expr, $dialect:expr) => {
+ paste::paste! {
+ #[test]
+ fn [<lints_ $filename _correctly>](){
+ let source = include_str!(
+ concat!(
+ "./test_sources/",
+ concat!(stringify!($filename), ".org")
+ )
+ );
+
+ let dict = FstDictionary::curated();
+ let document = Document::new(&source, &OrgMode, &dict);
+
+ let mut linter = LintGroup::new_curated(dict, $dialect);
+ let lints = linter.lint(&document);
+
+ dbg!(&lints);
+ assert_eq!(lints.len(), $correct_expected);
+
+ // Make sure that all generated tokens span real characters
+ for token in document.tokens(){
+ assert!(token.span.try_get_content(document.get_source()).is_some());
+ }
+ }
+ }
+ };
+}
+
create_test!(whack_bullets.md, 1, Dialect::American);
create_test!(preexisting.md, 0, Dialect::American);
create_test!(issue_109.md, 0, Dialect::American);
@@ -56,3 +89,6 @@ create_test!(yogurt_british_clean.md, 0, Dialect::British);
// Make sure it doesn't panic
create_test!(lukas_homework.md, 3, Dialect::American);
+
+// Org mode tests
+create_org_test!(index.org, 33, Dialect::American);
diff --git a/harper-core/tests/test_sources/index.org b/harper-core/tests/test_sources/index.org
new file mode 100644
index 000000000..d94c37baf
--- /dev/null
+++ b/harper-core/tests/test_sources/index.org
@@ -0,0 +1,81 @@
+#+OPTIONS: H:9 ^:nil
+* Nvim Orgmode
+
+Nvim orgmode is a clone of Emacs Orgmode for Neovim 0.10.3+.
+It aims to be a feature-complete implementation of Orgmode features in Neovim.
+
+Online version of this documentation is available at [[https://nvim-orgmode.github.io]].
+
+To view this documentation offline in Neovim, run =:Org help=. More info in [[#globals-and-commands][Globals and commands]] section.
+
+** Quick start
+:PROPERTIES:
+:CUSTOM_ID: quick-start
+:END:
+- Install with [[https://github.com/folke/lazy.nvim][lazy.nvim]]:
+ #+begin_src lua
+ {
+ 'nvim-orgmode/orgmode',
+ event = 'VeryLazy',
+ config = function()
+ -- Setup orgmode
+ require('orgmode').setup({
+ org_agenda_files = '~/orgfiles/**/*',
+ org_default_notes_file = '~/orgfiles/refile.org',
+ })
+ end,
+ }
+ #+end_src
+- Capture your first note with =<leader>oc=
+- Open up the prompt for agenda with =<leader>oa=
+
+For more details about the installation and usage, check [[./installation.org][Installation page]].
+
+To see all configuration options, check [[file:./configuration.org][Configuration page]].
+
+** Getting started with orgmode
+:PROPERTIES:
+:CUSTOM_ID: getting-started
+:END:
+To get a basic idea how Orgmode works, check our hands-on [[file:./tutorial.org][tutorial]].
+
+You can also check this screencast from [[https://github.com/dhruvasagar][@dhruvasagar]]
+that demonstrates how the similar Orgmode clone [[https://github.com/dhruvasagar/vim-dotoo][vim-dotoo]] works.
+
+[[https://www.youtube.com/watch?v=nsv33iOnH34]]
+
+** API docs
+:PROPERTIES:
+:CUSTOM_ID: api-docs
+:END:
+Nvim-orgmode exposes a Lua API that can be used to interact with the orgmode. To view it, check [[file:../doc/orgmode_api.txt][orgmode_api.txt]]
+or do =:h OrgApi= in Neovim.
+
+** Globals and commands
+:PROPERTIES:
+:CUSTOM_ID: globals-and-commands
+:END:
+There are 2 additional ways to interact with Orgmode:
+1. Through the =:Org= command
+2. Through =Org= Lua global variable
+
+List of available actions:
+- =:Org help= - Open this documentation in new tab, set working directory to the docs folder for the tab to allow browsing
+- =:Org helpgrep= - Open search agenda view that allows searching through the documentation
+- =:Org agenda {type?}= - Open agenda view by the shortcut, for example =:Org agenda M= will open =tags_todo= view. When =type= is omitted, it opens up Agenda view.
+- =:Org capture {type?}= - Open capture template by the shortcut, for example =:Org capture t=. When =type= is omitted, it opens up Capture prompt.
+- =:Org install_treesitter_grammar= - Install the treesitter grammar for Orgmode. If installed, prompt to reinstall. Grammar is installed automatically
+ on first run, but this is useful in case when there are issues with the grammar.
+- =:Org store_link= - [[file:./configuration.org::#org_store_link][Store link]] to the headline under cursor. Works in both agenda and org files.
+- =:Org indent_mode= - Toggle virtual indent mode in the current buffer. See [[file:./configuration.org::#org_startup_indented][osg_startup_indented]] for additional info.
+
+All of the commands above can be executed through the global Lua =Org= variable. Examples:
+- =Org.help()=
+- =Org.helpgrep()=
+- =Org.install_treesitter_grammar()=
+- =Org.store_link()=
+- =Org.indent_mode()=
+- =Org.agenda()= - Opens =agenda= prompt
+- =Org.agenda.m()= - Opens =tags= view
+- =Org.capture()= - Opens capture prompt
+- =Org.capture.t()= - Opens capture template for =t= shortcut
\ No newline at end of file
diff --git a/harper-ls/src/backend.rs b/harper-ls/src/backend.rs
index 73dd35497..952c25450 100644
--- a/harper-ls/src/backend.rs
+++ b/harper-ls/src/backend.rs
@@ -8,7 +8,9 @@ use anyhow::{Context, Result, anyhow};
use futures::future::join;
use harper_comments::CommentParser;
use harper_core::linting::{LintGroup, LintGroupConfig};
-use harper_core::parsers::{CollapseIdentifiers, IsolateEnglish, Markdown, Parser, PlainEnglish};
+use harper_core::parsers::{
+ CollapseIdentifiers, IsolateEnglish, Markdown, OrgMode, Parser, PlainEnglish,
+};
use harper_core::{
Dialect, Dictionary, Document, FstDictionary, IgnoredLints, MergedDictionary,
MutableDictionary, WordMetadata,
@@ -337,6 +339,7 @@ impl Backend {
"html" => Some(Box::new(HtmlParser::default())),
"mail" | "plaintext" | "text" => Some(Box::new(PlainEnglish)),
"typst" => Some(Box::new(Typst)),
+ "org" => Some(Box::new(OrgMode)),
_ => None,
};