Offline, privacy-first grammar checker. Fast, open-source, Rust-powered
fix: issues with structured newlines in JavaDoc lints
Elijah Potter04ea3e85dc72
diff --git a/harper-comments/src/comment_parser.rs b/harper-comments/src/comment_parser.rsindex 4062d5426..b46b38117 100644--- a/harper-comments/src/comment_parser.rs+++ b/harper-comments/src/comment_parser.rs@@ -9,7 +9,7 @@ use tree_sitter::Node;use crate::comment_parsers;pub struct CommentParser {- inner: parsers::Mask<TreeSitterMasker, Box<dyn Parser>>,+ inner: parsers::Mask<TreeSitterMasker, Box<dyn Parser>>}impl CommentParser {@@ -35,21 +35,21 @@ impl CommentParser {"lua" => tree_sitter_lua::language(),"sh" => tree_sitter_bash::language(),"java" => tree_sitter_java::language(),- _ => return None,+ _ => return None};let comment_parser: Box<dyn Parser> = match language_id {"javascriptreact" | "typescript" | "typescriptreact" | "javascript" => Box::new(JsDoc),"java" => Box::new(JavaDoc::default()),"go" => Box::new(Go),- _ => Box::new(Unit),+ _ => Box::new(Unit)};Some(Self {inner: parsers::Mask::new(TreeSitterMasker::new(language, Self::node_condition),- comment_parser,- ),+ comment_parser+ )})}@@ -82,7 +82,7 @@ impl CommentParser {"sh" => "sh","bash" => "sh","java" => "java",- _ => return None,+ _ => return None})}diff --git a/harper-comments/src/comment_parsers/javadoc.rs b/harper-comments/src/comment_parsers/javadoc.rsindex 2a15521a4..d3e6d42e6 100644--- a/harper-comments/src/comment_parsers/javadoc.rs+++ b/harper-comments/src/comment_parsers/javadoc.rs@@ -1,4 +1,7 @@-use harper_core::{parsers::Parser, Token};+use std::collections::VecDeque;++use harper_core::parsers::Parser;+use harper_core::{Punctuation, Token, TokenKind, VecExt};use harper_html::HtmlParser;use super::without_initiators;@@ -15,6 +18,41 @@ impl Parser for JavaDoc {let mut tokens = self.html_parser.parse(actual_source);+ // We need to remove leading spaces and stars from the block of tokens.+ let mut remove_these: VecDeque<usize> = VecDeque::new();++ let mut cursor = 0;++ while cursor < tokens.len() {+ let maybe_newline = tokens[cursor];++ if let TokenKind::Newline(_) = maybe_newline.kind {+ cursor += 1;++ loop {+ if cursor >= tokens.len() {+ break;+ }++ let maybe_removable = tokens[cursor];++ if matches!(+ maybe_removable.kind,+ TokenKind::Punctuation(Punctuation::Star) | TokenKind::Space(_)+ ) {+ remove_these.push_back(cursor);+ cursor += 1;+ } else {+ break;+ }+ }+ } else {+ cursor += 1;+ }+ }++ tokens.remove_indices(remove_these);+for token in tokens.iter_mut() {token.span.push_by(actual.start);}diff --git a/harper-comments/src/comment_parsers/jsdoc.rs b/harper-comments/src/comment_parsers/jsdoc.rsindex 3e9b42135..0948008aa 100644--- a/harper-comments/src/comment_parsers/jsdoc.rs+++ b/harper-comments/src/comment_parsers/jsdoc.rs@@ -17,7 +17,7 @@ impl Parser for JsDoc {new_tokens.push(Token::new(Span::new_with_len(line.len(), 1),- harper_core::TokenKind::Newline(1),+ harper_core::TokenKind::Newline(1)));new_tokensdiff --git a/harper-comments/src/comment_parsers/unit.rs b/harper-comments/src/comment_parsers/unit.rsindex 07eb144f6..2cd505401 100644--- a/harper-comments/src/comment_parsers/unit.rs+++ b/harper-comments/src/comment_parsers/unit.rs@@ -22,7 +22,7 @@ impl Parser for Unit {new_tokens.push(Token::new(Span::new_with_len(line.len(), 1),- harper_core::TokenKind::Newline(1),+ harper_core::TokenKind::Newline(1)));new_tokensdiff --git a/harper-core/src/document.rs b/harper-core/src/document.rsindex 1865c48b2..892854220 100644--- a/harper-core/src/document.rs+++ b/harper-core/src/document.rs@@ -10,6 +10,7 @@ use crate::parsers::{Markdown, Parser, PlainEnglish};use crate::punctuation::Punctuation;use crate::span::Span;use crate::token::NumberSuffix;+use crate::vec_ext::VecExt;use crate::{FatToken, Lrc, Token, TokenKind, TokenStringExt};pub struct Document {@@ -376,7 +377,7 @@ impl Document {cursor += 1;}- remove_indices(&mut self.tokens, remove_these);+ self.tokens.remove_indices(remove_these);}/// Searches for multiple sequential newline tokens and condenses them down@@ -414,7 +415,7 @@ impl Document {cursor += 1;}- remove_indices(&mut self.tokens, remove_these);+ self.tokens.remove_indices(remove_these);}/// Searches for contractions and condenses them down into single@@ -531,35 +532,9 @@ fn is_sentence_terminator(token: &TokenKind) -> bool {}}-/// Removes a list of indices from a Vector.-/// Assumes that the provided indices are already in sorted order.-fn remove_indices<T>(vec: &mut Vec<T>, mut to_remove: VecDeque<usize>) {- let mut i = 0;-- let mut next_remove = to_remove.pop_front();-- vec.retain(|_| {- let keep = if let Some(next_remove) = next_remove {- i != next_remove- } else {- true- };-- if !keep {- next_remove = to_remove.pop_front();- }-- i += 1;- keep- });-}-#[cfg(test)]mod tests {- use std::collections::VecDeque;-use super::Document;- use crate::document::remove_indices;use crate::parsers::{Markdown, PlainEnglish};use crate::token::TokenStringExt;use crate::{Span, Token, TokenKind};@@ -651,14 +626,4 @@ mod tests {18,);}-- #[test]- fn removes_requested_indices() {- let mut data: Vec<i32> = (0..10).collect();- let remove: VecDeque<usize> = vec![1, 4, 6].into_iter().collect();-- remove_indices(&mut data, remove);-- assert_eq!(data, vec![0, 2, 3, 5, 7, 8, 9])- }}diff --git a/harper-core/src/lib.rs b/harper-core/src/lib.rsindex c6f18df42..d69e87d8e 100644--- a/harper-core/src/lib.rs+++ b/harper-core/src/lib.rs@@ -14,6 +14,7 @@ mod span;mod spell;mod sync;mod token;+mod vec_ext;pub use char_string::{CharString, CharStringExt};pub use document::Document;@@ -24,6 +25,7 @@ pub use span::Span;pub use spell::{Dictionary, FullDictionary, MergedDictionary};pub use sync::Lrc;pub use token::{FatToken, Token, TokenKind, TokenStringExt};+pub use vec_ext::VecExt;/// A utility function that removes overlapping lints in a vector,/// keeping the more important ones.diff --git a/harper-core/src/parsers/mask.rs b/harper-core/src/parsers/mask.rsindex 2bd21aae6..fd1b21a2d 100644--- a/harper-core/src/parsers/mask.rs+++ b/harper-core/src/parsers/mask.rs@@ -6,16 +6,16 @@ use crate::{Span, Token, TokenKind};pub struct Mask<M, P>whereM: Masker,- P: Parser,+ P: Parser{pub masker: M,- pub parser: P,+ pub parser: P}impl<M, P> Mask<M, P>whereM: Masker,- P: Parser,+ P: Parser{pub fn new(masker: M, parser: P) -> Self {Self { masker, parser }@@ -25,7 +25,7 @@ whereimpl<M, P> Parser for Mask<M, P>whereM: Masker,- P: Parser,+ P: Parser{fn parse(&mut self, source: &[char]) -> Vec<Token> {let mask = self.masker.create_mask(source);diff --git a/harper-core/src/token.rs b/harper-core/src/token.rsindex 13b62ca69..a4f84db6a 100644--- a/harper-core/src/token.rs+++ b/harper-core/src/token.rs@@ -10,7 +10,7 @@ use crate::Quote;#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Default)]pub struct Token {pub span: Span,- pub kind: TokenKind,+ pub kind: TokenKind}impl Token {@@ -24,7 +24,7 @@ impl Token {FatToken {content,- kind: self.kind,+ kind: self.kind}}}@@ -34,7 +34,7 @@ impl Token {#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, PartialOrd)]pub struct FatToken {pub content: Vec<char>,- pub kind: TokenKind,+ pub kind: TokenKind}#[derive(Debug, Is, Clone, Copy, Serialize, Deserialize, PartialEq, Default, PartialOrd)]@@ -54,7 +54,7 @@ pub enum TokenKind {/// A special token used for things like inline code blocks that should be/// ignored by all linters.Unlintable,- ParagraphBreak,+ ParagraphBreak}#[derive(Debug, Serialize, Deserialize, Default, PartialEq, PartialOrd, Clone, Copy, Is)]@@ -63,7 +63,7 @@ pub enum NumberSuffix {Th,St,Nd,- Rd,+ Rd}impl NumberSuffix {@@ -89,7 +89,7 @@ impl NumberSuffix {7 => Some(Self::Th),8 => Some(Self::Th),9 => Some(Self::Th),- _ => None,+ _ => None}}@@ -98,7 +98,7 @@ impl NumberSuffix {NumberSuffix::Th => vec!['t', 'h'],NumberSuffix::St => vec!['s', 't'],NumberSuffix::Nd => vec!['n', 'd'],- NumberSuffix::Rd => vec!['r', 'd'],+ NumberSuffix::Rd => vec!['r', 'd']}}@@ -126,7 +126,7 @@ impl NumberSuffix {('R', 'd') => Some(NumberSuffix::Rd),('r', 'D') => Some(NumberSuffix::Rd),('R', 'D') => Some(NumberSuffix::Rd),- _ => None,+ _ => None}}}diff --git a/harper-core/src/vec_ext.rs b/harper-core/src/vec_ext.rsnew file mode 100644index 000000000..930913a4f--- /dev/null+++ b/harper-core/src/vec_ext.rs@@ -0,0 +1,47 @@+use std::collections::VecDeque;++pub trait VecExt {+ /// Removes a list of indices from a Vector.+ /// Assumes that the provided indices are already in sorted order.+ fn remove_indices(&mut self, to_remove: VecDeque<usize>);+}++impl<T> VecExt for Vec<T> {+ fn remove_indices(&mut self, mut to_remove: VecDeque<usize>) {+ let mut i = 0;++ let mut next_remove = to_remove.pop_front();++ self.retain(|_| {+ let keep = if let Some(next_remove) = next_remove {+ i != next_remove+ } else {+ true+ };++ if !keep {+ next_remove = to_remove.pop_front();+ }++ i += 1;+ keep+ });+ }+}++#[cfg(test)]+mod tests {+ use std::collections::VecDeque;++ use crate::vec_ext::VecExt;++ #[test]+ fn removes_requested_indices() {+ let mut data: Vec<i32> = (0..10).collect();+ let remove: VecDeque<usize> = vec![1, 4, 6].into_iter().collect();++ data.remove_indices(remove);++ assert_eq!(data, vec![0, 2, 3, 5, 7, 8, 9])+ }+}diff --git a/harper-html/src/lib.rs b/harper-html/src/lib.rsindex b4aefcf94..33dcfb90a 100644--- a/harper-html/src/lib.rs+++ b/harper-html/src/lib.rs@@ -4,7 +4,7 @@ use harper_tree_sitter::TreeSitterMasker;use tree_sitter::Node;pub struct HtmlParser {- inner: parsers::Mask<TreeSitterMasker, PlainEnglish>,+ inner: parsers::Mask<TreeSitterMasker, PlainEnglish>}impl HtmlParser {@@ -18,8 +18,8 @@ impl Default for HtmlParser {Self {inner: parsers::Mask::new(TreeSitterMasker::new(tree_sitter_html::language(), Self::node_condition),- PlainEnglish,- ),+ PlainEnglish+ )}}}diff --git a/harper-ls/src/backend.rs b/harper-ls/src/backend.rsindex 507ac9656..068709137 100644--- a/harper-ls/src/backend.rs+++ b/harper-ls/src/backend.rs@@ -5,7 +5,14 @@ use std::sync::Arc;use harper_comments::CommentParser;use harper_core::parsers::{Markdown, PlainEnglish};use harper_core::{- Dictionary, Document, FullDictionary, LintGroup, Linter, MergedDictionary, Token, TokenKind,+ Dictionary,+ Document,+ FullDictionary,+ LintGroup,+ Linter,+ MergedDictionary,+ Token,+ TokenKind};use harper_html::HtmlParser;use serde_json::Value;@@ -13,13 +20,31 @@ use tokio::sync::{Mutex, RwLock};use tower_lsp::jsonrpc::Result;use tower_lsp::lsp_types::notification::PublishDiagnostics;use tower_lsp::lsp_types::{- CodeActionOrCommand, CodeActionParams, CodeActionProviderCapability, CodeActionResponse,- Command, Diagnostic, DidChangeConfigurationParams, DidChangeTextDocumentParams,- DidCloseTextDocumentParams, DidOpenTextDocumentParams, DidSaveTextDocumentParams,- ExecuteCommandOptions, ExecuteCommandParams, InitializeParams, InitializeResult,- InitializedParams, MessageType, PublishDiagnosticsParams, Range, ServerCapabilities,- TextDocumentSyncCapability, TextDocumentSyncKind, TextDocumentSyncOptions,- TextDocumentSyncSaveOptions, Url,+ CodeActionOrCommand,+ CodeActionParams,+ CodeActionProviderCapability,+ CodeActionResponse,+ Command,+ Diagnostic,+ DidChangeConfigurationParams,+ DidChangeTextDocumentParams,+ DidCloseTextDocumentParams,+ DidOpenTextDocumentParams,+ DidSaveTextDocumentParams,+ ExecuteCommandOptions,+ ExecuteCommandParams,+ InitializeParams,+ InitializeResult,+ InitializedParams,+ MessageType,+ PublishDiagnosticsParams,+ Range,+ ServerCapabilities,+ TextDocumentSyncCapability,+ TextDocumentSyncKind,+ TextDocumentSyncOptions,+ TextDocumentSyncSaveOptions,+ Url};use tower_lsp::{Client, LanguageServer};use tracing::{error, info, instrument};@@ -35,7 +60,7 @@ struct DocumentState {document: Document,ident_dict: Arc<FullDictionary>,linter: LintGroup<MergedDictionary<FullDictionary>>,- language_id: Option<String>,+ language_id: Option<String>}/// Deallocate@@ -43,7 +68,7 @@ pub struct Backend {client: Client,static_dictionary: Arc<FullDictionary>,config: RwLock<Config>,- doc_state: Mutex<HashMap<Url, DocumentState>>,+ doc_state: Mutex<HashMap<Url, DocumentState>>}impl Backend {@@ -54,7 +79,7 @@ impl Backend {client,static_dictionary: dictionary.into(),doc_state: Mutex::new(HashMap::new()),- config: RwLock::new(config),+ config: RwLock::new(config)}}@@ -149,7 +174,7 @@ impl Backend {#[instrument(skip(self))]async fn generate_file_dictionary(&self,- url: &Url,+ url: &Url) -> anyhow::Result<MergedDictionary<FullDictionary>> {let (global_dictionary, file_dictionary) = tokio::join!(self.generate_global_dictionary(),@@ -166,11 +191,11 @@ impl Backend {async fn update_document_from_file(&self,url: &Url,- language_id: Option<&str>,+ language_id: Option<&str>) -> anyhow::Result<()> {let content = match tokio::fs::read_to_string(url.to_file_path()- .map_err(|_| anyhow::format_err!("Could not extract file path."))?,+ .map_err(|_| anyhow::format_err!("Could not extract file path."))?).await{@@ -189,7 +214,7 @@ impl Backend {&self,url: &Url,text: &str,- language_id: Option<&str>,+ language_id: Option<&str>) -> anyhow::Result<()> {let mut doc_lock = self.doc_state.lock().await;let config_lock = self.config.read().await;@@ -201,7 +226,7 @@ impl Backend {let mut doc_state = DocumentState {linter: LintGroup::new(config_lock.lint_config,- self.generate_file_dictionary(url).await?,+ self.generate_file_dictionary(url).await?),language_id: language_id.map(|v| v.to_string())@@ -254,7 +279,7 @@ impl Backend {async fn generate_code_actions(&self,url: &Url,- range: Range,+ range: Range) -> Result<Vec<CodeActionOrCommand>> {let (config, mut doc_states) = tokio::join!(self.config.read(), self.doc_state.lock());let Some(doc_state) = doc_states.get_mut(url) else {@@ -286,7 +311,7 @@ impl Backend {actions.push(CodeActionOrCommand::Command(Command::new("Open URL".to_string(),"HarperOpen".to_string(),- Some(vec![doc_state.document.get_span_content_str(span).into()]),+ Some(vec![doc_state.document.get_span_content_str(span).into()]))))}@@ -306,7 +331,7 @@ impl Backend {lints_to_diagnostics(doc_state.document.get_full_content(),&lints,- config.diagnostic_severity,+ config.diagnostic_severity)}@@ -317,7 +342,7 @@ impl Backend {let result = PublishDiagnosticsParams {uri: url.clone(),diagnostics,- version: None,+ version: None};self.client@@ -347,11 +372,11 @@ impl LanguageServer for Backend {change: Some(TextDocumentSyncKind::FULL),will_save: None,will_save_wait_until: None,- save: Some(TextDocumentSyncSaveOptions::Supported(true)),- },+ save: Some(TextDocumentSyncSaveOptions::Supported(true))+ })),..Default::default()- },+ }})}@@ -382,7 +407,7 @@ impl LanguageServer for Backend {.update_document(¶ms.text_document.uri,¶ms.text_document.text,- Some(¶ms.text_document.language_id),+ Some(¶ms.text_document.language_id)).await;@@ -471,7 +496,7 @@ impl LanguageServer for Backend {error!("Unable to open URL: {}", err);}},- _ => (),+ _ => ()}Ok(None)@@ -502,7 +527,7 @@ impl LanguageServer for Backend {// Update documents with new configfutures::future::join_all(keys.iter()- .map(|key| self.update_document_from_file(key, None)),+ .map(|key| self.update_document_from_file(key, None))).await;}