Skip to content

Offline, privacy-first grammar checker. Fast, open-source, Rust-powered

fix(CollapseIdentifiers): will no longer consume subsequent tokens

Elijah Potterc4a3a0c3136e
diff --git a/harper-core/src/parsers/collapse_identifiers.rs b/harper-core/src/parsers/collapse_identifiers.rs
index cc36774df..3b26dc67c 100644
--- a/harper-core/src/parsers/collapse_identifiers.rs
+++ b/harper-core/src/parsers/collapse_identifiers.rs
@@ -5,10 +5,10 @@ use itertools::Itertools;
use super::{Parser, TokenKind};
use crate::patterns::{PatternExt, SequencePattern};
-use crate::{Dictionary, FullDictionary, MergedDictionary, Span, Token, VecExt, WordMetadata};
+use crate::{Dictionary, FullDictionary, MergedDictionary, Span, Token, VecExt};
/// A parser that wraps any other parser to collapse token strings that match
-/// the pattern word_word or word-word.
+/// the pattern `word_word` or `word-word`.
pub struct CollapseIdentifiers {
inner: Box<dyn Parser>,
dict: Lrc<MergedDictionary<FullDictionary>>,
@@ -35,34 +35,23 @@ impl Parser for CollapseIdentifiers {
fn parse(&mut self, source: &[char]) -> Vec<Token> {
let mut tokens = self.inner.parse(source);
- let mut removal_indexes: VecDeque<usize> = VecDeque::default();
- let replacements = WORD_OR_NUMBER
+ let mut to_remove = VecDeque::default();
+
+ for tok_span in WORD_OR_NUMBER
.with(|v| v.clone())
.find_all_matches(&tokens, source)
- .into_iter()
- .map(|s| {
- let start_tok = tokens
- .get(s.start)
- .expect("Token not at expected position.");
- let end_tok = tokens
- .get(s.end - 1)
- .expect("Token not at expected position.");
- let char_span = Span::new(start_tok.span.start, end_tok.span.end);
- (
- s.start,
- s.end,
- Token::new(char_span, TokenKind::Word(WordMetadata::default())),
- char_span.get_content_string(source),
- )
- })
- .filter(|(_, _, _, st)| self.dict.contains_word_str(st))
- .collect_vec();
-
- replacements.into_iter().for_each(|(s, e, t, _)| {
- (s + 1..=e).for_each(|n| removal_indexes.push_front(n));
- tokens[s] = t;
- });
- tokens.remove_indices(removal_indexes.into_iter().sorted().unique().collect());
+ {
+ let start_tok = &tokens[tok_span.start];
+ let end_tok = &tokens[tok_span.end - 1];
+ let char_span = Span::new(start_tok.span.start, end_tok.span.end);
+
+ if self.dict.contains_word(char_span.get_content(source)) {
+ tokens[tok_span.start] = Token::new(char_span, TokenKind::blank_word());
+ to_remove.extend(tok_span.start + 1..tok_span.end);
+ }
+ }
+
+ tokens.remove_indices(to_remove.into_iter().sorted().unique().collect());
tokens
}
@@ -70,10 +59,26 @@ impl Parser for CollapseIdentifiers {
#[cfg(test)]
mod tests {
- use crate::parsers::{PlainEnglish, StrParser};
+ use crate::{
+ parsers::{PlainEnglish, StrParser},
+ WordMetadata,
+ };
use super::*;
+ #[test]
+ fn matches_kebab() {
+ let source: Vec<_> = "kebab-case".chars().collect();
+
+ assert_eq!(
+ WORD_OR_NUMBER
+ .with(|v| v.clone())
+ .find_all_matches(&PlainEnglish.parse(&source), &source)
+ .len(),
+ 1
+ );
+ }
+
#[test]
fn no_collapse() {
let dict = FullDictionary::curated();
@@ -97,17 +102,14 @@ mod tests {
assert_eq!(tokens.len(), 13);
let mut dict = FullDictionary::new();
- dict.append_word(
- "separated_identifier".chars().collect_vec(),
- WordMetadata::default(),
- );
+ dict.append_word_str("separated_identifier", WordMetadata::default());
let mut merged_dict = MergedDictionary::from(default_dict);
merged_dict.add_dictionary(Lrc::new(dict));
let tokens = CollapseIdentifiers::new(Box::new(PlainEnglish), &Lrc::new(merged_dict))
.parse_str(source);
- assert_eq!(tokens.len(), 10);
+ assert_eq!(tokens.len(), 11);
}
#[test]
@@ -120,20 +122,19 @@ mod tests {
&Lrc::new(default_dict.clone().into()),
)
.parse_str(source);
+
assert_eq!(tokens.len(), 13);
let mut dict = FullDictionary::new();
- dict.append_word(
- "separated-identifier".chars().collect_vec(),
- WordMetadata::default(),
- );
+ dict.append_word_str("separated-identifier", WordMetadata::default());
let mut merged_dict = MergedDictionary::from(default_dict);
merged_dict.add_dictionary(Lrc::new(dict));
let tokens = CollapseIdentifiers::new(Box::new(PlainEnglish), &Lrc::new(merged_dict))
.parse_str(source);
- assert_eq!(tokens.len(), 10);
+
+ assert_eq!(tokens.len(), 11);
}
#[test]
@@ -149,17 +150,14 @@ mod tests {
assert_eq!(tokens.len(), 15);
let mut dict = FullDictionary::new();
- dict.append_word(
- "separated_identifier_token".chars().collect_vec(),
- WordMetadata::default(),
- );
+ dict.append_word_str("separated_identifier_token", WordMetadata::default());
let mut merged_dict = MergedDictionary::from(default_dict);
merged_dict.add_dictionary(Lrc::new(dict));
let tokens = CollapseIdentifiers::new(Box::new(PlainEnglish), &Lrc::new(merged_dict))
.parse_str(source);
- assert_eq!(tokens.len(), 10);
+ assert_eq!(tokens.len(), 11);
}
#[test]
@@ -175,17 +173,14 @@ mod tests {
assert_eq!(tokens.len(), 17);
let mut dict = FullDictionary::new();
- dict.append_word(
- "separated_identifier".chars().collect_vec(),
- WordMetadata::default(),
- );
+ dict.append_word_str("separated_identifier", WordMetadata::default());
let mut merged_dict = MergedDictionary::from(default_dict);
merged_dict.add_dictionary(Lrc::new(dict));
let tokens = CollapseIdentifiers::new(Box::new(PlainEnglish), &Lrc::new(merged_dict))
.parse_str(source);
- assert_eq!(tokens.len(), 12);
+ assert_eq!(tokens.len(), 13);
}
#[test]
@@ -201,14 +196,8 @@ mod tests {
assert_eq!(tokens.len(), 15);
let mut dict = FullDictionary::new();
- dict.append_word(
- "separated_identifier".chars().collect_vec(),
- WordMetadata::default(),
- );
- dict.append_word(
- "identifier_token".chars().collect_vec(),
- WordMetadata::default(),
- );
+ dict.append_word_str("separated_identifier", WordMetadata::default());
+ dict.append_word_str("identifier_token", WordMetadata::default());
let mut merged_dict = MergedDictionary::from(default_dict);
merged_dict.add_dictionary(Lrc::new(dict));
@@ -231,20 +220,14 @@ mod tests {
assert_eq!(tokens.len(), 15);
let mut dict = FullDictionary::new();
- dict.append_word(
- "separated_identifier_token".chars().collect_vec(),
- WordMetadata::default(),
- );
- dict.append_word(
- "separated_identifier".chars().collect_vec(),
- WordMetadata::default(),
- );
+ dict.append_word_str("separated_identifier_token", WordMetadata::default());
+ dict.append_word_str("separated_identifier", WordMetadata::default());
let mut merged_dict = MergedDictionary::from(default_dict);
merged_dict.add_dictionary(Lrc::new(dict));
let tokens = CollapseIdentifiers::new(Box::new(PlainEnglish), &Lrc::new(merged_dict))
.parse_str(source);
- assert_eq!(tokens.len(), 10);
+ assert_eq!(tokens.len(), 11);
}
}
diff --git a/harper-core/src/spell/full_dictionary.rs b/harper-core/src/spell/full_dictionary.rs
index bec151f63..936bba3dc 100644
--- a/harper-core/src/spell/full_dictionary.rs
+++ b/harper-core/src/spell/full_dictionary.rs
@@ -91,6 +91,14 @@ impl FullDictionary {
self.extend_words(std::iter::once((word.as_ref(), metadata)))
}
+ /// Append a single string to the dictionary.
+ ///
+ /// If you are appending many words, consider using [`Self::extend_words`]
+ /// instead.
+ pub fn append_word_str(&mut self, word: &str, metadata: WordMetadata) {
+ self.append_word(word.chars().collect::<Vec<_>>(), metadata)
+ }
+
/// Create a lookup table for finding words of a specific length in a word
/// list. NOTE: This function will sort the original word list by its
/// length. If the word list's order is changed after creating the
diff --git a/harper-core/src/token.rs b/harper-core/src/token.rs
index 712d537c7..9f72ed30f 100644
--- a/harper-core/src/token.rs
+++ b/harper-core/src/token.rs
@@ -133,7 +133,7 @@ impl TokenKind {
}
impl TokenKind {
- /// Construct a [`TokenKind::Word`] with no metadata.
+ /// Construct a [`TokenKind::Word`] with no (default) metadata.
pub fn blank_word() -> Self {
Self::Word(WordMetadata::default())
}