Skip to content

Offline, privacy-first grammar checker. Fast, open-source, Rust-powered

feat: `Document` QoL functions for tokens & words (#1117)

Andrew Dunbarb9bdf1b51c8a
diff --git a/harper-core/src/document.rs b/harper-core/src/document.rs
index 9c0d31006..94e85249c 100644
--- a/harper-core/src/document.rs
+++ b/harper-core/src/document.rs
@@ -249,6 +249,14 @@ impl Document {
self.tokens.get(index)
}
+ /// Get a token at a signed offset from a base index, or None if out of bounds.
+ pub fn get_token_offset(&self, base: usize, offset: isize) -> Option<&Token> {
+ match base.checked_add_signed(offset) {
+ None => None,
+ Some(idx) => self.get_token(idx),
+ }
+ }
+
/// Get an iterator over all the tokens contained in the document.
pub fn tokens(&self) -> impl Iterator<Item = &Token> + '_ {
self.tokens.iter()
@@ -259,6 +267,19 @@ impl Document {
self.tokens().map(|token| token.to_fat(&self.source))
}
+ /// Get the next or previous word token relative to a base index, if separated by whitespace.
+ /// Returns None if the next/previous token is not a word or does not exist.
+ pub fn get_next_word_from_offset(&self, base: usize, offset: isize) -> Option<&Token> {
+ // Look for whitespace at the expected offset
+ if !self.get_token_offset(base, offset)?.kind.is_whitespace() {
+ return None;
+ }
+ // Now look beyond the whitespace for a word token
+ let word_token = self.get_token_offset(base, offset + offset.signum());
+ let word_token = word_token?;
+ word_token.kind.is_word().then_some(word_token)
+ }
+
/// Get an iterator over all the tokens contained in the document.
pub fn fat_string_tokens(&self) -> impl Iterator<Item = FatStringToken> + '_ {
self.fat_tokens().map(|t| t.into())
@@ -749,4 +770,76 @@ mod tests {
fn parses_short_ellipsis() {
assert_token_count("..", 1);
}
+
+ #[test]
+ fn selects_token_at_offset() {
+ let doc = Document::new_plain_english_curated("Foo bar baz");
+
+ let tok = doc.get_token_offset(1, -1).unwrap();
+
+ assert_eq!(tok.span, Span::new(0, 3));
+ }
+
+ #[test]
+ fn cant_select_token_before_start() {
+ let doc = Document::new_plain_english_curated("Foo bar baz");
+
+ let tok = doc.get_token_offset(0, -1);
+
+ assert!(tok.is_none());
+ }
+
+ #[test]
+ fn select_next_word_pos_offset() {
+ let doc = Document::new_plain_english_curated("Foo bar baz");
+
+ let bar = doc.get_next_word_from_offset(0, 1).unwrap();
+ let bar = doc.get_span_content(&bar.span);
+ assert_eq!(bar, ['b', 'a', 'r']);
+ }
+
+ #[test]
+ fn select_next_word_neg_offset() {
+ let doc = Document::new_plain_english_curated("Foo bar baz");
+
+ let bar = doc.get_next_word_from_offset(2, -1).unwrap();
+ let bar = doc.get_span_content(&bar.span);
+ assert_eq!(bar, ['F', 'o', 'o']);
+ }
+
+ #[test]
+ fn cant_select_next_word_not_from_whitespace() {
+ let doc = Document::new_plain_english_curated("Foo bar baz");
+
+ let tok = doc.get_next_word_from_offset(0, 2);
+
+ assert!(tok.is_none());
+ }
+
+ #[test]
+ fn cant_select_next_word_before_start() {
+ let doc = Document::new_plain_english_curated("Foo bar baz");
+
+ let tok = doc.get_next_word_from_offset(0, -1);
+
+ assert!(tok.is_none());
+ }
+
+ #[test]
+ fn cant_select_next_word_with_punctuation_instead_of_whitespace() {
+ let doc = Document::new_plain_english_curated("Foo, bar, baz");
+
+ let tok = doc.get_next_word_from_offset(0, 1);
+
+ assert!(tok.is_none());
+ }
+
+ #[test]
+ fn cant_select_next_word_with_punctuation_after_whitespace() {
+ let doc = Document::new_plain_english_curated("Foo \"bar\", baz");
+
+ let tok = doc.get_next_word_from_offset(0, 1);
+
+ assert!(tok.is_none());
+ }
}