Skip to content

Offline, privacy-first grammar checker. Fast, open-source, Rust-powered

refactor: typed spans (#1595)

Steve Kurch8ca4d84c95a6
diff --git a/harper-cli/src/annotate_tokens.rs b/harper-cli/src/annotate_tokens.rs
index f01fb2cf2..e7a2473db 100644
--- a/harper-cli/src/annotate_tokens.rs
+++ b/harper-cli/src/annotate_tokens.rs
@@ -6,7 +6,7 @@ use strum::IntoEnumIterator;
/// Represents an annotation.
pub(super) struct Annotation {
/// The range the annotation covers in the source. For instance, this might be a single word.
- span: Span,
+ span: Span<char>,
/// The message displayed by the annotation.
annotation_text: String,
/// The color of the annotation.
diff --git a/harper-comments/src/comment_parsers/mod.rs b/harper-comments/src/comment_parsers/mod.rs
index 8106b4a22..f8e53670b 100644
--- a/harper-comments/src/comment_parsers/mod.rs
+++ b/harper-comments/src/comment_parsers/mod.rs
@@ -13,7 +13,7 @@ pub use unit::Unit;
/// Get the span of a tree-sitter-produced comment that doesn't include the
/// comment openers and closers.
-fn without_initiators(source: &[char]) -> Span {
+fn without_initiators(source: &[char]) -> Span<char> {
// Skip over the comment start characters
let actual_start = source
.iter()
diff --git a/harper-core/src/document.rs b/harper-core/src/document.rs
index 23f37bb1f..b16c7eb74 100644
--- a/harper-core/src/document.rs
+++ b/harper-core/src/document.rs
@@ -31,7 +31,7 @@ impl Document {
/// Locate all the tokens that intersect a provided span.
///
/// Desperately needs optimization.
- pub fn token_indices_intersecting(&self, span: Span) -> Vec<usize> {
+ pub fn token_indices_intersecting(&self, span: Span<char>) -> Vec<usize> {
self.tokens()
.enumerate()
.filter_map(|(idx, tok)| tok.span.overlaps_with(span).then_some(idx))
@@ -41,7 +41,7 @@ impl Document {
/// Locate all the tokens that intersect a provided span and convert them to [`FatToken`]s.
///
/// Desperately needs optimization.
- pub fn fat_tokens_intersecting(&self, span: Span) -> Vec<FatToken> {
+ pub fn fat_tokens_intersecting(&self, span: Span<char>) -> Vec<FatToken> {
let indices = self.token_indices_intersecting(span);
indices
@@ -314,19 +314,16 @@ impl Document {
self.fat_tokens().map(|t| t.into())
}
- pub fn get_span_content(&self, span: &Span) -> &[char] {
+ pub fn get_span_content(&self, span: &Span<char>) -> &[char] {
span.get_content(&self.source)
}
- pub fn get_span_content_str(&self, span: &Span) -> String {
+ pub fn get_span_content_str(&self, span: &Span<char>) -> String {
String::from_iter(self.get_span_content(span))
}
pub fn get_full_string(&self) -> String {
- self.get_span_content_str(&Span {
- start: 0,
- end: self.source.len(),
- })
+ self.get_span_content_str(&Span::new(0, self.source.len()))
}
pub fn get_full_content(&self) -> &[char] {
@@ -724,7 +721,7 @@ impl TokenStringExt for Document {
self.tokens.first_non_whitespace()
}
- fn span(&self) -> Option<Span> {
+ fn span(&self) -> Option<Span<char>> {
self.tokens.span()
}
diff --git a/harper-core/src/expr/all.rs b/harper-core/src/expr/all.rs
index 2d6bcf46b..8a2e201e0 100644
--- a/harper-core/src/expr/all.rs
+++ b/harper-core/src/expr/all.rs
@@ -21,8 +21,8 @@ impl All {
}
impl Expr for All {
- fn run(&self, cursor: usize, tokens: &[Token], source: &[char]) -> Option<Span> {
- let mut longest: Option<Span> = None;
+ fn run(&self, cursor: usize, tokens: &[Token], source: &[char]) -> Option<Span<Token>> {
+ let mut longest: Option<Span<Token>> = None;
for expr in self.children.iter() {
let window = expr.run(cursor, tokens, source)?;
diff --git a/harper-core/src/expr/expr_map.rs b/harper-core/src/expr/expr_map.rs
index 369a10eca..f76e11abc 100644
--- a/harper-core/src/expr/expr_map.rs
+++ b/harper-core/src/expr/expr_map.rs
@@ -74,7 +74,7 @@ impl<T> Expr for ExprMap<T>
where
T: LSend,
{
- fn run(&self, cursor: usize, tokens: &[Token], source: &[char]) -> Option<Span> {
+ fn run(&self, cursor: usize, tokens: &[Token], source: &[char]) -> Option<Span<Token>> {
self.rows
.iter()
.filter_map(|row| row.key.run(cursor, tokens, source))
diff --git a/harper-core/src/expr/first_match_of.rs b/harper-core/src/expr/first_match_of.rs
index 6b0e144f6..0d21b29d1 100644
--- a/harper-core/src/expr/first_match_of.rs
+++ b/harper-core/src/expr/first_match_of.rs
@@ -21,7 +21,7 @@ impl FirstMatchOf {
}
impl Expr for FirstMatchOf {
- fn run(&self, cursor: usize, tokens: &[Token], source: &[char]) -> Option<Span> {
+ fn run(&self, cursor: usize, tokens: &[Token], source: &[char]) -> Option<Span<Token>> {
self.exprs
.iter()
.find_map(|p| p.run(cursor, tokens, source))
diff --git a/harper-core/src/expr/fixed_phrase.rs b/harper-core/src/expr/fixed_phrase.rs
index 78a77d6fc..8399bd50f 100644
--- a/harper-core/src/expr/fixed_phrase.rs
+++ b/harper-core/src/expr/fixed_phrase.rs
@@ -62,7 +62,7 @@ impl FixedPhrase {
}
impl Expr for FixedPhrase {
- fn run(&self, cursor: usize, tokens: &[Token], source: &[char]) -> Option<Span> {
+ fn run(&self, cursor: usize, tokens: &[Token], source: &[char]) -> Option<Span<Token>> {
self.inner.run(cursor, tokens, source)
}
}
diff --git a/harper-core/src/expr/longest_match_of.rs b/harper-core/src/expr/longest_match_of.rs
index ae6c982c1..9fd9aca73 100644
--- a/harper-core/src/expr/longest_match_of.rs
+++ b/harper-core/src/expr/longest_match_of.rs
@@ -17,8 +17,8 @@ impl LongestMatchOf {
}
impl Expr for LongestMatchOf {
- fn run(&self, cursor: usize, tokens: &[Token], source: &[char]) -> Option<Span> {
- let mut longest: Option<Span> = None;
+ fn run(&self, cursor: usize, tokens: &[Token], source: &[char]) -> Option<Span<Token>> {
+ let mut longest: Option<Span<Token>> = None;
for expr in self.exprs.iter() {
let Some(window) = expr.run(cursor, tokens, source) else {
diff --git a/harper-core/src/expr/mergeable_words.rs b/harper-core/src/expr/mergeable_words.rs
index d196af2ba..02934cc4f 100644
--- a/harper-core/src/expr/mergeable_words.rs
+++ b/harper-core/src/expr/mergeable_words.rs
@@ -60,7 +60,7 @@ impl MergeableWords {
}
impl Expr for MergeableWords {
- fn run(&self, cursor: usize, tokens: &[Token], source: &[char]) -> Option<Span> {
+ fn run(&self, cursor: usize, tokens: &[Token], source: &[char]) -> Option<Span<Token>> {
let inner_match = self.inner.run(cursor, tokens, source)?;
if inner_match.len() != 3 {
diff --git a/harper-core/src/expr/mod.rs b/harper-core/src/expr/mod.rs
index bb3dd249e..bab156b2d 100644
--- a/harper-core/src/expr/mod.rs
+++ b/harper-core/src/expr/mod.rs
@@ -60,14 +60,14 @@ pub use word_expr_group::WordExprGroup;
use crate::{Document, LSend, Span, Token};
pub trait Expr: LSend {
- fn run(&self, cursor: usize, tokens: &[Token], source: &[char]) -> Option<Span>;
+ fn run(&self, cursor: usize, tokens: &[Token], source: &[char]) -> Option<Span<Token>>;
}
impl<S> Expr for S
where
S: Step + ?Sized,
{
- fn run(&self, cursor: usize, tokens: &[Token], source: &[char]) -> Option<Span> {
+ fn run(&self, cursor: usize, tokens: &[Token], source: &[char]) -> Option<Span<Token>> {
self.step(tokens, cursor, source).map(|s| {
if s >= 0 {
Span::new_with_len(cursor, s as usize)
@@ -82,7 +82,7 @@ impl<E> Expr for Arc<E>
where
E: Expr,
{
- fn run(&self, cursor: usize, tokens: &[Token], source: &[char]) -> Option<Span> {
+ fn run(&self, cursor: usize, tokens: &[Token], source: &[char]) -> Option<Span<Token>> {
self.as_ref().run(cursor, tokens, source)
}
}
@@ -92,7 +92,7 @@ impl<E> Expr for Rc<E>
where
E: Expr,
{
- fn run(&self, cursor: usize, tokens: &[Token], source: &[char]) -> Option<Span> {
+ fn run(&self, cursor: usize, tokens: &[Token], source: &[char]) -> Option<Span<Token>> {
self.as_ref().run(cursor, tokens, source)
}
}
@@ -112,9 +112,12 @@ pub trait ExprExt {
&'a self,
tokens: &'a [Token],
source: &'a [char],
- ) -> Box<dyn Iterator<Item = Span> + 'a>;
+ ) -> Box<dyn Iterator<Item = Span<Token>> + 'a>;
- fn iter_matches_in_doc<'a>(&'a self, doc: &'a Document) -> Box<dyn Iterator<Item = Span> + 'a>;
+ fn iter_matches_in_doc<'a>(
+ &'a self,
+ doc: &'a Document,
+ ) -> Box<dyn Iterator<Item = Span<Token>> + 'a>;
}
impl<E: ?Sized> ExprExt for E
@@ -125,7 +128,7 @@ where
&'a self,
tokens: &'a [Token],
source: &'a [char],
- ) -> Box<(dyn Iterator<Item = Span> + 'a)> {
+ ) -> Box<(dyn Iterator<Item = Span<Token>> + 'a)> {
let mut last_end = 0usize;
Box::new((0..tokens.len()).filter_map(move |i| {
@@ -142,7 +145,7 @@ where
fn iter_matches_in_doc<'a>(
&'a self,
doc: &'a Document,
- ) -> Box<(dyn Iterator<Item = Span> + 'a)> {
+ ) -> Box<(dyn Iterator<Item = Span<Token>> + 'a)> {
Box::new(self.iter_matches(doc.get_tokens(), doc.get_source()))
}
}
diff --git a/harper-core/src/expr/optional.rs b/harper-core/src/expr/optional.rs
index f76775c99..2641dc4f7 100644
--- a/harper-core/src/expr/optional.rs
+++ b/harper-core/src/expr/optional.rs
@@ -18,7 +18,7 @@ impl Optional {
}
impl Expr for Optional {
- fn run(&self, cursor: usize, tokens: &[Token], source: &[char]) -> Option<Span> {
+ fn run(&self, cursor: usize, tokens: &[Token], source: &[char]) -> Option<Span<Token>> {
let res = self.inner.run(cursor, tokens, source);
if res.is_none() {
diff --git a/harper-core/src/expr/reflexive_pronoun.rs b/harper-core/src/expr/reflexive_pronoun.rs
index 81c856203..b3b54ab04 100644
--- a/harper-core/src/expr/reflexive_pronoun.rs
+++ b/harper-core/src/expr/reflexive_pronoun.rs
@@ -51,7 +51,7 @@ impl ReflexivePronoun {
}
impl Expr for ReflexivePronoun {
- fn run(&self, cursor: usize, tokens: &[Token], source: &[char]) -> Option<Span> {
+ fn run(&self, cursor: usize, tokens: &[Token], source: &[char]) -> Option<Span<Token>> {
let good_pronouns = |token: &Token, _: &[char]| token.kind.is_reflexive_pronoun();
let mut expr = FirstMatchOf::new(vec![Box::new(good_pronouns)]);
if self.include_common_errors {
diff --git a/harper-core/src/expr/repeating.rs b/harper-core/src/expr/repeating.rs
index 2274291ed..09f18afe3 100644
--- a/harper-core/src/expr/repeating.rs
+++ b/harper-core/src/expr/repeating.rs
@@ -19,7 +19,7 @@ impl Repeating {
}
impl Expr for Repeating {
- fn run(&self, mut cursor: usize, tokens: &[Token], source: &[char]) -> Option<Span> {
+ fn run(&self, mut cursor: usize, tokens: &[Token], source: &[char]) -> Option<Span<Token>> {
let mut window = Span::new_with_len(cursor, 0);
let mut repetition = 0;
diff --git a/harper-core/src/expr/sequence_expr.rs b/harper-core/src/expr/sequence_expr.rs
index bcf3d6b75..857874747 100644
--- a/harper-core/src/expr/sequence_expr.rs
+++ b/harper-core/src/expr/sequence_expr.rs
@@ -56,7 +56,7 @@ impl Expr for SequenceExpr {
/// Run the expression starting at an index, returning the total matched window.
///
/// If any step returns `None`, the entire expression does as well.
- fn run(&self, mut cursor: usize, tokens: &[Token], source: &[char]) -> Option<Span> {
+ fn run(&self, mut cursor: usize, tokens: &[Token], source: &[char]) -> Option<Span<Token>> {
let mut window = Span::new_with_len(cursor, 0);
for cur_expr in &self.exprs {
diff --git a/harper-core/src/expr/similar_to_phrase.rs b/harper-core/src/expr/similar_to_phrase.rs
index 08f59edaa..92fd810db 100644
--- a/harper-core/src/expr/similar_to_phrase.rs
+++ b/harper-core/src/expr/similar_to_phrase.rs
@@ -58,7 +58,7 @@ impl SimilarToPhrase {
}
impl Expr for SimilarToPhrase {
- fn run(&self, cursor: usize, tokens: &[Token], source: &[char]) -> Option<Span> {
+ fn run(&self, cursor: usize, tokens: &[Token], source: &[char]) -> Option<Span<Token>> {
if self.phrase.run(cursor, tokens, source).is_some() {
return None;
}
diff --git a/harper-core/src/expr/space_or_hyphen.rs b/harper-core/src/expr/space_or_hyphen.rs
index fb0986c76..a77481de3 100644
--- a/harper-core/src/expr/space_or_hyphen.rs
+++ b/harper-core/src/expr/space_or_hyphen.rs
@@ -9,7 +9,7 @@ use super::Expr;
pub struct SpaceOrHyphen;
impl Expr for SpaceOrHyphen {
- fn run(&self, cursor: usize, tokens: &[Token], source: &[char]) -> Option<Span> {
+ fn run(&self, cursor: usize, tokens: &[Token], source: &[char]) -> Option<Span<Token>> {
FirstMatchOf::new(vec![
Box::new(WhitespacePattern),
Box::new(|tok: &Token, _source: &[char]| tok.kind.is_hyphen()),
diff --git a/harper-core/src/expr/spelled_number_expr.rs b/harper-core/src/expr/spelled_number_expr.rs
index 0d95d2a1e..586313423 100644
--- a/harper-core/src/expr/spelled_number_expr.rs
+++ b/harper-core/src/expr/spelled_number_expr.rs
@@ -9,7 +9,7 @@ use super::{Expr, SequenceExpr};
pub struct SpelledNumberExpr;
impl Expr for SpelledNumberExpr {
- fn run(&self, cursor: usize, tokens: &[Token], source: &[char]) -> Option<Span> {
+ fn run(&self, cursor: usize, tokens: &[Token], source: &[char]) -> Option<Span<Token>> {
if tokens.is_empty() {
return None;
}
@@ -70,13 +70,13 @@ impl Expr for SpelledNumberExpr {
mod tests {
use super::SpelledNumberExpr;
use crate::expr::ExprExt;
- use crate::{Document, Span};
+ use crate::{Document, Span, Token};
trait SpanVecExt {
fn to_strings(&self, doc: &Document) -> Vec<String>;
}
- impl SpanVecExt for Vec<Span> {
+ impl SpanVecExt for Vec<Span<Token>> {
fn to_strings(&self, doc: &Document) -> Vec<String> {
self.iter()
.map(|sp| {
diff --git a/harper-core/src/expr/time_unit_expr.rs b/harper-core/src/expr/time_unit_expr.rs
index 2a941506b..fd794446a 100644
--- a/harper-core/src/expr/time_unit_expr.rs
+++ b/harper-core/src/expr/time_unit_expr.rs
@@ -15,7 +15,7 @@ use super::Expr;
pub struct TimeUnitExpr;
impl Expr for TimeUnitExpr {
- fn run(&self, cursor: usize, tokens: &[Token], source: &[char]) -> Option<Span> {
+ fn run(&self, cursor: usize, tokens: &[Token], source: &[char]) -> Option<Span<Token>> {
if tokens.is_empty() {
return None;
}
diff --git a/harper-core/src/expr/word_expr_group.rs b/harper-core/src/expr/word_expr_group.rs
index f8fbd4361..2e05a49d6 100644
--- a/harper-core/src/expr/word_expr_group.rs
+++ b/harper-core/src/expr/word_expr_group.rs
@@ -39,7 +39,7 @@ impl<E> Expr for WordExprGroup<E>
where
E: Expr,
{
- fn run(&self, cursor: usize, tokens: &[Token], source: &[char]) -> Option<Span> {
+ fn run(&self, cursor: usize, tokens: &[Token], source: &[char]) -> Option<Span<Token>> {
let first = tokens.get(cursor)?;
if !first.kind.is_word() {
return None;
diff --git a/harper-core/src/linting/lint.rs b/harper-core/src/linting/lint.rs
index ba33d2fdf..ff0c7ffda 100644
--- a/harper-core/src/linting/lint.rs
+++ b/harper-core/src/linting/lint.rs
@@ -11,7 +11,7 @@ use super::{LintKind, Suggestion};
pub struct Lint {
/// The location in the source text the error lies.
/// Important for automatic lint resolution through [`Self::suggestions`].
- pub span: Span,
+ pub span: Span<char>,
/// The general category the lint belongs to.
/// Mostly used for UI elements in integrations.
pub lint_kind: LintKind,
diff --git a/harper-core/src/linting/suggestion.rs b/harper-core/src/linting/suggestion.rs
index 310962759..4f8931e56 100644
--- a/harper-core/src/linting/suggestion.rs
+++ b/harper-core/src/linting/suggestion.rs
@@ -53,7 +53,7 @@ impl Suggestion {
}
/// Apply a suggestion to a given text.
- pub fn apply(&self, span: Span, source: &mut Vec<char>) {
+ pub fn apply(&self, span: Span<char>, source: &mut Vec<char>) {
match self {
Self::ReplaceWith(chars) => {
// Avoid allocation if possible
diff --git a/harper-core/src/mask/mod.rs b/harper-core/src/mask/mod.rs
index ee2a9c40b..a9b78ccc4 100644
--- a/harper-core/src/mask/mod.rs
+++ b/harper-core/src/mask/mod.rs
@@ -19,11 +19,11 @@ pub struct Mask {
// Right now, there aren't any use-cases where we can't treat this as a stack.
//
// Assumed that no elements overlap and exist in sorted order.
- pub(self) allowed: Vec<Span>,
+ pub(self) allowed: Vec<Span<char>>,
}
-impl FromIterator<Span> for Mask {
- fn from_iter<T: IntoIterator<Item = Span>>(iter: T) -> Self {
+impl FromIterator<Span<char>> for Mask {
+ fn from_iter<T: IntoIterator<Item = Span<char>>>(iter: T) -> Self {
let allowed = iter
.into_iter()
.sorted_by_key(|span| span.start)
@@ -49,12 +49,12 @@ impl Mask {
pub fn iter_allowed<'a>(
&'a self,
source: &'a [char],
- ) -> impl Iterator<Item = (Span, &'a [char])> {
+ ) -> impl Iterator<Item = (Span<char>, &'a [char])> {
self.allowed.iter().map(|s| (*s, s.get_content(source)))
}
/// Mark a span of the text as allowed.
- pub fn push_allowed(&mut self, allowed: Span) {
+ pub fn push_allowed(&mut self, allowed: Span<char>) {
if let Some(last) = self.allowed.last_mut() {
assert!(
allowed.start >= last.end,
diff --git a/harper-core/src/parsers/mask.rs b/harper-core/src/parsers/mask.rs
index d74393d66..20e532471 100644
--- a/harper-core/src/parsers/mask.rs
+++ b/harper-core/src/parsers/mask.rs
@@ -32,7 +32,7 @@ where
let mut tokens: Vec<Token> = Vec::new();
- let mut last_allowed: Option<Span> = None;
+ let mut last_allowed: Option<Span<char>> = None;
for (span, content) in mask.iter_allowed(source) {
// Check for a line break separating the current chunk from the preceding one.
diff --git a/harper-core/src/patterns/mod.rs b/harper-core/src/patterns/mod.rs
index 4198f8481..851488851 100644
--- a/harper-core/src/patterns/mod.rs
+++ b/harper-core/src/patterns/mod.rs
@@ -41,10 +41,10 @@ pub trait Pattern: LSend {
}
pub trait PatternExt {
- fn iter_matches(&self, tokens: &[Token], source: &[char]) -> impl Iterator<Item = Span>;
+ fn iter_matches(&self, tokens: &[Token], source: &[char]) -> impl Iterator<Item = Span<Token>>;
/// Search through all tokens to locate all non-overlapping pattern matches.
- fn find_all_matches(&self, tokens: &[Token], source: &[char]) -> Vec<Span> {
+ fn find_all_matches(&self, tokens: &[Token], source: &[char]) -> Vec<Span<Token>> {
self.iter_matches(tokens, source).collect()
}
}
@@ -53,7 +53,7 @@ impl<P> PatternExt for P
where
P: Pattern + ?Sized,
{
- fn iter_matches(&self, tokens: &[Token], source: &[char]) -> impl Iterator<Item = Span> {
+ fn iter_matches(&self, tokens: &[Token], source: &[char]) -> impl Iterator<Item = Span<Token>> {
MatchIter::new(self, tokens, source)
}
}
@@ -81,7 +81,7 @@ impl<P> Iterator for MatchIter<'_, '_, '_, P>
where
P: Pattern + ?Sized,
{
- type Item = Span;
+ type Item = Span<Token>;
fn next(&mut self) -> Option<Self::Item> {
while self.index < self.tokens.len() {
@@ -124,11 +124,11 @@ impl<F: LSend + Fn(&Token, &[char]) -> bool> SingleTokenPattern for F {
}
pub trait DocPattern {
- fn find_all_matches_in_doc(&self, document: &Document) -> Vec<Span>;
+ fn find_all_matches_in_doc(&self, document: &Document) -> Vec<Span<Token>>;
}
impl<P: PatternExt> DocPattern for P {
- fn find_all_matches_in_doc(&self, document: &Document) -> Vec<Span> {
+ fn find_all_matches_in_doc(&self, document: &Document) -> Vec<Span<Token>> {
self.find_all_matches(document.get_tokens(), document.get_source())
}
}
diff --git a/harper-core/src/patterns/nominal_phrase.rs b/harper-core/src/patterns/nominal_phrase.rs
index 70135fd74..33a7181e1 100644
--- a/harper-core/src/patterns/nominal_phrase.rs
+++ b/harper-core/src/patterns/nominal_phrase.rs
@@ -37,13 +37,13 @@ impl Pattern for NominalPhrase {
mod tests {
use super::super::DocPattern;
use super::NominalPhrase;
- use crate::{Document, Span, patterns::Pattern};
+ use crate::{Document, Span, Token, patterns::Pattern};
trait SpanVecExt {
fn to_strings(&self, doc: &Document) -> Vec<String>;
}
- impl SpanVecExt for Vec<Span> {
+ impl SpanVecExt for Vec<Span<Token>> {
fn to_strings(&self, doc: &Document) -> Vec<String> {
self.iter()
.map(|sp| {
@@ -104,7 +104,9 @@ mod tests {
dbg!(matches.to_strings(&doc));
for span in &matches {
- let gc = span.get_content(doc.get_source());
+ let gc = span
+ .to_char_span(doc.get_tokens())
+ .get_content(doc.get_source());
dbg!(gc);
}
diff --git a/harper-core/src/span.rs b/harper-core/src/span.rs
index 7e3c54dae..23ca9b495 100644
--- a/harper-core/src/span.rs
+++ b/harper-core/src/span.rs
@@ -1,54 +1,92 @@
-use std::ops::Range;
+use std::{fmt::Display, marker::PhantomData, ops::Range};
use serde::{Deserialize, Serialize};
-use crate::CharStringExt;
+use crate::Token;
-/// A window in a [`char`] sequence.
+/// A window in a [`T`] sequence.
+///
+/// Note that the range covered by a [`Span`] is end-exclusive, meaning that the end index is not
+/// included in the range covered by the [`Span`]. If you're familiar with the Rust range syntax,
+/// you could say the span covers the equivalent of `start..end`, *not* `start..=end`.
+///
+/// For a [`Span`] to be correct, its end index must be greater than or equal to its start
+/// index. Creating or using a [`Span`] which does not follow this rule may lead to unexpected
+/// behavior or panics.
///
/// Although specific to `harper.js`, [this page may clear up any questions you have](https://writewithharper.com/docs/harperjs/spans).
-#[derive(Debug, Clone, Copy, Serialize, Deserialize, Default, PartialEq, Eq)]
-pub struct Span {
+#[derive(Debug, Serialize, Deserialize, Default, PartialEq, Eq)]
+pub struct Span<T> {
+ /// The start index of the span.
pub start: usize,
+ /// The end index of the span.
+ ///
+ /// Note that [`Span`] represents an exclusive range. This means that a `Span::new(0, 5)` will
+ /// cover the values `0, 1, 2, 3, 4`; it will not cover the `5`.
pub end: usize,
+ span_type: PhantomData<T>,
}
-impl Span {
+impl<T> Span<T> {
+ /// An empty [`Span`].
+ pub const EMPTY: Self = Self {
+ start: 0,
+ end: 0,
+ span_type: PhantomData,
+ };
+
+ /// Creates a new [`Span`] with the provided start and end indices.
+ ///
+ /// # Panics
+ ///
+ /// This will panic if `start` is greater than `end`.
pub fn new(start: usize, end: usize) -> Self {
if start > end {
panic!("{start} > {end}");
}
- Self { start, end }
+ Self {
+ start,
+ end,
+ span_type: PhantomData,
+ }
}
+ /// Creates a new [`Span`] from the provided start position and length.
pub fn new_with_len(start: usize, len: usize) -> Self {
Self {
start,
end: start + len,
+ span_type: PhantomData,
}
}
+ /// The length of the [`Span`].
pub fn len(&self) -> usize {
self.end - self.start
}
+ /// Checks whether the [`Span`] is empty.
+ ///
+ /// A [`Span`] is considered empty if it has a length of 0.
pub fn is_empty(&self) -> bool {
self.len() == 0
}
+ /// Checks whether `idx` is within the range of the span.
pub fn contains(&self, idx: usize) -> bool {
assert!(self.start <= self.end);
self.start <= idx && idx < self.end
}
+ /// Checks whether this span's range overlaps with `other`.
pub fn overlaps_with(&self, other: Self) -> bool {
(self.start < other.end) && (other.start < self.end)
}
/// Get the associated content. Will return [`None`] if any aspect is
/// invalid.
- pub fn try_get_content<'a>(&self, source: &'a [char]) -> Option<&'a [char]> {
+ pub fn try_get_content<'a>(&self, source: &'a [T]) -> Option<&'a [T]> {
if (self.start > self.end) || (self.start >= source.len()) || (self.end > source.len()) {
if self.is_empty() {
return Some(&source[0..0]);
@@ -72,44 +110,38 @@ impl Span {
}
/// Get the associated content. Will panic if any aspect is invalid.
- pub fn get_content<'a>(&self, source: &'a [char]) -> &'a [char] {
+ pub fn get_content<'a>(&self, source: &'a [T]) -> &'a [T] {
match self.try_get_content(source) {
Some(v) => v,
- None => panic!(
- "Could not get position {:?} within \"{}\"",
- self,
- source.to_string()
- ),
+ None => panic!("Failed to get content for span."),
}
}
- pub fn get_content_string(&self, source: &[char]) -> String {
- String::from_iter(self.get_content(source))
- }
-
+ /// Set the span's length.
pub fn set_len(&mut self, length: usize) {
self.end = self.start + length;
}
+ /// Returns a copy of this [`Span`] with a new length.
pub fn with_len(&self, length: usize) -> Self {
let mut cloned = *self;
cloned.set_len(length);
cloned
}
- // Add an amount to both [`Self::start`] and [`Self::end`]
+ /// Add an amount to both [`Self::start`] and [`Self::end`]
pub fn push_by(&mut self, by: usize) {
self.start += by;
self.end += by;
}
- // Subtract an amount to both [`Self::start`] and [`Self::end`]
+ /// Subtract an amount from both [`Self::start`] and [`Self::end`]
pub fn pull_by(&mut self, by: usize) {
self.start -= by;
self.end -= by;
}
- // Add an amount to a copy of both [`Self::start`] and [`Self::end`]
+ /// Add an amount to a copy of both [`Self::start`] and [`Self::end`]
pub fn pushed_by(&self, by: usize) -> Self {
let mut clone = *self;
clone.start += by;
@@ -117,7 +149,7 @@ impl Span {
clone
}
- // Subtract an amount to a copy of both [`Self::start`] and [`Self::end`]
+ /// Subtract an amount to a copy of both [`Self::start`] and [`Self::end`]
pub fn pulled_by(&self, by: usize) -> Option<Self> {
if by > self.start {
return None;
@@ -129,7 +161,7 @@ impl Span {
Some(clone)
}
- // Add an amount a copy of both [`Self::start`] and [`Self::end`]
+ /// Add an amount to a copy of both [`Self::start`] and [`Self::end`]
pub fn with_offset(&self, by: usize) -> Self {
let mut clone = *self;
clone.push_by(by);
@@ -137,50 +169,127 @@ impl Span {
}
}
-impl From<Range<usize>> for Span {
+/// Additional functions for types that implement [`std::fmt::Debug`] and [`Display`].
+impl<T: Display + std::fmt::Debug> Span<T> {
+ /// Gets the content of this [`Span<T>`] as a [`String`].
+ pub fn get_content_string(&self, source: &[T]) -> String {
+ if let Some(content) = self.try_get_content(source) {
+ content.iter().map(|t| t.to_string()).collect()
+ } else {
+ panic!("Could not get position {self:?} within \"{source:?}\"")
+ }
+ }
+}
+
+/// Functionality specific to [`Token`] spans.
+impl Span<Token> {
+ /// Converts the [`Span<Token>`] into a [`Span<char>`].
+ ///
+ /// This requires knowing the character spans of the tokens covered by this
+ /// [`Span<Token>`]. Because of this, a reference to the source token sequence used to create
+ /// this span is required.
+ pub fn to_char_span(&self, source_document_tokens: &[Token]) -> Span<char> {
+ if self.is_empty() {
+ Span::EMPTY
+ } else {
+ let target_tokens = &source_document_tokens[self.start..self.end];
+ Span::new(
+ target_tokens.first().unwrap().span.start,
+ target_tokens.last().unwrap().span.end,
+ )
+ }
+ }
+}
+
+impl<T> From<Range<usize>> for Span<T> {
+ /// Reinterprets the provided [`std::ops::Range`] as a [`Span`].
fn from(value: Range<usize>) -> Self {
Self::new(value.start, value.end)
}
}
-impl From<Span> for Range<usize> {
- fn from(value: Span) -> Self {
+impl<T> From<Span<T>> for Range<usize> {
+ /// Converts the [`Span`] to an [`std::ops::Range`].
+ fn from(value: Span<T>) -> Self {
value.start..value.end
}
}
-impl IntoIterator for Span {
+impl<T> IntoIterator for Span<T> {
type Item = usize;
type IntoIter = Range<usize>;
+ /// Converts the [`Span`] into an iterator that yields the indices covered by its range.
+ ///
+ /// Note that [`Span`] is half-open, meaning that the value [`Self::end`] will not be yielded
+ /// by this iterator: it will stop at the index immediately preceding [`Self::end`].
fn into_iter(self) -> Self::IntoIter {
self.start..self.end
}
}
+impl<T> Clone for Span<T> {
+ // Note: manual implementation so we don't unnecessarily require `T` to impl `Clone`.
+ fn clone(&self) -> Self {
+ *self
+ }
+}
+impl<T> Copy for Span<T> {}
+
#[cfg(test)]
mod tests {
- use crate::Span;
+ use crate::{
+ Document,
+ expr::{ExprExt, SequenceExpr},
+ };
+
+ use super::Span;
+
+ type UntypedSpan = Span<()>;
#[test]
fn overlaps() {
- assert!(Span::new(0, 5).overlaps_with(Span::new(3, 6)));
- assert!(Span::new(0, 5).overlaps_with(Span::new(2, 3)));
- assert!(Span::new(0, 5).overlaps_with(Span::new(4, 5)));
- assert!(Span::new(0, 5).overlaps_with(Span::new(4, 4)));
+ assert!(UntypedSpan::new(0, 5).overlaps_with(UntypedSpan::new(3, 6)));
+ assert!(UntypedSpan::new(0, 5).overlaps_with(UntypedSpan::new(2, 3)));
+ assert!(UntypedSpan::new(0, 5).overlaps_with(UntypedSpan::new(4, 5)));
+ assert!(UntypedSpan::new(0, 5).overlaps_with(UntypedSpan::new(4, 4)));
- assert!(!Span::new(0, 3).overlaps_with(Span::new(3, 5)));
+ assert!(!UntypedSpan::new(0, 3).overlaps_with(UntypedSpan::new(3, 5)));
}
#[test]
fn expands_properly() {
- let mut span = Span::new(2, 2);
+ let mut span = UntypedSpan::new(2, 2);
span.expand_to_include(1);
- assert_eq!(span, Span::new(1, 2));
+ assert_eq!(span, UntypedSpan::new(1, 2));
span.expand_to_include(2);
- assert_eq!(span, Span::new(1, 3));
+ assert_eq!(span, UntypedSpan::new(1, 3));
+ }
+
+ #[test]
+ fn to_char_span_converts_correctly() {
+ let doc = Document::new_plain_english_curated("Hello world!");
+
+ // Empty span.
+ let token_span = Span::EMPTY;
+ let converted = token_span.to_char_span(doc.get_tokens());
+ assert!(converted.is_empty());
+
+ // Span from `Expr`.
+ let token_span = SequenceExpr::default()
+ .then_any_word()
+ .t_ws()
+ .then_any_word()
+ .iter_matches_in_doc(&doc)
+ .next()
+ .unwrap();
+ let converted = token_span.to_char_span(doc.get_tokens());
+ assert_eq!(
+ converted.get_content_string(doc.get_source()),
+ "Hello world"
+ );
}
}
diff --git a/harper-core/src/token.rs b/harper-core/src/token.rs
index 3d698ffce..8b30d2440 100644
--- a/harper-core/src/token.rs
+++ b/harper-core/src/token.rs
@@ -6,13 +6,13 @@ use crate::{FatToken, Span, TokenKind};
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Default)]
pub struct Token {
/// The characters the token represents.
- pub span: Span,
+ pub span: Span<char>,
/// The parsed value.
pub kind: TokenKind,
}
impl Token {
- pub fn new(span: Span, kind: TokenKind) -> Self {
+ pub fn new(span: Span<char>, kind: TokenKind) -> Self {
Self { span, kind }
}
diff --git a/harper-core/src/token_string_ext.rs b/harper-core/src/token_string_ext.rs
index 91abc782b..e136007ab 100644
--- a/harper-core/src/token_string_ext.rs
+++ b/harper-core/src/token_string_ext.rs
@@ -53,7 +53,7 @@ pub trait TokenStringExt {
fn first_non_whitespace(&self) -> Option<&Token>;
/// Grab the span that represents the beginning of the first element and the
/// end of the last element.
- fn span(&self) -> Option<Span>;
+ fn span(&self) -> Option<Span<char>>;
create_decl_for!(adjective);
create_decl_for!(apostrophe);
@@ -140,7 +140,7 @@ impl TokenStringExt for [Token] {
if w_idx < u_idx { Some(word) } else { None }
}
- fn span(&self) -> Option<Span> {
+ fn span(&self) -> Option<Span<char>> {
let min_max = self
.iter()
.flat_map(|v| [v.span.start, v.span.end].into_iter())
diff --git a/harper-ls/src/pos_conv.rs b/harper-ls/src/pos_conv.rs
index 783efc796..7be9d1e63 100644
--- a/harper-ls/src/pos_conv.rs
+++ b/harper-ls/src/pos_conv.rs
@@ -4,7 +4,7 @@
use harper_core::Span;
use tower_lsp_server::lsp_types::{Position, Range};
-pub fn span_to_range(source: &[char], span: Span) -> Range {
+pub fn span_to_range(source: &[char], span: Span<char>) -> Range {
let start = index_to_position(source, span.start);
let end = index_to_position(source, span.end);
@@ -76,7 +76,7 @@ fn position_to_index(source: &[char], position: Position) -> usize {
(target_char_pointer as usize - source.as_ptr() as usize) / size_of::<char>()
}
-pub fn range_to_span(source: &[char], range: Range) -> Span {
+pub fn range_to_span(source: &[char], range: Range) -> Span<char> {
let start = position_to_index(source, range.start);
let end = position_to_index(source, range.end);
diff --git a/harper-tree-sitter/src/lib.rs b/harper-tree-sitter/src/lib.rs
index c249a8ae5..d5c37e2b0 100644
--- a/harper-tree-sitter/src/lib.rs
+++ b/harper-tree-sitter/src/lib.rs
@@ -62,7 +62,7 @@ impl TreeSitterMasker {
/// Visits the children of a TreeSitter node, searching for comments.
///
/// Returns the BYTE spans of the comment position.
- fn extract_comments(&self, cursor: &mut TreeCursor, comments: &mut Vec<Span>) {
+ fn extract_comments(&self, cursor: &mut TreeCursor, comments: &mut Vec<Span<u8>>) {
Self::visit_nodes(cursor, &mut |node: &Node| {
if (self.node_condition)(node) {
comments.push(node.byte_range().into());
@@ -120,7 +120,7 @@ impl Masker for TreeSitterMasker {
/// NOTE: Will sort the given slice by their [`Span::start`].
///
/// If any spans overlap, it will merge them.
-fn byte_spans_to_char_spans(mut byte_spans: Vec<Span>, source: &str) -> Vec<Span> {
+fn byte_spans_to_char_spans(mut byte_spans: Vec<Span<u8>>, source: &str) -> Vec<Span<char>> {
byte_spans.sort_unstable_by_key(|s| s.start);
// merge overlapping spans
@@ -136,18 +136,20 @@ fn byte_spans_to_char_spans(mut byte_spans: Vec<Span>, source: &str) -> Vec<Span
}
}
- let mut last_byte_pos = 0;
- let mut last_char_pos = 0;
- spans.iter_mut().for_each(|span| {
- let byte_span = *span;
+ // Convert byte spans to char spans.
+ spans
+ .iter()
+ .scan((0, 0), |(last_byte_pos, last_char_pos), span| {
+ let byte_span = *span;
- last_char_pos += source[last_byte_pos..byte_span.start].chars().count();
- span.start = last_char_pos;
+ *last_char_pos += source[*last_byte_pos..byte_span.start].chars().count();
+ let start = *last_char_pos;
- last_char_pos += source[byte_span.start..byte_span.end].chars().count();
- span.end = last_char_pos;
+ *last_char_pos += source[byte_span.start..byte_span.end].chars().count();
+ let end = *last_char_pos;
- last_byte_pos = byte_span.end;
- });
- spans
+ *last_byte_pos = byte_span.end;
+ Some(Span::new(start, end))
+ })
+ .collect()
}
diff --git a/harper-typst/src/typst_translator.rs b/harper-typst/src/typst_translator.rs
index cea52005d..2c2db3c84 100644
--- a/harper-typst/src/typst_translator.rs
+++ b/harper-typst/src/typst_translator.rs
@@ -20,10 +20,7 @@ macro_rules! def_token {
let end_char_loc = start.push_to(range.end).char;
Some(vec![Token {
- span: harper_core::Span {
- start: start.char,
- end: end_char_loc,
- },
+ span: harper_core::Span::new(start.char, end_char_loc),
kind: $kind,
}])
}};
diff --git a/harper-wasm/src/lib.rs b/harper-wasm/src/lib.rs
index 85b9a3039..a017ff7be 100644
--- a/harper-wasm/src/lib.rs
+++ b/harper-wasm/src/lib.rs
@@ -531,18 +531,18 @@ impl Span {
}
pub fn len(&self) -> usize {
- Into::<harper_core::Span>::into(*self).len()
+ Into::<harper_core::Span<char>>::into(*self).len()
}
}
-impl From<Span> for harper_core::Span {
+impl From<Span> for harper_core::Span<char> {
fn from(value: Span) -> Self {
harper_core::Span::new(value.start, value.end)
}
}
-impl From<harper_core::Span> for Span {
- fn from(value: harper_core::Span) -> Self {
+impl From<harper_core::Span<char>> for Span {
+ fn from(value: harper_core::Span<char>) -> Self {
Span::new(value.start, value.end)
}
}