Offline, privacy-first grammar checker. Fast, open-source, Rust-powered
refactor: typed spans (#1595)
Steve Kurch8ca4d84c95a6
diff --git a/harper-cli/src/annotate_tokens.rs b/harper-cli/src/annotate_tokens.rsindex f01fb2cf2..e7a2473db 100644--- a/harper-cli/src/annotate_tokens.rs+++ b/harper-cli/src/annotate_tokens.rs@@ -6,7 +6,7 @@ use strum::IntoEnumIterator;/// Represents an annotation.pub(super) struct Annotation {/// The range the annotation covers in the source. For instance, this might be a single word.- span: Span,+ span: Span<char>,/// The message displayed by the annotation.annotation_text: String,/// The color of the annotation.diff --git a/harper-comments/src/comment_parsers/mod.rs b/harper-comments/src/comment_parsers/mod.rsindex 8106b4a22..f8e53670b 100644--- a/harper-comments/src/comment_parsers/mod.rs+++ b/harper-comments/src/comment_parsers/mod.rs@@ -13,7 +13,7 @@ pub use unit::Unit;/// Get the span of a tree-sitter-produced comment that doesn't include the/// comment openers and closers.-fn without_initiators(source: &[char]) -> Span {+fn without_initiators(source: &[char]) -> Span<char> {// Skip over the comment start characterslet actual_start = source.iter()diff --git a/harper-core/src/document.rs b/harper-core/src/document.rsindex 23f37bb1f..b16c7eb74 100644--- a/harper-core/src/document.rs+++ b/harper-core/src/document.rs@@ -31,7 +31,7 @@ impl Document {/// Locate all the tokens that intersect a provided span.////// Desperately needs optimization.- pub fn token_indices_intersecting(&self, span: Span) -> Vec<usize> {+ pub fn token_indices_intersecting(&self, span: Span<char>) -> Vec<usize> {self.tokens().enumerate().filter_map(|(idx, tok)| tok.span.overlaps_with(span).then_some(idx))@@ -41,7 +41,7 @@ impl Document {/// Locate all the tokens that intersect a provided span and convert them to [`FatToken`]s.////// Desperately needs optimization.- pub fn fat_tokens_intersecting(&self, span: Span) -> Vec<FatToken> {+ pub fn fat_tokens_intersecting(&self, span: Span<char>) -> Vec<FatToken> {let indices = self.token_indices_intersecting(span);indices@@ -314,19 +314,16 @@ impl Document {self.fat_tokens().map(|t| t.into())}- pub fn get_span_content(&self, span: &Span) -> &[char] {+ pub fn get_span_content(&self, span: &Span<char>) -> &[char] {span.get_content(&self.source)}- pub fn get_span_content_str(&self, span: &Span) -> String {+ pub fn get_span_content_str(&self, span: &Span<char>) -> String {String::from_iter(self.get_span_content(span))}pub fn get_full_string(&self) -> String {- self.get_span_content_str(&Span {- start: 0,- end: self.source.len(),- })+ self.get_span_content_str(&Span::new(0, self.source.len()))}pub fn get_full_content(&self) -> &[char] {@@ -724,7 +721,7 @@ impl TokenStringExt for Document {self.tokens.first_non_whitespace()}- fn span(&self) -> Option<Span> {+ fn span(&self) -> Option<Span<char>> {self.tokens.span()}diff --git a/harper-core/src/expr/all.rs b/harper-core/src/expr/all.rsindex 2d6bcf46b..8a2e201e0 100644--- a/harper-core/src/expr/all.rs+++ b/harper-core/src/expr/all.rs@@ -21,8 +21,8 @@ impl All {}impl Expr for All {- fn run(&self, cursor: usize, tokens: &[Token], source: &[char]) -> Option<Span> {- let mut longest: Option<Span> = None;+ fn run(&self, cursor: usize, tokens: &[Token], source: &[char]) -> Option<Span<Token>> {+ let mut longest: Option<Span<Token>> = None;for expr in self.children.iter() {let window = expr.run(cursor, tokens, source)?;diff --git a/harper-core/src/expr/expr_map.rs b/harper-core/src/expr/expr_map.rsindex 369a10eca..f76e11abc 100644--- a/harper-core/src/expr/expr_map.rs+++ b/harper-core/src/expr/expr_map.rs@@ -74,7 +74,7 @@ impl<T> Expr for ExprMap<T>whereT: LSend,{- fn run(&self, cursor: usize, tokens: &[Token], source: &[char]) -> Option<Span> {+ fn run(&self, cursor: usize, tokens: &[Token], source: &[char]) -> Option<Span<Token>> {self.rows.iter().filter_map(|row| row.key.run(cursor, tokens, source))diff --git a/harper-core/src/expr/first_match_of.rs b/harper-core/src/expr/first_match_of.rsindex 6b0e144f6..0d21b29d1 100644--- a/harper-core/src/expr/first_match_of.rs+++ b/harper-core/src/expr/first_match_of.rs@@ -21,7 +21,7 @@ impl FirstMatchOf {}impl Expr for FirstMatchOf {- fn run(&self, cursor: usize, tokens: &[Token], source: &[char]) -> Option<Span> {+ fn run(&self, cursor: usize, tokens: &[Token], source: &[char]) -> Option<Span<Token>> {self.exprs.iter().find_map(|p| p.run(cursor, tokens, source))diff --git a/harper-core/src/expr/fixed_phrase.rs b/harper-core/src/expr/fixed_phrase.rsindex 78a77d6fc..8399bd50f 100644--- a/harper-core/src/expr/fixed_phrase.rs+++ b/harper-core/src/expr/fixed_phrase.rs@@ -62,7 +62,7 @@ impl FixedPhrase {}impl Expr for FixedPhrase {- fn run(&self, cursor: usize, tokens: &[Token], source: &[char]) -> Option<Span> {+ fn run(&self, cursor: usize, tokens: &[Token], source: &[char]) -> Option<Span<Token>> {self.inner.run(cursor, tokens, source)}}diff --git a/harper-core/src/expr/longest_match_of.rs b/harper-core/src/expr/longest_match_of.rsindex ae6c982c1..9fd9aca73 100644--- a/harper-core/src/expr/longest_match_of.rs+++ b/harper-core/src/expr/longest_match_of.rs@@ -17,8 +17,8 @@ impl LongestMatchOf {}impl Expr for LongestMatchOf {- fn run(&self, cursor: usize, tokens: &[Token], source: &[char]) -> Option<Span> {- let mut longest: Option<Span> = None;+ fn run(&self, cursor: usize, tokens: &[Token], source: &[char]) -> Option<Span<Token>> {+ let mut longest: Option<Span<Token>> = None;for expr in self.exprs.iter() {let Some(window) = expr.run(cursor, tokens, source) else {diff --git a/harper-core/src/expr/mergeable_words.rs b/harper-core/src/expr/mergeable_words.rsindex d196af2ba..02934cc4f 100644--- a/harper-core/src/expr/mergeable_words.rs+++ b/harper-core/src/expr/mergeable_words.rs@@ -60,7 +60,7 @@ impl MergeableWords {}impl Expr for MergeableWords {- fn run(&self, cursor: usize, tokens: &[Token], source: &[char]) -> Option<Span> {+ fn run(&self, cursor: usize, tokens: &[Token], source: &[char]) -> Option<Span<Token>> {let inner_match = self.inner.run(cursor, tokens, source)?;if inner_match.len() != 3 {diff --git a/harper-core/src/expr/mod.rs b/harper-core/src/expr/mod.rsindex bb3dd249e..bab156b2d 100644--- a/harper-core/src/expr/mod.rs+++ b/harper-core/src/expr/mod.rs@@ -60,14 +60,14 @@ pub use word_expr_group::WordExprGroup;use crate::{Document, LSend, Span, Token};pub trait Expr: LSend {- fn run(&self, cursor: usize, tokens: &[Token], source: &[char]) -> Option<Span>;+ fn run(&self, cursor: usize, tokens: &[Token], source: &[char]) -> Option<Span<Token>>;}impl<S> Expr for SwhereS: Step + ?Sized,{- fn run(&self, cursor: usize, tokens: &[Token], source: &[char]) -> Option<Span> {+ fn run(&self, cursor: usize, tokens: &[Token], source: &[char]) -> Option<Span<Token>> {self.step(tokens, cursor, source).map(|s| {if s >= 0 {Span::new_with_len(cursor, s as usize)@@ -82,7 +82,7 @@ impl<E> Expr for Arc<E>whereE: Expr,{- fn run(&self, cursor: usize, tokens: &[Token], source: &[char]) -> Option<Span> {+ fn run(&self, cursor: usize, tokens: &[Token], source: &[char]) -> Option<Span<Token>> {self.as_ref().run(cursor, tokens, source)}}@@ -92,7 +92,7 @@ impl<E> Expr for Rc<E>whereE: Expr,{- fn run(&self, cursor: usize, tokens: &[Token], source: &[char]) -> Option<Span> {+ fn run(&self, cursor: usize, tokens: &[Token], source: &[char]) -> Option<Span<Token>> {self.as_ref().run(cursor, tokens, source)}}@@ -112,9 +112,12 @@ pub trait ExprExt {&'a self,tokens: &'a [Token],source: &'a [char],- ) -> Box<dyn Iterator<Item = Span> + 'a>;+ ) -> Box<dyn Iterator<Item = Span<Token>> + 'a>;- fn iter_matches_in_doc<'a>(&'a self, doc: &'a Document) -> Box<dyn Iterator<Item = Span> + 'a>;+ fn iter_matches_in_doc<'a>(+ &'a self,+ doc: &'a Document,+ ) -> Box<dyn Iterator<Item = Span<Token>> + 'a>;}impl<E: ?Sized> ExprExt for E@@ -125,7 +128,7 @@ where&'a self,tokens: &'a [Token],source: &'a [char],- ) -> Box<(dyn Iterator<Item = Span> + 'a)> {+ ) -> Box<(dyn Iterator<Item = Span<Token>> + 'a)> {let mut last_end = 0usize;Box::new((0..tokens.len()).filter_map(move |i| {@@ -142,7 +145,7 @@ wherefn iter_matches_in_doc<'a>(&'a self,doc: &'a Document,- ) -> Box<(dyn Iterator<Item = Span> + 'a)> {+ ) -> Box<(dyn Iterator<Item = Span<Token>> + 'a)> {Box::new(self.iter_matches(doc.get_tokens(), doc.get_source()))}}diff --git a/harper-core/src/expr/optional.rs b/harper-core/src/expr/optional.rsindex f76775c99..2641dc4f7 100644--- a/harper-core/src/expr/optional.rs+++ b/harper-core/src/expr/optional.rs@@ -18,7 +18,7 @@ impl Optional {}impl Expr for Optional {- fn run(&self, cursor: usize, tokens: &[Token], source: &[char]) -> Option<Span> {+ fn run(&self, cursor: usize, tokens: &[Token], source: &[char]) -> Option<Span<Token>> {let res = self.inner.run(cursor, tokens, source);if res.is_none() {diff --git a/harper-core/src/expr/reflexive_pronoun.rs b/harper-core/src/expr/reflexive_pronoun.rsindex 81c856203..b3b54ab04 100644--- a/harper-core/src/expr/reflexive_pronoun.rs+++ b/harper-core/src/expr/reflexive_pronoun.rs@@ -51,7 +51,7 @@ impl ReflexivePronoun {}impl Expr for ReflexivePronoun {- fn run(&self, cursor: usize, tokens: &[Token], source: &[char]) -> Option<Span> {+ fn run(&self, cursor: usize, tokens: &[Token], source: &[char]) -> Option<Span<Token>> {let good_pronouns = |token: &Token, _: &[char]| token.kind.is_reflexive_pronoun();let mut expr = FirstMatchOf::new(vec![Box::new(good_pronouns)]);if self.include_common_errors {diff --git a/harper-core/src/expr/repeating.rs b/harper-core/src/expr/repeating.rsindex 2274291ed..09f18afe3 100644--- a/harper-core/src/expr/repeating.rs+++ b/harper-core/src/expr/repeating.rs@@ -19,7 +19,7 @@ impl Repeating {}impl Expr for Repeating {- fn run(&self, mut cursor: usize, tokens: &[Token], source: &[char]) -> Option<Span> {+ fn run(&self, mut cursor: usize, tokens: &[Token], source: &[char]) -> Option<Span<Token>> {let mut window = Span::new_with_len(cursor, 0);let mut repetition = 0;diff --git a/harper-core/src/expr/sequence_expr.rs b/harper-core/src/expr/sequence_expr.rsindex bcf3d6b75..857874747 100644--- a/harper-core/src/expr/sequence_expr.rs+++ b/harper-core/src/expr/sequence_expr.rs@@ -56,7 +56,7 @@ impl Expr for SequenceExpr {/// Run the expression starting at an index, returning the total matched window.////// If any step returns `None`, the entire expression does as well.- fn run(&self, mut cursor: usize, tokens: &[Token], source: &[char]) -> Option<Span> {+ fn run(&self, mut cursor: usize, tokens: &[Token], source: &[char]) -> Option<Span<Token>> {let mut window = Span::new_with_len(cursor, 0);for cur_expr in &self.exprs {diff --git a/harper-core/src/expr/similar_to_phrase.rs b/harper-core/src/expr/similar_to_phrase.rsindex 08f59edaa..92fd810db 100644--- a/harper-core/src/expr/similar_to_phrase.rs+++ b/harper-core/src/expr/similar_to_phrase.rs@@ -58,7 +58,7 @@ impl SimilarToPhrase {}impl Expr for SimilarToPhrase {- fn run(&self, cursor: usize, tokens: &[Token], source: &[char]) -> Option<Span> {+ fn run(&self, cursor: usize, tokens: &[Token], source: &[char]) -> Option<Span<Token>> {if self.phrase.run(cursor, tokens, source).is_some() {return None;}diff --git a/harper-core/src/expr/space_or_hyphen.rs b/harper-core/src/expr/space_or_hyphen.rsindex fb0986c76..a77481de3 100644--- a/harper-core/src/expr/space_or_hyphen.rs+++ b/harper-core/src/expr/space_or_hyphen.rs@@ -9,7 +9,7 @@ use super::Expr;pub struct SpaceOrHyphen;impl Expr for SpaceOrHyphen {- fn run(&self, cursor: usize, tokens: &[Token], source: &[char]) -> Option<Span> {+ fn run(&self, cursor: usize, tokens: &[Token], source: &[char]) -> Option<Span<Token>> {FirstMatchOf::new(vec![Box::new(WhitespacePattern),Box::new(|tok: &Token, _source: &[char]| tok.kind.is_hyphen()),diff --git a/harper-core/src/expr/spelled_number_expr.rs b/harper-core/src/expr/spelled_number_expr.rsindex 0d95d2a1e..586313423 100644--- a/harper-core/src/expr/spelled_number_expr.rs+++ b/harper-core/src/expr/spelled_number_expr.rs@@ -9,7 +9,7 @@ use super::{Expr, SequenceExpr};pub struct SpelledNumberExpr;impl Expr for SpelledNumberExpr {- fn run(&self, cursor: usize, tokens: &[Token], source: &[char]) -> Option<Span> {+ fn run(&self, cursor: usize, tokens: &[Token], source: &[char]) -> Option<Span<Token>> {if tokens.is_empty() {return None;}@@ -70,13 +70,13 @@ impl Expr for SpelledNumberExpr {mod tests {use super::SpelledNumberExpr;use crate::expr::ExprExt;- use crate::{Document, Span};+ use crate::{Document, Span, Token};trait SpanVecExt {fn to_strings(&self, doc: &Document) -> Vec<String>;}- impl SpanVecExt for Vec<Span> {+ impl SpanVecExt for Vec<Span<Token>> {fn to_strings(&self, doc: &Document) -> Vec<String> {self.iter().map(|sp| {diff --git a/harper-core/src/expr/time_unit_expr.rs b/harper-core/src/expr/time_unit_expr.rsindex 2a941506b..fd794446a 100644--- a/harper-core/src/expr/time_unit_expr.rs+++ b/harper-core/src/expr/time_unit_expr.rs@@ -15,7 +15,7 @@ use super::Expr;pub struct TimeUnitExpr;impl Expr for TimeUnitExpr {- fn run(&self, cursor: usize, tokens: &[Token], source: &[char]) -> Option<Span> {+ fn run(&self, cursor: usize, tokens: &[Token], source: &[char]) -> Option<Span<Token>> {if tokens.is_empty() {return None;}diff --git a/harper-core/src/expr/word_expr_group.rs b/harper-core/src/expr/word_expr_group.rsindex f8fbd4361..2e05a49d6 100644--- a/harper-core/src/expr/word_expr_group.rs+++ b/harper-core/src/expr/word_expr_group.rs@@ -39,7 +39,7 @@ impl<E> Expr for WordExprGroup<E>whereE: Expr,{- fn run(&self, cursor: usize, tokens: &[Token], source: &[char]) -> Option<Span> {+ fn run(&self, cursor: usize, tokens: &[Token], source: &[char]) -> Option<Span<Token>> {let first = tokens.get(cursor)?;if !first.kind.is_word() {return None;diff --git a/harper-core/src/linting/lint.rs b/harper-core/src/linting/lint.rsindex ba33d2fdf..ff0c7ffda 100644--- a/harper-core/src/linting/lint.rs+++ b/harper-core/src/linting/lint.rs@@ -11,7 +11,7 @@ use super::{LintKind, Suggestion};pub struct Lint {/// The location in the source text the error lies./// Important for automatic lint resolution through [`Self::suggestions`].- pub span: Span,+ pub span: Span<char>,/// The general category the lint belongs to./// Mostly used for UI elements in integrations.pub lint_kind: LintKind,diff --git a/harper-core/src/linting/suggestion.rs b/harper-core/src/linting/suggestion.rsindex 310962759..4f8931e56 100644--- a/harper-core/src/linting/suggestion.rs+++ b/harper-core/src/linting/suggestion.rs@@ -53,7 +53,7 @@ impl Suggestion {}/// Apply a suggestion to a given text.- pub fn apply(&self, span: Span, source: &mut Vec<char>) {+ pub fn apply(&self, span: Span<char>, source: &mut Vec<char>) {match self {Self::ReplaceWith(chars) => {// Avoid allocation if possiblediff --git a/harper-core/src/mask/mod.rs b/harper-core/src/mask/mod.rsindex ee2a9c40b..a9b78ccc4 100644--- a/harper-core/src/mask/mod.rs+++ b/harper-core/src/mask/mod.rs@@ -19,11 +19,11 @@ pub struct Mask {// Right now, there aren't any use-cases where we can't treat this as a stack.//// Assumed that no elements overlap and exist in sorted order.- pub(self) allowed: Vec<Span>,+ pub(self) allowed: Vec<Span<char>>,}-impl FromIterator<Span> for Mask {- fn from_iter<T: IntoIterator<Item = Span>>(iter: T) -> Self {+impl FromIterator<Span<char>> for Mask {+ fn from_iter<T: IntoIterator<Item = Span<char>>>(iter: T) -> Self {let allowed = iter.into_iter().sorted_by_key(|span| span.start)@@ -49,12 +49,12 @@ impl Mask {pub fn iter_allowed<'a>(&'a self,source: &'a [char],- ) -> impl Iterator<Item = (Span, &'a [char])> {+ ) -> impl Iterator<Item = (Span<char>, &'a [char])> {self.allowed.iter().map(|s| (*s, s.get_content(source)))}/// Mark a span of the text as allowed.- pub fn push_allowed(&mut self, allowed: Span) {+ pub fn push_allowed(&mut self, allowed: Span<char>) {if let Some(last) = self.allowed.last_mut() {assert!(allowed.start >= last.end,diff --git a/harper-core/src/parsers/mask.rs b/harper-core/src/parsers/mask.rsindex d74393d66..20e532471 100644--- a/harper-core/src/parsers/mask.rs+++ b/harper-core/src/parsers/mask.rs@@ -32,7 +32,7 @@ wherelet mut tokens: Vec<Token> = Vec::new();- let mut last_allowed: Option<Span> = None;+ let mut last_allowed: Option<Span<char>> = None;for (span, content) in mask.iter_allowed(source) {// Check for a line break separating the current chunk from the preceding one.diff --git a/harper-core/src/patterns/mod.rs b/harper-core/src/patterns/mod.rsindex 4198f8481..851488851 100644--- a/harper-core/src/patterns/mod.rs+++ b/harper-core/src/patterns/mod.rs@@ -41,10 +41,10 @@ pub trait Pattern: LSend {}pub trait PatternExt {- fn iter_matches(&self, tokens: &[Token], source: &[char]) -> impl Iterator<Item = Span>;+ fn iter_matches(&self, tokens: &[Token], source: &[char]) -> impl Iterator<Item = Span<Token>>;/// Search through all tokens to locate all non-overlapping pattern matches.- fn find_all_matches(&self, tokens: &[Token], source: &[char]) -> Vec<Span> {+ fn find_all_matches(&self, tokens: &[Token], source: &[char]) -> Vec<Span<Token>> {self.iter_matches(tokens, source).collect()}}@@ -53,7 +53,7 @@ impl<P> PatternExt for PwhereP: Pattern + ?Sized,{- fn iter_matches(&self, tokens: &[Token], source: &[char]) -> impl Iterator<Item = Span> {+ fn iter_matches(&self, tokens: &[Token], source: &[char]) -> impl Iterator<Item = Span<Token>> {MatchIter::new(self, tokens, source)}}@@ -81,7 +81,7 @@ impl<P> Iterator for MatchIter<'_, '_, '_, P>whereP: Pattern + ?Sized,{- type Item = Span;+ type Item = Span<Token>;fn next(&mut self) -> Option<Self::Item> {while self.index < self.tokens.len() {@@ -124,11 +124,11 @@ impl<F: LSend + Fn(&Token, &[char]) -> bool> SingleTokenPattern for F {}pub trait DocPattern {- fn find_all_matches_in_doc(&self, document: &Document) -> Vec<Span>;+ fn find_all_matches_in_doc(&self, document: &Document) -> Vec<Span<Token>>;}impl<P: PatternExt> DocPattern for P {- fn find_all_matches_in_doc(&self, document: &Document) -> Vec<Span> {+ fn find_all_matches_in_doc(&self, document: &Document) -> Vec<Span<Token>> {self.find_all_matches(document.get_tokens(), document.get_source())}}diff --git a/harper-core/src/patterns/nominal_phrase.rs b/harper-core/src/patterns/nominal_phrase.rsindex 70135fd74..33a7181e1 100644--- a/harper-core/src/patterns/nominal_phrase.rs+++ b/harper-core/src/patterns/nominal_phrase.rs@@ -37,13 +37,13 @@ impl Pattern for NominalPhrase {mod tests {use super::super::DocPattern;use super::NominalPhrase;- use crate::{Document, Span, patterns::Pattern};+ use crate::{Document, Span, Token, patterns::Pattern};trait SpanVecExt {fn to_strings(&self, doc: &Document) -> Vec<String>;}- impl SpanVecExt for Vec<Span> {+ impl SpanVecExt for Vec<Span<Token>> {fn to_strings(&self, doc: &Document) -> Vec<String> {self.iter().map(|sp| {@@ -104,7 +104,9 @@ mod tests {dbg!(matches.to_strings(&doc));for span in &matches {- let gc = span.get_content(doc.get_source());+ let gc = span+ .to_char_span(doc.get_tokens())+ .get_content(doc.get_source());dbg!(gc);}diff --git a/harper-core/src/span.rs b/harper-core/src/span.rsindex 7e3c54dae..23ca9b495 100644--- a/harper-core/src/span.rs+++ b/harper-core/src/span.rs@@ -1,54 +1,92 @@-use std::ops::Range;+use std::{fmt::Display, marker::PhantomData, ops::Range};use serde::{Deserialize, Serialize};-use crate::CharStringExt;+use crate::Token;-/// A window in a [`char`] sequence.+/// A window in a [`T`] sequence.+///+/// Note that the range covered by a [`Span`] is end-exclusive, meaning that the end index is not+/// included in the range covered by the [`Span`]. If you're familiar with the Rust range syntax,+/// you could say the span covers the equivalent of `start..end`, *not* `start..=end`.+///+/// For a [`Span`] to be correct, its end index must be greater than or equal to its start+/// index. Creating or using a [`Span`] which does not follow this rule may lead to unexpected+/// behavior or panics.////// Although specific to `harper.js`, [this page may clear up any questions you have](https://writewithharper.com/docs/harperjs/spans).-#[derive(Debug, Clone, Copy, Serialize, Deserialize, Default, PartialEq, Eq)]-pub struct Span {+#[derive(Debug, Serialize, Deserialize, Default, PartialEq, Eq)]+pub struct Span<T> {+ /// The start index of the span.pub start: usize,+ /// The end index of the span.+ ///+ /// Note that [`Span`] represents an exclusive range. This means that a `Span::new(0, 5)` will+ /// cover the values `0, 1, 2, 3, 4`; it will not cover the `5`.pub end: usize,+ span_type: PhantomData<T>,}-impl Span {+impl<T> Span<T> {+ /// An empty [`Span`].+ pub const EMPTY: Self = Self {+ start: 0,+ end: 0,+ span_type: PhantomData,+ };++ /// Creates a new [`Span`] with the provided start and end indices.+ ///+ /// # Panics+ ///+ /// This will panic if `start` is greater than `end`.pub fn new(start: usize, end: usize) -> Self {if start > end {panic!("{start} > {end}");}- Self { start, end }+ Self {+ start,+ end,+ span_type: PhantomData,+ }}+ /// Creates a new [`Span`] from the provided start position and length.pub fn new_with_len(start: usize, len: usize) -> Self {Self {start,end: start + len,+ span_type: PhantomData,}}+ /// The length of the [`Span`].pub fn len(&self) -> usize {self.end - self.start}+ /// Checks whether the [`Span`] is empty.+ ///+ /// A [`Span`] is considered empty if it has a length of 0.pub fn is_empty(&self) -> bool {self.len() == 0}+ /// Checks whether `idx` is within the range of the span.pub fn contains(&self, idx: usize) -> bool {assert!(self.start <= self.end);self.start <= idx && idx < self.end}+ /// Checks whether this span's range overlaps with `other`.pub fn overlaps_with(&self, other: Self) -> bool {(self.start < other.end) && (other.start < self.end)}/// Get the associated content. Will return [`None`] if any aspect is/// invalid.- pub fn try_get_content<'a>(&self, source: &'a [char]) -> Option<&'a [char]> {+ pub fn try_get_content<'a>(&self, source: &'a [T]) -> Option<&'a [T]> {if (self.start > self.end) || (self.start >= source.len()) || (self.end > source.len()) {if self.is_empty() {return Some(&source[0..0]);@@ -72,44 +110,38 @@ impl Span {}/// Get the associated content. Will panic if any aspect is invalid.- pub fn get_content<'a>(&self, source: &'a [char]) -> &'a [char] {+ pub fn get_content<'a>(&self, source: &'a [T]) -> &'a [T] {match self.try_get_content(source) {Some(v) => v,- None => panic!(- "Could not get position {:?} within \"{}\"",- self,- source.to_string()- ),+ None => panic!("Failed to get content for span."),}}- pub fn get_content_string(&self, source: &[char]) -> String {- String::from_iter(self.get_content(source))- }-+ /// Set the span's length.pub fn set_len(&mut self, length: usize) {self.end = self.start + length;}+ /// Returns a copy of this [`Span`] with a new length.pub fn with_len(&self, length: usize) -> Self {let mut cloned = *self;cloned.set_len(length);cloned}- // Add an amount to both [`Self::start`] and [`Self::end`]+ /// Add an amount to both [`Self::start`] and [`Self::end`]pub fn push_by(&mut self, by: usize) {self.start += by;self.end += by;}- // Subtract an amount to both [`Self::start`] and [`Self::end`]+ /// Subtract an amount from both [`Self::start`] and [`Self::end`]pub fn pull_by(&mut self, by: usize) {self.start -= by;self.end -= by;}- // Add an amount to a copy of both [`Self::start`] and [`Self::end`]+ /// Add an amount to a copy of both [`Self::start`] and [`Self::end`]pub fn pushed_by(&self, by: usize) -> Self {let mut clone = *self;clone.start += by;@@ -117,7 +149,7 @@ impl Span {clone}- // Subtract an amount to a copy of both [`Self::start`] and [`Self::end`]+ /// Subtract an amount to a copy of both [`Self::start`] and [`Self::end`]pub fn pulled_by(&self, by: usize) -> Option<Self> {if by > self.start {return None;@@ -129,7 +161,7 @@ impl Span {Some(clone)}- // Add an amount a copy of both [`Self::start`] and [`Self::end`]+ /// Add an amount to a copy of both [`Self::start`] and [`Self::end`]pub fn with_offset(&self, by: usize) -> Self {let mut clone = *self;clone.push_by(by);@@ -137,50 +169,127 @@ impl Span {}}-impl From<Range<usize>> for Span {+/// Additional functions for types that implement [`std::fmt::Debug`] and [`Display`].+impl<T: Display + std::fmt::Debug> Span<T> {+ /// Gets the content of this [`Span<T>`] as a [`String`].+ pub fn get_content_string(&self, source: &[T]) -> String {+ if let Some(content) = self.try_get_content(source) {+ content.iter().map(|t| t.to_string()).collect()+ } else {+ panic!("Could not get position {self:?} within \"{source:?}\"")+ }+ }+}++/// Functionality specific to [`Token`] spans.+impl Span<Token> {+ /// Converts the [`Span<Token>`] into a [`Span<char>`].+ ///+ /// This requires knowing the character spans of the tokens covered by this+ /// [`Span<Token>`]. Because of this, a reference to the source token sequence used to create+ /// this span is required.+ pub fn to_char_span(&self, source_document_tokens: &[Token]) -> Span<char> {+ if self.is_empty() {+ Span::EMPTY+ } else {+ let target_tokens = &source_document_tokens[self.start..self.end];+ Span::new(+ target_tokens.first().unwrap().span.start,+ target_tokens.last().unwrap().span.end,+ )+ }+ }+}++impl<T> From<Range<usize>> for Span<T> {+ /// Reinterprets the provided [`std::ops::Range`] as a [`Span`].fn from(value: Range<usize>) -> Self {Self::new(value.start, value.end)}}-impl From<Span> for Range<usize> {- fn from(value: Span) -> Self {+impl<T> From<Span<T>> for Range<usize> {+ /// Converts the [`Span`] to an [`std::ops::Range`].+ fn from(value: Span<T>) -> Self {value.start..value.end}}-impl IntoIterator for Span {+impl<T> IntoIterator for Span<T> {type Item = usize;type IntoIter = Range<usize>;+ /// Converts the [`Span`] into an iterator that yields the indices covered by its range.+ ///+ /// Note that [`Span`] is half-open, meaning that the value [`Self::end`] will not be yielded+ /// by this iterator: it will stop at the index immediately preceding [`Self::end`].fn into_iter(self) -> Self::IntoIter {self.start..self.end}}+impl<T> Clone for Span<T> {+ // Note: manual implementation so we don't unnecessarily require `T` to impl `Clone`.+ fn clone(&self) -> Self {+ *self+ }+}+impl<T> Copy for Span<T> {}+#[cfg(test)]mod tests {- use crate::Span;+ use crate::{+ Document,+ expr::{ExprExt, SequenceExpr},+ };++ use super::Span;++ type UntypedSpan = Span<()>;#[test]fn overlaps() {- assert!(Span::new(0, 5).overlaps_with(Span::new(3, 6)));- assert!(Span::new(0, 5).overlaps_with(Span::new(2, 3)));- assert!(Span::new(0, 5).overlaps_with(Span::new(4, 5)));- assert!(Span::new(0, 5).overlaps_with(Span::new(4, 4)));+ assert!(UntypedSpan::new(0, 5).overlaps_with(UntypedSpan::new(3, 6)));+ assert!(UntypedSpan::new(0, 5).overlaps_with(UntypedSpan::new(2, 3)));+ assert!(UntypedSpan::new(0, 5).overlaps_with(UntypedSpan::new(4, 5)));+ assert!(UntypedSpan::new(0, 5).overlaps_with(UntypedSpan::new(4, 4)));- assert!(!Span::new(0, 3).overlaps_with(Span::new(3, 5)));+ assert!(!UntypedSpan::new(0, 3).overlaps_with(UntypedSpan::new(3, 5)));}#[test]fn expands_properly() {- let mut span = Span::new(2, 2);+ let mut span = UntypedSpan::new(2, 2);span.expand_to_include(1);- assert_eq!(span, Span::new(1, 2));+ assert_eq!(span, UntypedSpan::new(1, 2));span.expand_to_include(2);- assert_eq!(span, Span::new(1, 3));+ assert_eq!(span, UntypedSpan::new(1, 3));+ }++ #[test]+ fn to_char_span_converts_correctly() {+ let doc = Document::new_plain_english_curated("Hello world!");++ // Empty span.+ let token_span = Span::EMPTY;+ let converted = token_span.to_char_span(doc.get_tokens());+ assert!(converted.is_empty());++ // Span from `Expr`.+ let token_span = SequenceExpr::default()+ .then_any_word()+ .t_ws()+ .then_any_word()+ .iter_matches_in_doc(&doc)+ .next()+ .unwrap();+ let converted = token_span.to_char_span(doc.get_tokens());+ assert_eq!(+ converted.get_content_string(doc.get_source()),+ "Hello world"+ );}}diff --git a/harper-core/src/token.rs b/harper-core/src/token.rsindex 3d698ffce..8b30d2440 100644--- a/harper-core/src/token.rs+++ b/harper-core/src/token.rs@@ -6,13 +6,13 @@ use crate::{FatToken, Span, TokenKind};#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Default)]pub struct Token {/// The characters the token represents.- pub span: Span,+ pub span: Span<char>,/// The parsed value.pub kind: TokenKind,}impl Token {- pub fn new(span: Span, kind: TokenKind) -> Self {+ pub fn new(span: Span<char>, kind: TokenKind) -> Self {Self { span, kind }}diff --git a/harper-core/src/token_string_ext.rs b/harper-core/src/token_string_ext.rsindex 91abc782b..e136007ab 100644--- a/harper-core/src/token_string_ext.rs+++ b/harper-core/src/token_string_ext.rs@@ -53,7 +53,7 @@ pub trait TokenStringExt {fn first_non_whitespace(&self) -> Option<&Token>;/// Grab the span that represents the beginning of the first element and the/// end of the last element.- fn span(&self) -> Option<Span>;+ fn span(&self) -> Option<Span<char>>;create_decl_for!(adjective);create_decl_for!(apostrophe);@@ -140,7 +140,7 @@ impl TokenStringExt for [Token] {if w_idx < u_idx { Some(word) } else { None }}- fn span(&self) -> Option<Span> {+ fn span(&self) -> Option<Span<char>> {let min_max = self.iter().flat_map(|v| [v.span.start, v.span.end].into_iter())diff --git a/harper-ls/src/pos_conv.rs b/harper-ls/src/pos_conv.rsindex 783efc796..7be9d1e63 100644--- a/harper-ls/src/pos_conv.rs+++ b/harper-ls/src/pos_conv.rs@@ -4,7 +4,7 @@use harper_core::Span;use tower_lsp_server::lsp_types::{Position, Range};-pub fn span_to_range(source: &[char], span: Span) -> Range {+pub fn span_to_range(source: &[char], span: Span<char>) -> Range {let start = index_to_position(source, span.start);let end = index_to_position(source, span.end);@@ -76,7 +76,7 @@ fn position_to_index(source: &[char], position: Position) -> usize {(target_char_pointer as usize - source.as_ptr() as usize) / size_of::<char>()}-pub fn range_to_span(source: &[char], range: Range) -> Span {+pub fn range_to_span(source: &[char], range: Range) -> Span<char> {let start = position_to_index(source, range.start);let end = position_to_index(source, range.end);diff --git a/harper-tree-sitter/src/lib.rs b/harper-tree-sitter/src/lib.rsindex c249a8ae5..d5c37e2b0 100644--- a/harper-tree-sitter/src/lib.rs+++ b/harper-tree-sitter/src/lib.rs@@ -62,7 +62,7 @@ impl TreeSitterMasker {/// Visits the children of a TreeSitter node, searching for comments.////// Returns the BYTE spans of the comment position.- fn extract_comments(&self, cursor: &mut TreeCursor, comments: &mut Vec<Span>) {+ fn extract_comments(&self, cursor: &mut TreeCursor, comments: &mut Vec<Span<u8>>) {Self::visit_nodes(cursor, &mut |node: &Node| {if (self.node_condition)(node) {comments.push(node.byte_range().into());@@ -120,7 +120,7 @@ impl Masker for TreeSitterMasker {/// NOTE: Will sort the given slice by their [`Span::start`].////// If any spans overlap, it will merge them.-fn byte_spans_to_char_spans(mut byte_spans: Vec<Span>, source: &str) -> Vec<Span> {+fn byte_spans_to_char_spans(mut byte_spans: Vec<Span<u8>>, source: &str) -> Vec<Span<char>> {byte_spans.sort_unstable_by_key(|s| s.start);// merge overlapping spans@@ -136,18 +136,20 @@ fn byte_spans_to_char_spans(mut byte_spans: Vec<Span>, source: &str) -> Vec<Span}}- let mut last_byte_pos = 0;- let mut last_char_pos = 0;- spans.iter_mut().for_each(|span| {- let byte_span = *span;+ // Convert byte spans to char spans.+ spans+ .iter()+ .scan((0, 0), |(last_byte_pos, last_char_pos), span| {+ let byte_span = *span;- last_char_pos += source[last_byte_pos..byte_span.start].chars().count();- span.start = last_char_pos;+ *last_char_pos += source[*last_byte_pos..byte_span.start].chars().count();+ let start = *last_char_pos;- last_char_pos += source[byte_span.start..byte_span.end].chars().count();- span.end = last_char_pos;+ *last_char_pos += source[byte_span.start..byte_span.end].chars().count();+ let end = *last_char_pos;- last_byte_pos = byte_span.end;- });- spans+ *last_byte_pos = byte_span.end;+ Some(Span::new(start, end))+ })+ .collect()}diff --git a/harper-typst/src/typst_translator.rs b/harper-typst/src/typst_translator.rsindex cea52005d..2c2db3c84 100644--- a/harper-typst/src/typst_translator.rs+++ b/harper-typst/src/typst_translator.rs@@ -20,10 +20,7 @@ macro_rules! def_token {let end_char_loc = start.push_to(range.end).char;Some(vec![Token {- span: harper_core::Span {- start: start.char,- end: end_char_loc,- },+ span: harper_core::Span::new(start.char, end_char_loc),kind: $kind,}])}};diff --git a/harper-wasm/src/lib.rs b/harper-wasm/src/lib.rsindex 85b9a3039..a017ff7be 100644--- a/harper-wasm/src/lib.rs+++ b/harper-wasm/src/lib.rs@@ -531,18 +531,18 @@ impl Span {}pub fn len(&self) -> usize {- Into::<harper_core::Span>::into(*self).len()+ Into::<harper_core::Span<char>>::into(*self).len()}}-impl From<Span> for harper_core::Span {+impl From<Span> for harper_core::Span<char> {fn from(value: Span) -> Self {harper_core::Span::new(value.start, value.end)}}-impl From<harper_core::Span> for Span {- fn from(value: harper_core::Span) -> Self {+impl From<harper_core::Span<char>> for Span {+ fn from(value: harper_core::Span<char>) -> Self {Span::new(value.start, value.end)}}