Skip to content

web search done right

generalize 'SplitWhitespaceWithRangeIter' to 'SplitWithRangeIter' that takes accepts a predicate for where to create the splits

Mikkel Denker0f9c0bbd87a2
diff --git a/crates/core/src/tokenizer/mod.rs b/crates/core/src/tokenizer/mod.rs
index 769489c5..3386ec31 100644
--- a/crates/core/src/tokenizer/mod.rs
+++ b/crates/core/src/tokenizer/mod.rs
@@ -21,7 +21,7 @@ mod script;
mod script_tokenizer;
mod segmenter;
mod split_preserve;
-mod split_whitespace_with_range;
+mod split_with_range;
mod stemmer;
use std::borrow::{Borrow, Cow};
diff --git a/crates/core/src/tokenizer/script_tokenizer.rs b/crates/core/src/tokenizer/script_tokenizer.rs
index 9c0c3483..886f2530 100644
--- a/crates/core/src/tokenizer/script_tokenizer.rs
+++ b/crates/core/src/tokenizer/script_tokenizer.rs
@@ -15,8 +15,7 @@
// along with this program. If not, see <https://www.gnu.org/licenses/>.
use super::{
- split_preserve::StrSplitPreserveWithRange,
- split_whitespace_with_range::SplitWhitespaceWithRange, Token,
+ split_preserve::StrSplitPreserveWithRange, split_with_range::SplitWhitespaceWithRange, Token,
};
pub trait ScriptTokenizer {
diff --git a/crates/core/src/tokenizer/split_whitespace_with_range.rs b/crates/core/src/tokenizer/split_with_range.rs
similarity index 72%
rename from crates/core/src/tokenizer/split_whitespace_with_range.rs
rename to crates/core/src/tokenizer/split_with_range.rs
index 150f4deb..79106941 100644
--- a/crates/core/src/tokenizer/split_whitespace_with_range.rs
+++ b/crates/core/src/tokenizer/split_with_range.rs
@@ -14,27 +14,37 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
+pub trait SplitWithRange {
+ fn split_with_range<P>(&self, pred: P) -> impl Iterator<Item = (&str, std::ops::Range<usize>)>
+ where
+ P: Fn(char) -> bool;
+}
+
pub trait SplitWhitespaceWithRange {
- fn split_whitespace_with_range(&self) -> SplitWhitespaceWithRangeIter;
+ fn split_whitespace_with_range(&self) -> impl Iterator<Item = (&str, std::ops::Range<usize>)>;
}
-pub struct SplitWhitespaceWithRangeIter<'a> {
+pub struct SplitWithRangeIter<'a, P> {
s: &'a str,
+ pred: P,
start: usize,
}
-impl<'a> SplitWhitespaceWithRangeIter<'a> {
- fn new(s: &'a str) -> Self {
- Self { s, start: 0 }
+impl<'a, P> SplitWithRangeIter<'a, P> {
+ fn new(s: &'a str, pred: P) -> Self {
+ Self { s, pred, start: 0 }
}
}
-impl<'a> Iterator for SplitWhitespaceWithRangeIter<'a> {
+impl<'a, P> Iterator for SplitWithRangeIter<'a, P>
+where
+ P: Fn(char) -> bool,
+{
type Item = (&'a str, std::ops::Range<usize>);
fn next(&mut self) -> Option<Self::Item> {
for c in self.s[self.start..].chars() {
- if !c.is_whitespace() {
+ if !(self.pred)(c) {
break;
}
self.start += c.len_utf8();
@@ -44,10 +54,10 @@ impl<'a> Iterator for SplitWhitespaceWithRangeIter<'a> {
return None;
}
- let start = self.s[self.start..].find(|c: char| !c.is_whitespace())?;
+ let start = self.s[self.start..].find(|c: char| !(self.pred)(c))?;
let start = self.start + start;
let end = self.s[start..]
- .find(char::is_whitespace)
+ .find(|c| (self.pred)(c))
.map(|end| start + end)
.unwrap_or(self.s.len());
let range = start..end;
@@ -57,14 +67,32 @@ impl<'a> Iterator for SplitWhitespaceWithRangeIter<'a> {
}
impl SplitWhitespaceWithRange for str {
- fn split_whitespace_with_range(&self) -> SplitWhitespaceWithRangeIter {
- SplitWhitespaceWithRangeIter::new(self)
+ fn split_whitespace_with_range(&self) -> impl Iterator<Item = (&str, std::ops::Range<usize>)> {
+ self.split_with_range(char::is_whitespace)
}
}
impl SplitWhitespaceWithRange for String {
- fn split_whitespace_with_range(&self) -> SplitWhitespaceWithRangeIter {
- SplitWhitespaceWithRangeIter::new(self)
+ fn split_whitespace_with_range(&self) -> impl Iterator<Item = (&str, std::ops::Range<usize>)> {
+ self.split_with_range(char::is_whitespace)
+ }
+}
+
+impl SplitWithRange for str {
+ fn split_with_range<P>(&self, pred: P) -> impl Iterator<Item = (&str, std::ops::Range<usize>)>
+ where
+ P: Fn(char) -> bool,
+ {
+ SplitWithRangeIter::new(self, pred)
+ }
+}
+
+impl SplitWithRange for String {
+ fn split_with_range<P>(&self, pred: P) -> impl Iterator<Item = (&str, std::ops::Range<usize>)>
+ where
+ P: Fn(char) -> bool,
+ {
+ SplitWithRangeIter::new(self, pred)
}
}