Skip to content

web search done right

move some of the 'stream.next()' functionality into traits in a lending-iter crate so we can implement and re-use adapters

Mikkel Denkerfa5282a800cc
diff --git a/Cargo.lock b/Cargo.lock
index 95d3f4cd..2c2c489f 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -2630,6 +2630,10 @@ version = "0.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "03087c2bad5e1034e8cace5926dec053fb3790248370865f5117a7d0213354c8"
+[[package]]
+name = "lending-iter"
+version = "0.1.0"
+
[[package]]
name = "levenshtein_automata"
version = "0.2.1"
@@ -4946,6 +4950,7 @@ dependencies = [
"insta",
"itertools 0.13.0",
"kuchiki",
+ "lending-iter",
"log",
"logos",
"lz4_flex",
@@ -5216,6 +5221,7 @@ dependencies = [
"futures-util",
"htmlescape",
"itertools 0.13.0",
+ "lending-iter",
"levenshtein_automata",
"log",
"lru",
diff --git a/Cargo.toml b/Cargo.toml
index 27cd984c..2bdd00ec 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -10,6 +10,7 @@ members = [
"crates/optics",
"crates/speedy-kv",
"crates/zimba",
+ "crates/lending-iter",
"crates/robotstxt",
"crates/tantivy",
"fuzz",
@@ -88,6 +89,7 @@ itertools = "0.13.0"
js-sys = "0.3.60"
lalrpop = {version = "0.20.0", features = ["lexer"]}
lalrpop-util = {version = "0.20.0", features = ["lexer"]}
+lending-iter = {path = "crates/lending-iter"}
levenshtein_automata = "0.2.1"
libc = "0.2.142"
log = {version = "0.4", features = ["release_max_level_info"]}
diff --git a/assets/licenses.html b/assets/licenses.html
index c643b33b..8908872a 100644
--- a/assets/licenses.html
+++ b/assets/licenses.html
@@ -48,7 +48,7 @@
<li><a href="#MIT">MIT License</a> (187)</li>
<li><a href="#BSD-3-Clause">BSD 3-Clause &quot;New&quot; or &quot;Revised&quot; License</a> (9)</li>
<li><a href="#MPL-2.0">Mozilla Public License 2.0</a> (8)</li>
- <li><a href="#AGPL-3.0">GNU Affero General Public License v3.0</a> (5)</li>
+ <li><a href="#AGPL-3.0">GNU Affero General Public License v3.0</a> (6)</li>
<li><a href="#ISC">ISC License</a> (4)</li>
<li><a href="#Unicode-3.0">Unicode License v3</a> (4)</li>
<li><a href="#BSD-2-Clause">BSD 2-Clause &quot;Simplified&quot; License</a> (3)</li>
@@ -70,6 +70,7 @@
<ul class="license-used-by">
<li><a href=" https://crates.io/crates/bloom ">bloom 0.1.0</a></li>
<li><a href=" https://crates.io/crates/file_store ">file_store 0.1.0</a></li>
+ <li><a href=" https://crates.io/crates/lending-iter ">lending-iter 0.1.0</a></li>
<li><a href=" https://crates.io/crates/ownedbytes ">ownedbytes 0.1.0</a></li>
<li><a href=" https://crates.io/crates/robotstxt ">robotstxt 0.1.0</a></li>
<li><a href=" https://crates.io/crates/speedy_kv ">speedy_kv 0.1.0</a></li>
diff --git a/crates/core/Cargo.toml b/crates/core/Cargo.toml
index e729659a..2aa3cdac 100644
--- a/crates/core/Cargo.toml
+++ b/crates/core/Cargo.toml
@@ -58,6 +58,7 @@ image.workspace = true
indicatif.workspace = true
itertools.workspace = true
kuchiki = {path = "../kuchiki"}
+lending-iter.workspace = true
log.workspace = true
logos.workspace = true
lz4_flex.workspace = true
diff --git a/crates/core/src/collector/approx_count.rs b/crates/core/src/collector/approx_count.rs
index 20afc0bf..85d95b51 100644
--- a/crates/core/src/collector/approx_count.rs
+++ b/crates/core/src/collector/approx_count.rs
@@ -17,6 +17,7 @@
use std::ops::Add;
use itertools::Itertools;
+use lending_iter::LendingIterator;
use tantivy::{
collector::{Collector, SegmentCollector},
tokenizer::Tokenizer,
@@ -113,8 +114,9 @@ impl Collector for ApproxCount {
for term in &self.terms {
let mut term_freq = 0;
let mut stream = tokenizer.token_stream(term.as_str());
+ let mut it = tantivy::tokenizer::TokenStream::iter(&mut stream);
- while let Some(term) = stream.next() {
+ while let Some(term) = it.next() {
term_freq += inverted_index
.doc_freq(&tantivy::Term::from_field_text(field, term.text.as_str()))?;
}
diff --git a/crates/core/src/entity_index/mod.rs b/crates/core/src/entity_index/mod.rs
index c27022dd..88187572 100644
--- a/crates/core/src/entity_index/mod.rs
+++ b/crates/core/src/entity_index/mod.rs
@@ -23,6 +23,7 @@ use std::{
use base64::{prelude::BASE64_STANDARD as BASE64_ENGINE, Engine};
+use lending_iter::LendingIterator;
use tantivy::{
collector::TopDocs,
query::{BooleanQuery, BoostQuery, MoreLikeThisQuery, Occur, QueryClone, TermQuery},
@@ -272,7 +273,8 @@ impl EntityIndex {
let mut term_queries = Vec::new();
let mut tokenizer = DefaultTokenizer::default();
let mut stream = tokenizer.token_stream(query);
- while let Some(token) = stream.next() {
+ let mut it = tantivy::tokenizer::TokenStream::iter(&mut stream);
+ while let Some(token) = it.next() {
if self.stopwords.contains(&token.text) {
continue;
}
diff --git a/crates/core/src/feed/index.rs b/crates/core/src/feed/index.rs
index 9593d23b..6b0ca1e8 100644
--- a/crates/core/src/feed/index.rs
+++ b/crates/core/src/feed/index.rs
@@ -25,6 +25,7 @@ use crate::{
};
use anyhow::Result;
use hashbrown::HashSet;
+use lending_iter::LendingIterator;
use tantivy::{
merge_policy::NoMergePolicy,
query::{PhraseQuery, TermQuery},
@@ -131,8 +132,9 @@ impl FeedIndex {
let tv_field = self.schema.get_field("url").unwrap();
let mut stream = tokenizer.token_stream(query);
+ let mut it = tantivy::tokenizer::TokenStream::iter(&mut stream);
- while let Some(token) = stream.next() {
+ while let Some(token) = it.next() {
res.push(tantivy::Term::from_field_text(tv_field, &token.text));
}
diff --git a/crates/core/src/inverted_index/key_phrase.rs b/crates/core/src/inverted_index/key_phrase.rs
index 1d6c29b5..5b5ad38e 100644
--- a/crates/core/src/inverted_index/key_phrase.rs
+++ b/crates/core/src/inverted_index/key_phrase.rs
@@ -15,6 +15,7 @@
// along with this program. If not, see <https://www.gnu.org/licenses/>.
use itertools::Itertools;
+use lending_iter::LendingIterator;
use std::str;
use std::{cmp::Reverse, collections::HashMap};
diff --git a/crates/core/src/query/pattern_query/mod.rs b/crates/core/src/query/pattern_query/mod.rs
index d9139b01..75f76659 100644
--- a/crates/core/src/query/pattern_query/mod.rs
+++ b/crates/core/src/query/pattern_query/mod.rs
@@ -17,6 +17,7 @@
mod scorer;
mod weight;
+use lending_iter::LendingIterator;
use optics::PatternPart;
use tantivy::tokenizer::Tokenizer;
@@ -101,8 +102,9 @@ impl PatternQuery {
.unwrap()
.tokenizer(Some(&whatlang::Lang::Eng));
let mut stream = tokenizer.token_stream(text);
+ let mut it = tantivy::tokenizer::TokenStream::iter(&mut stream);
- while let Some(token) = stream.next() {
+ while let Some(token) = it.next() {
new_patterns.push(PatternPart::Raw(token.text.clone()));
let term = tantivy::Term::from_field_text(tv_field, &token.text);
raw_terms.push(term);
diff --git a/crates/core/src/ranking/signal/computer/mod.rs b/crates/core/src/ranking/signal/computer/mod.rs
index 97928ffd..9a38fc04 100644
--- a/crates/core/src/ranking/signal/computer/mod.rs
+++ b/crates/core/src/ranking/signal/computer/mod.rs
@@ -26,6 +26,7 @@ use std::cell::RefCell;
use std::sync::Arc;
use itertools::Itertools;
+use lending_iter::LendingIterator;
use tantivy::fieldnorm::FieldNormReader;
use tantivy::postings::SegmentPostings;
use tantivy::query::{Query as _, Scorer};
@@ -266,8 +267,9 @@ impl SignalComputer {
let mut terms = Vec::new();
let mut tokenizer = text_field.tokenizer(query.lang.as_ref());
let mut stream = tokenizer.token_stream(&simple_query);
+ let mut it = tantivy::tokenizer::TokenStream::iter(&mut stream);
- while let Some(token) = stream.next() {
+ while let Some(token) = it.next() {
let term = tantivy::Term::from_field_text(tv_field, &token.text);
terms.push(term);
}
diff --git a/crates/core/src/simhash.rs b/crates/core/src/simhash.rs
index 20e15561..c5b0c423 100644
--- a/crates/core/src/simhash.rs
+++ b/crates/core/src/simhash.rs
@@ -19,6 +19,8 @@ use std::{
hash::{Hash, Hasher},
};
+use lending_iter::LendingIterator;
+
use crate::tokenizer::FieldTokenizer;
pub type HashType = u64;
@@ -33,10 +35,11 @@ pub fn hash(text: &str) -> HashType {
let mut tokenizer = FieldTokenizer::default();
let mut stream = tantivy::tokenizer::Tokenizer::token_stream(&mut tokenizer, text);
+ let mut it = tantivy::tokenizer::TokenStream::iter(&mut stream);
let mut v = [0i64; HashType::BITS as usize];
- while let Some(token) = stream.next() {
+ while let Some(token) = it.next() {
let h = hash_token(token);
for (i, item) in v.iter_mut().enumerate() {
diff --git a/crates/core/src/snippet.rs b/crates/core/src/snippet.rs
index 9ce581f3..0615f07e 100644
--- a/crates/core/src/snippet.rs
+++ b/crates/core/src/snippet.rs
@@ -25,6 +25,7 @@ use crate::tokenizer::fields::{
use crate::web_spell::sentence_ranges;
use crate::webpage::region::Region;
use hashbrown::{HashMap, HashSet};
+use lending_iter::LendingIterator;
use utoipa::ToSchema;
use itertools::Itertools;
@@ -90,7 +91,8 @@ impl SnippetBuilder {
] {
let mut stream =
tantivy::tokenizer::Tokenizer::token_stream(&mut tokenizer, &self.fragment);
- while let Some(tok) = stream.next() {
+ let mut it = tantivy::tokenizer::TokenStream::iter(&mut stream);
+ while let Some(tok) = it.next() {
if terms.contains(&tok.text) {
self.highlights.push(tok.offset_from..tok.offset_to);
}
@@ -161,7 +163,8 @@ fn passages(
{
let mut stream =
tantivy::tokenizer::Tokenizer::token_stream(&mut tokenizer, &sentence);
- while let Some(tok) = stream.next() {
+ let mut it = tantivy::tokenizer::TokenStream::iter(&mut stream);
+ while let Some(tok) = it.next() {
*doc_terms.entry(tok.text.clone()).or_insert(0) += 1;
}
}
@@ -230,7 +233,8 @@ fn snippet_string_builder(
let mut stream = tantivy::tokenizer::Tokenizer::token_stream(&mut tokenizer, term);
let mut res = Vec::new();
- while let Some(tok) = stream.next() {
+ let mut it = tantivy::tokenizer::TokenStream::iter(&mut stream);
+ while let Some(tok) = it.next() {
res.push(tok.text.clone());
}
diff --git a/crates/core/src/tokenizer/fields/bigram.rs b/crates/core/src/tokenizer/fields/bigram.rs
index 548ff74e..aa63bf37 100644
--- a/crates/core/src/tokenizer/fields/bigram.rs
+++ b/crates/core/src/tokenizer/fields/bigram.rs
@@ -48,6 +48,7 @@ impl tantivy::tokenizer::Tokenizer for BigramTokenizer {
#[cfg(test)]
mod tests {
+ use lending_iter::LendingIterator;
use tantivy::tokenizer::Tokenizer;
use super::*;
@@ -56,7 +57,8 @@ mod tests {
let mut tokenizer = BigramTokenizer::default();
let mut stream = tokenizer.token_stream(s);
- while let Some(token) = stream.next() {
+ let mut it = tantivy::tokenizer::TokenStream::iter(&mut stream);
+ while let Some(token) = it.next() {
res.push(token.text.clone());
}
diff --git a/crates/core/src/tokenizer/fields/default.rs b/crates/core/src/tokenizer/fields/default.rs
index 1d012cab..50328620 100644
--- a/crates/core/src/tokenizer/fields/default.rs
+++ b/crates/core/src/tokenizer/fields/default.rs
@@ -122,6 +122,7 @@ impl<'a> NormalTokenStream<'a> {
#[cfg(test)]
mod tests {
use super::*;
+ use lending_iter::LendingIterator;
use proptest::prelude::*;
use tantivy::tokenizer::Tokenizer as _;
@@ -129,8 +130,9 @@ mod tests {
let mut res = Vec::new();
let mut tokenizer = DefaultTokenizer::default();
let mut stream = tokenizer.token_stream(s);
+ let mut it = tantivy::tokenizer::TokenStream::iter(&mut stream);
- while let Some(token) = stream.next() {
+ while let Some(token) = it.next() {
res.push(token.text.clone());
}
diff --git a/crates/core/src/tokenizer/fields/identity.rs b/crates/core/src/tokenizer/fields/identity.rs
index 957b28e6..8a642363 100644
--- a/crates/core/src/tokenizer/fields/identity.rs
+++ b/crates/core/src/tokenizer/fields/identity.rs
@@ -73,14 +73,16 @@ impl tantivy::tokenizer::TokenStream for IdentityTokenStream {
#[cfg(test)]
mod tests {
use super::*;
+ use lending_iter::LendingIterator;
use tantivy::tokenizer::Tokenizer as _;
fn tokenize_identity(s: &str) -> Vec<String> {
let mut res = Vec::new();
let mut tokenizer = Identity {};
let mut stream = tokenizer.token_stream(s);
+ let mut it = tantivy::tokenizer::TokenStream::iter(&mut stream);
- while let Some(token) = stream.next() {
+ while let Some(token) = it.next() {
res.push(token.text.clone());
}
diff --git a/crates/core/src/tokenizer/fields/json.rs b/crates/core/src/tokenizer/fields/json.rs
index 5dd551d1..86c612b8 100644
--- a/crates/core/src/tokenizer/fields/json.rs
+++ b/crates/core/src/tokenizer/fields/json.rs
@@ -202,6 +202,7 @@ impl<'a> tantivy::tokenizer::TokenStream for JsonFieldTokenStream<'a> {
#[cfg(test)]
mod tests {
+ use lending_iter::LendingIterator;
use tantivy::tokenizer::Tokenizer;
use super::*;
@@ -211,7 +212,8 @@ mod tests {
let mut tokenizer = JsonField;
let mut stream = tokenizer.token_stream(s);
- while let Some(token) = stream.next() {
+ let mut it = tantivy::tokenizer::TokenStream::iter(&mut stream);
+ while let Some(token) = it.next() {
res.push(token.text.clone());
}
diff --git a/crates/core/src/tokenizer/fields/split_newlines.rs b/crates/core/src/tokenizer/fields/split_newlines.rs
index 5d01008d..d37844e6 100644
--- a/crates/core/src/tokenizer/fields/split_newlines.rs
+++ b/crates/core/src/tokenizer/fields/split_newlines.rs
@@ -108,14 +108,16 @@ impl<'a> NewlineTokenStream<'a> {
#[cfg(test)]
mod tests {
use super::*;
+ use lending_iter::LendingIterator;
use tantivy::tokenizer::Tokenizer as _;
fn tokenize_newline(s: &str) -> Vec<String> {
let mut res = Vec::new();
let mut tokenizer = NewlineTokenizer::default();
let mut stream = tokenizer.token_stream(s);
+ let mut it = tantivy::tokenizer::TokenStream::iter(&mut stream);
- while let Some(token) = stream.next() {
+ while let Some(token) = it.next() {
res.push(token.text.clone());
}
diff --git a/crates/core/src/tokenizer/fields/trigram.rs b/crates/core/src/tokenizer/fields/trigram.rs
index 54fc0884..024742d8 100644
--- a/crates/core/src/tokenizer/fields/trigram.rs
+++ b/crates/core/src/tokenizer/fields/trigram.rs
@@ -47,6 +47,7 @@ impl tantivy::tokenizer::Tokenizer for TrigramTokenizer {
#[cfg(test)]
mod tests {
+ use lending_iter::LendingIterator;
use tantivy::tokenizer::Tokenizer;
use super::*;
@@ -56,8 +57,9 @@ mod tests {
let mut tokenizer = TrigramTokenizer::default();
let mut stream = tokenizer.token_stream(s);
+ let mut it = tantivy::tokenizer::TokenStream::iter(&mut stream);
- while let Some(token) = stream.next() {
+ while let Some(token) = it.next() {
res.push(token.text.clone());
}
diff --git a/crates/core/src/tokenizer/fields/url.rs b/crates/core/src/tokenizer/fields/url.rs
index d127fa15..86070059 100644
--- a/crates/core/src/tokenizer/fields/url.rs
+++ b/crates/core/src/tokenizer/fields/url.rs
@@ -196,6 +196,7 @@ impl tantivy::tokenizer::TokenStream for SiteOperatorUrlTokenStream {
#[cfg(test)]
mod tests {
use super::*;
+ use lending_iter::LendingIterator;
use proptest::prelude::*;
use tantivy::tokenizer::Tokenizer as _;
@@ -203,8 +204,9 @@ mod tests {
let mut res = Vec::new();
let mut tokenizer = UrlTokenizer;
let mut stream = tokenizer.token_stream(s);
+ let mut it = tantivy::tokenizer::TokenStream::iter(&mut stream);
- while let Some(token) = stream.next() {
+ while let Some(token) = it.next() {
res.push(token.text.clone());
}
diff --git a/crates/core/src/webpage/html/fn_cache.rs b/crates/core/src/webpage/html/fn_cache.rs
index daf66c59..3bb35abd 100644
--- a/crates/core/src/webpage/html/fn_cache.rs
+++ b/crates/core/src/webpage/html/fn_cache.rs
@@ -15,6 +15,7 @@
// along with this program. If not, see <https://www.gnu.org/licenses/>.
use crate::{webpage::schema_org, Result};
+use lending_iter::LendingIterator;
use tantivy::tokenizer::PreTokenizedString;
use super::{find_recipe_first_ingredient_tag_id, Html};
@@ -112,8 +113,9 @@ impl<'a> FnCache<'a> {
{
let mut stream = f.token_stream();
+ let mut it = tantivy::tokenizer::TokenStream::iter(&mut stream);
- while let Some(token) = stream.next() {
+ while let Some(token) = it.next() {
tokens.push(token.clone());
}
}
diff --git a/crates/core/src/webpage/html/into_tantivy.rs b/crates/core/src/webpage/html/into_tantivy.rs
index f289a5af..f3a7100b 100644
--- a/crates/core/src/webpage/html/into_tantivy.rs
+++ b/crates/core/src/webpage/html/into_tantivy.rs
@@ -28,6 +28,7 @@ use crate::{
Error, Result,
};
use bloom::split_u128;
+use lending_iter::LendingIterator;
use tantivy::{
tokenizer::{PreTokenizedString, Tokenizer},
TantivyDocument,
@@ -122,7 +123,8 @@ impl Html {
{
let mut stream = tokenizer.token_stream(&text);
- while let Some(token) = stream.next() {
+ let mut it = tantivy::tokenizer::TokenStream::iter(&mut stream);
+ while let Some(token) = it.next() {
tokens.push(token.clone());
}
}
diff --git a/crates/lending-iter/Cargo.toml b/crates/lending-iter/Cargo.toml
new file mode 100644
index 00000000..44ce5405
--- /dev/null
+++ b/crates/lending-iter/Cargo.toml
@@ -0,0 +1,9 @@
+[package]
+edition = "2021"
+license = "AGPL-3.0"
+name = "lending-iter"
+version = "0.1.0"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
diff --git a/crates/lending-iter/src/adapters/cloned.rs b/crates/lending-iter/src/adapters/cloned.rs
new file mode 100644
index 00000000..449a8224
--- /dev/null
+++ b/crates/lending-iter/src/adapters/cloned.rs
@@ -0,0 +1,48 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2024 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use std::ops::Deref;
+
+use crate::LendingIterator;
+
+pub struct Cloned<I> {
+ iter: I,
+}
+
+impl<I> Cloned<I> {
+ pub fn new(iter: I) -> Self {
+ Self { iter }
+ }
+}
+
+impl<I> From<I> for Cloned<I> {
+ fn from(iter: I) -> Self {
+ Self::new(iter)
+ }
+}
+
+impl<I, T> Iterator for Cloned<I>
+where
+ I: LendingIterator,
+ for<'a> I::Item<'a>: Deref<Target = T>,
+ T: Clone,
+{
+ type Item = T;
+
+ fn next(&mut self) -> Option<Self::Item> {
+ self.iter.next().map(|item| item.deref().clone())
+ }
+}
diff --git a/crates/lending-iter/src/adapters/enumerate.rs b/crates/lending-iter/src/adapters/enumerate.rs
new file mode 100644
index 00000000..5b321f8b
--- /dev/null
+++ b/crates/lending-iter/src/adapters/enumerate.rs
@@ -0,0 +1,51 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2024 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use crate::LendingIterator;
+
+pub struct Enumerate<I> {
+ iter: I,
+ index: usize,
+}
+
+impl<I> Enumerate<I> {
+ pub fn new(iter: I) -> Self {
+ Self { iter, index: 0 }
+ }
+}
+
+impl<I> From<I> for Enumerate<I> {
+ fn from(iter: I) -> Self {
+ Self::new(iter)
+ }
+}
+
+impl<I> LendingIterator for Enumerate<I>
+where
+ I: LendingIterator,
+{
+ type Item<'a> = (usize, I::Item<'a>)
+ where
+ I: 'a;
+
+ fn next(&mut self) -> Option<Self::Item<'_>> {
+ self.iter.next().map(|item| {
+ let index = self.index;
+ self.index += 1;
+ (index, item)
+ })
+ }
+}
diff --git a/crates/lending-iter/src/adapters/filter.rs b/crates/lending-iter/src/adapters/filter.rs
new file mode 100644
index 00000000..9eea0bc2
--- /dev/null
+++ b/crates/lending-iter/src/adapters/filter.rs
@@ -0,0 +1,57 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2024 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use crate::LendingIterator;
+
+pub struct Filter<I, P> {
+ iter: I,
+ predicate: P,
+}
+
+impl<I, P> Filter<I, P> {
+ pub fn new(iter: I, predicate: P) -> Self {
+ Self { iter, predicate }
+ }
+}
+
+impl<I, P> LendingIterator for Filter<I, P>
+where
+ I: LendingIterator,
+ for<'a> P: FnMut(&I::Item<'a>) -> bool,
+{
+ type Item<'a> = I::Item<'a>
+ where
+ Self: 'a;
+
+ fn next(&mut self) -> Option<Self::Item<'_>> {
+ // SAFETY (polonius):
+ // - `self` is a mutable reference, so it is the only reference to this instance.
+ // - we have a mutable reference to `self.iter`, so it is the only reference to this field.
+ // - the reference to `item` is only returned in previous loop iff. the predicate is true.
+ // Otherwise, the reference is dropped before the next iteration.
+
+ loop {
+ let self_ = unsafe { &mut *(self as *mut Self) };
+ if let Some(item) = self_.iter.next() {
+ if (self_.predicate)(&item) {
+ return Some(item);
+ }
+ } else {
+ return None;
+ }
+ }
+ }
+}
diff --git a/crates/lending-iter/src/adapters/flatten.rs b/crates/lending-iter/src/adapters/flatten.rs
new file mode 100644
index 00000000..f07d38ae
--- /dev/null
+++ b/crates/lending-iter/src/adapters/flatten.rs
@@ -0,0 +1,72 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2024 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use crate::LendingIterator;
+
+pub struct Flatten<'a, I>
+where
+ I: LendingIterator,
+ I::Item<'a>: LendingIterator,
+ Self: 'a,
+{
+ iter: I,
+ current: Option<I::Item<'a>>,
+}
+
+impl<'a, I> Flatten<'a, I>
+where
+ I: LendingIterator,
+ I::Item<'a>: LendingIterator,
+{
+ pub fn new(iter: I) -> Self {
+ Self {
+ iter,
+ current: None,
+ }
+ }
+}
+
+impl<'a, I> LendingIterator for Flatten<'a, I>
+where
+ I: LendingIterator,
+ I::Item<'a>: LendingIterator,
+ Self: 'a,
+{
+ type Item<'b> = <I::Item<'a> as LendingIterator>::Item<'b>
+ where
+ Self: 'b;
+
+ fn next(&mut self) -> Option<Self::Item<'_>> {
+ // SAFETY (polonius):
+ // - `self` is a mutable reference, so it is the only reference to this instance.
+ // - we have a mutable reference to `self.current`, so it is the only reference to this field.
+ // - `self.iter` is not accessed while `self.current` is borrowed.
+
+ loop {
+ let self_ = unsafe { &mut *(self as *mut Self) };
+
+ if let Some(current) = unsafe { &mut *(&mut self_.current as *mut Option<I::Item<'a>>) }
+ {
+ if let Some(item) = current.next() {
+ return Some(item);
+ }
+ }
+
+ self_.current = self_.iter.next();
+ self_.current.as_ref()?;
+ }
+ }
+}
diff --git a/crates/lending-iter/src/adapters/map.rs b/crates/lending-iter/src/adapters/map.rs
new file mode 100644
index 00000000..ec775b64
--- /dev/null
+++ b/crates/lending-iter/src/adapters/map.rs
@@ -0,0 +1,42 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2024 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use crate::LendingIterator;
+
+pub struct Map<I, F> {
+ iter: I,
+ f: F,
+}
+
+impl<I, F> Map<I, F> {
+ pub fn new(iter: I, f: F) -> Self {
+ Self { iter, f }
+ }
+}
+
+impl<I, F, T> LendingIterator for Map<I, F>
+where
+ I: LendingIterator,
+ F: for<'a> FnMut(I::Item<'a>) -> T,
+{
+ type Item<'a> = T
+ where
+ Self: 'a;
+
+ fn next(&mut self) -> Option<Self::Item<'_>> {
+ self.iter.next().map(&mut self.f)
+ }
+}
diff --git a/crates/lending-iter/src/adapters/mod.rs b/crates/lending-iter/src/adapters/mod.rs
new file mode 100644
index 00000000..d9cab115
--- /dev/null
+++ b/crates/lending-iter/src/adapters/mod.rs
@@ -0,0 +1,27 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2024 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+mod cloned;
+mod enumerate;
+mod filter;
+mod flatten;
+mod map;
+
+pub use cloned::Cloned;
+pub use enumerate::Enumerate;
+pub use filter::Filter;
+pub use flatten::Flatten;
+pub use map::Map;
diff --git a/crates/lending-iter/src/lib.rs b/crates/lending-iter/src/lib.rs
new file mode 100644
index 00000000..f1380e24
--- /dev/null
+++ b/crates/lending-iter/src/lib.rs
@@ -0,0 +1,274 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2024 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+mod adapters;
+
+pub trait LendingIterator {
+ type Item<'a>
+ where
+ Self: 'a;
+
+ fn next(&mut self) -> Option<Self::Item<'_>>;
+
+ fn enumerate(self) -> adapters::Enumerate<Self>
+ where
+ Self: Sized,
+ {
+ adapters::Enumerate::new(self)
+ }
+
+ fn cloned<'a, T>(self) -> adapters::Cloned<Self>
+ where
+ Self: Sized,
+ for<'b> Self::Item<'b>: std::ops::Deref<Target = T>,
+ T: Clone,
+ {
+ adapters::Cloned::new(self)
+ }
+
+ fn fold<B, F>(self, init: B, f: F) -> B
+ where
+ Self: Sized,
+ F: FnMut(B, Self::Item<'_>) -> B,
+ {
+ let mut f = f;
+ let mut acc = init;
+ let mut iter = self;
+
+ while let Some(item) = iter.next() {
+ acc = f(acc, item);
+ }
+
+ acc
+ }
+
+ fn count(self) -> usize
+ where
+ Self: Sized,
+ {
+ self.fold(0, |acc, _| acc + 1)
+ }
+
+ fn map<B, F>(self, f: F) -> adapters::Map<Self, F>
+ where
+ Self: Sized,
+ F: FnMut(Self::Item<'_>) -> B,
+ {
+ adapters::Map::new(self, f)
+ }
+
+ fn filter<F>(self, f: F) -> adapters::Filter<Self, F>
+ where
+ Self: Sized,
+ F: FnMut(&Self::Item<'_>) -> bool,
+ {
+ adapters::Filter::new(self, f)
+ }
+
+ fn flatten<'a>(self) -> adapters::Flatten<'a, Self>
+ where
+ Self: Sized,
+ Self::Item<'a>: LendingIterator,
+ {
+ adapters::Flatten::new(self)
+ }
+}
+
+impl<'a, I> LendingIterator for &'a mut I
+where
+ I: LendingIterator,
+{
+ type Item<'b> = I::Item<'b>
+ where
+ I: 'b,
+ 'a: 'b;
+
+ fn next(&mut self) -> Option<Self::Item<'_>> {
+ (*self).next()
+ }
+}
+
+pub trait IntoLendingIterator: Sized {
+ fn lending(self) -> IntoLending<Self>;
+}
+
+pub struct IntoLending<I> {
+ iter: I,
+}
+
+impl<I> LendingIterator for IntoLending<I>
+where
+ I: Iterator,
+{
+ type Item<'a> = I::Item
+ where
+ Self: 'a;
+
+ fn next(&mut self) -> Option<Self::Item<'_>> {
+ self.iter.next()
+ }
+}
+
+impl<I> IntoLendingIterator for I
+where
+ I: Iterator,
+{
+ fn lending(self) -> IntoLending<Self> {
+ IntoLending { iter: self }
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ struct Iter {
+ items: Vec<i32>,
+ index: usize,
+ }
+
+ impl Iter {
+ fn new(items: Vec<i32>) -> Self {
+ Self { items, index: 0 }
+ }
+ }
+
+ impl LendingIterator for Iter {
+ type Item<'a> = &'a i32;
+
+ fn next(&mut self) -> Option<Self::Item<'_>> {
+ if self.index < self.items.len() {
+ let item = &self.items[self.index];
+ self.index += 1;
+ Some(item)
+ } else {
+ None
+ }
+ }
+ }
+
+ impl From<Vec<i32>> for Iter {
+ fn from(items: Vec<i32>) -> Self {
+ Self::new(items)
+ }
+ }
+
+ #[test]
+ fn test_lending_iterator() {
+ let items = vec![1, 2, 3];
+ let mut iter = Iter::new(items);
+ assert_eq!(iter.next(), Some(&1));
+ assert_eq!(iter.next(), Some(&2));
+ assert_eq!(iter.next(), Some(&3));
+ assert_eq!(iter.next(), None);
+ }
+
+ #[test]
+ fn test_enumerate() {
+ let items = vec![1, 2, 3];
+ let mut iter = Iter::new(items).enumerate();
+ assert_eq!(iter.next(), Some((0, &1)));
+ assert_eq!(iter.next(), Some((1, &2)));
+ assert_eq!(iter.next(), Some((2, &3)));
+ assert_eq!(iter.next(), None);
+ }
+
+ #[test]
+ fn test_cloned() {
+ let items = vec![1, 2, 3];
+ let mut iter = Iter::new(items).cloned();
+ assert_eq!(iter.next(), Some(1));
+ assert_eq!(iter.next(), Some(2));
+ assert_eq!(iter.next(), Some(3));
+ assert_eq!(iter.next(), None);
+ }
+
+ #[test]
+ fn test_fold() {
+ let items = vec![1, 2, 3];
+ let iter = Iter::new(items);
+ let sum = iter.fold(0, |acc, item| acc + item);
+ assert_eq!(sum, 6);
+ }
+
+ #[test]
+ fn test_count() {
+ let items = vec![1, 2, 3];
+ let iter = Iter::new(items);
+ let count = iter.count();
+ assert_eq!(count, 3);
+ }
+
+ #[test]
+ fn test_map() {
+ let items = vec![1, 2, 3];
+ let mut iter = Iter::new(items).map(|item| item * 2);
+ assert_eq!(iter.next(), Some(2));
+ assert_eq!(iter.next(), Some(4));
+ assert_eq!(iter.next(), Some(6));
+ assert_eq!(iter.next(), None);
+ }
+
+ #[test]
+ fn test_filter() {
+ let items = vec![1, 2, 3];
+ let mut iter = Iter::new(items).filter(|item| **item % 2 == 0);
+ assert_eq!(iter.next(), Some(&2));
+ assert_eq!(iter.next(), None);
+ }
+
+ struct NestedIter {
+ items: Vec<Iter>,
+ index: usize,
+ }
+
+ impl NestedIter {
+ fn new(items: Vec<Iter>) -> Self {
+ Self { items, index: 0 }
+ }
+ }
+
+ impl LendingIterator for NestedIter {
+ type Item<'a> = &'a mut Iter;
+
+ fn next(&mut self) -> Option<Self::Item<'_>> {
+ if self.index < self.items.len() {
+ let item = &mut self.items[self.index];
+ self.index += 1;
+ Some(item)
+ } else {
+ None
+ }
+ }
+ }
+
+ #[test]
+ fn test_flatten() {
+ let items = vec![
+ Iter::new(vec![1, 2]),
+ Iter::new(vec![3, 4]),
+ Iter::new(vec![5, 6]),
+ ];
+ let mut iter = NestedIter::new(items).flatten();
+ assert_eq!(iter.next(), Some(&1));
+ assert_eq!(iter.next(), Some(&2));
+ assert_eq!(iter.next(), Some(&3));
+ assert_eq!(iter.next(), Some(&4));
+ assert_eq!(iter.next(), Some(&5));
+ assert_eq!(iter.next(), Some(&6));
+ assert_eq!(iter.next(), None);
+ }
+}
diff --git a/crates/tantivy/Cargo.toml b/crates/tantivy/Cargo.toml
index 27d47866..07f1498f 100644
--- a/crates/tantivy/Cargo.toml
+++ b/crates/tantivy/Cargo.toml
@@ -31,6 +31,7 @@ fst.workspace = true
futures-util = {workspace = true, optional = true}
htmlescape.workspace = true
itertools.workspace = true
+lending-iter.workspace = true
levenshtein_automata.workspace = true
log.workspace = true
lru.workspace = true
diff --git a/crates/tantivy/src/index/inverted_index_reader.rs b/crates/tantivy/src/index/inverted_index_reader.rs
index 11fca48b..543f6dc0 100644
--- a/crates/tantivy/src/index/inverted_index_reader.rs
+++ b/crates/tantivy/src/index/inverted_index_reader.rs
@@ -3,6 +3,7 @@ use std::io;
use crate::common::json_path_writer::JSON_END_OF_PATH;
use crate::common::BinarySerializable;
use fnv::FnvHashSet;
+use lending_iter::LendingIterator;
use crate::directory::FileSlice;
use crate::positions::PositionReader;
diff --git a/crates/tantivy/src/snippet/mod.rs b/crates/tantivy/src/snippet/mod.rs
index 16d54953..38059a1a 100644
--- a/crates/tantivy/src/snippet/mod.rs
+++ b/crates/tantivy/src/snippet/mod.rs
@@ -19,6 +19,7 @@ use std::collections::{BTreeMap, BTreeSet};
use std::ops::Range;
use htmlescape::encode_minimal;
+use lending_iter::LendingIterator;
use crate::query::Query;
use crate::schema::document::{Document, Value};
@@ -168,9 +169,10 @@ fn search_fragments(
max_num_chars: usize,
) -> Vec<FragmentCandidate> {
let mut token_stream = tokenizer.token_stream(text);
+ let mut it = crate::tokenizer::TokenStream::iter(&mut token_stream);
let mut fragment = FragmentCandidate::new(0);
let mut fragments: Vec<FragmentCandidate> = vec![];
- while let Some(next) = token_stream.next() {
+ while let Some(next) = it.next() {
if (next.offset_to - fragment.start_offset) > max_num_chars {
if fragment.score > 0.0 {
fragments.push(fragment)
diff --git a/crates/tantivy/src/termdict/fst_termdict/streamer.rs b/crates/tantivy/src/termdict/fst_termdict/streamer.rs
index e13f30e6..a6093009 100644
--- a/crates/tantivy/src/termdict/fst_termdict/streamer.rs
+++ b/crates/tantivy/src/termdict/fst_termdict/streamer.rs
@@ -3,6 +3,7 @@ use std::io;
use fst::automaton::AlwaysMatch;
use fst::map::{Stream, StreamBuilder};
use fst::{Automaton, IntoStreamer, Streamer};
+use lending_iter::LendingIterator;
use super::TermDictionary;
use crate::postings::TermInfo;
@@ -132,10 +133,17 @@ where
pub fn value(&self) -> &TermInfo {
&self.current_value
}
+}
+
+impl<'a, A> LendingIterator for TermStreamer<'a, A>
+where
+ A: Automaton,
+{
+ type Item<'b> = (&'b [u8], &'b TermInfo)
+ where
+ Self: 'b;
- /// Return the next `(key, value)` pair.
- #[allow(clippy::should_implement_trait)]
- pub fn next(&mut self) -> Option<(&[u8], &TermInfo)> {
+ fn next(&mut self) -> Option<Self::Item<'_>> {
if self.advance() {
Some((self.key(), self.value()))
} else {
diff --git a/crates/tantivy/src/termdict/tests.rs b/crates/tantivy/src/termdict/tests.rs
index 3aec04e7..47466dec 100644
--- a/crates/tantivy/src/termdict/tests.rs
+++ b/crates/tantivy/src/termdict/tests.rs
@@ -1,6 +1,8 @@
use std::path::PathBuf;
use std::{io, str};
+use lending_iter::LendingIterator;
+
use super::{TermDictionary, TermDictionaryBuilder, TermStreamer};
use crate::directory::{Directory, FileSlice, RamDirectory, TerminatingWrite};
use crate::postings::TermInfo;
diff --git a/crates/tantivy/src/tokenizer/alphanum_only.rs b/crates/tantivy/src/tokenizer/alphanum_only.rs
index b40731fd..f5e6ffa3 100644
--- a/crates/tantivy/src/tokenizer/alphanum_only.rs
+++ b/crates/tantivy/src/tokenizer/alphanum_only.rs
@@ -7,19 +7,21 @@
//! .build();
//!
//! let mut stream = tokenizer.token_stream("hello there");
+//! let mut it = TokenStream::iter(&mut stream);
//! // is none because the raw filter emits one token that
//! // contains a space
-//! assert!(stream.next().is_none());
+//! assert!(it.next().is_none());
//!
//! let mut tokenizer = TextAnalyzer::builder(SimpleTokenizer::default())
//! .filter(AlphaNumOnlyFilter)
//! .build();
//!
//! let mut stream = tokenizer.token_stream("hello there 💣");
-//! assert!(stream.next().is_some());
-//! assert!(stream.next().is_some());
+//! let mut it = TokenStream::iter(&mut stream);
+//! assert!(it.next().is_some());
+//! assert!(it.next().is_some());
//! // the "emoji" is dropped because its not an alphanum
-//! assert!(stream.next().is_none());
+//! assert!(it.next().is_none());
//! ```
use super::{Token, TokenFilter, TokenStream, Tokenizer};
diff --git a/crates/tantivy/src/tokenizer/mod.rs b/crates/tantivy/src/tokenizer/mod.rs
index b2d30319..a77ee883 100644
--- a/crates/tantivy/src/tokenizer/mod.rs
+++ b/crates/tantivy/src/tokenizer/mod.rs
@@ -155,6 +155,7 @@ pub use self::tokenized_string::{PreTokenizedStream, PreTokenizedString};
pub use self::tokenizer::{TextAnalyzer, TextAnalyzerBuilder};
pub use self::tokenizer_manager::TokenizerManager;
pub use self::whitespace_tokenizer::WhitespaceTokenizer;
+pub use lending_iter::LendingIterator;
/// Maximum authorized len (in bytes) for a token.
///
diff --git a/crates/tantivy/src/tokenizer/ngram_tokenizer.rs b/crates/tantivy/src/tokenizer/ngram_tokenizer.rs
index e98c5ff6..691a6ed0 100644
--- a/crates/tantivy/src/tokenizer/ngram_tokenizer.rs
+++ b/crates/tantivy/src/tokenizer/ngram_tokenizer.rs
@@ -36,49 +36,50 @@ use crate::TantivyError;
///
/// let mut tokenizer = NgramTokenizer::new(2, 3, false).unwrap();
/// let mut stream = tokenizer.token_stream("hello");
+/// let mut it = TokenStream::iter(&mut stream);
/// {
-/// let token = stream.next().unwrap();
+/// let token = it.next().unwrap();
/// assert_eq!(token.text, "he");
/// assert_eq!(token.offset_from, 0);
/// assert_eq!(token.offset_to, 2);
/// }
/// {
-/// let token = stream.next().unwrap();
+/// let token = it.next().unwrap();
/// assert_eq!(token.text, "hel");
/// assert_eq!(token.offset_from, 0);
/// assert_eq!(token.offset_to, 3);
/// }
/// {
-/// let token = stream.next().unwrap();
+/// let token = it.next().unwrap();
/// assert_eq!(token.text, "el");
/// assert_eq!(token.offset_from, 1);
/// assert_eq!(token.offset_to, 3);
/// }
/// {
-/// let token = stream.next().unwrap();
+/// let token = it.next().unwrap();
/// assert_eq!(token.text, "ell");
/// assert_eq!(token.offset_from, 1);
/// assert_eq!(token.offset_to, 4);
/// }
/// {
-/// let token = stream.next().unwrap();
+/// let token = it.next().unwrap();
/// assert_eq!(token.text, "ll");
/// assert_eq!(token.offset_from, 2);
/// assert_eq!(token.offset_to, 4);
/// }
/// {
-/// let token = stream.next().unwrap();
+/// let token = it.next().unwrap();
/// assert_eq!(token.text, "llo");
/// assert_eq!(token.offset_from, 2);
/// assert_eq!(token.offset_to, 5);
/// }
/// {
-/// let token = stream.next().unwrap();
+/// let token = it.next().unwrap();
/// assert_eq!(token.text, "lo");
/// assert_eq!(token.offset_from, 3);
/// assert_eq!(token.offset_to, 5);
/// }
-/// assert!(stream.next().is_none());
+/// assert!(it.next().is_none());
/// ```
#[derive(Clone, Debug)]
pub struct NgramTokenizer {
@@ -205,7 +206,8 @@ struct StutteringIterator<T> {
}
impl<T> StutteringIterator<T>
-where T: Iterator<Item = usize>
+where
+ T: Iterator<Item = usize>,
{
pub fn new(mut underlying: T, min_gram: usize, max_gram: usize) -> StutteringIterator<T> {
assert!(min_gram > 0);
@@ -234,7 +236,8 @@ where T: Iterator<Item = usize>
}
impl<T> Iterator for StutteringIterator<T>
-where T: Iterator<Item = usize>
+where
+ T: Iterator<Item = usize>,
{
type Item = (usize, usize);
diff --git a/crates/tantivy/src/tokenizer/regex_tokenizer.rs b/crates/tantivy/src/tokenizer/regex_tokenizer.rs
index f9a10ad2..f4a608b3 100644
--- a/crates/tantivy/src/tokenizer/regex_tokenizer.rs
+++ b/crates/tantivy/src/tokenizer/regex_tokenizer.rs
@@ -24,25 +24,26 @@ use crate::TantivyError;
///
/// let mut tokenizer = RegexTokenizer::new(r"'(?:\w*)'").unwrap();
/// let mut stream = tokenizer.token_stream("'aaa' bbb 'ccc' 'ddd'");
+/// let mut it = TokenStream::iter(&mut stream);
/// {
-/// let token = stream.next().unwrap();
+/// let token = it.next().unwrap();
/// assert_eq!(token.text, "'aaa'");
/// assert_eq!(token.offset_from, 0);
/// assert_eq!(token.offset_to, 5);
/// }
/// {
-/// let token = stream.next().unwrap();
+/// let token = it.next().unwrap();
/// assert_eq!(token.text, "'ccc'");
/// assert_eq!(token.offset_from, 10);
/// assert_eq!(token.offset_to, 15);
/// }
/// {
-/// let token = stream.next().unwrap();
+/// let token = it.next().unwrap();
/// assert_eq!(token.text, "'ddd'");
/// assert_eq!(token.offset_from, 16);
/// assert_eq!(token.offset_to, 21);
/// }
-/// assert!(stream.next().is_none());
+/// assert!(it.next().is_none());
/// ```
#[derive(Clone)]
diff --git a/crates/tantivy/src/tokenizer/remove_long.rs b/crates/tantivy/src/tokenizer/remove_long.rs
index 78f3e731..9cd64e3c 100644
--- a/crates/tantivy/src/tokenizer/remove_long.rs
+++ b/crates/tantivy/src/tokenizer/remove_long.rs
@@ -7,10 +7,11 @@
//! .build();
//!
//! let mut stream = tokenizer.token_stream("toolong nice");
+//! let mut it = TokenStream::iter(&mut stream);
//! // because `toolong` is more than 5 characters, it is filtered
//! // out of the token stream.
-//! assert_eq!(stream.next().unwrap().text, "nice");
-//! assert!(stream.next().is_none());
+//! assert_eq!(it.next().unwrap().text, "nice");
+//! assert!(it.next().is_none());
//! ```
use super::{Token, TokenFilter, TokenStream, Tokenizer};
diff --git a/crates/tantivy/src/tokenizer/split_compound_words.rs b/crates/tantivy/src/tokenizer/split_compound_words.rs
index 5ee244f0..bd729910 100644
--- a/crates/tantivy/src/tokenizer/split_compound_words.rs
+++ b/crates/tantivy/src/tokenizer/split_compound_words.rs
@@ -18,7 +18,7 @@ use super::{Token, TokenFilter, TokenStream, Tokenizer};
/// is not split in the following example.
///
/// ```rust
-/// use tantivy::tokenizer::{SimpleTokenizer, SplitCompoundWords, TextAnalyzer};
+/// use tantivy::tokenizer::*;
///
/// let mut tokenizer =
/// TextAnalyzer::builder(SimpleTokenizer::default())
@@ -31,14 +31,16 @@ use super::{Token, TokenFilter, TokenStream, Tokenizer};
/// .build();
/// {
/// let mut stream = tokenizer.token_stream("dampfschifffahrt");
-/// assert_eq!(stream.next().unwrap().text, "dampf");
-/// assert_eq!(stream.next().unwrap().text, "schiff");
-/// assert_eq!(stream.next().unwrap().text, "fahrt");
-/// assert_eq!(stream.next(), None);
+/// let mut it = TokenStream::iter(&mut stream);
+/// assert_eq!(it.next().unwrap().text, "dampf");
+/// assert_eq!(it.next().unwrap().text, "schiff");
+/// assert_eq!(it.next().unwrap().text, "fahrt");
+/// assert_eq!(it.next(), None);
/// }
/// let mut stream = tokenizer.token_stream("brotbackautomat");
-/// assert_eq!(stream.next().unwrap().text, "brotbackautomat");
-/// assert_eq!(stream.next(), None);
+/// let mut it = TokenStream::iter(&mut stream);
+/// assert_eq!(it.next().unwrap().text, "brotbackautomat");
+/// assert_eq!(it.next(), None);
/// ```
///
/// [compound]: https://en.wikipedia.org/wiki/Compound_(linguistics)
@@ -189,6 +191,8 @@ impl<'a, T: TokenStream> TokenStream for SplitCompoundWordsTokenStream<'a, T> {
#[cfg(test)]
mod tests {
+ use lending_iter::LendingIterator;
+
use super::*;
use crate::tokenizer::{SimpleTokenizer, TextAnalyzer};
@@ -200,83 +204,93 @@ mod tests {
{
let mut stream = tokenizer.token_stream("");
- assert_eq!(stream.next(), None);
+ let mut it = stream.iter();
+ assert_eq!(it.next(), None);
}
{
let mut stream = tokenizer.token_stream("foo bar");
- assert_eq!(stream.next().unwrap().text, "foo");
- assert_eq!(stream.next().unwrap().text, "bar");
- assert_eq!(stream.next(), None);
+ let mut it = stream.iter();
+ assert_eq!(it.next().unwrap().text, "foo");
+ assert_eq!(it.next().unwrap().text, "bar");
+ assert_eq!(it.next(), None);
}
{
let mut stream = tokenizer.token_stream("foobar");
- assert_eq!(stream.next().unwrap().text, "foo");
- assert_eq!(stream.next().unwrap().text, "bar");
- assert_eq!(stream.next(), None);
+ let mut it = stream.iter();
+ assert_eq!(it.next().unwrap().text, "foo");
+ assert_eq!(it.next().unwrap().text, "bar");
+ assert_eq!(it.next(), None);
}
{
let mut stream = tokenizer.token_stream("foobarbaz");
- assert_eq!(stream.next().unwrap().text, "foobarbaz");
- assert_eq!(stream.next(), None);
+ let mut it = stream.iter();
+ assert_eq!(it.next().unwrap().text, "foobarbaz");
+ assert_eq!(it.next(), None);
}
{
let mut stream = tokenizer.token_stream("baz foobar qux");
- assert_eq!(stream.next().unwrap().text, "baz");
- assert_eq!(stream.next().unwrap().text, "foo");
- assert_eq!(stream.next().unwrap().text, "bar");
- assert_eq!(stream.next().unwrap().text, "qux");
- assert_eq!(stream.next(), None);
+ let mut it = stream.iter();
+ assert_eq!(it.next().unwrap().text, "baz");
+ assert_eq!(it.next().unwrap().text, "foo");
+ assert_eq!(it.next().unwrap().text, "bar");
+ assert_eq!(it.next().unwrap().text, "qux");
+ assert_eq!(it.next(), None);
}
{
let mut stream = tokenizer.token_stream("foobar foobar");
- assert_eq!(stream.next().unwrap().text, "foo");
- assert_eq!(stream.next().unwrap().text, "bar");
- assert_eq!(stream.next().unwrap().text, "foo");
- assert_eq!(stream.next().unwrap().text, "bar");
- assert_eq!(stream.next(), None);
+ let mut it = stream.iter();
+ assert_eq!(it.next().unwrap().text, "foo");
+ assert_eq!(it.next().unwrap().text, "bar");
+ assert_eq!(it.next().unwrap().text, "foo");
+ assert_eq!(it.next().unwrap().text, "bar");
+ assert_eq!(it.next(), None);
}
{
let mut stream = tokenizer.token_stream("foobar foo bar foobar");
- assert_eq!(stream.next().unwrap().text, "foo");
- assert_eq!(stream.next().unwrap().text, "bar");
- assert_eq!(stream.next().unwrap().text, "foo");
- assert_eq!(stream.next().unwrap().text, "bar");
- assert_eq!(stream.next().unwrap().text, "foo");
- assert_eq!(stream.next().unwrap().text, "bar");
- assert_eq!(stream.next(), None);
+ let mut it = stream.iter();
+ assert_eq!(it.next().unwrap().text, "foo");
+ assert_eq!(it.next().unwrap().text, "bar");
+ assert_eq!(it.next().unwrap().text, "foo");
+ assert_eq!(it.next().unwrap().text, "bar");
+ assert_eq!(it.next().unwrap().text, "foo");
+ assert_eq!(it.next().unwrap().text, "bar");
+ assert_eq!(it.next(), None);
}
{
let mut stream = tokenizer.token_stream("foobazbar foo bar foobar");
- assert_eq!(stream.next().unwrap().text, "foobazbar");
- assert_eq!(stream.next().unwrap().text, "foo");
- assert_eq!(stream.next().unwrap().text, "bar");
- assert_eq!(stream.next().unwrap().text, "foo");
- assert_eq!(stream.next().unwrap().text, "bar");
- assert_eq!(stream.next(), None);
+ let mut it = stream.iter();
+ assert_eq!(it.next().unwrap().text, "foobazbar");
+ assert_eq!(it.next().unwrap().text, "foo");
+ assert_eq!(it.next().unwrap().text, "bar");
+ assert_eq!(it.next().unwrap().text, "foo");
+ assert_eq!(it.next().unwrap().text, "bar");
+ assert_eq!(it.next(), None);
}
{
let mut stream = tokenizer.token_stream("foobar qux foobar");
- assert_eq!(stream.next().unwrap().text, "foo");
- assert_eq!(stream.next().unwrap().text, "bar");
- assert_eq!(stream.next().unwrap().text, "qux");
- assert_eq!(stream.next().unwrap().text, "foo");
- assert_eq!(stream.next().unwrap().text, "bar");
- assert_eq!(stream.next(), None);
+ let mut it = stream.iter();
+ assert_eq!(it.next().unwrap().text, "foo");
+ assert_eq!(it.next().unwrap().text, "bar");
+ assert_eq!(it.next().unwrap().text, "qux");
+ assert_eq!(it.next().unwrap().text, "foo");
+ assert_eq!(it.next().unwrap().text, "bar");
+ assert_eq!(it.next(), None);
}
{
let mut stream = tokenizer.token_stream("barfoo");
- assert_eq!(stream.next().unwrap().text, "bar");
- assert_eq!(stream.next().unwrap().text, "foo");
- assert_eq!(stream.next(), None);
+ let mut it = stream.iter();
+ assert_eq!(it.next().unwrap().text, "bar");
+ assert_eq!(it.next().unwrap().text, "foo");
+ assert_eq!(it.next(), None);
}
}
}
diff --git a/crates/tantivy/src/tokenizer/stop_word_filter/mod.rs b/crates/tantivy/src/tokenizer/stop_word_filter/mod.rs
index 3217af71..ad2e97fd 100644
--- a/crates/tantivy/src/tokenizer/stop_word_filter/mod.rs
+++ b/crates/tantivy/src/tokenizer/stop_word_filter/mod.rs
@@ -1,5 +1,6 @@
//! # Example
//! ```rust
+//! use lending_iter::LendingIterator;
//! use tantivy::tokenizer::*;
//!
//! let mut tokenizer = TextAnalyzer::builder(SimpleTokenizer::default())
@@ -7,9 +8,10 @@
//! .build();
//!
//! let mut stream = tokenizer.token_stream("the fox is crafty");
-//! assert_eq!(stream.next().unwrap().text, "fox");
-//! assert_eq!(stream.next().unwrap().text, "crafty");
-//! assert!(stream.next().is_none());
+//! let mut it = TokenStream::iter(&mut stream);
+//! assert_eq!(it.next().unwrap().text, "fox");
+//! assert_eq!(it.next().unwrap().text, "crafty");
+//! assert!(it.next().is_none());
//! ```
#[cfg(feature = "stopwords")]
#[rustfmt::skip]
diff --git a/crates/tantivy/src/tokenizer/tokenizer.rs b/crates/tantivy/src/tokenizer/tokenizer.rs
index 96ffa400..f028132a 100644
--- a/crates/tantivy/src/tokenizer/tokenizer.rs
+++ b/crates/tantivy/src/tokenizer/tokenizer.rs
@@ -4,6 +4,8 @@ use crate::tokenizer_api::{BoxTokenStream, TokenFilter, Tokenizer};
use crate::tokenizer::empty_tokenizer::EmptyTokenizer;
+pub use lending_iter::LendingIterator;
+
/// `TextAnalyzer` tokenizes an input text into tokens and modifies the resulting `TokenStream`.
#[derive(Clone)]
pub struct TextAnalyzer {
@@ -123,6 +125,8 @@ impl<T: Tokenizer> TextAnalyzerBuilder<T> {
#[cfg(test)]
mod tests {
+ use lending_iter::LendingIterator;
+
use super::*;
use crate::tokenizer::{LowerCaser, RemoveLongFilter, SimpleTokenizer};
@@ -133,8 +137,9 @@ mod tests {
.filter(LowerCaser)
.build();
let mut stream = analyzer.token_stream("- first bullet point");
- assert_eq!(stream.next().unwrap().text, "first");
- assert_eq!(stream.next().unwrap().text, "bullet");
+ let mut it = crate::tokenizer::TokenStream::iter(&mut stream);
+ assert_eq!(it.next().unwrap().text, "first");
+ assert_eq!(it.next().unwrap().text, "bullet");
}
#[test]
@@ -169,7 +174,8 @@ mod tests {
}
let mut analyzer = analyzer_builder.build();
let mut stream = analyzer.token_stream("first bullet point");
- assert_eq!(stream.next().unwrap().text, "first");
- assert_eq!(stream.next().unwrap().text, "bullet");
+ let mut it = crate::tokenizer::TokenStream::iter(&mut stream);
+ assert_eq!(it.next().unwrap().text, "first");
+ assert_eq!(it.next().unwrap().text, "bullet");
}
}
diff --git a/crates/tantivy/src/tokenizer_api/mod.rs b/crates/tantivy/src/tokenizer_api/mod.rs
index 1ed4c9b8..c788ea36 100644
--- a/crates/tantivy/src/tokenizer_api/mod.rs
+++ b/crates/tantivy/src/tokenizer_api/mod.rs
@@ -8,6 +8,7 @@
use std::borrow::{Borrow, BorrowMut};
use std::ops::{Deref, DerefMut};
+use lending_iter::LendingIterator;
use serde::{Deserialize, Serialize};
/// Token
@@ -128,17 +129,6 @@ pub trait TokenStream {
/// Returns a mutable reference to the current token.
fn token_mut(&mut self) -> &mut Token;
- /// Helper to iterate over tokens. It
- /// simply combines a call to `.advance()`
- /// and `.token()`.
- fn next(&mut self) -> Option<&Token> {
- if self.advance() {
- Some(self.token())
- } else {
- None
- }
- }
-
/// Helper function to consume the entire `TokenStream`
/// and push the tokens to a sink function.
fn process(&mut self, sink: &mut dyn FnMut(&Token)) {
@@ -146,6 +136,36 @@ pub trait TokenStream {
sink(self.token());
}
}
+
+ /// Helper function to iterate over the tokens of the `TokenStream`
+ /// using a lending iterator.
+ fn iter<'a>(&'a mut self) -> TokenIter<'a, Self>
+ where
+ Self: 'a + Sized,
+ {
+ TokenIter { token_stream: self }
+ }
+}
+
+pub struct TokenIter<'a, T> {
+ token_stream: &'a mut T,
+}
+
+impl<'a, T> LendingIterator for TokenIter<'a, T>
+where
+ T: TokenStream,
+{
+ type Item<'b> = &'b Token
+ where
+ 'a: 'b;
+
+ fn next(&mut self) -> Option<Self::Item<'_>> {
+ if self.token_stream.advance() {
+ Some(self.token_stream.token())
+ } else {
+ None
+ }
+ }
}
/// Trait for the pluggable components of `Tokenizer`s.