web search done right
move some of the 'stream.next()' functionality into traits in a lending-iter crate so we can implement and re-use adapters
Mikkel Denkerfa5282a800cc
diff --git a/Cargo.lock b/Cargo.lockindex 95d3f4cd..2c2c489f 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -2630,6 +2630,10 @@ version = "0.5.2"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "03087c2bad5e1034e8cace5926dec053fb3790248370865f5117a7d0213354c8"+[[package]]+name = "lending-iter"+version = "0.1.0"+[[package]]name = "levenshtein_automata"version = "0.2.1"@@ -4946,6 +4950,7 @@ dependencies = ["insta","itertools 0.13.0","kuchiki",+ "lending-iter","log","logos","lz4_flex",@@ -5216,6 +5221,7 @@ dependencies = ["futures-util","htmlescape","itertools 0.13.0",+ "lending-iter","levenshtein_automata","log","lru",diff --git a/Cargo.toml b/Cargo.tomlindex 27cd984c..2bdd00ec 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -10,6 +10,7 @@ members = ["crates/optics","crates/speedy-kv","crates/zimba",+ "crates/lending-iter","crates/robotstxt","crates/tantivy","fuzz",@@ -88,6 +89,7 @@ itertools = "0.13.0"js-sys = "0.3.60"lalrpop = {version = "0.20.0", features = ["lexer"]}lalrpop-util = {version = "0.20.0", features = ["lexer"]}+lending-iter = {path = "crates/lending-iter"}levenshtein_automata = "0.2.1"libc = "0.2.142"log = {version = "0.4", features = ["release_max_level_info"]}diff --git a/assets/licenses.html b/assets/licenses.htmlindex c643b33b..8908872a 100644--- a/assets/licenses.html+++ b/assets/licenses.html@@ -48,7 +48,7 @@<li><a href="#MIT">MIT License</a> (187)</li><li><a href="#BSD-3-Clause">BSD 3-Clause "New" or "Revised" License</a> (9)</li><li><a href="#MPL-2.0">Mozilla Public License 2.0</a> (8)</li>- <li><a href="#AGPL-3.0">GNU Affero General Public License v3.0</a> (5)</li>+ <li><a href="#AGPL-3.0">GNU Affero General Public License v3.0</a> (6)</li><li><a href="#ISC">ISC License</a> (4)</li><li><a href="#Unicode-3.0">Unicode License v3</a> (4)</li><li><a href="#BSD-2-Clause">BSD 2-Clause "Simplified" License</a> (3)</li>@@ -70,6 +70,7 @@<ul class="license-used-by"><li><a href=" https://crates.io/crates/bloom ">bloom 0.1.0</a></li><li><a href=" https://crates.io/crates/file_store ">file_store 0.1.0</a></li>+ <li><a href=" https://crates.io/crates/lending-iter ">lending-iter 0.1.0</a></li><li><a href=" https://crates.io/crates/ownedbytes ">ownedbytes 0.1.0</a></li><li><a href=" https://crates.io/crates/robotstxt ">robotstxt 0.1.0</a></li><li><a href=" https://crates.io/crates/speedy_kv ">speedy_kv 0.1.0</a></li>diff --git a/crates/core/Cargo.toml b/crates/core/Cargo.tomlindex e729659a..2aa3cdac 100644--- a/crates/core/Cargo.toml+++ b/crates/core/Cargo.toml@@ -58,6 +58,7 @@ image.workspace = trueindicatif.workspace = trueitertools.workspace = truekuchiki = {path = "../kuchiki"}+lending-iter.workspace = truelog.workspace = truelogos.workspace = truelz4_flex.workspace = truediff --git a/crates/core/src/collector/approx_count.rs b/crates/core/src/collector/approx_count.rsindex 20afc0bf..85d95b51 100644--- a/crates/core/src/collector/approx_count.rs+++ b/crates/core/src/collector/approx_count.rs@@ -17,6 +17,7 @@use std::ops::Add;use itertools::Itertools;+use lending_iter::LendingIterator;use tantivy::{collector::{Collector, SegmentCollector},tokenizer::Tokenizer,@@ -113,8 +114,9 @@ impl Collector for ApproxCount {for term in &self.terms {let mut term_freq = 0;let mut stream = tokenizer.token_stream(term.as_str());+ let mut it = tantivy::tokenizer::TokenStream::iter(&mut stream);- while let Some(term) = stream.next() {+ while let Some(term) = it.next() {term_freq += inverted_index.doc_freq(&tantivy::Term::from_field_text(field, term.text.as_str()))?;}diff --git a/crates/core/src/entity_index/mod.rs b/crates/core/src/entity_index/mod.rsindex c27022dd..88187572 100644--- a/crates/core/src/entity_index/mod.rs+++ b/crates/core/src/entity_index/mod.rs@@ -23,6 +23,7 @@ use std::{use base64::{prelude::BASE64_STANDARD as BASE64_ENGINE, Engine};+use lending_iter::LendingIterator;use tantivy::{collector::TopDocs,query::{BooleanQuery, BoostQuery, MoreLikeThisQuery, Occur, QueryClone, TermQuery},@@ -272,7 +273,8 @@ impl EntityIndex {let mut term_queries = Vec::new();let mut tokenizer = DefaultTokenizer::default();let mut stream = tokenizer.token_stream(query);- while let Some(token) = stream.next() {+ let mut it = tantivy::tokenizer::TokenStream::iter(&mut stream);+ while let Some(token) = it.next() {if self.stopwords.contains(&token.text) {continue;}diff --git a/crates/core/src/feed/index.rs b/crates/core/src/feed/index.rsindex 9593d23b..6b0ca1e8 100644--- a/crates/core/src/feed/index.rs+++ b/crates/core/src/feed/index.rs@@ -25,6 +25,7 @@ use crate::{};use anyhow::Result;use hashbrown::HashSet;+use lending_iter::LendingIterator;use tantivy::{merge_policy::NoMergePolicy,query::{PhraseQuery, TermQuery},@@ -131,8 +132,9 @@ impl FeedIndex {let tv_field = self.schema.get_field("url").unwrap();let mut stream = tokenizer.token_stream(query);+ let mut it = tantivy::tokenizer::TokenStream::iter(&mut stream);- while let Some(token) = stream.next() {+ while let Some(token) = it.next() {res.push(tantivy::Term::from_field_text(tv_field, &token.text));}diff --git a/crates/core/src/inverted_index/key_phrase.rs b/crates/core/src/inverted_index/key_phrase.rsindex 1d6c29b5..5b5ad38e 100644--- a/crates/core/src/inverted_index/key_phrase.rs+++ b/crates/core/src/inverted_index/key_phrase.rs@@ -15,6 +15,7 @@// along with this program. If not, see <https://www.gnu.org/licenses/>.use itertools::Itertools;+use lending_iter::LendingIterator;use std::str;use std::{cmp::Reverse, collections::HashMap};diff --git a/crates/core/src/query/pattern_query/mod.rs b/crates/core/src/query/pattern_query/mod.rsindex d9139b01..75f76659 100644--- a/crates/core/src/query/pattern_query/mod.rs+++ b/crates/core/src/query/pattern_query/mod.rs@@ -17,6 +17,7 @@mod scorer;mod weight;+use lending_iter::LendingIterator;use optics::PatternPart;use tantivy::tokenizer::Tokenizer;@@ -101,8 +102,9 @@ impl PatternQuery {.unwrap().tokenizer(Some(&whatlang::Lang::Eng));let mut stream = tokenizer.token_stream(text);+ let mut it = tantivy::tokenizer::TokenStream::iter(&mut stream);- while let Some(token) = stream.next() {+ while let Some(token) = it.next() {new_patterns.push(PatternPart::Raw(token.text.clone()));let term = tantivy::Term::from_field_text(tv_field, &token.text);raw_terms.push(term);diff --git a/crates/core/src/ranking/signal/computer/mod.rs b/crates/core/src/ranking/signal/computer/mod.rsindex 97928ffd..9a38fc04 100644--- a/crates/core/src/ranking/signal/computer/mod.rs+++ b/crates/core/src/ranking/signal/computer/mod.rs@@ -26,6 +26,7 @@ use std::cell::RefCell;use std::sync::Arc;use itertools::Itertools;+use lending_iter::LendingIterator;use tantivy::fieldnorm::FieldNormReader;use tantivy::postings::SegmentPostings;use tantivy::query::{Query as _, Scorer};@@ -266,8 +267,9 @@ impl SignalComputer {let mut terms = Vec::new();let mut tokenizer = text_field.tokenizer(query.lang.as_ref());let mut stream = tokenizer.token_stream(&simple_query);+ let mut it = tantivy::tokenizer::TokenStream::iter(&mut stream);- while let Some(token) = stream.next() {+ while let Some(token) = it.next() {let term = tantivy::Term::from_field_text(tv_field, &token.text);terms.push(term);}diff --git a/crates/core/src/simhash.rs b/crates/core/src/simhash.rsindex 20e15561..c5b0c423 100644--- a/crates/core/src/simhash.rs+++ b/crates/core/src/simhash.rs@@ -19,6 +19,8 @@ use std::{hash::{Hash, Hasher},};+use lending_iter::LendingIterator;+use crate::tokenizer::FieldTokenizer;pub type HashType = u64;@@ -33,10 +35,11 @@ pub fn hash(text: &str) -> HashType {let mut tokenizer = FieldTokenizer::default();let mut stream = tantivy::tokenizer::Tokenizer::token_stream(&mut tokenizer, text);+ let mut it = tantivy::tokenizer::TokenStream::iter(&mut stream);let mut v = [0i64; HashType::BITS as usize];- while let Some(token) = stream.next() {+ while let Some(token) = it.next() {let h = hash_token(token);for (i, item) in v.iter_mut().enumerate() {diff --git a/crates/core/src/snippet.rs b/crates/core/src/snippet.rsindex 9ce581f3..0615f07e 100644--- a/crates/core/src/snippet.rs+++ b/crates/core/src/snippet.rs@@ -25,6 +25,7 @@ use crate::tokenizer::fields::{use crate::web_spell::sentence_ranges;use crate::webpage::region::Region;use hashbrown::{HashMap, HashSet};+use lending_iter::LendingIterator;use utoipa::ToSchema;use itertools::Itertools;@@ -90,7 +91,8 @@ impl SnippetBuilder {] {let mut stream =tantivy::tokenizer::Tokenizer::token_stream(&mut tokenizer, &self.fragment);- while let Some(tok) = stream.next() {+ let mut it = tantivy::tokenizer::TokenStream::iter(&mut stream);+ while let Some(tok) = it.next() {if terms.contains(&tok.text) {self.highlights.push(tok.offset_from..tok.offset_to);}@@ -161,7 +163,8 @@ fn passages({let mut stream =tantivy::tokenizer::Tokenizer::token_stream(&mut tokenizer, &sentence);- while let Some(tok) = stream.next() {+ let mut it = tantivy::tokenizer::TokenStream::iter(&mut stream);+ while let Some(tok) = it.next() {*doc_terms.entry(tok.text.clone()).or_insert(0) += 1;}}@@ -230,7 +233,8 @@ fn snippet_string_builder(let mut stream = tantivy::tokenizer::Tokenizer::token_stream(&mut tokenizer, term);let mut res = Vec::new();- while let Some(tok) = stream.next() {+ let mut it = tantivy::tokenizer::TokenStream::iter(&mut stream);+ while let Some(tok) = it.next() {res.push(tok.text.clone());}diff --git a/crates/core/src/tokenizer/fields/bigram.rs b/crates/core/src/tokenizer/fields/bigram.rsindex 548ff74e..aa63bf37 100644--- a/crates/core/src/tokenizer/fields/bigram.rs+++ b/crates/core/src/tokenizer/fields/bigram.rs@@ -48,6 +48,7 @@ impl tantivy::tokenizer::Tokenizer for BigramTokenizer {#[cfg(test)]mod tests {+ use lending_iter::LendingIterator;use tantivy::tokenizer::Tokenizer;use super::*;@@ -56,7 +57,8 @@ mod tests {let mut tokenizer = BigramTokenizer::default();let mut stream = tokenizer.token_stream(s);- while let Some(token) = stream.next() {+ let mut it = tantivy::tokenizer::TokenStream::iter(&mut stream);+ while let Some(token) = it.next() {res.push(token.text.clone());}diff --git a/crates/core/src/tokenizer/fields/default.rs b/crates/core/src/tokenizer/fields/default.rsindex 1d012cab..50328620 100644--- a/crates/core/src/tokenizer/fields/default.rs+++ b/crates/core/src/tokenizer/fields/default.rs@@ -122,6 +122,7 @@ impl<'a> NormalTokenStream<'a> {#[cfg(test)]mod tests {use super::*;+ use lending_iter::LendingIterator;use proptest::prelude::*;use tantivy::tokenizer::Tokenizer as _;@@ -129,8 +130,9 @@ mod tests {let mut res = Vec::new();let mut tokenizer = DefaultTokenizer::default();let mut stream = tokenizer.token_stream(s);+ let mut it = tantivy::tokenizer::TokenStream::iter(&mut stream);- while let Some(token) = stream.next() {+ while let Some(token) = it.next() {res.push(token.text.clone());}diff --git a/crates/core/src/tokenizer/fields/identity.rs b/crates/core/src/tokenizer/fields/identity.rsindex 957b28e6..8a642363 100644--- a/crates/core/src/tokenizer/fields/identity.rs+++ b/crates/core/src/tokenizer/fields/identity.rs@@ -73,14 +73,16 @@ impl tantivy::tokenizer::TokenStream for IdentityTokenStream {#[cfg(test)]mod tests {use super::*;+ use lending_iter::LendingIterator;use tantivy::tokenizer::Tokenizer as _;fn tokenize_identity(s: &str) -> Vec<String> {let mut res = Vec::new();let mut tokenizer = Identity {};let mut stream = tokenizer.token_stream(s);+ let mut it = tantivy::tokenizer::TokenStream::iter(&mut stream);- while let Some(token) = stream.next() {+ while let Some(token) = it.next() {res.push(token.text.clone());}diff --git a/crates/core/src/tokenizer/fields/json.rs b/crates/core/src/tokenizer/fields/json.rsindex 5dd551d1..86c612b8 100644--- a/crates/core/src/tokenizer/fields/json.rs+++ b/crates/core/src/tokenizer/fields/json.rs@@ -202,6 +202,7 @@ impl<'a> tantivy::tokenizer::TokenStream for JsonFieldTokenStream<'a> {#[cfg(test)]mod tests {+ use lending_iter::LendingIterator;use tantivy::tokenizer::Tokenizer;use super::*;@@ -211,7 +212,8 @@ mod tests {let mut tokenizer = JsonField;let mut stream = tokenizer.token_stream(s);- while let Some(token) = stream.next() {+ let mut it = tantivy::tokenizer::TokenStream::iter(&mut stream);+ while let Some(token) = it.next() {res.push(token.text.clone());}diff --git a/crates/core/src/tokenizer/fields/split_newlines.rs b/crates/core/src/tokenizer/fields/split_newlines.rsindex 5d01008d..d37844e6 100644--- a/crates/core/src/tokenizer/fields/split_newlines.rs+++ b/crates/core/src/tokenizer/fields/split_newlines.rs@@ -108,14 +108,16 @@ impl<'a> NewlineTokenStream<'a> {#[cfg(test)]mod tests {use super::*;+ use lending_iter::LendingIterator;use tantivy::tokenizer::Tokenizer as _;fn tokenize_newline(s: &str) -> Vec<String> {let mut res = Vec::new();let mut tokenizer = NewlineTokenizer::default();let mut stream = tokenizer.token_stream(s);+ let mut it = tantivy::tokenizer::TokenStream::iter(&mut stream);- while let Some(token) = stream.next() {+ while let Some(token) = it.next() {res.push(token.text.clone());}diff --git a/crates/core/src/tokenizer/fields/trigram.rs b/crates/core/src/tokenizer/fields/trigram.rsindex 54fc0884..024742d8 100644--- a/crates/core/src/tokenizer/fields/trigram.rs+++ b/crates/core/src/tokenizer/fields/trigram.rs@@ -47,6 +47,7 @@ impl tantivy::tokenizer::Tokenizer for TrigramTokenizer {#[cfg(test)]mod tests {+ use lending_iter::LendingIterator;use tantivy::tokenizer::Tokenizer;use super::*;@@ -56,8 +57,9 @@ mod tests {let mut tokenizer = TrigramTokenizer::default();let mut stream = tokenizer.token_stream(s);+ let mut it = tantivy::tokenizer::TokenStream::iter(&mut stream);- while let Some(token) = stream.next() {+ while let Some(token) = it.next() {res.push(token.text.clone());}diff --git a/crates/core/src/tokenizer/fields/url.rs b/crates/core/src/tokenizer/fields/url.rsindex d127fa15..86070059 100644--- a/crates/core/src/tokenizer/fields/url.rs+++ b/crates/core/src/tokenizer/fields/url.rs@@ -196,6 +196,7 @@ impl tantivy::tokenizer::TokenStream for SiteOperatorUrlTokenStream {#[cfg(test)]mod tests {use super::*;+ use lending_iter::LendingIterator;use proptest::prelude::*;use tantivy::tokenizer::Tokenizer as _;@@ -203,8 +204,9 @@ mod tests {let mut res = Vec::new();let mut tokenizer = UrlTokenizer;let mut stream = tokenizer.token_stream(s);+ let mut it = tantivy::tokenizer::TokenStream::iter(&mut stream);- while let Some(token) = stream.next() {+ while let Some(token) = it.next() {res.push(token.text.clone());}diff --git a/crates/core/src/webpage/html/fn_cache.rs b/crates/core/src/webpage/html/fn_cache.rsindex daf66c59..3bb35abd 100644--- a/crates/core/src/webpage/html/fn_cache.rs+++ b/crates/core/src/webpage/html/fn_cache.rs@@ -15,6 +15,7 @@// along with this program. If not, see <https://www.gnu.org/licenses/>.use crate::{webpage::schema_org, Result};+use lending_iter::LendingIterator;use tantivy::tokenizer::PreTokenizedString;use super::{find_recipe_first_ingredient_tag_id, Html};@@ -112,8 +113,9 @@ impl<'a> FnCache<'a> {{let mut stream = f.token_stream();+ let mut it = tantivy::tokenizer::TokenStream::iter(&mut stream);- while let Some(token) = stream.next() {+ while let Some(token) = it.next() {tokens.push(token.clone());}}diff --git a/crates/core/src/webpage/html/into_tantivy.rs b/crates/core/src/webpage/html/into_tantivy.rsindex f289a5af..f3a7100b 100644--- a/crates/core/src/webpage/html/into_tantivy.rs+++ b/crates/core/src/webpage/html/into_tantivy.rs@@ -28,6 +28,7 @@ use crate::{Error, Result,};use bloom::split_u128;+use lending_iter::LendingIterator;use tantivy::{tokenizer::{PreTokenizedString, Tokenizer},TantivyDocument,@@ -122,7 +123,8 @@ impl Html {{let mut stream = tokenizer.token_stream(&text);- while let Some(token) = stream.next() {+ let mut it = tantivy::tokenizer::TokenStream::iter(&mut stream);+ while let Some(token) = it.next() {tokens.push(token.clone());}}diff --git a/crates/lending-iter/Cargo.toml b/crates/lending-iter/Cargo.tomlnew file mode 100644index 00000000..44ce5405--- /dev/null+++ b/crates/lending-iter/Cargo.toml@@ -0,0 +1,9 @@+[package]+edition = "2021"+license = "AGPL-3.0"+name = "lending-iter"+version = "0.1.0"++# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html++[dependencies]diff --git a/crates/lending-iter/src/adapters/cloned.rs b/crates/lending-iter/src/adapters/cloned.rsnew file mode 100644index 00000000..449a8224--- /dev/null+++ b/crates/lending-iter/src/adapters/cloned.rs@@ -0,0 +1,48 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use std::ops::Deref;++use crate::LendingIterator;++pub struct Cloned<I> {+ iter: I,+}++impl<I> Cloned<I> {+ pub fn new(iter: I) -> Self {+ Self { iter }+ }+}++impl<I> From<I> for Cloned<I> {+ fn from(iter: I) -> Self {+ Self::new(iter)+ }+}++impl<I, T> Iterator for Cloned<I>+where+ I: LendingIterator,+ for<'a> I::Item<'a>: Deref<Target = T>,+ T: Clone,+{+ type Item = T;++ fn next(&mut self) -> Option<Self::Item> {+ self.iter.next().map(|item| item.deref().clone())+ }+}diff --git a/crates/lending-iter/src/adapters/enumerate.rs b/crates/lending-iter/src/adapters/enumerate.rsnew file mode 100644index 00000000..5b321f8b--- /dev/null+++ b/crates/lending-iter/src/adapters/enumerate.rs@@ -0,0 +1,51 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use crate::LendingIterator;++pub struct Enumerate<I> {+ iter: I,+ index: usize,+}++impl<I> Enumerate<I> {+ pub fn new(iter: I) -> Self {+ Self { iter, index: 0 }+ }+}++impl<I> From<I> for Enumerate<I> {+ fn from(iter: I) -> Self {+ Self::new(iter)+ }+}++impl<I> LendingIterator for Enumerate<I>+where+ I: LendingIterator,+{+ type Item<'a> = (usize, I::Item<'a>)+ where+ I: 'a;++ fn next(&mut self) -> Option<Self::Item<'_>> {+ self.iter.next().map(|item| {+ let index = self.index;+ self.index += 1;+ (index, item)+ })+ }+}diff --git a/crates/lending-iter/src/adapters/filter.rs b/crates/lending-iter/src/adapters/filter.rsnew file mode 100644index 00000000..9eea0bc2--- /dev/null+++ b/crates/lending-iter/src/adapters/filter.rs@@ -0,0 +1,57 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use crate::LendingIterator;++pub struct Filter<I, P> {+ iter: I,+ predicate: P,+}++impl<I, P> Filter<I, P> {+ pub fn new(iter: I, predicate: P) -> Self {+ Self { iter, predicate }+ }+}++impl<I, P> LendingIterator for Filter<I, P>+where+ I: LendingIterator,+ for<'a> P: FnMut(&I::Item<'a>) -> bool,+{+ type Item<'a> = I::Item<'a>+ where+ Self: 'a;++ fn next(&mut self) -> Option<Self::Item<'_>> {+ // SAFETY (polonius):+ // - `self` is a mutable reference, so it is the only reference to this instance.+ // - we have a mutable reference to `self.iter`, so it is the only reference to this field.+ // - the reference to `item` is only returned in previous loop iff. the predicate is true.+ // Otherwise, the reference is dropped before the next iteration.++ loop {+ let self_ = unsafe { &mut *(self as *mut Self) };+ if let Some(item) = self_.iter.next() {+ if (self_.predicate)(&item) {+ return Some(item);+ }+ } else {+ return None;+ }+ }+ }+}diff --git a/crates/lending-iter/src/adapters/flatten.rs b/crates/lending-iter/src/adapters/flatten.rsnew file mode 100644index 00000000..f07d38ae--- /dev/null+++ b/crates/lending-iter/src/adapters/flatten.rs@@ -0,0 +1,72 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use crate::LendingIterator;++pub struct Flatten<'a, I>+where+ I: LendingIterator,+ I::Item<'a>: LendingIterator,+ Self: 'a,+{+ iter: I,+ current: Option<I::Item<'a>>,+}++impl<'a, I> Flatten<'a, I>+where+ I: LendingIterator,+ I::Item<'a>: LendingIterator,+{+ pub fn new(iter: I) -> Self {+ Self {+ iter,+ current: None,+ }+ }+}++impl<'a, I> LendingIterator for Flatten<'a, I>+where+ I: LendingIterator,+ I::Item<'a>: LendingIterator,+ Self: 'a,+{+ type Item<'b> = <I::Item<'a> as LendingIterator>::Item<'b>+ where+ Self: 'b;++ fn next(&mut self) -> Option<Self::Item<'_>> {+ // SAFETY (polonius):+ // - `self` is a mutable reference, so it is the only reference to this instance.+ // - we have a mutable reference to `self.current`, so it is the only reference to this field.+ // - `self.iter` is not accessed while `self.current` is borrowed.++ loop {+ let self_ = unsafe { &mut *(self as *mut Self) };++ if let Some(current) = unsafe { &mut *(&mut self_.current as *mut Option<I::Item<'a>>) }+ {+ if let Some(item) = current.next() {+ return Some(item);+ }+ }++ self_.current = self_.iter.next();+ self_.current.as_ref()?;+ }+ }+}diff --git a/crates/lending-iter/src/adapters/map.rs b/crates/lending-iter/src/adapters/map.rsnew file mode 100644index 00000000..ec775b64--- /dev/null+++ b/crates/lending-iter/src/adapters/map.rs@@ -0,0 +1,42 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use crate::LendingIterator;++pub struct Map<I, F> {+ iter: I,+ f: F,+}++impl<I, F> Map<I, F> {+ pub fn new(iter: I, f: F) -> Self {+ Self { iter, f }+ }+}++impl<I, F, T> LendingIterator for Map<I, F>+where+ I: LendingIterator,+ F: for<'a> FnMut(I::Item<'a>) -> T,+{+ type Item<'a> = T+ where+ Self: 'a;++ fn next(&mut self) -> Option<Self::Item<'_>> {+ self.iter.next().map(&mut self.f)+ }+}diff --git a/crates/lending-iter/src/adapters/mod.rs b/crates/lending-iter/src/adapters/mod.rsnew file mode 100644index 00000000..d9cab115--- /dev/null+++ b/crates/lending-iter/src/adapters/mod.rs@@ -0,0 +1,27 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++mod cloned;+mod enumerate;+mod filter;+mod flatten;+mod map;++pub use cloned::Cloned;+pub use enumerate::Enumerate;+pub use filter::Filter;+pub use flatten::Flatten;+pub use map::Map;diff --git a/crates/lending-iter/src/lib.rs b/crates/lending-iter/src/lib.rsnew file mode 100644index 00000000..f1380e24--- /dev/null+++ b/crates/lending-iter/src/lib.rs@@ -0,0 +1,274 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++mod adapters;++pub trait LendingIterator {+ type Item<'a>+ where+ Self: 'a;++ fn next(&mut self) -> Option<Self::Item<'_>>;++ fn enumerate(self) -> adapters::Enumerate<Self>+ where+ Self: Sized,+ {+ adapters::Enumerate::new(self)+ }++ fn cloned<'a, T>(self) -> adapters::Cloned<Self>+ where+ Self: Sized,+ for<'b> Self::Item<'b>: std::ops::Deref<Target = T>,+ T: Clone,+ {+ adapters::Cloned::new(self)+ }++ fn fold<B, F>(self, init: B, f: F) -> B+ where+ Self: Sized,+ F: FnMut(B, Self::Item<'_>) -> B,+ {+ let mut f = f;+ let mut acc = init;+ let mut iter = self;++ while let Some(item) = iter.next() {+ acc = f(acc, item);+ }++ acc+ }++ fn count(self) -> usize+ where+ Self: Sized,+ {+ self.fold(0, |acc, _| acc + 1)+ }++ fn map<B, F>(self, f: F) -> adapters::Map<Self, F>+ where+ Self: Sized,+ F: FnMut(Self::Item<'_>) -> B,+ {+ adapters::Map::new(self, f)+ }++ fn filter<F>(self, f: F) -> adapters::Filter<Self, F>+ where+ Self: Sized,+ F: FnMut(&Self::Item<'_>) -> bool,+ {+ adapters::Filter::new(self, f)+ }++ fn flatten<'a>(self) -> adapters::Flatten<'a, Self>+ where+ Self: Sized,+ Self::Item<'a>: LendingIterator,+ {+ adapters::Flatten::new(self)+ }+}++impl<'a, I> LendingIterator for &'a mut I+where+ I: LendingIterator,+{+ type Item<'b> = I::Item<'b>+ where+ I: 'b,+ 'a: 'b;++ fn next(&mut self) -> Option<Self::Item<'_>> {+ (*self).next()+ }+}++pub trait IntoLendingIterator: Sized {+ fn lending(self) -> IntoLending<Self>;+}++pub struct IntoLending<I> {+ iter: I,+}++impl<I> LendingIterator for IntoLending<I>+where+ I: Iterator,+{+ type Item<'a> = I::Item+ where+ Self: 'a;++ fn next(&mut self) -> Option<Self::Item<'_>> {+ self.iter.next()+ }+}++impl<I> IntoLendingIterator for I+where+ I: Iterator,+{+ fn lending(self) -> IntoLending<Self> {+ IntoLending { iter: self }+ }+}++#[cfg(test)]+mod tests {+ use super::*;++ struct Iter {+ items: Vec<i32>,+ index: usize,+ }++ impl Iter {+ fn new(items: Vec<i32>) -> Self {+ Self { items, index: 0 }+ }+ }++ impl LendingIterator for Iter {+ type Item<'a> = &'a i32;++ fn next(&mut self) -> Option<Self::Item<'_>> {+ if self.index < self.items.len() {+ let item = &self.items[self.index];+ self.index += 1;+ Some(item)+ } else {+ None+ }+ }+ }++ impl From<Vec<i32>> for Iter {+ fn from(items: Vec<i32>) -> Self {+ Self::new(items)+ }+ }++ #[test]+ fn test_lending_iterator() {+ let items = vec![1, 2, 3];+ let mut iter = Iter::new(items);+ assert_eq!(iter.next(), Some(&1));+ assert_eq!(iter.next(), Some(&2));+ assert_eq!(iter.next(), Some(&3));+ assert_eq!(iter.next(), None);+ }++ #[test]+ fn test_enumerate() {+ let items = vec![1, 2, 3];+ let mut iter = Iter::new(items).enumerate();+ assert_eq!(iter.next(), Some((0, &1)));+ assert_eq!(iter.next(), Some((1, &2)));+ assert_eq!(iter.next(), Some((2, &3)));+ assert_eq!(iter.next(), None);+ }++ #[test]+ fn test_cloned() {+ let items = vec![1, 2, 3];+ let mut iter = Iter::new(items).cloned();+ assert_eq!(iter.next(), Some(1));+ assert_eq!(iter.next(), Some(2));+ assert_eq!(iter.next(), Some(3));+ assert_eq!(iter.next(), None);+ }++ #[test]+ fn test_fold() {+ let items = vec![1, 2, 3];+ let iter = Iter::new(items);+ let sum = iter.fold(0, |acc, item| acc + item);+ assert_eq!(sum, 6);+ }++ #[test]+ fn test_count() {+ let items = vec![1, 2, 3];+ let iter = Iter::new(items);+ let count = iter.count();+ assert_eq!(count, 3);+ }++ #[test]+ fn test_map() {+ let items = vec![1, 2, 3];+ let mut iter = Iter::new(items).map(|item| item * 2);+ assert_eq!(iter.next(), Some(2));+ assert_eq!(iter.next(), Some(4));+ assert_eq!(iter.next(), Some(6));+ assert_eq!(iter.next(), None);+ }++ #[test]+ fn test_filter() {+ let items = vec![1, 2, 3];+ let mut iter = Iter::new(items).filter(|item| **item % 2 == 0);+ assert_eq!(iter.next(), Some(&2));+ assert_eq!(iter.next(), None);+ }++ struct NestedIter {+ items: Vec<Iter>,+ index: usize,+ }++ impl NestedIter {+ fn new(items: Vec<Iter>) -> Self {+ Self { items, index: 0 }+ }+ }++ impl LendingIterator for NestedIter {+ type Item<'a> = &'a mut Iter;++ fn next(&mut self) -> Option<Self::Item<'_>> {+ if self.index < self.items.len() {+ let item = &mut self.items[self.index];+ self.index += 1;+ Some(item)+ } else {+ None+ }+ }+ }++ #[test]+ fn test_flatten() {+ let items = vec![+ Iter::new(vec![1, 2]),+ Iter::new(vec![3, 4]),+ Iter::new(vec![5, 6]),+ ];+ let mut iter = NestedIter::new(items).flatten();+ assert_eq!(iter.next(), Some(&1));+ assert_eq!(iter.next(), Some(&2));+ assert_eq!(iter.next(), Some(&3));+ assert_eq!(iter.next(), Some(&4));+ assert_eq!(iter.next(), Some(&5));+ assert_eq!(iter.next(), Some(&6));+ assert_eq!(iter.next(), None);+ }+}diff --git a/crates/tantivy/Cargo.toml b/crates/tantivy/Cargo.tomlindex 27d47866..07f1498f 100644--- a/crates/tantivy/Cargo.toml+++ b/crates/tantivy/Cargo.toml@@ -31,6 +31,7 @@ fst.workspace = truefutures-util = {workspace = true, optional = true}htmlescape.workspace = trueitertools.workspace = true+lending-iter.workspace = truelevenshtein_automata.workspace = truelog.workspace = truelru.workspace = truediff --git a/crates/tantivy/src/index/inverted_index_reader.rs b/crates/tantivy/src/index/inverted_index_reader.rsindex 11fca48b..543f6dc0 100644--- a/crates/tantivy/src/index/inverted_index_reader.rs+++ b/crates/tantivy/src/index/inverted_index_reader.rs@@ -3,6 +3,7 @@ use std::io;use crate::common::json_path_writer::JSON_END_OF_PATH;use crate::common::BinarySerializable;use fnv::FnvHashSet;+use lending_iter::LendingIterator;use crate::directory::FileSlice;use crate::positions::PositionReader;diff --git a/crates/tantivy/src/snippet/mod.rs b/crates/tantivy/src/snippet/mod.rsindex 16d54953..38059a1a 100644--- a/crates/tantivy/src/snippet/mod.rs+++ b/crates/tantivy/src/snippet/mod.rs@@ -19,6 +19,7 @@ use std::collections::{BTreeMap, BTreeSet};use std::ops::Range;use htmlescape::encode_minimal;+use lending_iter::LendingIterator;use crate::query::Query;use crate::schema::document::{Document, Value};@@ -168,9 +169,10 @@ fn search_fragments(max_num_chars: usize,) -> Vec<FragmentCandidate> {let mut token_stream = tokenizer.token_stream(text);+ let mut it = crate::tokenizer::TokenStream::iter(&mut token_stream);let mut fragment = FragmentCandidate::new(0);let mut fragments: Vec<FragmentCandidate> = vec![];- while let Some(next) = token_stream.next() {+ while let Some(next) = it.next() {if (next.offset_to - fragment.start_offset) > max_num_chars {if fragment.score > 0.0 {fragments.push(fragment)diff --git a/crates/tantivy/src/termdict/fst_termdict/streamer.rs b/crates/tantivy/src/termdict/fst_termdict/streamer.rsindex e13f30e6..a6093009 100644--- a/crates/tantivy/src/termdict/fst_termdict/streamer.rs+++ b/crates/tantivy/src/termdict/fst_termdict/streamer.rs@@ -3,6 +3,7 @@ use std::io;use fst::automaton::AlwaysMatch;use fst::map::{Stream, StreamBuilder};use fst::{Automaton, IntoStreamer, Streamer};+use lending_iter::LendingIterator;use super::TermDictionary;use crate::postings::TermInfo;@@ -132,10 +133,17 @@ wherepub fn value(&self) -> &TermInfo {&self.current_value}+}++impl<'a, A> LendingIterator for TermStreamer<'a, A>+where+ A: Automaton,+{+ type Item<'b> = (&'b [u8], &'b TermInfo)+ where+ Self: 'b;- /// Return the next `(key, value)` pair.- #[allow(clippy::should_implement_trait)]- pub fn next(&mut self) -> Option<(&[u8], &TermInfo)> {+ fn next(&mut self) -> Option<Self::Item<'_>> {if self.advance() {Some((self.key(), self.value()))} else {diff --git a/crates/tantivy/src/termdict/tests.rs b/crates/tantivy/src/termdict/tests.rsindex 3aec04e7..47466dec 100644--- a/crates/tantivy/src/termdict/tests.rs+++ b/crates/tantivy/src/termdict/tests.rs@@ -1,6 +1,8 @@use std::path::PathBuf;use std::{io, str};+use lending_iter::LendingIterator;+use super::{TermDictionary, TermDictionaryBuilder, TermStreamer};use crate::directory::{Directory, FileSlice, RamDirectory, TerminatingWrite};use crate::postings::TermInfo;diff --git a/crates/tantivy/src/tokenizer/alphanum_only.rs b/crates/tantivy/src/tokenizer/alphanum_only.rsindex b40731fd..f5e6ffa3 100644--- a/crates/tantivy/src/tokenizer/alphanum_only.rs+++ b/crates/tantivy/src/tokenizer/alphanum_only.rs@@ -7,19 +7,21 @@//! .build();//!//! let mut stream = tokenizer.token_stream("hello there");+//! let mut it = TokenStream::iter(&mut stream);//! // is none because the raw filter emits one token that//! // contains a space-//! assert!(stream.next().is_none());+//! assert!(it.next().is_none());//!//! let mut tokenizer = TextAnalyzer::builder(SimpleTokenizer::default())//! .filter(AlphaNumOnlyFilter)//! .build();//!//! let mut stream = tokenizer.token_stream("hello there 💣");-//! assert!(stream.next().is_some());-//! assert!(stream.next().is_some());+//! let mut it = TokenStream::iter(&mut stream);+//! assert!(it.next().is_some());+//! assert!(it.next().is_some());//! // the "emoji" is dropped because its not an alphanum-//! assert!(stream.next().is_none());+//! assert!(it.next().is_none());//! ```use super::{Token, TokenFilter, TokenStream, Tokenizer};diff --git a/crates/tantivy/src/tokenizer/mod.rs b/crates/tantivy/src/tokenizer/mod.rsindex b2d30319..a77ee883 100644--- a/crates/tantivy/src/tokenizer/mod.rs+++ b/crates/tantivy/src/tokenizer/mod.rs@@ -155,6 +155,7 @@ pub use self::tokenized_string::{PreTokenizedStream, PreTokenizedString};pub use self::tokenizer::{TextAnalyzer, TextAnalyzerBuilder};pub use self::tokenizer_manager::TokenizerManager;pub use self::whitespace_tokenizer::WhitespaceTokenizer;+pub use lending_iter::LendingIterator;/// Maximum authorized len (in bytes) for a token.///diff --git a/crates/tantivy/src/tokenizer/ngram_tokenizer.rs b/crates/tantivy/src/tokenizer/ngram_tokenizer.rsindex e98c5ff6..691a6ed0 100644--- a/crates/tantivy/src/tokenizer/ngram_tokenizer.rs+++ b/crates/tantivy/src/tokenizer/ngram_tokenizer.rs@@ -36,49 +36,50 @@ use crate::TantivyError;////// let mut tokenizer = NgramTokenizer::new(2, 3, false).unwrap();/// let mut stream = tokenizer.token_stream("hello");+/// let mut it = TokenStream::iter(&mut stream);/// {-/// let token = stream.next().unwrap();+/// let token = it.next().unwrap();/// assert_eq!(token.text, "he");/// assert_eq!(token.offset_from, 0);/// assert_eq!(token.offset_to, 2);/// }/// {-/// let token = stream.next().unwrap();+/// let token = it.next().unwrap();/// assert_eq!(token.text, "hel");/// assert_eq!(token.offset_from, 0);/// assert_eq!(token.offset_to, 3);/// }/// {-/// let token = stream.next().unwrap();+/// let token = it.next().unwrap();/// assert_eq!(token.text, "el");/// assert_eq!(token.offset_from, 1);/// assert_eq!(token.offset_to, 3);/// }/// {-/// let token = stream.next().unwrap();+/// let token = it.next().unwrap();/// assert_eq!(token.text, "ell");/// assert_eq!(token.offset_from, 1);/// assert_eq!(token.offset_to, 4);/// }/// {-/// let token = stream.next().unwrap();+/// let token = it.next().unwrap();/// assert_eq!(token.text, "ll");/// assert_eq!(token.offset_from, 2);/// assert_eq!(token.offset_to, 4);/// }/// {-/// let token = stream.next().unwrap();+/// let token = it.next().unwrap();/// assert_eq!(token.text, "llo");/// assert_eq!(token.offset_from, 2);/// assert_eq!(token.offset_to, 5);/// }/// {-/// let token = stream.next().unwrap();+/// let token = it.next().unwrap();/// assert_eq!(token.text, "lo");/// assert_eq!(token.offset_from, 3);/// assert_eq!(token.offset_to, 5);/// }-/// assert!(stream.next().is_none());+/// assert!(it.next().is_none());/// ```#[derive(Clone, Debug)]pub struct NgramTokenizer {@@ -205,7 +206,8 @@ struct StutteringIterator<T> {}impl<T> StutteringIterator<T>-where T: Iterator<Item = usize>+where+ T: Iterator<Item = usize>,{pub fn new(mut underlying: T, min_gram: usize, max_gram: usize) -> StutteringIterator<T> {assert!(min_gram > 0);@@ -234,7 +236,8 @@ where T: Iterator<Item = usize>}impl<T> Iterator for StutteringIterator<T>-where T: Iterator<Item = usize>+where+ T: Iterator<Item = usize>,{type Item = (usize, usize);diff --git a/crates/tantivy/src/tokenizer/regex_tokenizer.rs b/crates/tantivy/src/tokenizer/regex_tokenizer.rsindex f9a10ad2..f4a608b3 100644--- a/crates/tantivy/src/tokenizer/regex_tokenizer.rs+++ b/crates/tantivy/src/tokenizer/regex_tokenizer.rs@@ -24,25 +24,26 @@ use crate::TantivyError;////// let mut tokenizer = RegexTokenizer::new(r"'(?:\w*)'").unwrap();/// let mut stream = tokenizer.token_stream("'aaa' bbb 'ccc' 'ddd'");+/// let mut it = TokenStream::iter(&mut stream);/// {-/// let token = stream.next().unwrap();+/// let token = it.next().unwrap();/// assert_eq!(token.text, "'aaa'");/// assert_eq!(token.offset_from, 0);/// assert_eq!(token.offset_to, 5);/// }/// {-/// let token = stream.next().unwrap();+/// let token = it.next().unwrap();/// assert_eq!(token.text, "'ccc'");/// assert_eq!(token.offset_from, 10);/// assert_eq!(token.offset_to, 15);/// }/// {-/// let token = stream.next().unwrap();+/// let token = it.next().unwrap();/// assert_eq!(token.text, "'ddd'");/// assert_eq!(token.offset_from, 16);/// assert_eq!(token.offset_to, 21);/// }-/// assert!(stream.next().is_none());+/// assert!(it.next().is_none());/// ```#[derive(Clone)]diff --git a/crates/tantivy/src/tokenizer/remove_long.rs b/crates/tantivy/src/tokenizer/remove_long.rsindex 78f3e731..9cd64e3c 100644--- a/crates/tantivy/src/tokenizer/remove_long.rs+++ b/crates/tantivy/src/tokenizer/remove_long.rs@@ -7,10 +7,11 @@//! .build();//!//! let mut stream = tokenizer.token_stream("toolong nice");+//! let mut it = TokenStream::iter(&mut stream);//! // because `toolong` is more than 5 characters, it is filtered//! // out of the token stream.-//! assert_eq!(stream.next().unwrap().text, "nice");-//! assert!(stream.next().is_none());+//! assert_eq!(it.next().unwrap().text, "nice");+//! assert!(it.next().is_none());//! ```use super::{Token, TokenFilter, TokenStream, Tokenizer};diff --git a/crates/tantivy/src/tokenizer/split_compound_words.rs b/crates/tantivy/src/tokenizer/split_compound_words.rsindex 5ee244f0..bd729910 100644--- a/crates/tantivy/src/tokenizer/split_compound_words.rs+++ b/crates/tantivy/src/tokenizer/split_compound_words.rs@@ -18,7 +18,7 @@ use super::{Token, TokenFilter, TokenStream, Tokenizer};/// is not split in the following example.////// ```rust-/// use tantivy::tokenizer::{SimpleTokenizer, SplitCompoundWords, TextAnalyzer};+/// use tantivy::tokenizer::*;////// let mut tokenizer =/// TextAnalyzer::builder(SimpleTokenizer::default())@@ -31,14 +31,16 @@ use super::{Token, TokenFilter, TokenStream, Tokenizer};/// .build();/// {/// let mut stream = tokenizer.token_stream("dampfschifffahrt");-/// assert_eq!(stream.next().unwrap().text, "dampf");-/// assert_eq!(stream.next().unwrap().text, "schiff");-/// assert_eq!(stream.next().unwrap().text, "fahrt");-/// assert_eq!(stream.next(), None);+/// let mut it = TokenStream::iter(&mut stream);+/// assert_eq!(it.next().unwrap().text, "dampf");+/// assert_eq!(it.next().unwrap().text, "schiff");+/// assert_eq!(it.next().unwrap().text, "fahrt");+/// assert_eq!(it.next(), None);/// }/// let mut stream = tokenizer.token_stream("brotbackautomat");-/// assert_eq!(stream.next().unwrap().text, "brotbackautomat");-/// assert_eq!(stream.next(), None);+/// let mut it = TokenStream::iter(&mut stream);+/// assert_eq!(it.next().unwrap().text, "brotbackautomat");+/// assert_eq!(it.next(), None);/// ```////// [compound]: https://en.wikipedia.org/wiki/Compound_(linguistics)@@ -189,6 +191,8 @@ impl<'a, T: TokenStream> TokenStream for SplitCompoundWordsTokenStream<'a, T> {#[cfg(test)]mod tests {+ use lending_iter::LendingIterator;+use super::*;use crate::tokenizer::{SimpleTokenizer, TextAnalyzer};@@ -200,83 +204,93 @@ mod tests {{let mut stream = tokenizer.token_stream("");- assert_eq!(stream.next(), None);+ let mut it = stream.iter();+ assert_eq!(it.next(), None);}{let mut stream = tokenizer.token_stream("foo bar");- assert_eq!(stream.next().unwrap().text, "foo");- assert_eq!(stream.next().unwrap().text, "bar");- assert_eq!(stream.next(), None);+ let mut it = stream.iter();+ assert_eq!(it.next().unwrap().text, "foo");+ assert_eq!(it.next().unwrap().text, "bar");+ assert_eq!(it.next(), None);}{let mut stream = tokenizer.token_stream("foobar");- assert_eq!(stream.next().unwrap().text, "foo");- assert_eq!(stream.next().unwrap().text, "bar");- assert_eq!(stream.next(), None);+ let mut it = stream.iter();+ assert_eq!(it.next().unwrap().text, "foo");+ assert_eq!(it.next().unwrap().text, "bar");+ assert_eq!(it.next(), None);}{let mut stream = tokenizer.token_stream("foobarbaz");- assert_eq!(stream.next().unwrap().text, "foobarbaz");- assert_eq!(stream.next(), None);+ let mut it = stream.iter();+ assert_eq!(it.next().unwrap().text, "foobarbaz");+ assert_eq!(it.next(), None);}{let mut stream = tokenizer.token_stream("baz foobar qux");- assert_eq!(stream.next().unwrap().text, "baz");- assert_eq!(stream.next().unwrap().text, "foo");- assert_eq!(stream.next().unwrap().text, "bar");- assert_eq!(stream.next().unwrap().text, "qux");- assert_eq!(stream.next(), None);+ let mut it = stream.iter();+ assert_eq!(it.next().unwrap().text, "baz");+ assert_eq!(it.next().unwrap().text, "foo");+ assert_eq!(it.next().unwrap().text, "bar");+ assert_eq!(it.next().unwrap().text, "qux");+ assert_eq!(it.next(), None);}{let mut stream = tokenizer.token_stream("foobar foobar");- assert_eq!(stream.next().unwrap().text, "foo");- assert_eq!(stream.next().unwrap().text, "bar");- assert_eq!(stream.next().unwrap().text, "foo");- assert_eq!(stream.next().unwrap().text, "bar");- assert_eq!(stream.next(), None);+ let mut it = stream.iter();+ assert_eq!(it.next().unwrap().text, "foo");+ assert_eq!(it.next().unwrap().text, "bar");+ assert_eq!(it.next().unwrap().text, "foo");+ assert_eq!(it.next().unwrap().text, "bar");+ assert_eq!(it.next(), None);}{let mut stream = tokenizer.token_stream("foobar foo bar foobar");- assert_eq!(stream.next().unwrap().text, "foo");- assert_eq!(stream.next().unwrap().text, "bar");- assert_eq!(stream.next().unwrap().text, "foo");- assert_eq!(stream.next().unwrap().text, "bar");- assert_eq!(stream.next().unwrap().text, "foo");- assert_eq!(stream.next().unwrap().text, "bar");- assert_eq!(stream.next(), None);+ let mut it = stream.iter();+ assert_eq!(it.next().unwrap().text, "foo");+ assert_eq!(it.next().unwrap().text, "bar");+ assert_eq!(it.next().unwrap().text, "foo");+ assert_eq!(it.next().unwrap().text, "bar");+ assert_eq!(it.next().unwrap().text, "foo");+ assert_eq!(it.next().unwrap().text, "bar");+ assert_eq!(it.next(), None);}{let mut stream = tokenizer.token_stream("foobazbar foo bar foobar");- assert_eq!(stream.next().unwrap().text, "foobazbar");- assert_eq!(stream.next().unwrap().text, "foo");- assert_eq!(stream.next().unwrap().text, "bar");- assert_eq!(stream.next().unwrap().text, "foo");- assert_eq!(stream.next().unwrap().text, "bar");- assert_eq!(stream.next(), None);+ let mut it = stream.iter();+ assert_eq!(it.next().unwrap().text, "foobazbar");+ assert_eq!(it.next().unwrap().text, "foo");+ assert_eq!(it.next().unwrap().text, "bar");+ assert_eq!(it.next().unwrap().text, "foo");+ assert_eq!(it.next().unwrap().text, "bar");+ assert_eq!(it.next(), None);}{let mut stream = tokenizer.token_stream("foobar qux foobar");- assert_eq!(stream.next().unwrap().text, "foo");- assert_eq!(stream.next().unwrap().text, "bar");- assert_eq!(stream.next().unwrap().text, "qux");- assert_eq!(stream.next().unwrap().text, "foo");- assert_eq!(stream.next().unwrap().text, "bar");- assert_eq!(stream.next(), None);+ let mut it = stream.iter();+ assert_eq!(it.next().unwrap().text, "foo");+ assert_eq!(it.next().unwrap().text, "bar");+ assert_eq!(it.next().unwrap().text, "qux");+ assert_eq!(it.next().unwrap().text, "foo");+ assert_eq!(it.next().unwrap().text, "bar");+ assert_eq!(it.next(), None);}{let mut stream = tokenizer.token_stream("barfoo");- assert_eq!(stream.next().unwrap().text, "bar");- assert_eq!(stream.next().unwrap().text, "foo");- assert_eq!(stream.next(), None);+ let mut it = stream.iter();+ assert_eq!(it.next().unwrap().text, "bar");+ assert_eq!(it.next().unwrap().text, "foo");+ assert_eq!(it.next(), None);}}}diff --git a/crates/tantivy/src/tokenizer/stop_word_filter/mod.rs b/crates/tantivy/src/tokenizer/stop_word_filter/mod.rsindex 3217af71..ad2e97fd 100644--- a/crates/tantivy/src/tokenizer/stop_word_filter/mod.rs+++ b/crates/tantivy/src/tokenizer/stop_word_filter/mod.rs@@ -1,5 +1,6 @@//! # Example//! ```rust+//! use lending_iter::LendingIterator;//! use tantivy::tokenizer::*;//!//! let mut tokenizer = TextAnalyzer::builder(SimpleTokenizer::default())@@ -7,9 +8,10 @@//! .build();//!//! let mut stream = tokenizer.token_stream("the fox is crafty");-//! assert_eq!(stream.next().unwrap().text, "fox");-//! assert_eq!(stream.next().unwrap().text, "crafty");-//! assert!(stream.next().is_none());+//! let mut it = TokenStream::iter(&mut stream);+//! assert_eq!(it.next().unwrap().text, "fox");+//! assert_eq!(it.next().unwrap().text, "crafty");+//! assert!(it.next().is_none());//! ```#[cfg(feature = "stopwords")]#[rustfmt::skip]diff --git a/crates/tantivy/src/tokenizer/tokenizer.rs b/crates/tantivy/src/tokenizer/tokenizer.rsindex 96ffa400..f028132a 100644--- a/crates/tantivy/src/tokenizer/tokenizer.rs+++ b/crates/tantivy/src/tokenizer/tokenizer.rs@@ -4,6 +4,8 @@ use crate::tokenizer_api::{BoxTokenStream, TokenFilter, Tokenizer};use crate::tokenizer::empty_tokenizer::EmptyTokenizer;+pub use lending_iter::LendingIterator;+/// `TextAnalyzer` tokenizes an input text into tokens and modifies the resulting `TokenStream`.#[derive(Clone)]pub struct TextAnalyzer {@@ -123,6 +125,8 @@ impl<T: Tokenizer> TextAnalyzerBuilder<T> {#[cfg(test)]mod tests {+ use lending_iter::LendingIterator;+use super::*;use crate::tokenizer::{LowerCaser, RemoveLongFilter, SimpleTokenizer};@@ -133,8 +137,9 @@ mod tests {.filter(LowerCaser).build();let mut stream = analyzer.token_stream("- first bullet point");- assert_eq!(stream.next().unwrap().text, "first");- assert_eq!(stream.next().unwrap().text, "bullet");+ let mut it = crate::tokenizer::TokenStream::iter(&mut stream);+ assert_eq!(it.next().unwrap().text, "first");+ assert_eq!(it.next().unwrap().text, "bullet");}#[test]@@ -169,7 +174,8 @@ mod tests {}let mut analyzer = analyzer_builder.build();let mut stream = analyzer.token_stream("first bullet point");- assert_eq!(stream.next().unwrap().text, "first");- assert_eq!(stream.next().unwrap().text, "bullet");+ let mut it = crate::tokenizer::TokenStream::iter(&mut stream);+ assert_eq!(it.next().unwrap().text, "first");+ assert_eq!(it.next().unwrap().text, "bullet");}}diff --git a/crates/tantivy/src/tokenizer_api/mod.rs b/crates/tantivy/src/tokenizer_api/mod.rsindex 1ed4c9b8..c788ea36 100644--- a/crates/tantivy/src/tokenizer_api/mod.rs+++ b/crates/tantivy/src/tokenizer_api/mod.rs@@ -8,6 +8,7 @@use std::borrow::{Borrow, BorrowMut};use std::ops::{Deref, DerefMut};+use lending_iter::LendingIterator;use serde::{Deserialize, Serialize};/// Token@@ -128,17 +129,6 @@ pub trait TokenStream {/// Returns a mutable reference to the current token.fn token_mut(&mut self) -> &mut Token;- /// Helper to iterate over tokens. It- /// simply combines a call to `.advance()`- /// and `.token()`.- fn next(&mut self) -> Option<&Token> {- if self.advance() {- Some(self.token())- } else {- None- }- }-/// Helper function to consume the entire `TokenStream`/// and push the tokens to a sink function.fn process(&mut self, sink: &mut dyn FnMut(&Token)) {@@ -146,6 +136,36 @@ pub trait TokenStream {sink(self.token());}}++ /// Helper function to iterate over the tokens of the `TokenStream`+ /// using a lending iterator.+ fn iter<'a>(&'a mut self) -> TokenIter<'a, Self>+ where+ Self: 'a + Sized,+ {+ TokenIter { token_stream: self }+ }+}++pub struct TokenIter<'a, T> {+ token_stream: &'a mut T,+}++impl<'a, T> LendingIterator for TokenIter<'a, T>+where+ T: TokenStream,+{+ type Item<'b> = &'b Token+ where+ 'a: 'b;++ fn next(&mut self) -> Option<Self::Item<'_>> {+ if self.token_stream.advance() {+ Some(self.token_stream.token())+ } else {+ None+ }+ }}/// Trait for the pluggable components of `Tokenizer`s.