Skip to content

web search done right

Custom tokenizer that stems multiple languages and fixed snippet generation as a result of the changed tokenizer

Mikkel Denkerc8b51db9efa0
diff --git a/src/snippet.rs b/src/snippet.rs
index f83ede79..99d6b514 100644
--- a/src/snippet.rs
+++ b/src/snippet.rs
@@ -14,14 +14,270 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
-use tantivy::SnippetGenerator;
+use std::cmp::Ordering;
+use std::collections::BTreeMap;
+use std::ops::Range;
-use crate::{query::Query, schema::Field, Result};
+use htmlescape::encode_minimal;
+
+use tantivy::tokenizer::{TextAnalyzer, Token};
+use tantivy::{Document, Score, Searcher};
-/// For now we use the snippet generator from tantivy.
+/// For now we use the snippet generator from tantivy with a minor modification to support our TokenStreamMerger.
/// In the future we want to implement something closer to the method described in https://cs.pomona.edu/~dkauchak/ir_project/whitepapers/Snippet-IL.pdf.
/// This will require us to store each paragraph of the webpage separately to get adequate performance.
/// Implementing SnippetIL will also allow us to correctly add "..." to the snippet.
+
+const DEFAULT_MAX_NUM_CHARS: usize = 150;
+
+#[derive(Debug)]
+struct FragmentCandidate {
+ score: Score,
+ start_offset: usize,
+ stop_offset: usize,
+ highlighted: Vec<Range<usize>>,
+}
+
+impl FragmentCandidate {
+ /// Create a basic `FragmentCandidate`
+ ///
+ /// `score`, `num_chars` are set to 0
+ /// and `highlighted` is set to empty vec
+ /// stop_offset is set to start_offset, which is taken as a param.
+ fn new(start_offset: usize) -> FragmentCandidate {
+ FragmentCandidate {
+ score: 0.0,
+ start_offset,
+ stop_offset: start_offset,
+ highlighted: vec![],
+ }
+ }
+
+ /// Updates `score` and `highlighted` fields of the objects.
+ ///
+ /// taking the token and terms, the token is added to the fragment.
+ /// if the token is one of the terms, the score
+ /// and highlighted fields are updated in the fragment.
+ fn try_add_token(&mut self, token: &Token, terms: &BTreeMap<String, Score>) {
+ self.stop_offset = token.offset_to;
+
+ if let Some(&score) = terms.get(&token.text.to_lowercase()) {
+ self.score += score;
+ self.highlighted.push(token.offset_from..token.offset_to);
+ }
+ }
+}
+
+/// `Snippet`
+/// Contains a fragment of a document, and some highlighed parts inside it.
+#[derive(Debug)]
+struct Snippet {
+ fragment: String,
+ highlighted: Vec<Range<usize>>,
+}
+
+const HIGHLIGHTEN_PREFIX: &str = "<b>";
+const HIGHLIGHTEN_POSTFIX: &str = "</b>";
+
+impl Snippet {
+ /// Create a new, empty, `Snippet`
+ fn empty() -> Snippet {
+ Snippet {
+ fragment: String::new(),
+ highlighted: Vec::new(),
+ }
+ }
+
+ /// Returns a hignlightned html from the `Snippet`.
+ fn to_html(&self) -> String {
+ let mut html = String::new();
+ let mut start_from: usize = 0;
+
+ for item in self.highlighted.iter() {
+ if item.start < start_from {
+ start_from = item.end;
+ continue;
+ }
+ html.push_str(&encode_minimal(&self.fragment[start_from..item.start]));
+ html.push_str(HIGHLIGHTEN_PREFIX);
+ html.push_str(&encode_minimal(&self.fragment[item.clone()]));
+ html.push_str(HIGHLIGHTEN_POSTFIX);
+ start_from = item.end;
+ }
+ html.push_str(&encode_minimal(
+ &self.fragment[start_from..self.fragment.len()],
+ ));
+ html
+ }
+
+ /// Returns the fragment of text used in the snippet.
+ fn fragment(&self) -> &str {
+ &self.fragment
+ }
+
+ /// Returns a list of higlighted positions from the `Snippet`.
+ fn highlighted(&self) -> &[Range<usize>] {
+ &self.highlighted
+ }
+}
+
+/// Returns a non-empty list of "good" fragments.
+///
+/// If no target term is within the text, then the function
+/// should return an empty Vec.
+///
+/// If a target term is within the text, then the returned
+/// list is required to be non-empty.
+///
+/// The returned list is non-empty and contain less
+/// than 12 possibly overlapping fragments.
+///
+/// All fragments should contain at least one target term
+/// and have at most `max_num_chars` characters (not bytes).
+///
+/// It is ok to emit non-overlapping fragments, for instance,
+/// one short and one long containing the same keyword, in order
+/// to leave optimization opportunity to the fragment selector
+/// upstream.
+///
+/// Fragments must be valid in the sense that `&text[fragment.start..fragment.stop]`\
+/// has to be a valid string.
+fn search_fragments<'a>(
+ tokenizer: &TextAnalyzer,
+ text: &'a str,
+ terms: &BTreeMap<String, Score>,
+ max_num_chars: usize,
+) -> Vec<FragmentCandidate> {
+ let mut token_stream = tokenizer.token_stream(text);
+ let mut fragment = FragmentCandidate::new(0);
+ let mut fragments: Vec<FragmentCandidate> = vec![];
+ while let Some(next) = token_stream.next() {
+ if (next.offset_to - fragment.start_offset) > max_num_chars {
+ if fragment.score > 0.0 {
+ fragments.push(fragment)
+ };
+ fragment = FragmentCandidate::new(next.offset_from);
+ }
+ fragment.try_add_token(next, terms);
+ }
+ if fragment.score > 0.0 {
+ fragments.push(fragment)
+ }
+
+ fragments
+}
+
+/// Returns a Snippet
+///
+/// Takes a vector of `FragmentCandidate`s and the text.
+/// Figures out the best fragment from it and creates a snippet.
+fn select_best_fragment_combination(fragments: &[FragmentCandidate], text: &str) -> Snippet {
+ let best_fragment_opt = fragments.iter().max_by(|left, right| {
+ let cmp_score = left
+ .score
+ .partial_cmp(&right.score)
+ .unwrap_or(Ordering::Equal);
+ if cmp_score == Ordering::Equal {
+ (right.start_offset, right.stop_offset).cmp(&(left.start_offset, left.stop_offset))
+ } else {
+ cmp_score
+ }
+ });
+ if let Some(fragment) = best_fragment_opt {
+ let fragment_text = &text[fragment.start_offset..fragment.stop_offset];
+ let highlighted = fragment
+ .highlighted
+ .iter()
+ .map(|item| item.start - fragment.start_offset..item.end - fragment.start_offset)
+ .collect();
+ Snippet {
+ fragment: fragment_text.to_string(),
+ highlighted,
+ }
+ } else {
+ // when there no fragments to chose from,
+ // for now create a empty snippet
+ Snippet {
+ fragment: String::new(),
+ highlighted: vec![],
+ }
+ }
+}
+
+struct SnippetGenerator {
+ terms_text: BTreeMap<String, Score>,
+ tokenizer: TextAnalyzer,
+ field: tantivy::schema::Field,
+ max_num_chars: usize,
+}
+
+impl SnippetGenerator {
+ /// Creates a new snippet generator
+ fn create(
+ searcher: &Searcher,
+ query: &dyn tantivy::query::Query,
+ field: tantivy::schema::Field,
+ ) -> crate::Result<SnippetGenerator> {
+ let mut terms = BTreeMap::new();
+ query.query_terms(&mut terms);
+ let mut terms_text: BTreeMap<String, Score> = Default::default();
+ for (term, _) in terms {
+ if term.field() != field {
+ continue;
+ }
+ let term_str = if let Some(term_str) = term.as_str() {
+ term_str
+ } else {
+ continue;
+ };
+ let doc_freq = searcher.doc_freq(&term)?;
+ if doc_freq > 0 {
+ let score = 1.0 / (1.0 + doc_freq as Score);
+ terms_text.insert(term_str.to_string(), score);
+ }
+ }
+ let tokenizer = searcher.index().tokenizer_for_field(field)?;
+ Ok(SnippetGenerator {
+ terms_text,
+ tokenizer,
+ field,
+ max_num_chars: DEFAULT_MAX_NUM_CHARS,
+ })
+ }
+
+ /// Sets a maximum number of chars.
+ fn set_max_num_chars(&mut self, max_num_chars: usize) {
+ self.max_num_chars = max_num_chars;
+ }
+
+ #[cfg(test)]
+ fn terms_text(&self) -> &BTreeMap<String, Score> {
+ &self.terms_text
+ }
+
+ /// Generates a snippet for the given `Document`.
+ ///
+ /// This method extract the text associated to the `SnippetGenerator`'s field
+ /// and computes a snippet.
+ fn snippet_from_doc(&self, doc: &Document) -> Snippet {
+ let text: String = doc
+ .get_all(self.field)
+ .flat_map(tantivy::schema::Value::as_text)
+ .collect::<Vec<&str>>()
+ .join(" ");
+ self.snippet(&text)
+ }
+
+ /// Generates a snippet for the given text.
+ fn snippet(&self, text: &str) -> Snippet {
+ let fragment_candidates =
+ search_fragments(&self.tokenizer, text, &self.terms_text, self.max_num_chars);
+ select_best_fragment_combination(&fragment_candidates[..], text)
+ }
+}
+
+use crate::{query::Query, schema::Field, Result};
+
pub fn generate(query: &Query, text: &str, searcher: &tantivy::Searcher) -> Result<String> {
let generator = SnippetGenerator::create(
searcher,
@@ -40,7 +296,10 @@ pub fn generate(query: &Query, text: &str, searcher: &tantivy::Searcher) -> Resu
#[cfg(test)]
mod tests {
+ use super::*;
use crate::{index::Index, searcher::Searcher, webpage::Webpage};
+ use maplit::btreemap;
+ use tantivy::tokenizer::SimpleTokenizer;
const TEST_TEXT: &str = r#"Rust is a systems programming language sponsored by
Mozilla which describes it as a "safe, concurrent, practical language", supporting functional and
@@ -90,4 +349,260 @@ Survey in 2016, 2017, and 2018."#;
.to_string()
);
}
+
+ #[test]
+ fn test_snippet() {
+ let terms = btreemap! {
+ String::from("rust") => 1.0,
+ String::from("language") => 0.9
+ };
+ let fragments = search_fragments(&From::from(SimpleTokenizer), TEST_TEXT, &terms, 100);
+ assert_eq!(fragments.len(), 7);
+ {
+ let first = &fragments[0];
+ assert_eq!(first.score, 1.9);
+ assert_eq!(first.stop_offset, 89);
+ }
+ let snippet = select_best_fragment_combination(&fragments[..], TEST_TEXT);
+ assert_eq!(
+ snippet.fragment,
+ "Rust is a systems programming language sponsored by\nMozilla which describes it as a \
+ \"safe"
+ );
+ assert_eq!(
+ snippet.to_html(),
+ "<b>Rust</b> is a systems programming <b>language</b> sponsored by\nMozilla which \
+ describes it as a &quot;safe"
+ )
+ }
+
+ #[test]
+ fn test_snippet_scored_fragment() {
+ {
+ let terms = btreemap! {
+ String::from("rust") =>1.0,
+ String::from("language") => 0.9
+ };
+ let fragments = search_fragments(&From::from(SimpleTokenizer), TEST_TEXT, &terms, 20);
+ {
+ let first = &fragments[0];
+ assert_eq!(first.score, 1.0);
+ assert_eq!(first.stop_offset, 17);
+ }
+ let snippet = select_best_fragment_combination(&fragments[..], TEST_TEXT);
+ assert_eq!(snippet.to_html(), "<b>Rust</b> is a systems")
+ }
+ {
+ let terms = btreemap! {
+ String::from("rust") =>0.9,
+ String::from("language") => 1.0
+ };
+ let fragments = search_fragments(&From::from(SimpleTokenizer), TEST_TEXT, &terms, 20);
+ // assert_eq!(fragments.len(), 7);
+ {
+ let first = &fragments[0];
+ assert_eq!(first.score, 0.9);
+ assert_eq!(first.stop_offset, 17);
+ }
+ let snippet = select_best_fragment_combination(&fragments[..], TEST_TEXT);
+ assert_eq!(snippet.to_html(), "programming <b>language</b>")
+ }
+ }
+
+ #[test]
+ fn test_snippet_in_second_fragment() {
+ let text = "a b c d e f g";
+
+ let mut terms = BTreeMap::new();
+ terms.insert(String::from("c"), 1.0);
+
+ let fragments = search_fragments(&From::from(SimpleTokenizer), text, &terms, 3);
+
+ assert_eq!(fragments.len(), 1);
+ {
+ let first = &fragments[0];
+ assert_eq!(first.score, 1.0);
+ assert_eq!(first.start_offset, 4);
+ assert_eq!(first.stop_offset, 7);
+ }
+
+ let snippet = select_best_fragment_combination(&fragments[..], text);
+ assert_eq!(snippet.fragment, "c d");
+ assert_eq!(snippet.to_html(), "<b>c</b> d");
+ }
+
+ #[test]
+ fn test_snippet_with_term_at_the_end_of_fragment() {
+ let text = "a b c d e f f g";
+
+ let mut terms = BTreeMap::new();
+ terms.insert(String::from("f"), 1.0);
+
+ let fragments = search_fragments(&From::from(SimpleTokenizer), text, &terms, 3);
+
+ assert_eq!(fragments.len(), 2);
+ {
+ let first = &fragments[0];
+ assert_eq!(first.score, 1.0);
+ assert_eq!(first.stop_offset, 11);
+ assert_eq!(first.start_offset, 8);
+ }
+
+ let snippet = select_best_fragment_combination(&fragments[..], text);
+ assert_eq!(snippet.fragment, "e f");
+ assert_eq!(snippet.to_html(), "e <b>f</b>");
+ }
+
+ #[test]
+ fn test_snippet_with_second_fragment_has_the_highest_score() {
+ let text = "a b c d e f g";
+
+ let mut terms = BTreeMap::new();
+ terms.insert(String::from("f"), 1.0);
+ terms.insert(String::from("a"), 0.9);
+
+ let fragments = search_fragments(&From::from(SimpleTokenizer), text, &terms, 7);
+
+ assert_eq!(fragments.len(), 2);
+ {
+ let first = &fragments[0];
+ assert_eq!(first.score, 0.9);
+ assert_eq!(first.stop_offset, 7);
+ assert_eq!(first.start_offset, 0);
+ }
+
+ let snippet = select_best_fragment_combination(&fragments[..], text);
+ assert_eq!(snippet.fragment, "e f g");
+ assert_eq!(snippet.to_html(), "e <b>f</b> g");
+ }
+
+ #[test]
+ fn test_snippet_with_term_not_in_text() {
+ let text = "a b c d";
+
+ let mut terms = BTreeMap::new();
+ terms.insert(String::from("z"), 1.0);
+
+ let fragments = search_fragments(&From::from(SimpleTokenizer), text, &terms, 3);
+
+ assert_eq!(fragments.len(), 0);
+
+ let snippet = select_best_fragment_combination(&fragments[..], text);
+ assert_eq!(snippet.fragment, "");
+ assert_eq!(snippet.to_html(), "");
+ }
+
+ #[test]
+ fn test_snippet_with_no_terms() {
+ let text = "a b c d";
+
+ let terms = BTreeMap::new();
+ let fragments = search_fragments(&From::from(SimpleTokenizer), text, &terms, 3);
+ assert_eq!(fragments.len(), 0);
+
+ let snippet = select_best_fragment_combination(&fragments[..], text);
+ assert_eq!(snippet.fragment, "");
+ assert_eq!(snippet.to_html(), "");
+ }
+
+ #[test]
+ fn test_snippet_generator_term_score() {
+ let mut schema_builder = tantivy::schema::Schema::builder();
+ let text_field = schema_builder.add_text_field("text", tantivy::schema::TEXT);
+ let schema = schema_builder.build();
+ let index = tantivy::Index::create_in_ram(schema);
+ {
+ // writing the segment
+ let mut index_writer = index.writer_with_num_threads(1, 10_000_000).unwrap();
+ index_writer
+ .add_document(tantivy::doc!(text_field => "a"))
+ .unwrap();
+ index_writer
+ .add_document(tantivy::doc!(text_field => "a"))
+ .unwrap();
+ index_writer
+ .add_document(tantivy::doc!(text_field => "a b"))
+ .unwrap();
+ index_writer.commit().unwrap();
+ }
+ let searcher = index.reader().unwrap().searcher();
+ let query_parser = tantivy::query::QueryParser::for_index(&index, vec![text_field]);
+ {
+ let query = query_parser.parse_query("e").unwrap();
+ let snippet_generator =
+ SnippetGenerator::create(&searcher, &*query, text_field).unwrap();
+ assert!(snippet_generator.terms_text().is_empty());
+ }
+ {
+ let query = query_parser.parse_query("a").unwrap();
+ let snippet_generator =
+ SnippetGenerator::create(&searcher, &*query, text_field).unwrap();
+ assert_eq!(
+ &btreemap!("a".to_string() => 0.25),
+ snippet_generator.terms_text()
+ );
+ }
+ {
+ let query = query_parser.parse_query("a b").unwrap();
+ let snippet_generator =
+ SnippetGenerator::create(&searcher, &*query, text_field).unwrap();
+ assert_eq!(
+ &btreemap!("a".to_string() => 0.25, "b".to_string() => 0.5),
+ snippet_generator.terms_text()
+ );
+ }
+ {
+ let query = query_parser.parse_query("a b c").unwrap();
+ let snippet_generator =
+ SnippetGenerator::create(&searcher, &*query, text_field).unwrap();
+ assert_eq!(
+ &btreemap!("a".to_string() => 0.25, "b".to_string() => 0.5),
+ snippet_generator.terms_text()
+ );
+ }
+ }
+
+ #[test]
+ fn test_snippet_generator() -> crate::Result<()> {
+ let mut schema_builder = tantivy::schema::Schema::builder();
+ let text_options = tantivy::schema::TextOptions::default().set_indexing_options(
+ tantivy::schema::TextFieldIndexing::default()
+ .set_tokenizer("en_stem")
+ .set_index_option(tantivy::schema::IndexRecordOption::Basic),
+ );
+ let text_field = schema_builder.add_text_field("text", text_options);
+ let schema = schema_builder.build();
+ let index = tantivy::Index::create_in_ram(schema);
+ {
+ // writing the segment
+ let mut index_writer = index.writer_with_num_threads(1, 10_000_000)?;
+ let doc = tantivy::doc!(text_field => TEST_TEXT);
+ index_writer.add_document(doc)?;
+ index_writer.commit()?;
+ }
+ let searcher = index.reader().unwrap().searcher();
+ let query_parser = tantivy::query::QueryParser::for_index(&index, vec![text_field]);
+ let query = query_parser.parse_query("rust design").unwrap();
+ let mut snippet_generator =
+ SnippetGenerator::create(&searcher, &*query, text_field).unwrap();
+ {
+ let snippet = snippet_generator.snippet(TEST_TEXT);
+ assert_eq!(
+ snippet.to_html(),
+ "imperative-procedural paradigms. <b>Rust</b> is syntactically similar to \
+ C++[according to whom?],\nbut its <b>designers</b> intend it to provide better \
+ memory safety"
+ );
+ }
+ {
+ snippet_generator.set_max_num_chars(90);
+ let snippet = snippet_generator.snippet(TEST_TEXT);
+ assert_eq!(
+ snippet.to_html(),
+ "<b>Rust</b> is syntactically similar to C++[according to whom?],\nbut its \
+ <b>designers</b> intend it to"
+ );
+ }
+ Ok(())
+ }
}
diff --git a/src/tokenizer.rs b/src/tokenizer.rs
index 6df7ccf3..12c8d2e8 100644
--- a/src/tokenizer.rs
+++ b/src/tokenizer.rs
@@ -14,24 +14,108 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
-use tantivy::tokenizer::{Language, LowerCaser, SimpleTokenizer, Stemmer, TextAnalyzer};
+use tantivy::tokenizer::{
+ Language, LowerCaser, PreTokenizedStream, PreTokenizedString, SimpleTokenizer, Stemmer,
+ TextAnalyzer, TokenStream,
+};
+use whatlang::Lang;
+
+struct TokenStreamMerger<'a> {
+ streams: Vec<tantivy::tokenizer::BoxTokenStream<'a>>,
+ current_stream: usize,
+}
+
+impl<'a> TokenStreamMerger<'a> {
+ fn merge(streams: Vec<tantivy::tokenizer::BoxTokenStream<'a>>) -> Self {
+ Self {
+ streams,
+ current_stream: 0,
+ }
+ }
+}
+
+impl<'a> tantivy::tokenizer::TokenStream for TokenStreamMerger<'a> {
+ fn advance(&mut self) -> bool {
+ if let Some(stream) = self.streams.get_mut(self.current_stream) {
+ let mut has_more = stream.advance();
+
+ if !has_more {
+ self.current_stream += 1;
+ has_more = self.current_stream < self.streams.len()
+ && self.streams[self.current_stream].advance();
+ }
+
+ has_more
+ } else {
+ false
+ }
+ }
+
+ fn token(&self) -> &tantivy::tokenizer::Token {
+ self.streams[self.current_stream].token()
+ }
+
+ fn token_mut(&mut self) -> &mut tantivy::tokenizer::Token {
+ self.streams[self.current_stream].token_mut()
+ }
+}
+
+struct MyStemmer(Stemmer);
+
+impl From<Lang> for MyStemmer {
+ fn from(lang: Lang) -> Self {
+ match lang {
+ Lang::Dan => MyStemmer(Stemmer::new(Language::Danish)),
+ Lang::Ara => MyStemmer(Stemmer::new(Language::Arabic)),
+ Lang::Nld => MyStemmer(Stemmer::new(Language::Dutch)),
+ Lang::Eng => MyStemmer(Stemmer::new(Language::English)),
+ Lang::Fin => MyStemmer(Stemmer::new(Language::Finnish)),
+ Lang::Fra => MyStemmer(Stemmer::new(Language::French)),
+ Lang::Deu => MyStemmer(Stemmer::new(Language::German)),
+ Lang::Hun => MyStemmer(Stemmer::new(Language::Hungarian)),
+ Lang::Ita => MyStemmer(Stemmer::new(Language::Italian)),
+ Lang::Por => MyStemmer(Stemmer::new(Language::Portuguese)),
+ Lang::Ron => MyStemmer(Stemmer::new(Language::Romanian)),
+ Lang::Rus => MyStemmer(Stemmer::new(Language::Russian)),
+ Lang::Spa => MyStemmer(Stemmer::new(Language::Spanish)),
+ Lang::Swe => MyStemmer(Stemmer::new(Language::Swedish)),
+ Lang::Tam => MyStemmer(Stemmer::new(Language::Tamil)),
+ Lang::Tur => MyStemmer(Stemmer::new(Language::Turkish)),
+ _ => MyStemmer(Stemmer::new(Language::English)),
+ }
+ }
+}
+
#[derive(Clone, Default)]
pub struct Tokenizer {}
impl tantivy::tokenizer::Tokenizer for Tokenizer {
fn token_stream<'a>(&self, text: &'a str) -> tantivy::tokenizer::BoxTokenStream<'a> {
- let mut analyzer = TextAnalyzer::from(SimpleTokenizer).filter(LowerCaser);
-
- // TODO:
- // Use whatlang to determine which stemmer to use.
- // We will somehow need to search for the non-stemmed version
- // of the string, as whatlang might be wrong which will result in
- // wrong stemming. I tried merging multiple token streams but it
- // caused tantivy to crash. A solution might be to create a separate index
- // for stem and non-stemmed and do merging in the searcher. This
- // might use too much space.
- analyzer = analyzer.filter(Stemmer::new(Language::English));
-
- analyzer.token_stream(text.clone())
+ let analyzer = TextAnalyzer::from(SimpleTokenizer).filter(LowerCaser);
+
+ let mut streams = vec![analyzer.token_stream(text)];
+
+ if let Some(lang) = whatlang::detect_lang(text) {
+ let stemmed_analyser = analyzer.clone().filter(MyStemmer::from(lang).0);
+ streams.push(stemmed_analyser.token_stream(text));
+ }
+
+ let mut merger = TokenStreamMerger::merge(streams);
+
+ let mut tokens = Vec::new();
+ while let Some(token) = merger.next() {
+ tokens.push(token.clone());
+ }
+
+ tokens.sort_by(|a, b| {
+ a.offset_from
+ .partial_cmp(&b.offset_from)
+ .unwrap_or(a.offset_to.cmp(&b.offset_to))
+ });
+
+ tantivy::tokenizer::BoxTokenStream::from(PreTokenizedStream::from(PreTokenizedString {
+ text: text.to_string(),
+ tokens,
+ }))
}
}