Skip to content

web search done right

host-level graph

Mikkel Denkerd90888d52f2a
diff --git a/src/tokenizer.rs b/src/tokenizer.rs
index a3da085b..6df7ccf3 100644
--- a/src/tokenizer.rs
+++ b/src/tokenizer.rs
@@ -29,9 +29,9 @@ impl tantivy::tokenizer::Tokenizer for Tokenizer {
// wrong stemming. I tried merging multiple token streams but it
// caused tantivy to crash. A solution might be to create a separate index
// for stem and non-stemmed and do merging in the searcher. This
- // might cause too much space.
+ // might use too much space.
analyzer = analyzer.filter(Stemmer::new(Language::English));
- tantivy::tokenizer::BoxTokenStream::from(analyzer.clone().token_stream(text.clone()))
+ analyzer.token_stream(text.clone())
}
}