web search done right
avoid string alloc in tokenizer if possible
Mikkel Denkercb1c66f0e8ae
diff --git a/crates/core/src/snippet.rs b/crates/core/src/snippet.rsindex 8dfac08a..c1282b28 100644--- a/crates/core/src/snippet.rs+++ b/crates/core/src/snippet.rs@@ -571,7 +571,7 @@ Survey in 2016, 2017, and 2018."#;proptest! {#[test]- fn prop_snippet_gen(text: String, query: String) {+ fn prop_snippet_gen(text in ".{1,100}", query in ".{1,100}") {let terms = query.split_whitespace().map(|s| s.to_string()).collect::<Vec<_>>();let _ = snippet_string(&text, &terms, whatlang::Lang::Eng, SnippetConfig::default());}diff --git a/crates/core/src/tokenizer/fields/default.rs b/crates/core/src/tokenizer/fields/default.rsindex 5c0476bd..1f299d2f 100644--- a/crates/core/src/tokenizer/fields/default.rs+++ b/crates/core/src/tokenizer/fields/default.rs@@ -60,7 +60,7 @@ impl tantivy::tokenizer::Tokenizer for DefaultTokenizer {pub struct Normal;pub struct NormalTokenStream<'a> {- stream: Box<dyn Iterator<Item = tokenizer::Token> + 'a>,+ stream: Box<dyn Iterator<Item = tokenizer::Token<'a>> + 'a>,token: Option<tantivy::tokenizer::Token>,next_position: usize,}@@ -103,7 +103,9 @@ impl<'a> tantivy::tokenizer::TokenStream for NormalTokenStream<'a> {}impl<'a> NormalTokenStream<'a> {- fn new_boxed(stream: Box<dyn Iterator<Item = tokenizer::Token> + 'a>) -> BoxTokenStream<'a> {+ fn new_boxed(+ stream: Box<dyn Iterator<Item = tokenizer::Token<'a>> + 'a>,+ ) -> BoxTokenStream<'a> {BoxTokenStream::new(Self {stream,token: None,diff --git a/crates/core/src/tokenizer/mod.rs b/crates/core/src/tokenizer/mod.rsindex 74e2fb3a..a9107ec5 100644--- a/crates/core/src/tokenizer/mod.rs+++ b/crates/core/src/tokenizer/mod.rs@@ -23,23 +23,31 @@ mod split_preserve;mod split_whitespace_with_range;mod stemmer;+use std::borrow::{Borrow, Cow};+pub use fields::FieldTokenizer;use self::segmenter::Segmenter;#[derive(Debug)]-pub struct Token {- text: String,+pub struct Token<'a> {+ text: Cow<'a, str>,span: std::ops::Range<usize>,}-impl Token {- pub fn new(text: String, span: std::ops::Range<usize>) -> Self {- Token { text, span }+impl<'a> Token<'a> {+ pub fn new<S>(text: S, span: std::ops::Range<usize>) -> Self+ where+ S: Into<Cow<'a, str>>,+ {+ Token {+ text: text.into(),+ span,+ }}pub fn text(&self) -> &str {- &self.text+ self.text.borrow()}pub fn span(&self) -> std::ops::Range<usize> {diff --git a/crates/core/src/tokenizer/script_tokenizer.rs b/crates/core/src/tokenizer/script_tokenizer.rsindex 00e2f756..9c0c3483 100644--- a/crates/core/src/tokenizer/script_tokenizer.rs+++ b/crates/core/src/tokenizer/script_tokenizer.rs@@ -20,13 +20,13 @@ use super::{};pub trait ScriptTokenizer {- fn tokenize<'a>(&self, text: &'a str) -> Box<dyn Iterator<Item = Token> + 'a>;+ fn tokenize<'a>(&self, text: &'a str) -> Box<dyn Iterator<Item = Token<'a>> + 'a>;}pub struct Latin;impl ScriptTokenizer for Latin {- fn tokenize<'a>(&self, text: &'a str) -> Box<dyn Iterator<Item = Token> + 'a> {+ fn tokenize<'a>(&self, text: &'a str) -> Box<dyn Iterator<Item = Token<'a>> + 'a> {Box::new(text.split_whitespace_with_range().flat_map(|(txt, span)| {@@ -37,7 +37,7 @@ impl ScriptTokenizer for Latin {(txt, span)})})- .map(|(txt, span)| Token::new(txt.to_string(), span)),+ .map(|(txt, span)| Token::new(txt, span)),)}}diff --git a/crates/core/src/tokenizer/segmenter.rs b/crates/core/src/tokenizer/segmenter.rsindex efa302ec..42409f14 100644--- a/crates/core/src/tokenizer/segmenter.rs+++ b/crates/core/src/tokenizer/segmenter.rs@@ -29,7 +29,7 @@ impl<'a> Segment<'a> {&self.full_text[self.span.clone()]}- pub fn tokenize(&self) -> impl Iterator<Item = Token> + 'a {+ pub fn tokenize(&self) -> impl Iterator<Item = Token<'a>> + 'a {let offset = self.span.start;let script = self.script;