Skip to content

web search done right

avoid string alloc in tokenizer if possible

Mikkel Denkercb1c66f0e8ae
diff --git a/crates/core/src/snippet.rs b/crates/core/src/snippet.rs
index 8dfac08a..c1282b28 100644
--- a/crates/core/src/snippet.rs
+++ b/crates/core/src/snippet.rs
@@ -571,7 +571,7 @@ Survey in 2016, 2017, and 2018."#;
proptest! {
#[test]
- fn prop_snippet_gen(text: String, query: String) {
+ fn prop_snippet_gen(text in ".{1,100}", query in ".{1,100}") {
let terms = query.split_whitespace().map(|s| s.to_string()).collect::<Vec<_>>();
let _ = snippet_string(&text, &terms, whatlang::Lang::Eng, SnippetConfig::default());
}
diff --git a/crates/core/src/tokenizer/fields/default.rs b/crates/core/src/tokenizer/fields/default.rs
index 5c0476bd..1f299d2f 100644
--- a/crates/core/src/tokenizer/fields/default.rs
+++ b/crates/core/src/tokenizer/fields/default.rs
@@ -60,7 +60,7 @@ impl tantivy::tokenizer::Tokenizer for DefaultTokenizer {
pub struct Normal;
pub struct NormalTokenStream<'a> {
- stream: Box<dyn Iterator<Item = tokenizer::Token> + 'a>,
+ stream: Box<dyn Iterator<Item = tokenizer::Token<'a>> + 'a>,
token: Option<tantivy::tokenizer::Token>,
next_position: usize,
}
@@ -103,7 +103,9 @@ impl<'a> tantivy::tokenizer::TokenStream for NormalTokenStream<'a> {
}
impl<'a> NormalTokenStream<'a> {
- fn new_boxed(stream: Box<dyn Iterator<Item = tokenizer::Token> + 'a>) -> BoxTokenStream<'a> {
+ fn new_boxed(
+ stream: Box<dyn Iterator<Item = tokenizer::Token<'a>> + 'a>,
+ ) -> BoxTokenStream<'a> {
BoxTokenStream::new(Self {
stream,
token: None,
diff --git a/crates/core/src/tokenizer/mod.rs b/crates/core/src/tokenizer/mod.rs
index 74e2fb3a..a9107ec5 100644
--- a/crates/core/src/tokenizer/mod.rs
+++ b/crates/core/src/tokenizer/mod.rs
@@ -23,23 +23,31 @@ mod split_preserve;
mod split_whitespace_with_range;
mod stemmer;
+use std::borrow::{Borrow, Cow};
+
pub use fields::FieldTokenizer;
use self::segmenter::Segmenter;
#[derive(Debug)]
-pub struct Token {
- text: String,
+pub struct Token<'a> {
+ text: Cow<'a, str>,
span: std::ops::Range<usize>,
}
-impl Token {
- pub fn new(text: String, span: std::ops::Range<usize>) -> Self {
- Token { text, span }
+impl<'a> Token<'a> {
+ pub fn new<S>(text: S, span: std::ops::Range<usize>) -> Self
+ where
+ S: Into<Cow<'a, str>>,
+ {
+ Token {
+ text: text.into(),
+ span,
+ }
}
pub fn text(&self) -> &str {
- &self.text
+ self.text.borrow()
}
pub fn span(&self) -> std::ops::Range<usize> {
diff --git a/crates/core/src/tokenizer/script_tokenizer.rs b/crates/core/src/tokenizer/script_tokenizer.rs
index 00e2f756..9c0c3483 100644
--- a/crates/core/src/tokenizer/script_tokenizer.rs
+++ b/crates/core/src/tokenizer/script_tokenizer.rs
@@ -20,13 +20,13 @@ use super::{
};
pub trait ScriptTokenizer {
- fn tokenize<'a>(&self, text: &'a str) -> Box<dyn Iterator<Item = Token> + 'a>;
+ fn tokenize<'a>(&self, text: &'a str) -> Box<dyn Iterator<Item = Token<'a>> + 'a>;
}
pub struct Latin;
impl ScriptTokenizer for Latin {
- fn tokenize<'a>(&self, text: &'a str) -> Box<dyn Iterator<Item = Token> + 'a> {
+ fn tokenize<'a>(&self, text: &'a str) -> Box<dyn Iterator<Item = Token<'a>> + 'a> {
Box::new(
text.split_whitespace_with_range()
.flat_map(|(txt, span)| {
@@ -37,7 +37,7 @@ impl ScriptTokenizer for Latin {
(txt, span)
})
})
- .map(|(txt, span)| Token::new(txt.to_string(), span)),
+ .map(|(txt, span)| Token::new(txt, span)),
)
}
}
diff --git a/crates/core/src/tokenizer/segmenter.rs b/crates/core/src/tokenizer/segmenter.rs
index efa302ec..42409f14 100644
--- a/crates/core/src/tokenizer/segmenter.rs
+++ b/crates/core/src/tokenizer/segmenter.rs
@@ -29,7 +29,7 @@ impl<'a> Segment<'a> {
&self.full_text[self.span.clone()]
}
- pub fn tokenize(&self) -> impl Iterator<Item = Token> + 'a {
+ pub fn tokenize(&self) -> impl Iterator<Item = Token<'a>> + 'a {
let offset = self.span.start;
let script = self.script;