Skip to content

web search done right

use words tokenizer at query time for all *NoTokenizer fields to ensure that e.g. the query 'example query site.com' matches site.com against the field instead of the entire query

Mikkel Denker38f33c80b5d0
diff --git a/crates/core/src/query/plan/mod.rs b/crates/core/src/query/plan/mod.rs
index 0eca5598..ed01658f 100644
--- a/crates/core/src/query/plan/mod.rs
+++ b/crates/core/src/query/plan/mod.rs
@@ -217,6 +217,10 @@ fn process_tantivy_term<T: TextField>(
});
}
+ if terms.is_empty() {
+ dbg!(field.name());
+ }
+
terms
}
diff --git a/crates/core/src/schema/text_field.rs b/crates/core/src/schema/text_field.rs
index 9c24a24c..8b9aee30 100644
--- a/crates/core/src/schema/text_field.rs
+++ b/crates/core/src/schema/text_field.rs
@@ -33,7 +33,7 @@ use crate::{
self,
fields::{
BigramTokenizer, FieldTokenizer, Identity, JsonField, NewlineTokenizer,
- TrigramTokenizer, UrlTokenizer,
+ TrigramTokenizer, UrlTokenizer, WordTokenizer,
},
},
webpage::Html,
@@ -577,6 +577,10 @@ impl TextField for UrlNoTokenizer {
FieldTokenizer::Identity(Identity {})
}
+ fn query_tokenizer(&self, _: Option<&whatlang::Lang>) -> FieldTokenizer {
+ FieldTokenizer::Words(WordTokenizer::default())
+ }
+
fn is_searchable(&self) -> bool {
true
}
@@ -707,6 +711,10 @@ impl TextField for SiteNoTokenizer {
FieldTokenizer::Identity(Identity {})
}
+ fn query_tokenizer(&self, _: Option<&whatlang::Lang>) -> FieldTokenizer {
+ FieldTokenizer::Words(WordTokenizer::default())
+ }
+
fn is_searchable(&self) -> bool {
true
}
@@ -750,6 +758,10 @@ impl TextField for DomainNoTokenizer {
FieldTokenizer::Identity(Identity {})
}
+ fn query_tokenizer(&self, _: Option<&whatlang::Lang>) -> FieldTokenizer {
+ FieldTokenizer::Words(WordTokenizer::default())
+ }
+
fn is_searchable(&self) -> bool {
true
}
@@ -793,9 +805,8 @@ impl TextField for DomainNameNoTokenizer {
FieldTokenizer::Identity(Identity {})
}
- fn query_tokenizer(&self, lang: Option<&whatlang::Lang>) -> FieldTokenizer {
- // TODO: change this to word tokenizer
- FieldTokenizer::default()
+ fn query_tokenizer(&self, _: Option<&whatlang::Lang>) -> FieldTokenizer {
+ FieldTokenizer::Words(WordTokenizer::default())
}
fn is_searchable(&self) -> bool {
@@ -841,6 +852,10 @@ impl TextField for SiteIfHomepageNoTokenizer {
FieldTokenizer::Identity(Identity {})
}
+ fn query_tokenizer(&self, _: Option<&whatlang::Lang>) -> FieldTokenizer {
+ FieldTokenizer::Words(WordTokenizer::default())
+ }
+
fn add_html_tantivy(
&self,
html: &Html,
@@ -921,6 +936,10 @@ impl TextField for DomainNameIfHomepageNoTokenizer {
FieldTokenizer::Identity(Identity {})
}
+ fn query_tokenizer(&self, _: Option<&whatlang::Lang>) -> FieldTokenizer {
+ FieldTokenizer::Words(WordTokenizer::default())
+ }
+
fn add_html_tantivy(
&self,
html: &Html,
@@ -968,6 +987,10 @@ impl TextField for DomainIfHomepageNoTokenizer {
FieldTokenizer::Identity(Identity {})
}
+ fn query_tokenizer(&self, _: Option<&whatlang::Lang>) -> FieldTokenizer {
+ FieldTokenizer::Words(WordTokenizer::default())
+ }
+
fn add_html_tantivy(
&self,
html: &Html,
diff --git a/crates/core/src/tokenizer/fields/bigram.rs b/crates/core/src/tokenizer/fields/bigram.rs
index aa63bf37..5f0f80d5 100644
--- a/crates/core/src/tokenizer/fields/bigram.rs
+++ b/crates/core/src/tokenizer/fields/bigram.rs
@@ -68,7 +68,7 @@ mod tests {
#[test]
fn bigram_tokenizer() {
assert!(tokenize_bigram("").is_empty());
- assert!(tokenize_bigram("test").is_empty());
+ assert_eq!(tokenize_bigram("test"), vec!["test"]);
assert_eq!(tokenize_bigram("this is"), vec!["thisis"]);
assert_eq!(tokenize_bigram("this is a"), vec!["thisis", "isa",]);
diff --git a/crates/core/src/tokenizer/fields/mod.rs b/crates/core/src/tokenizer/fields/mod.rs
index ebaf04b9..f4f6781f 100644
--- a/crates/core/src/tokenizer/fields/mod.rs
+++ b/crates/core/src/tokenizer/fields/mod.rs
@@ -19,15 +19,17 @@ use tantivy::tokenizer::BoxTokenStream;
pub use self::{
bigram::BigramTokenizer, default::DefaultTokenizer, identity::Identity, json::FlattenedJson,
json::JsonField, split_newlines::NewlineTokenizer, stemmed::Stemmed, trigram::TrigramTokenizer,
- url::UrlTokenizer,
+ url::UrlTokenizer, words::WordTokenizer,
};
mod default;
mod identity;
mod json;
+mod pred;
mod split_newlines;
mod stemmed;
mod url;
+mod words;
mod bigram;
mod ngram;
@@ -43,6 +45,7 @@ pub enum FieldTokenizer {
Json(JsonField),
Url(UrlTokenizer),
Newline(NewlineTokenizer),
+ Words(WordTokenizer),
}
impl FieldTokenizer {
@@ -56,6 +59,7 @@ impl FieldTokenizer {
FieldTokenizer::Json(_) => JsonField::as_str(),
FieldTokenizer::Url(_) => UrlTokenizer::as_str(),
FieldTokenizer::Newline(_) => NewlineTokenizer::as_str(),
+ FieldTokenizer::Words(_) => WordTokenizer::as_str(),
}
}
}
@@ -84,6 +88,7 @@ impl tantivy::tokenizer::Tokenizer for FieldTokenizer {
FieldTokenizer::Trigram(tokenizer) => tokenizer.token_stream(text),
FieldTokenizer::Url(tokenizer) => tokenizer.token_stream(text),
FieldTokenizer::Newline(tokenizer) => tokenizer.token_stream(text),
+ FieldTokenizer::Words(tokenizer) => tokenizer.token_stream(text),
}
}
}
diff --git a/crates/core/src/tokenizer/fields/ngram.rs b/crates/core/src/tokenizer/fields/ngram.rs
index 369972b2..01b4e7e0 100644
--- a/crates/core/src/tokenizer/fields/ngram.rs
+++ b/crates/core/src/tokenizer/fields/ngram.rs
@@ -54,7 +54,7 @@ impl<'a, const N: usize> tantivy::tokenizer::TokenStream for NGramTokenStream<'a
while self.token_window[0].text.is_empty() {
if !self.inner.advance() {
- return false;
+ break;
}
self.token_window.rotate_left(1);
diff --git a/crates/core/src/tokenizer/fields/pred.rs b/crates/core/src/tokenizer/fields/pred.rs
new file mode 100644
index 00000000..bfc776a8
--- /dev/null
+++ b/crates/core/src/tokenizer/fields/pred.rs
@@ -0,0 +1,88 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2024 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use tantivy::tokenizer::BoxTokenStream;
+
+use crate::tokenizer::{self, normalizer, split_with_range::SplitWithRange, Normalize};
+
+#[derive(Clone)]
+pub struct PredTokenizer<P>(pub P)
+where
+ P: Fn(char) -> bool + Send + Sync + Clone + 'static;
+
+impl<P> tantivy::tokenizer::Tokenizer for PredTokenizer<P>
+where
+ P: Fn(char) -> bool + Send + Sync + Clone + 'static,
+{
+ type TokenStream<'a> = BoxTokenStream<'a>;
+
+ fn token_stream<'a>(&'a mut self, text: &'a str) -> Self::TokenStream<'a> {
+ let stream = Box::new(
+ text.split_with_range(|c| self.0(c))
+ .map(|(s, range)| tokenizer::Token::new(s, range))
+ .normalize(&normalizer::Lowercase)
+ .normalize(&normalizer::UnicodeNFKD)
+ .normalize(&normalizer::UnicodeDiacritics),
+ );
+
+ BoxTokenStream::new(PredTokenStream::new_boxed(stream))
+ }
+}
+
+pub struct PredTokenStream<'a> {
+ stream: Box<dyn Iterator<Item = tokenizer::Token<'a>> + 'a>,
+ token: Option<tantivy::tokenizer::Token>,
+ next_position: usize,
+}
+
+impl<'a> tantivy::tokenizer::TokenStream for PredTokenStream<'a> {
+ fn advance(&mut self) -> bool {
+ self.token = self.stream.next().map(|token| {
+ let span = token.span();
+ let pos = self.next_position;
+ self.next_position += 1;
+ tantivy::tokenizer::Token {
+ offset_from: span.start,
+ offset_to: span.end,
+ position: pos,
+ text: token.text().to_string(),
+ ..Default::default()
+ }
+ });
+
+ self.token.is_some()
+ }
+
+ fn token(&self) -> &tantivy::tokenizer::Token {
+ self.token.as_ref().unwrap()
+ }
+
+ fn token_mut(&mut self) -> &mut tantivy::tokenizer::Token {
+ self.token.as_mut().unwrap()
+ }
+}
+
+impl<'a> PredTokenStream<'a> {
+ fn new_boxed(
+ stream: Box<dyn Iterator<Item = tokenizer::Token<'a>> + 'a>,
+ ) -> BoxTokenStream<'a> {
+ BoxTokenStream::new(Self {
+ stream,
+ token: None,
+ next_position: 0,
+ })
+ }
+}
diff --git a/crates/core/src/tokenizer/fields/split_newlines.rs b/crates/core/src/tokenizer/fields/split_newlines.rs
index d37844e6..64d44e08 100644
--- a/crates/core/src/tokenizer/fields/split_newlines.rs
+++ b/crates/core/src/tokenizer/fields/split_newlines.rs
@@ -16,7 +16,7 @@
use tantivy::tokenizer::{BoxTokenStream, TextAnalyzer};
-use crate::tokenizer::{self, normalizer, split_with_range::SplitWithRange, Normalize};
+use super::pred::PredTokenizer;
#[derive(Clone, Default)]
pub struct NewlineTokenizer {
@@ -33,7 +33,7 @@ impl tantivy::tokenizer::Tokenizer for NewlineTokenizer {
type TokenStream<'a> = BoxTokenStream<'a>;
fn token_stream<'a>(&'a mut self, text: &'a str) -> Self::TokenStream<'a> {
- let builder = TextAnalyzer::builder(Newline);
+ let builder = TextAnalyzer::builder(PredTokenizer(|c| c == '\n' || c == '\r'));
self.analyzer = Some(builder.build());
@@ -41,70 +41,6 @@ impl tantivy::tokenizer::Tokenizer for NewlineTokenizer {
}
}
-#[derive(Clone)]
-pub struct Newline;
-
-pub struct NewlineTokenStream<'a> {
- stream: Box<dyn Iterator<Item = tokenizer::Token<'a>> + 'a>,
- token: Option<tantivy::tokenizer::Token>,
- next_position: usize,
-}
-
-impl tantivy::tokenizer::Tokenizer for Newline {
- type TokenStream<'a> = BoxTokenStream<'a>;
-
- fn token_stream<'a>(&mut self, text: &'a str) -> Self::TokenStream<'a> {
- let stream = Box::new(
- text.split_with_range(|c| c == '\n' || c == '\r')
- .map(|(s, range)| tokenizer::Token::new(s, range))
- .normalize(&normalizer::Lowercase)
- .normalize(&normalizer::UnicodeNFKD)
- .normalize(&normalizer::UnicodeDiacritics),
- );
-
- BoxTokenStream::new(NewlineTokenStream::new_boxed(stream))
- }
-}
-
-impl<'a> tantivy::tokenizer::TokenStream for NewlineTokenStream<'a> {
- fn advance(&mut self) -> bool {
- self.token = self.stream.next().map(|token| {
- let span = token.span();
- let pos = self.next_position;
- self.next_position += 1;
- tantivy::tokenizer::Token {
- offset_from: span.start,
- offset_to: span.end,
- position: pos,
- text: token.text().to_string(),
- ..Default::default()
- }
- });
-
- self.token.is_some()
- }
-
- fn token(&self) -> &tantivy::tokenizer::Token {
- self.token.as_ref().unwrap()
- }
-
- fn token_mut(&mut self) -> &mut tantivy::tokenizer::Token {
- self.token.as_mut().unwrap()
- }
-}
-
-impl<'a> NewlineTokenStream<'a> {
- fn new_boxed(
- stream: Box<dyn Iterator<Item = tokenizer::Token<'a>> + 'a>,
- ) -> BoxTokenStream<'a> {
- BoxTokenStream::new(Self {
- stream,
- token: None,
- next_position: 0,
- })
- }
-}
-
#[cfg(test)]
mod tests {
use super::*;
diff --git a/crates/core/src/tokenizer/fields/trigram.rs b/crates/core/src/tokenizer/fields/trigram.rs
index 024742d8..60693127 100644
--- a/crates/core/src/tokenizer/fields/trigram.rs
+++ b/crates/core/src/tokenizer/fields/trigram.rs
@@ -69,8 +69,8 @@ mod tests {
#[test]
fn trigram_tokenizer() {
assert!(tokenize_trigram("").is_empty());
- assert!(tokenize_trigram("test").is_empty());
- assert!(tokenize_trigram("this is").is_empty());
+ assert_eq!(tokenize_trigram("test"), vec!["test"]);
+ assert_eq!(tokenize_trigram("this is"), vec!["thisis"]);
assert_eq!(tokenize_trigram("this is a"), vec!["thisisa",]);
assert_eq!(
diff --git a/crates/core/src/tokenizer/fields/words.rs b/crates/core/src/tokenizer/fields/words.rs
new file mode 100644
index 00000000..41425189
--- /dev/null
+++ b/crates/core/src/tokenizer/fields/words.rs
@@ -0,0 +1,71 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2024 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use tantivy::tokenizer::{BoxTokenStream, TextAnalyzer};
+
+use super::pred::PredTokenizer;
+
+#[derive(Clone, Default)]
+pub struct WordTokenizer {
+ analyzer: Option<TextAnalyzer>,
+}
+
+impl WordTokenizer {
+ pub fn as_str() -> &'static str {
+ "word"
+ }
+}
+
+impl tantivy::tokenizer::Tokenizer for WordTokenizer {
+ type TokenStream<'a> = BoxTokenStream<'a>;
+
+ fn token_stream<'a>(&'a mut self, text: &'a str) -> Self::TokenStream<'a> {
+ let builder = TextAnalyzer::builder(PredTokenizer(|c| c.is_whitespace()));
+
+ self.analyzer = Some(builder.build());
+
+ self.analyzer.as_mut().unwrap().token_stream(text)
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use lending_iter::LendingIterator;
+ use tantivy::tokenizer::Tokenizer as _;
+
+ fn tokenize(s: &str) -> Vec<String> {
+ let mut res = Vec::new();
+ let mut tokenizer = WordTokenizer::default();
+ let mut stream = tokenizer.token_stream(s);
+ let mut it = tantivy::tokenizer::TokenStream::iter(&mut stream);
+
+ while let Some(token) = it.next() {
+ res.push(token.text.clone());
+ }
+
+ res
+ }
+
+ #[test]
+ fn test_words_tokenizer() {
+ assert!(tokenize("").is_empty());
+ assert_eq!(tokenize("a b"), vec!["a", "b"]);
+ assert_eq!(tokenize("a b "), vec!["a", "b"]);
+ assert_eq!(tokenize(" a b "), vec!["a", "b"]);
+ assert_eq!(tokenize("a b c"), vec!["a", "b", "c"]);
+ }
+}