web search done right
use words tokenizer at query time for all *NoTokenizer fields to ensure that e.g. the query 'example query site.com' matches site.com against the field instead of the entire query
Mikkel Denker38f33c80b5d0
diff --git a/crates/core/src/query/plan/mod.rs b/crates/core/src/query/plan/mod.rsindex 0eca5598..ed01658f 100644--- a/crates/core/src/query/plan/mod.rs+++ b/crates/core/src/query/plan/mod.rs@@ -217,6 +217,10 @@ fn process_tantivy_term<T: TextField>(});}+ if terms.is_empty() {+ dbg!(field.name());+ }+terms}diff --git a/crates/core/src/schema/text_field.rs b/crates/core/src/schema/text_field.rsindex 9c24a24c..8b9aee30 100644--- a/crates/core/src/schema/text_field.rs+++ b/crates/core/src/schema/text_field.rs@@ -33,7 +33,7 @@ use crate::{self,fields::{BigramTokenizer, FieldTokenizer, Identity, JsonField, NewlineTokenizer,- TrigramTokenizer, UrlTokenizer,+ TrigramTokenizer, UrlTokenizer, WordTokenizer,},},webpage::Html,@@ -577,6 +577,10 @@ impl TextField for UrlNoTokenizer {FieldTokenizer::Identity(Identity {})}+ fn query_tokenizer(&self, _: Option<&whatlang::Lang>) -> FieldTokenizer {+ FieldTokenizer::Words(WordTokenizer::default())+ }+fn is_searchable(&self) -> bool {true}@@ -707,6 +711,10 @@ impl TextField for SiteNoTokenizer {FieldTokenizer::Identity(Identity {})}+ fn query_tokenizer(&self, _: Option<&whatlang::Lang>) -> FieldTokenizer {+ FieldTokenizer::Words(WordTokenizer::default())+ }+fn is_searchable(&self) -> bool {true}@@ -750,6 +758,10 @@ impl TextField for DomainNoTokenizer {FieldTokenizer::Identity(Identity {})}+ fn query_tokenizer(&self, _: Option<&whatlang::Lang>) -> FieldTokenizer {+ FieldTokenizer::Words(WordTokenizer::default())+ }+fn is_searchable(&self) -> bool {true}@@ -793,9 +805,8 @@ impl TextField for DomainNameNoTokenizer {FieldTokenizer::Identity(Identity {})}- fn query_tokenizer(&self, lang: Option<&whatlang::Lang>) -> FieldTokenizer {- // TODO: change this to word tokenizer- FieldTokenizer::default()+ fn query_tokenizer(&self, _: Option<&whatlang::Lang>) -> FieldTokenizer {+ FieldTokenizer::Words(WordTokenizer::default())}fn is_searchable(&self) -> bool {@@ -841,6 +852,10 @@ impl TextField for SiteIfHomepageNoTokenizer {FieldTokenizer::Identity(Identity {})}+ fn query_tokenizer(&self, _: Option<&whatlang::Lang>) -> FieldTokenizer {+ FieldTokenizer::Words(WordTokenizer::default())+ }+fn add_html_tantivy(&self,html: &Html,@@ -921,6 +936,10 @@ impl TextField for DomainNameIfHomepageNoTokenizer {FieldTokenizer::Identity(Identity {})}+ fn query_tokenizer(&self, _: Option<&whatlang::Lang>) -> FieldTokenizer {+ FieldTokenizer::Words(WordTokenizer::default())+ }+fn add_html_tantivy(&self,html: &Html,@@ -968,6 +987,10 @@ impl TextField for DomainIfHomepageNoTokenizer {FieldTokenizer::Identity(Identity {})}+ fn query_tokenizer(&self, _: Option<&whatlang::Lang>) -> FieldTokenizer {+ FieldTokenizer::Words(WordTokenizer::default())+ }+fn add_html_tantivy(&self,html: &Html,diff --git a/crates/core/src/tokenizer/fields/bigram.rs b/crates/core/src/tokenizer/fields/bigram.rsindex aa63bf37..5f0f80d5 100644--- a/crates/core/src/tokenizer/fields/bigram.rs+++ b/crates/core/src/tokenizer/fields/bigram.rs@@ -68,7 +68,7 @@ mod tests {#[test]fn bigram_tokenizer() {assert!(tokenize_bigram("").is_empty());- assert!(tokenize_bigram("test").is_empty());+ assert_eq!(tokenize_bigram("test"), vec!["test"]);assert_eq!(tokenize_bigram("this is"), vec!["thisis"]);assert_eq!(tokenize_bigram("this is a"), vec!["thisis", "isa",]);diff --git a/crates/core/src/tokenizer/fields/mod.rs b/crates/core/src/tokenizer/fields/mod.rsindex ebaf04b9..f4f6781f 100644--- a/crates/core/src/tokenizer/fields/mod.rs+++ b/crates/core/src/tokenizer/fields/mod.rs@@ -19,15 +19,17 @@ use tantivy::tokenizer::BoxTokenStream;pub use self::{bigram::BigramTokenizer, default::DefaultTokenizer, identity::Identity, json::FlattenedJson,json::JsonField, split_newlines::NewlineTokenizer, stemmed::Stemmed, trigram::TrigramTokenizer,- url::UrlTokenizer,+ url::UrlTokenizer, words::WordTokenizer,};mod default;mod identity;mod json;+mod pred;mod split_newlines;mod stemmed;mod url;+mod words;mod bigram;mod ngram;@@ -43,6 +45,7 @@ pub enum FieldTokenizer {Json(JsonField),Url(UrlTokenizer),Newline(NewlineTokenizer),+ Words(WordTokenizer),}impl FieldTokenizer {@@ -56,6 +59,7 @@ impl FieldTokenizer {FieldTokenizer::Json(_) => JsonField::as_str(),FieldTokenizer::Url(_) => UrlTokenizer::as_str(),FieldTokenizer::Newline(_) => NewlineTokenizer::as_str(),+ FieldTokenizer::Words(_) => WordTokenizer::as_str(),}}}@@ -84,6 +88,7 @@ impl tantivy::tokenizer::Tokenizer for FieldTokenizer {FieldTokenizer::Trigram(tokenizer) => tokenizer.token_stream(text),FieldTokenizer::Url(tokenizer) => tokenizer.token_stream(text),FieldTokenizer::Newline(tokenizer) => tokenizer.token_stream(text),+ FieldTokenizer::Words(tokenizer) => tokenizer.token_stream(text),}}}diff --git a/crates/core/src/tokenizer/fields/ngram.rs b/crates/core/src/tokenizer/fields/ngram.rsindex 369972b2..01b4e7e0 100644--- a/crates/core/src/tokenizer/fields/ngram.rs+++ b/crates/core/src/tokenizer/fields/ngram.rs@@ -54,7 +54,7 @@ impl<'a, const N: usize> tantivy::tokenizer::TokenStream for NGramTokenStream<'awhile self.token_window[0].text.is_empty() {if !self.inner.advance() {- return false;+ break;}self.token_window.rotate_left(1);diff --git a/crates/core/src/tokenizer/fields/pred.rs b/crates/core/src/tokenizer/fields/pred.rsnew file mode 100644index 00000000..bfc776a8--- /dev/null+++ b/crates/core/src/tokenizer/fields/pred.rs@@ -0,0 +1,88 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use tantivy::tokenizer::BoxTokenStream;++use crate::tokenizer::{self, normalizer, split_with_range::SplitWithRange, Normalize};++#[derive(Clone)]+pub struct PredTokenizer<P>(pub P)+where+ P: Fn(char) -> bool + Send + Sync + Clone + 'static;++impl<P> tantivy::tokenizer::Tokenizer for PredTokenizer<P>+where+ P: Fn(char) -> bool + Send + Sync + Clone + 'static,+{+ type TokenStream<'a> = BoxTokenStream<'a>;++ fn token_stream<'a>(&'a mut self, text: &'a str) -> Self::TokenStream<'a> {+ let stream = Box::new(+ text.split_with_range(|c| self.0(c))+ .map(|(s, range)| tokenizer::Token::new(s, range))+ .normalize(&normalizer::Lowercase)+ .normalize(&normalizer::UnicodeNFKD)+ .normalize(&normalizer::UnicodeDiacritics),+ );++ BoxTokenStream::new(PredTokenStream::new_boxed(stream))+ }+}++pub struct PredTokenStream<'a> {+ stream: Box<dyn Iterator<Item = tokenizer::Token<'a>> + 'a>,+ token: Option<tantivy::tokenizer::Token>,+ next_position: usize,+}++impl<'a> tantivy::tokenizer::TokenStream for PredTokenStream<'a> {+ fn advance(&mut self) -> bool {+ self.token = self.stream.next().map(|token| {+ let span = token.span();+ let pos = self.next_position;+ self.next_position += 1;+ tantivy::tokenizer::Token {+ offset_from: span.start,+ offset_to: span.end,+ position: pos,+ text: token.text().to_string(),+ ..Default::default()+ }+ });++ self.token.is_some()+ }++ fn token(&self) -> &tantivy::tokenizer::Token {+ self.token.as_ref().unwrap()+ }++ fn token_mut(&mut self) -> &mut tantivy::tokenizer::Token {+ self.token.as_mut().unwrap()+ }+}++impl<'a> PredTokenStream<'a> {+ fn new_boxed(+ stream: Box<dyn Iterator<Item = tokenizer::Token<'a>> + 'a>,+ ) -> BoxTokenStream<'a> {+ BoxTokenStream::new(Self {+ stream,+ token: None,+ next_position: 0,+ })+ }+}diff --git a/crates/core/src/tokenizer/fields/split_newlines.rs b/crates/core/src/tokenizer/fields/split_newlines.rsindex d37844e6..64d44e08 100644--- a/crates/core/src/tokenizer/fields/split_newlines.rs+++ b/crates/core/src/tokenizer/fields/split_newlines.rs@@ -16,7 +16,7 @@use tantivy::tokenizer::{BoxTokenStream, TextAnalyzer};-use crate::tokenizer::{self, normalizer, split_with_range::SplitWithRange, Normalize};+use super::pred::PredTokenizer;#[derive(Clone, Default)]pub struct NewlineTokenizer {@@ -33,7 +33,7 @@ impl tantivy::tokenizer::Tokenizer for NewlineTokenizer {type TokenStream<'a> = BoxTokenStream<'a>;fn token_stream<'a>(&'a mut self, text: &'a str) -> Self::TokenStream<'a> {- let builder = TextAnalyzer::builder(Newline);+ let builder = TextAnalyzer::builder(PredTokenizer(|c| c == '\n' || c == '\r'));self.analyzer = Some(builder.build());@@ -41,70 +41,6 @@ impl tantivy::tokenizer::Tokenizer for NewlineTokenizer {}}-#[derive(Clone)]-pub struct Newline;--pub struct NewlineTokenStream<'a> {- stream: Box<dyn Iterator<Item = tokenizer::Token<'a>> + 'a>,- token: Option<tantivy::tokenizer::Token>,- next_position: usize,-}--impl tantivy::tokenizer::Tokenizer for Newline {- type TokenStream<'a> = BoxTokenStream<'a>;-- fn token_stream<'a>(&mut self, text: &'a str) -> Self::TokenStream<'a> {- let stream = Box::new(- text.split_with_range(|c| c == '\n' || c == '\r')- .map(|(s, range)| tokenizer::Token::new(s, range))- .normalize(&normalizer::Lowercase)- .normalize(&normalizer::UnicodeNFKD)- .normalize(&normalizer::UnicodeDiacritics),- );-- BoxTokenStream::new(NewlineTokenStream::new_boxed(stream))- }-}--impl<'a> tantivy::tokenizer::TokenStream for NewlineTokenStream<'a> {- fn advance(&mut self) -> bool {- self.token = self.stream.next().map(|token| {- let span = token.span();- let pos = self.next_position;- self.next_position += 1;- tantivy::tokenizer::Token {- offset_from: span.start,- offset_to: span.end,- position: pos,- text: token.text().to_string(),- ..Default::default()- }- });-- self.token.is_some()- }-- fn token(&self) -> &tantivy::tokenizer::Token {- self.token.as_ref().unwrap()- }-- fn token_mut(&mut self) -> &mut tantivy::tokenizer::Token {- self.token.as_mut().unwrap()- }-}--impl<'a> NewlineTokenStream<'a> {- fn new_boxed(- stream: Box<dyn Iterator<Item = tokenizer::Token<'a>> + 'a>,- ) -> BoxTokenStream<'a> {- BoxTokenStream::new(Self {- stream,- token: None,- next_position: 0,- })- }-}-#[cfg(test)]mod tests {use super::*;diff --git a/crates/core/src/tokenizer/fields/trigram.rs b/crates/core/src/tokenizer/fields/trigram.rsindex 024742d8..60693127 100644--- a/crates/core/src/tokenizer/fields/trigram.rs+++ b/crates/core/src/tokenizer/fields/trigram.rs@@ -69,8 +69,8 @@ mod tests {#[test]fn trigram_tokenizer() {assert!(tokenize_trigram("").is_empty());- assert!(tokenize_trigram("test").is_empty());- assert!(tokenize_trigram("this is").is_empty());+ assert_eq!(tokenize_trigram("test"), vec!["test"]);+ assert_eq!(tokenize_trigram("this is"), vec!["thisis"]);assert_eq!(tokenize_trigram("this is a"), vec!["thisisa",]);assert_eq!(diff --git a/crates/core/src/tokenizer/fields/words.rs b/crates/core/src/tokenizer/fields/words.rsnew file mode 100644index 00000000..41425189--- /dev/null+++ b/crates/core/src/tokenizer/fields/words.rs@@ -0,0 +1,71 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use tantivy::tokenizer::{BoxTokenStream, TextAnalyzer};++use super::pred::PredTokenizer;++#[derive(Clone, Default)]+pub struct WordTokenizer {+ analyzer: Option<TextAnalyzer>,+}++impl WordTokenizer {+ pub fn as_str() -> &'static str {+ "word"+ }+}++impl tantivy::tokenizer::Tokenizer for WordTokenizer {+ type TokenStream<'a> = BoxTokenStream<'a>;++ fn token_stream<'a>(&'a mut self, text: &'a str) -> Self::TokenStream<'a> {+ let builder = TextAnalyzer::builder(PredTokenizer(|c| c.is_whitespace()));++ self.analyzer = Some(builder.build());++ self.analyzer.as_mut().unwrap().token_stream(text)+ }+}++#[cfg(test)]+mod tests {+ use super::*;+ use lending_iter::LendingIterator;+ use tantivy::tokenizer::Tokenizer as _;++ fn tokenize(s: &str) -> Vec<String> {+ let mut res = Vec::new();+ let mut tokenizer = WordTokenizer::default();+ let mut stream = tokenizer.token_stream(s);+ let mut it = tantivy::tokenizer::TokenStream::iter(&mut stream);++ while let Some(token) = it.next() {+ res.push(token.text.clone());+ }++ res+ }++ #[test]+ fn test_words_tokenizer() {+ assert!(tokenize("").is_empty());+ assert_eq!(tokenize("a b"), vec!["a", "b"]);+ assert_eq!(tokenize("a b "), vec!["a", "b"]);+ assert_eq!(tokenize(" a b "), vec!["a", "b"]);+ assert_eq!(tokenize("a b c"), vec!["a", "b", "c"]);+ }+}