Skip to content

web search done right

Ability to have different tokenizer for field during query and indexing. This allows us to match compound words in the bigram and trigram field while still keeping ranking signals intact. Before we had modified the bigram and trigram indexers to also output monograms at the start and end, but this turned out to introduce too much noise during ranking.

Mikkel Denkere534cd0d8376
diff --git a/core/src/query/mod.rs b/core/src/query/mod.rs
index e4039212..8e61c923 100644
--- a/core/src/query/mod.rs
+++ b/core/src/query/mod.rs
@@ -70,14 +70,12 @@ impl Query {
}
let schema = index.schema();
- let tokenizer_manager = index.tokenizers();
- let fields: Vec<(tantivy::schema::Field, &tantivy::schema::FieldEntry)> =
- schema.fields().collect();
+ let fields: Vec<tantivy::schema::Field> = schema.fields().map(|(field, _)| field).collect();
let mut queries: Vec<(Occur, Box<dyn tantivy::query::Query + 'static>)> = terms
.iter()
- .map(|term| term.as_tantivy_query(&fields, tokenizer_manager))
+ .map(|term| term.as_tantivy_query(&fields))
.collect();
if query.safe_search {
diff --git a/core/src/query/parser.rs b/core/src/query/parser.rs
index 124bcb10..5688f2eb 100644
--- a/core/src/query/parser.rs
+++ b/core/src/query/parser.rs
@@ -16,7 +16,7 @@
use tantivy::{
query::{BooleanQuery, Occur, PhraseQuery, TermQuery},
- tokenizer::{TextAnalyzer, TokenizerManager},
+ tokenizer::{TextAnalyzer, Tokenizer},
};
use crate::{
@@ -54,16 +54,11 @@ impl ToString for Term {
fn simple_into_tantivy(
term: &str,
- fields: &[(tantivy::schema::Field, &tantivy::schema::FieldEntry)],
- tokenizer_manager: &TokenizerManager,
+ fields: &[tantivy::schema::Field],
) -> (Occur, Box<dyn tantivy::query::Query + 'static>) {
(
Occur::Must,
- Box::new(BooleanQuery::new(Term::into_tantivy_simple(
- term,
- fields,
- tokenizer_manager,
- ))),
+ Box::new(BooleanQuery::new(Term::into_tantivy_simple(term, fields))),
)
}
@@ -83,41 +78,37 @@ impl Term {
pub fn as_tantivy_query(
&self,
- fields: &[(tantivy::schema::Field, &tantivy::schema::FieldEntry)],
- tokenizer_manager: &TokenizerManager,
+ fields: &[tantivy::schema::Field],
) -> (Occur, Box<dyn tantivy::query::Query + 'static>) {
match self {
- Term::Simple(term) => simple_into_tantivy(term, fields, tokenizer_manager),
+ Term::Simple(term) => simple_into_tantivy(term, fields),
Term::Phrase(phrase) => {
let mut phrases = Vec::with_capacity(fields.len());
- for (field, entry) in fields
+ for field in fields
.iter()
- .filter(|(field, _)| ALL_FIELDS[field.field_id() as usize].is_searchable())
- .filter(|(field, _)| ALL_FIELDS[field.field_id() as usize].has_pos())
+ .filter(|field| ALL_FIELDS[field.field_id() as usize].is_searchable())
+ .filter(|field| ALL_FIELDS[field.field_id() as usize].has_pos())
{
- if let Some(analyzer) = Term::get_tantivy_analyzer(entry, tokenizer_manager) {
- let mut processed_terms =
- Term::process_tantivy_term(phrase, Some(analyzer), *field);
-
- if processed_terms.len() == 1 {
- let options = ALL_FIELDS[field.field_id() as usize]
- .as_text()
- .unwrap()
- .index_option();
-
- phrases.push((
- Occur::Should,
- Box::new(TermQuery::new(processed_terms.pop().unwrap(), options))
- as Box<dyn tantivy::query::Query>,
- ));
- } else {
- phrases.push((
- Occur::Should,
- Box::new(PhraseQuery::new(processed_terms))
- as Box<dyn tantivy::query::Query>,
- ));
- }
+ let mut processed_terms = Term::process_tantivy_term(phrase, *field);
+
+ if processed_terms.len() == 1 {
+ let options = ALL_FIELDS[field.field_id() as usize]
+ .as_text()
+ .unwrap()
+ .index_option();
+
+ phrases.push((
+ Occur::Should,
+ Box::new(TermQuery::new(processed_terms.pop().unwrap(), options))
+ as Box<dyn tantivy::query::Query>,
+ ));
+ } else {
+ phrases.push((
+ Occur::Should,
+ Box::new(PhraseQuery::new(processed_terms))
+ as Box<dyn tantivy::query::Query>,
+ ));
}
}
@@ -125,22 +116,16 @@ impl Term {
}
Term::Not(subterm) => (
Occur::MustNot,
- Box::new(BooleanQuery::new(vec![
- subterm.as_tantivy_query(fields, tokenizer_manager)
- ])),
+ Box::new(BooleanQuery::new(vec![subterm.as_tantivy_query(fields)])),
),
Term::Site(site) => (
Occur::Must,
- Box::new(BooleanQuery::new(Term::into_tantivy_site(
- site,
- fields,
- tokenizer_manager,
- ))),
+ Box::new(BooleanQuery::new(Term::into_tantivy_site(site, fields))),
),
Term::Title(title) => {
- let (field, entry) = fields
+ let field = fields
.iter()
- .find(|(field, _)| {
+ .find(|field| {
matches!(
ALL_FIELDS[field.field_id() as usize],
Field::Text(TextField::Title)
@@ -148,15 +133,12 @@ impl Term {
})
.unwrap();
- (
- Occur::Must,
- Term::tantivy_text_query(field, entry, tokenizer_manager, title),
- )
+ (Occur::Must, Term::tantivy_text_query(field, title))
}
Term::Body(body) => {
- let (field, entry) = fields
+ let field = fields
.iter()
- .find(|(field, _)| {
+ .find(|field| {
matches!(
ALL_FIELDS[field.field_id() as usize],
Field::Text(TextField::AllBody)
@@ -164,15 +146,12 @@ impl Term {
})
.unwrap();
- (
- Occur::Must,
- Term::tantivy_text_query(field, entry, tokenizer_manager, body),
- )
+ (Occur::Must, Term::tantivy_text_query(field, body))
}
Term::Url(url) => {
- let (field, entry) = fields
+ let field = fields
.iter()
- .find(|(field, _)| {
+ .find(|field| {
matches!(
ALL_FIELDS[field.field_id() as usize],
Field::Text(TextField::Url)
@@ -180,10 +159,7 @@ impl Term {
})
.unwrap();
- (
- Occur::Must,
- Term::tantivy_text_query(field, entry, tokenizer_manager, url),
- )
+ (Occur::Must, Term::tantivy_text_query(field, url))
}
Term::PossibleBang(text) => {
let mut term = String::new();
@@ -191,59 +167,44 @@ impl Term {
term.push(BANG_PREFIX);
term.push_str(text);
- simple_into_tantivy(&term, fields, tokenizer_manager)
+ simple_into_tantivy(&term, fields)
}
}
}
fn into_tantivy_simple(
term: &str,
- fields: &[(tantivy::schema::Field, &tantivy::schema::FieldEntry)],
- tokenizer_manager: &TokenizerManager,
+ fields: &[tantivy::schema::Field],
) -> Vec<(Occur, Box<dyn tantivy::query::Query + 'static>)> {
fields
.iter()
- .filter(|(field, _)| ALL_FIELDS[field.field_id() as usize].is_searchable())
- .map(|(field, entry)| {
- (
- Occur::Should,
- Term::tantivy_text_query(field, entry, tokenizer_manager, term),
- )
- })
+ .filter(|field| ALL_FIELDS[field.field_id() as usize].is_searchable())
+ .map(|field| (Occur::Should, Term::tantivy_text_query(field, term)))
.collect()
}
fn into_tantivy_site(
term: &str,
- fields: &[(tantivy::schema::Field, &tantivy::schema::FieldEntry)],
- tokenizer_manager: &TokenizerManager,
+ fields: &[tantivy::schema::Field],
) -> Vec<(Occur, Box<dyn tantivy::query::Query + 'static>)> {
fields
.iter()
- .filter(|(field, _)| {
+ .filter(|field| {
matches!(
ALL_FIELDS[field.field_id() as usize],
Field::Text(TextField::DomainNoTokenizer)
| Field::Text(TextField::SiteNoTokenizer)
)
})
- .map(|(field, entry)| {
- (
- Occur::Should,
- Term::tantivy_text_query(field, entry, tokenizer_manager, term),
- )
- })
+ .map(|field| (Occur::Should, Term::tantivy_text_query(field, term)))
.collect()
}
pub fn tantivy_text_query(
field: &tantivy::schema::Field,
- entry: &tantivy::schema::FieldEntry,
- tokenizer_manager: &TokenizerManager,
term: &str,
) -> Box<dyn tantivy::query::Query + 'static> {
- let analyzer = Term::get_tantivy_analyzer(entry, tokenizer_manager);
- let mut processed_terms = Term::process_tantivy_term(term, analyzer, *field);
+ let mut processed_terms = Term::process_tantivy_term(term, *field);
let option = ALL_FIELDS[field.field_id() as usize]
.as_text()
@@ -288,14 +249,13 @@ impl Term {
pub fn process_tantivy_term(
term: &str,
- analyzer: Option<TextAnalyzer>,
tantivy_field: tantivy::schema::Field,
) -> Vec<tantivy::Term> {
- match analyzer {
- None => vec![tantivy::Term::from_field_text(tantivy_field, term)],
- Some(tokenizer) => {
+ match ALL_FIELDS[tantivy_field.field_id() as usize] {
+ Field::Fast(_) => vec![tantivy::Term::from_field_text(tantivy_field, term)],
+ Field::Text(text_field) => {
let mut terms: Vec<tantivy::Term> = Vec::new();
- let mut token_stream = tokenizer.token_stream(term);
+ let mut token_stream = text_field.query_tokenizer().token_stream(term);
token_stream.process(&mut |token| {
let term = tantivy::Term::from_field_text(tantivy_field, &token.text);
terms.push(term);
diff --git a/core/src/query/pattern_query.rs b/core/src/query/pattern_query.rs
index 319aca9a..9f91b9f2 100644
--- a/core/src/query/pattern_query.rs
+++ b/core/src/query/pattern_query.rs
@@ -101,7 +101,11 @@ impl PatternQuery {
for pattern in &patterns {
match pattern {
PatternPart::Raw(text) => {
- let mut stream = field.as_text().unwrap().tokenizer().token_stream(text);
+ let mut stream = field
+ .as_text()
+ .unwrap()
+ .indexing_tokenizer()
+ .token_stream(text);
while let Some(token) = stream.next() {
new_patterns.push(PatternPart::Raw(token.text.clone()));
diff --git a/core/src/ranking/signal.rs b/core/src/ranking/signal.rs
index 40fc27b1..31fa7979 100644
--- a/core/src/ranking/signal.rs
+++ b/core/src/ranking/signal.rs
@@ -849,7 +849,8 @@ impl SignalAggregator {
.collect::<String>();
let mut terms = Vec::new();
- let mut stream = text_field.tokenizer().token_stream(&simple_query);
+ let mut stream =
+ text_field.indexing_tokenizer().token_stream(&simple_query);
while let Some(token) = stream.next() {
let term = tantivy::Term::from_field_text(tv_field, &token.text);
@@ -941,7 +942,7 @@ impl SignalAggregator {
let mut queries = Vec::new();
for field in proximity_fields {
let tv_field = schema.get_field(field.name()).unwrap();
- let tokenizer = field.tokenizer();
+ let tokenizer = field.indexing_tokenizer();
let mut terms = Vec::with_capacity(query.simple_terms.len());
for term in &query.simple_terms {
diff --git a/core/src/schema.rs b/core/src/schema.rs
index 7edcd657..8de93764 100644
--- a/core/src/schema.rs
+++ b/core/src/schema.rs
@@ -63,7 +63,17 @@ impl From<TextField> for usize {
}
impl TextField {
- pub fn tokenizer(&self) -> Tokenizer {
+ pub fn query_tokenizer(&self) -> Tokenizer {
+ match self {
+ TextField::TitleBigrams => Tokenizer::default(),
+ TextField::CleanBodyBigrams => Tokenizer::default(),
+ TextField::TitleTrigrams => Tokenizer::default(),
+ TextField::CleanBodyTrigrams => Tokenizer::default(),
+ _ => self.indexing_tokenizer(),
+ }
+ }
+
+ pub fn indexing_tokenizer(&self) -> Tokenizer {
match self {
TextField::Title => Tokenizer::default(),
TextField::CleanBody => Tokenizer::default(),
@@ -313,7 +323,7 @@ pub static ALL_FIELDS: [Field; 57] = [
impl Field {
fn default_text_options(&self) -> tantivy::schema::TextOptions {
- let tokenizer = self.as_text().unwrap().tokenizer();
+ let tokenizer = self.as_text().unwrap().indexing_tokenizer();
let option = self.as_text().unwrap().index_option();
TextOptions::default().set_indexing_options(
diff --git a/core/src/tokenizer.rs b/core/src/tokenizer.rs
index b8192516..ce65dfcc 100644
--- a/core/src/tokenizer.rs
+++ b/core/src/tokenizer.rs
@@ -50,7 +50,7 @@ impl From<Lang> for MyStemmer {
}
}
-#[derive(Clone)]
+#[derive(Clone, Debug)]
pub enum Tokenizer {
Normal(Normal),
Identity(Identity),
@@ -83,7 +83,7 @@ impl Default for Tokenizer {
}
}
-#[derive(Clone, Default)]
+#[derive(Clone, Default, Debug)]
pub struct Normal {
stopwords: Option<Vec<String>>,
}
@@ -100,7 +100,7 @@ impl Normal {
}
}
-#[derive(Clone, Default)]
+#[derive(Clone, Default, Debug)]
pub struct BigramTokenizer {}
impl BigramTokenizer {
@@ -109,7 +109,7 @@ impl BigramTokenizer {
}
}
-#[derive(Clone, Default)]
+#[derive(Clone, Default, Debug)]
pub struct TrigramTokenizer {}
impl TrigramTokenizer {
@@ -118,7 +118,7 @@ impl TrigramTokenizer {
}
}
-#[derive(Clone, Default)]
+#[derive(Clone, Default, Debug)]
pub struct Stemmed {
force_language: Option<Lang>,
}
@@ -134,7 +134,7 @@ impl Stemmed {
}
}
-#[derive(Clone, Default)]
+#[derive(Clone, Default, Debug)]
pub struct Identity {}
impl Identity {
@@ -334,23 +334,17 @@ fn reuse_token_alloc(token: &mut tantivy::tokenizer::Token, new_token: &tantivy:
impl<'a, const N: usize> tantivy::tokenizer::TokenStream for NGramTokenStream<'a, N> {
fn advance(&mut self) -> bool {
if !self.inner.advance() {
- while self.token_window[0].text.is_empty()
- && !self.token_window.iter().all(|token| token.text.is_empty())
- {
- self.token_window.rotate_left(1);
- }
-
- self.token_window.rotate_left(1);
+ return false;
+ }
- reuse_token_alloc(
- &mut self.token_window[N - 1],
- &tantivy::tokenizer::Token::default(),
- );
+ self.token_window.rotate_left(1);
+ reuse_token_alloc(&mut self.token_window[N - 1], self.inner.token());
- if self.token_window.iter().all(|token| token.text.is_empty()) {
+ while self.token_window[0].text.is_empty() {
+ if !self.inner.advance() {
return false;
}
- } else {
+
self.token_window.rotate_left(1);
reuse_token_alloc(&mut self.token_window[N - 1], self.inner.token());
}
@@ -469,7 +463,7 @@ impl FlattenedJson {
}
}
-#[derive(Clone)]
+#[derive(Clone, Debug)]
pub struct JsonField;
impl JsonField {
@@ -815,72 +809,34 @@ key1.key2="this\" is @ a # test""#;
#[test]
fn bigram_tokenizer() {
assert!(tokenize_bigram("").is_empty());
- assert_eq!(tokenize_bigram("test"), vec!["test".to_string()]);
+ assert!(tokenize_bigram("test").is_empty());
- assert_eq!(
- tokenize_bigram("this is"),
- vec!["this".to_string(), "thisis".to_string(), "is".to_string()]
- );
+ assert_eq!(tokenize_bigram("this is"), vec!["thisis".to_string()]);
assert_eq!(
tokenize_bigram("this is a"),
- vec![
- "this".to_string(),
- "thisis".to_string(),
- "isa".to_string(),
- "a".to_string()
- ]
+ vec!["thisis".to_string(), "isa".to_string(),]
);
assert_eq!(
tokenize_bigram("this is a test"),
- vec![
- "this".to_string(),
- "thisis".to_string(),
- "isa".to_string(),
- "atest".to_string(),
- "test".to_string()
- ]
+ vec!["thisis".to_string(), "isa".to_string(), "atest".to_string(),]
);
assert_eq!(
tokenize_bigram("this.is"),
- vec![
- "this".to_string(),
- "this.".to_string(),
- ".is".to_string(),
- "is".to_string()
- ]
+ vec!["this.".to_string(), ".is".to_string(),]
);
}
#[test]
fn trigram_tokenizer() {
assert!(tokenize_trigram("").is_empty());
- assert_eq!(tokenize_trigram("test"), vec!["test".to_string()]);
- assert_eq!(
- tokenize_trigram("this is"),
- vec!["this".to_string(), "thisis".to_string(), "is".to_string()]
- );
+ assert!(tokenize_trigram("test").is_empty());
+ assert!(tokenize_trigram("this is").is_empty());
- assert_eq!(
- tokenize_trigram("this is a"),
- vec![
- "this".to_string(),
- "thisis".to_string(),
- "thisisa".to_string(),
- "isa".to_string(),
- "a".to_string()
- ]
- );
+ assert_eq!(tokenize_trigram("this is a"), vec!["thisisa".to_string(),]);
assert_eq!(
tokenize_trigram("this is a test"),
- vec![
- "this".to_string(),
- "thisis".to_string(),
- "thisisa".to_string(),
- "isatest".to_string(),
- "atest".to_string(),
- "test".to_string()
- ]
+ vec!["thisisa".to_string(), "isatest".to_string(),]
);
}