Skip to content

web search done right

reuse language detection calls by passing whatlang::Lang into tokenizer function for fields

Mikkel Denker8d80ce285cff
diff --git a/crates/core/src/collector/approx_count.rs b/crates/core/src/collector/approx_count.rs
index cb67f11e..20afc0bf 100644
--- a/crates/core/src/collector/approx_count.rs
+++ b/crates/core/src/collector/approx_count.rs
@@ -104,7 +104,7 @@ impl Collector for ApproxCount {
segment: &tantivy::SegmentReader,
) -> tantivy::Result<Self::Child> {
let field = text_field::AllBody.tantivy_field(segment.schema()).unwrap();
- let mut tokenizer = text_field::AllBody.query_tokenizer();
+ let mut tokenizer = text_field::AllBody.query_tokenizer(None);
let inverted_index = segment.inverted_index(field)?;
let num_docs = segment.max_doc() as u64;
diff --git a/crates/core/src/inverted_index/mod.rs b/crates/core/src/inverted_index/mod.rs
index 37b155ad..38960001 100644
--- a/crates/core/src/inverted_index/mod.rs
+++ b/crates/core/src/inverted_index/mod.rs
@@ -49,7 +49,7 @@ use crate::schema::text_field::TextField;
use crate::schema::{fast_field, text_field, FastFieldEnum, Field, TextFieldEnum};
use crate::snippet::TextSnippet;
use crate::tokenizer::{
- BigramTokenizer, Identity, JsonField, SiteOperatorUrlTokenizer, TrigramTokenizer,
+ BigramTokenizer, Identity, JsonField, SiteOperatorUrlTokenizer, Stemmed, TrigramTokenizer,
};
use crate::webpage::region::Region;
@@ -112,7 +112,7 @@ fn register_tokenizers(manager: &TokenizerManager) {
let tokenizer = Tokenizer::default();
manager.register(tokenizer.as_str(), tokenizer);
- let tokenizer = Tokenizer::new_stemmed();
+ let tokenizer = Tokenizer::Stemmed(Stemmed::default());
manager.register(tokenizer.as_str(), tokenizer);
let tokenizer = Tokenizer::Identity(Identity::default());
diff --git a/crates/core/src/query/mod.rs b/crates/core/src/query/mod.rs
index b79baf56..226332aa 100644
--- a/crates/core/src/query/mod.rs
+++ b/crates/core/src/query/mod.rs
@@ -54,6 +54,7 @@ pub struct Query {
top_n: usize,
count_results_exact: bool,
signal_coefficients: SignalCoefficient,
+ lang: Option<whatlang::Lang>,
}
impl Clone for Query {
@@ -68,12 +69,15 @@ impl Clone for Query {
top_n: self.top_n,
count_results_exact: self.count_results_exact,
signal_coefficients: self.signal_coefficients.clone(),
+ lang: self.lang,
}
}
}
impl Query {
pub fn parse(ctx: &Ctx, query: &SearchQuery, index: &InvertedIndex) -> Result<Query> {
+ let lang = whatlang::detect_lang(&query.query);
+
let parsed_terms = parser::truncate(parser::parse(&query.query)?);
if parsed_terms.is_empty() {
@@ -115,7 +119,7 @@ impl Query {
let mut tantivy_query = plan
.into_query()
- .as_tantivy(&schema)
+ .as_tantivy(lang.as_ref(), &schema)
.expect("there should at least be one field in the index");
let mut optics = Vec::new();
@@ -146,6 +150,7 @@ impl Query {
top_n: query.num_results,
count_results_exact: query.count_results_exact,
signal_coefficients: query.signal_coefficients(),
+ lang,
})
}
@@ -180,6 +185,10 @@ impl Query {
pub fn signal_coefficients(&self) -> SignalCoefficient {
self.signal_coefficients.clone()
}
+
+ pub fn lang(&self) -> Option<whatlang::Lang> {
+ self.lang
+ }
}
impl tantivy::query::Query for Query {
diff --git a/crates/core/src/query/pattern_query/mod.rs b/crates/core/src/query/pattern_query/mod.rs
index cb7866d9..8b420330 100644
--- a/crates/core/src/query/pattern_query/mod.rs
+++ b/crates/core/src/query/pattern_query/mod.rs
@@ -96,7 +96,10 @@ impl PatternQuery {
for pattern in &patterns {
match pattern {
PatternPart::Raw(text) => {
- let mut tokenizer = field.as_text().unwrap().tokenizer();
+ let mut tokenizer = field
+ .as_text()
+ .unwrap()
+ .tokenizer(Some(&whatlang::Lang::Eng));
let mut stream = tokenizer.token_stream(text);
while let Some(token) = stream.next() {
diff --git a/crates/core/src/query/plan/mod.rs b/crates/core/src/query/plan/mod.rs
index ce0d0d24..c84b52d7 100644
--- a/crates/core/src/query/plan/mod.rs
+++ b/crates/core/src/query/plan/mod.rs
@@ -136,12 +136,13 @@ impl Query {
pub fn as_tantivy(
&self,
+ lang: Option<&whatlang::Lang>,
schema: &tantivy::schema::Schema,
) -> Option<Box<dyn tantivy::query::Query>> {
match self {
Query::Term(Term { text, field }) => match text {
SimpleOrPhrase::Simple(s) => {
- let mut terms = process_tantivy_term(s.as_str(), *field, schema);
+ let mut terms = process_tantivy_term(s.as_str(), *field, lang, schema);
let option = field.record_option();
if terms.len() == 1 {
@@ -166,7 +167,7 @@ impl Query {
}
SimpleOrPhrase::Phrase(p) => {
let phrase = p.join(" ");
- let mut processed_terms = process_tantivy_term(&phrase, *field, schema);
+ let mut processed_terms = process_tantivy_term(&phrase, *field, lang, schema);
if processed_terms.is_empty() {
return None;
@@ -188,7 +189,7 @@ impl Query {
Query::Boolean { clauses } => {
let mut t_clauses = Vec::new();
for (occur, query) in clauses {
- if let Some(query) = query.as_tantivy(schema) {
+ if let Some(query) = query.as_tantivy(lang, schema) {
t_clauses.push(((*occur).into(), query));
}
}
@@ -202,10 +203,11 @@ impl Query {
fn process_tantivy_term<T: TextField>(
term: &str,
field: T,
+ lang: Option<&whatlang::Lang>,
schema: &tantivy::schema::Schema,
) -> Vec<tantivy::Term> {
let mut terms: Vec<tantivy::Term> = Vec::new();
- let mut tokenizer = field.query_tokenizer();
+ let mut tokenizer = field.query_tokenizer(lang);
let mut token_stream = tokenizer.token_stream(term);
if let Some(tantivy_field) = field.tantivy_field(schema) {
diff --git a/crates/core/src/ranking/signal/computer/mod.rs b/crates/core/src/ranking/signal/computer/mod.rs
index ec054890..73affc26 100644
--- a/crates/core/src/ranking/signal/computer/mod.rs
+++ b/crates/core/src/ranking/signal/computer/mod.rs
@@ -79,6 +79,7 @@ pub struct QueryData {
simple_terms: Vec<String>,
optic_rules: Vec<optics::Rule>,
selected_region: Option<crate::webpage::Region>,
+ lang: Option<whatlang::Lang>,
}
impl QueryData {
pub fn selected_region(&self) -> Option<crate::webpage::Region> {
@@ -154,6 +155,7 @@ impl SignalComputer {
.cloned()
.collect(),
selected_region: q.region().cloned(),
+ lang: q.lang(),
});
let mut s = Self {
@@ -202,7 +204,7 @@ impl SignalComputer {
.collect::<String>();
let mut terms = Vec::new();
- let mut tokenizer = text_field.tokenizer();
+ let mut tokenizer = text_field.tokenizer(query.lang.as_ref());
let mut stream = tokenizer.token_stream(&simple_query);
while let Some(token) = stream.next() {
diff --git a/crates/core/src/schema/text_field.rs b/crates/core/src/schema/text_field.rs
index 8b112306..eeb5b356 100644
--- a/crates/core/src/schema/text_field.rs
+++ b/crates/core/src/schema/text_field.rs
@@ -28,7 +28,8 @@ use crate::{
enum_dispatch_from_discriminant,
enum_map::InsertEnumMapKey,
tokenizer::{
- BigramTokenizer, Identity, JsonField, SiteOperatorUrlTokenizer, Tokenizer, TrigramTokenizer,
+ self, BigramTokenizer, Identity, JsonField, SiteOperatorUrlTokenizer, Tokenizer,
+ TrigramTokenizer,
},
webpage::Html,
Result,
@@ -51,21 +52,23 @@ pub trait TextField:
schema: &tantivy::schema::Schema,
) -> Result<()>;
+ #[allow(unused_variables)]
fn add_webpage_tantivy(
&self,
- _webpage: &crate::webpage::Webpage,
- _doc: &mut TantivyDocument,
- _schema: &tantivy::schema::Schema,
+ webpage: &crate::webpage::Webpage,
+ doc: &mut TantivyDocument,
+ schema: &tantivy::schema::Schema,
) -> Result<()> {
Ok(())
}
- fn tokenizer(&self) -> Tokenizer {
+ #[allow(unused_variables)]
+ fn tokenizer(&self, lang: Option<&whatlang::Lang>) -> Tokenizer {
Tokenizer::default()
}
- fn query_tokenizer(&self) -> Tokenizer {
- self.tokenizer()
+ fn query_tokenizer(&self, lang: Option<&whatlang::Lang>) -> Tokenizer {
+ self.tokenizer(lang)
}
fn ngram_size(&self) -> usize {
@@ -118,7 +121,7 @@ pub trait TextField:
}
fn indexing_option(&self) -> IndexingOption {
- let tokenizer = self.tokenizer();
+ let tokenizer = self.tokenizer(None);
let option = self.record_option();
let mut opt = TextOptions::default().set_indexing_options(
@@ -362,8 +365,11 @@ impl TextField for StemmedTitle {
"stemmed_title"
}
- fn tokenizer(&self) -> Tokenizer {
- Tokenizer::new_stemmed()
+ fn tokenizer(&self, lang: Option<&whatlang::Lang>) -> Tokenizer {
+ match lang {
+ Some(lang) => tokenizer::Stemmed::with_forced_language(*lang).into(),
+ None => tokenizer::Stemmed::default().into(),
+ }
}
fn is_searchable(&self) -> bool {
@@ -405,8 +411,11 @@ impl TextField for StemmedCleanBody {
"stemmed_body"
}
- fn tokenizer(&self) -> Tokenizer {
- Tokenizer::new_stemmed()
+ fn tokenizer(&self, lang: Option<&whatlang::Lang>) -> Tokenizer {
+ match lang {
+ Some(lang) => tokenizer::Stemmed::with_forced_language(*lang).into(),
+ None => tokenizer::Stemmed::default().into(),
+ }
}
fn is_stored(&self) -> bool {
@@ -527,7 +536,7 @@ impl TextField for UrlNoTokenizer {
"url_no_tokenizer"
}
- fn tokenizer(&self) -> Tokenizer {
+ fn tokenizer(&self, _: Option<&whatlang::Lang>) -> Tokenizer {
Tokenizer::Identity(Identity {})
}
@@ -574,7 +583,7 @@ impl TextField for UrlForSiteOperator {
true
}
- fn tokenizer(&self) -> Tokenizer {
+ fn tokenizer(&self, _: Option<&whatlang::Lang>) -> Tokenizer {
Tokenizer::SiteOperator(SiteOperatorUrlTokenizer)
}
@@ -657,7 +666,7 @@ impl TextField for SiteNoTokenizer {
"site_no_tokenizer"
}
- fn tokenizer(&self) -> Tokenizer {
+ fn tokenizer(&self, _: Option<&whatlang::Lang>) -> Tokenizer {
Tokenizer::Identity(Identity {})
}
@@ -700,7 +709,7 @@ impl TextField for DomainNoTokenizer {
"domain_no_tokenizer"
}
- fn tokenizer(&self) -> Tokenizer {
+ fn tokenizer(&self, _: Option<&whatlang::Lang>) -> Tokenizer {
Tokenizer::Identity(Identity {})
}
@@ -743,7 +752,7 @@ impl TextField for DomainNameNoTokenizer {
"domain_name_no_tokenizer"
}
- fn tokenizer(&self) -> Tokenizer {
+ fn tokenizer(&self, _: Option<&whatlang::Lang>) -> Tokenizer {
Tokenizer::Identity(Identity {})
}
@@ -786,7 +795,7 @@ impl TextField for SiteIfHomepageNoTokenizer {
"site_if_homepage_no_tokenizer"
}
- fn tokenizer(&self) -> Tokenizer {
+ fn tokenizer(&self, _: Option<&whatlang::Lang>) -> Tokenizer {
Tokenizer::Identity(Identity {})
}
@@ -866,7 +875,7 @@ impl TextField for DomainNameIfHomepageNoTokenizer {
"domain_name_if_homepage_no_tokenizer"
}
- fn tokenizer(&self) -> Tokenizer {
+ fn tokenizer(&self, _: Option<&whatlang::Lang>) -> Tokenizer {
Tokenizer::Identity(Identity {})
}
@@ -913,7 +922,7 @@ impl TextField for DomainIfHomepageNoTokenizer {
"domain_if_homepage_no_tokenizer"
}
- fn tokenizer(&self) -> Tokenizer {
+ fn tokenizer(&self, _: Option<&whatlang::Lang>) -> Tokenizer {
Tokenizer::Identity(Identity {})
}
@@ -1109,7 +1118,7 @@ impl TextField for SchemaOrgJson {
"schema_org_json"
}
- fn tokenizer(&self) -> Tokenizer {
+ fn tokenizer(&self, _: Option<&whatlang::Lang>) -> Tokenizer {
Tokenizer::Identity(Identity {})
}
@@ -1145,7 +1154,7 @@ impl TextField for FlattenedSchemaOrgJson {
true
}
- fn tokenizer(&self) -> Tokenizer {
+ fn tokenizer(&self, _: Option<&whatlang::Lang>) -> Tokenizer {
Tokenizer::Json(JsonField)
}
@@ -1181,11 +1190,11 @@ impl TextField for CleanBodyBigrams {
CleanBody.into()
}
- fn tokenizer(&self) -> Tokenizer {
+ fn tokenizer(&self, _: Option<&whatlang::Lang>) -> Tokenizer {
Tokenizer::Bigram(BigramTokenizer::default())
}
- fn query_tokenizer(&self) -> Tokenizer {
+ fn query_tokenizer(&self, _: Option<&whatlang::Lang>) -> Tokenizer {
Tokenizer::default()
}
@@ -1225,11 +1234,11 @@ impl TextField for TitleBigrams {
Title.into()
}
- fn tokenizer(&self) -> Tokenizer {
+ fn tokenizer(&self, _: Option<&whatlang::Lang>) -> Tokenizer {
Tokenizer::Bigram(BigramTokenizer::default())
}
- fn query_tokenizer(&self) -> Tokenizer {
+ fn query_tokenizer(&self, _: Option<&whatlang::Lang>) -> Tokenizer {
Tokenizer::default()
}
@@ -1275,11 +1284,11 @@ impl TextField for CleanBodyTrigrams {
CleanBody.into()
}
- fn tokenizer(&self) -> Tokenizer {
+ fn tokenizer(&self, _: Option<&whatlang::Lang>) -> Tokenizer {
Tokenizer::Trigram(TrigramTokenizer::default())
}
- fn query_tokenizer(&self) -> Tokenizer {
+ fn query_tokenizer(&self, _: Option<&whatlang::Lang>) -> Tokenizer {
Tokenizer::default()
}
@@ -1319,11 +1328,11 @@ impl TextField for TitleTrigrams {
Title.into()
}
- fn tokenizer(&self) -> Tokenizer {
+ fn tokenizer(&self, _: Option<&whatlang::Lang>) -> Tokenizer {
Tokenizer::Trigram(TrigramTokenizer::default())
}
- fn query_tokenizer(&self) -> Tokenizer {
+ fn query_tokenizer(&self, _: Option<&whatlang::Lang>) -> Tokenizer {
Tokenizer::default()
}
@@ -1389,7 +1398,7 @@ impl TextField for SafetyClassification {
"safety_classification"
}
- fn tokenizer(&self) -> Tokenizer {
+ fn tokenizer(&self, _: Option<&whatlang::Lang>) -> Tokenizer {
Tokenizer::Identity(Identity {})
}
@@ -1431,7 +1440,7 @@ impl TextField for InsertionTimestamp {
"insertion_timestamp"
}
- fn tokenizer(&self) -> Tokenizer {
+ fn tokenizer(&self, _: Option<&whatlang::Lang>) -> Tokenizer {
Tokenizer::Identity(Identity {})
}
@@ -1474,7 +1483,7 @@ impl TextField for RecipeFirstIngredientTagId {
"recipe_first_ingredient_tag_id"
}
- fn tokenizer(&self) -> Tokenizer {
+ fn tokenizer(&self, _: Option<&whatlang::Lang>) -> Tokenizer {
Tokenizer::Identity(Identity {})
}
diff --git a/crates/core/src/tokenizer/mod.rs b/crates/core/src/tokenizer/mod.rs
index 8985fe7e..89a7b061 100644
--- a/crates/core/src/tokenizer/mod.rs
+++ b/crates/core/src/tokenizer/mod.rs
@@ -67,10 +67,6 @@ pub enum Tokenizer {
}
impl Tokenizer {
- pub fn new_stemmed() -> Self {
- Self::Stemmed(Stemmed::default())
- }
-
pub fn as_str(&self) -> &'static str {
match self {
Tokenizer::Normal(_) => Normal::as_str(),
@@ -84,6 +80,12 @@ impl Tokenizer {
}
}
+impl From<Stemmed> for Tokenizer {
+ fn from(stemmed: Stemmed) -> Self {
+ Self::Stemmed(stemmed)
+ }
+}
+
impl Default for Tokenizer {
fn default() -> Self {
Self::Normal(Normal::default())
diff --git a/crates/core/src/webpage/html/into_tantivy.rs b/crates/core/src/webpage/html/into_tantivy.rs
index 6bd3dd24..9bd58d38 100644
--- a/crates/core/src/webpage/html/into_tantivy.rs
+++ b/crates/core/src/webpage/html/into_tantivy.rs
@@ -108,7 +108,7 @@ impl Html {
}
fn pretokenize_string(&self, text: String, field: TextFieldEnum) -> PreTokenizedString {
- self.pretokenize_string_with(text, field.tokenizer())
+ self.pretokenize_string_with(text, field.tokenizer(self.lang()))
}
fn pretokenize_string_with(