web search done right
reuse language detection calls by passing whatlang::Lang into tokenizer function for fields
Mikkel Denker8d80ce285cff
diff --git a/crates/core/src/collector/approx_count.rs b/crates/core/src/collector/approx_count.rsindex cb67f11e..20afc0bf 100644--- a/crates/core/src/collector/approx_count.rs+++ b/crates/core/src/collector/approx_count.rs@@ -104,7 +104,7 @@ impl Collector for ApproxCount {segment: &tantivy::SegmentReader,) -> tantivy::Result<Self::Child> {let field = text_field::AllBody.tantivy_field(segment.schema()).unwrap();- let mut tokenizer = text_field::AllBody.query_tokenizer();+ let mut tokenizer = text_field::AllBody.query_tokenizer(None);let inverted_index = segment.inverted_index(field)?;let num_docs = segment.max_doc() as u64;diff --git a/crates/core/src/inverted_index/mod.rs b/crates/core/src/inverted_index/mod.rsindex 37b155ad..38960001 100644--- a/crates/core/src/inverted_index/mod.rs+++ b/crates/core/src/inverted_index/mod.rs@@ -49,7 +49,7 @@ use crate::schema::text_field::TextField;use crate::schema::{fast_field, text_field, FastFieldEnum, Field, TextFieldEnum};use crate::snippet::TextSnippet;use crate::tokenizer::{- BigramTokenizer, Identity, JsonField, SiteOperatorUrlTokenizer, TrigramTokenizer,+ BigramTokenizer, Identity, JsonField, SiteOperatorUrlTokenizer, Stemmed, TrigramTokenizer,};use crate::webpage::region::Region;@@ -112,7 +112,7 @@ fn register_tokenizers(manager: &TokenizerManager) {let tokenizer = Tokenizer::default();manager.register(tokenizer.as_str(), tokenizer);- let tokenizer = Tokenizer::new_stemmed();+ let tokenizer = Tokenizer::Stemmed(Stemmed::default());manager.register(tokenizer.as_str(), tokenizer);let tokenizer = Tokenizer::Identity(Identity::default());diff --git a/crates/core/src/query/mod.rs b/crates/core/src/query/mod.rsindex b79baf56..226332aa 100644--- a/crates/core/src/query/mod.rs+++ b/crates/core/src/query/mod.rs@@ -54,6 +54,7 @@ pub struct Query {top_n: usize,count_results_exact: bool,signal_coefficients: SignalCoefficient,+ lang: Option<whatlang::Lang>,}impl Clone for Query {@@ -68,12 +69,15 @@ impl Clone for Query {top_n: self.top_n,count_results_exact: self.count_results_exact,signal_coefficients: self.signal_coefficients.clone(),+ lang: self.lang,}}}impl Query {pub fn parse(ctx: &Ctx, query: &SearchQuery, index: &InvertedIndex) -> Result<Query> {+ let lang = whatlang::detect_lang(&query.query);+let parsed_terms = parser::truncate(parser::parse(&query.query)?);if parsed_terms.is_empty() {@@ -115,7 +119,7 @@ impl Query {let mut tantivy_query = plan.into_query()- .as_tantivy(&schema)+ .as_tantivy(lang.as_ref(), &schema).expect("there should at least be one field in the index");let mut optics = Vec::new();@@ -146,6 +150,7 @@ impl Query {top_n: query.num_results,count_results_exact: query.count_results_exact,signal_coefficients: query.signal_coefficients(),+ lang,})}@@ -180,6 +185,10 @@ impl Query {pub fn signal_coefficients(&self) -> SignalCoefficient {self.signal_coefficients.clone()}++ pub fn lang(&self) -> Option<whatlang::Lang> {+ self.lang+ }}impl tantivy::query::Query for Query {diff --git a/crates/core/src/query/pattern_query/mod.rs b/crates/core/src/query/pattern_query/mod.rsindex cb7866d9..8b420330 100644--- a/crates/core/src/query/pattern_query/mod.rs+++ b/crates/core/src/query/pattern_query/mod.rs@@ -96,7 +96,10 @@ impl PatternQuery {for pattern in &patterns {match pattern {PatternPart::Raw(text) => {- let mut tokenizer = field.as_text().unwrap().tokenizer();+ let mut tokenizer = field+ .as_text()+ .unwrap()+ .tokenizer(Some(&whatlang::Lang::Eng));let mut stream = tokenizer.token_stream(text);while let Some(token) = stream.next() {diff --git a/crates/core/src/query/plan/mod.rs b/crates/core/src/query/plan/mod.rsindex ce0d0d24..c84b52d7 100644--- a/crates/core/src/query/plan/mod.rs+++ b/crates/core/src/query/plan/mod.rs@@ -136,12 +136,13 @@ impl Query {pub fn as_tantivy(&self,+ lang: Option<&whatlang::Lang>,schema: &tantivy::schema::Schema,) -> Option<Box<dyn tantivy::query::Query>> {match self {Query::Term(Term { text, field }) => match text {SimpleOrPhrase::Simple(s) => {- let mut terms = process_tantivy_term(s.as_str(), *field, schema);+ let mut terms = process_tantivy_term(s.as_str(), *field, lang, schema);let option = field.record_option();if terms.len() == 1 {@@ -166,7 +167,7 @@ impl Query {}SimpleOrPhrase::Phrase(p) => {let phrase = p.join(" ");- let mut processed_terms = process_tantivy_term(&phrase, *field, schema);+ let mut processed_terms = process_tantivy_term(&phrase, *field, lang, schema);if processed_terms.is_empty() {return None;@@ -188,7 +189,7 @@ impl Query {Query::Boolean { clauses } => {let mut t_clauses = Vec::new();for (occur, query) in clauses {- if let Some(query) = query.as_tantivy(schema) {+ if let Some(query) = query.as_tantivy(lang, schema) {t_clauses.push(((*occur).into(), query));}}@@ -202,10 +203,11 @@ impl Query {fn process_tantivy_term<T: TextField>(term: &str,field: T,+ lang: Option<&whatlang::Lang>,schema: &tantivy::schema::Schema,) -> Vec<tantivy::Term> {let mut terms: Vec<tantivy::Term> = Vec::new();- let mut tokenizer = field.query_tokenizer();+ let mut tokenizer = field.query_tokenizer(lang);let mut token_stream = tokenizer.token_stream(term);if let Some(tantivy_field) = field.tantivy_field(schema) {diff --git a/crates/core/src/ranking/signal/computer/mod.rs b/crates/core/src/ranking/signal/computer/mod.rsindex ec054890..73affc26 100644--- a/crates/core/src/ranking/signal/computer/mod.rs+++ b/crates/core/src/ranking/signal/computer/mod.rs@@ -79,6 +79,7 @@ pub struct QueryData {simple_terms: Vec<String>,optic_rules: Vec<optics::Rule>,selected_region: Option<crate::webpage::Region>,+ lang: Option<whatlang::Lang>,}impl QueryData {pub fn selected_region(&self) -> Option<crate::webpage::Region> {@@ -154,6 +155,7 @@ impl SignalComputer {.cloned().collect(),selected_region: q.region().cloned(),+ lang: q.lang(),});let mut s = Self {@@ -202,7 +204,7 @@ impl SignalComputer {.collect::<String>();let mut terms = Vec::new();- let mut tokenizer = text_field.tokenizer();+ let mut tokenizer = text_field.tokenizer(query.lang.as_ref());let mut stream = tokenizer.token_stream(&simple_query);while let Some(token) = stream.next() {diff --git a/crates/core/src/schema/text_field.rs b/crates/core/src/schema/text_field.rsindex 8b112306..eeb5b356 100644--- a/crates/core/src/schema/text_field.rs+++ b/crates/core/src/schema/text_field.rs@@ -28,7 +28,8 @@ use crate::{enum_dispatch_from_discriminant,enum_map::InsertEnumMapKey,tokenizer::{- BigramTokenizer, Identity, JsonField, SiteOperatorUrlTokenizer, Tokenizer, TrigramTokenizer,+ self, BigramTokenizer, Identity, JsonField, SiteOperatorUrlTokenizer, Tokenizer,+ TrigramTokenizer,},webpage::Html,Result,@@ -51,21 +52,23 @@ pub trait TextField:schema: &tantivy::schema::Schema,) -> Result<()>;+ #[allow(unused_variables)]fn add_webpage_tantivy(&self,- _webpage: &crate::webpage::Webpage,- _doc: &mut TantivyDocument,- _schema: &tantivy::schema::Schema,+ webpage: &crate::webpage::Webpage,+ doc: &mut TantivyDocument,+ schema: &tantivy::schema::Schema,) -> Result<()> {Ok(())}- fn tokenizer(&self) -> Tokenizer {+ #[allow(unused_variables)]+ fn tokenizer(&self, lang: Option<&whatlang::Lang>) -> Tokenizer {Tokenizer::default()}- fn query_tokenizer(&self) -> Tokenizer {- self.tokenizer()+ fn query_tokenizer(&self, lang: Option<&whatlang::Lang>) -> Tokenizer {+ self.tokenizer(lang)}fn ngram_size(&self) -> usize {@@ -118,7 +121,7 @@ pub trait TextField:}fn indexing_option(&self) -> IndexingOption {- let tokenizer = self.tokenizer();+ let tokenizer = self.tokenizer(None);let option = self.record_option();let mut opt = TextOptions::default().set_indexing_options(@@ -362,8 +365,11 @@ impl TextField for StemmedTitle {"stemmed_title"}- fn tokenizer(&self) -> Tokenizer {- Tokenizer::new_stemmed()+ fn tokenizer(&self, lang: Option<&whatlang::Lang>) -> Tokenizer {+ match lang {+ Some(lang) => tokenizer::Stemmed::with_forced_language(*lang).into(),+ None => tokenizer::Stemmed::default().into(),+ }}fn is_searchable(&self) -> bool {@@ -405,8 +411,11 @@ impl TextField for StemmedCleanBody {"stemmed_body"}- fn tokenizer(&self) -> Tokenizer {- Tokenizer::new_stemmed()+ fn tokenizer(&self, lang: Option<&whatlang::Lang>) -> Tokenizer {+ match lang {+ Some(lang) => tokenizer::Stemmed::with_forced_language(*lang).into(),+ None => tokenizer::Stemmed::default().into(),+ }}fn is_stored(&self) -> bool {@@ -527,7 +536,7 @@ impl TextField for UrlNoTokenizer {"url_no_tokenizer"}- fn tokenizer(&self) -> Tokenizer {+ fn tokenizer(&self, _: Option<&whatlang::Lang>) -> Tokenizer {Tokenizer::Identity(Identity {})}@@ -574,7 +583,7 @@ impl TextField for UrlForSiteOperator {true}- fn tokenizer(&self) -> Tokenizer {+ fn tokenizer(&self, _: Option<&whatlang::Lang>) -> Tokenizer {Tokenizer::SiteOperator(SiteOperatorUrlTokenizer)}@@ -657,7 +666,7 @@ impl TextField for SiteNoTokenizer {"site_no_tokenizer"}- fn tokenizer(&self) -> Tokenizer {+ fn tokenizer(&self, _: Option<&whatlang::Lang>) -> Tokenizer {Tokenizer::Identity(Identity {})}@@ -700,7 +709,7 @@ impl TextField for DomainNoTokenizer {"domain_no_tokenizer"}- fn tokenizer(&self) -> Tokenizer {+ fn tokenizer(&self, _: Option<&whatlang::Lang>) -> Tokenizer {Tokenizer::Identity(Identity {})}@@ -743,7 +752,7 @@ impl TextField for DomainNameNoTokenizer {"domain_name_no_tokenizer"}- fn tokenizer(&self) -> Tokenizer {+ fn tokenizer(&self, _: Option<&whatlang::Lang>) -> Tokenizer {Tokenizer::Identity(Identity {})}@@ -786,7 +795,7 @@ impl TextField for SiteIfHomepageNoTokenizer {"site_if_homepage_no_tokenizer"}- fn tokenizer(&self) -> Tokenizer {+ fn tokenizer(&self, _: Option<&whatlang::Lang>) -> Tokenizer {Tokenizer::Identity(Identity {})}@@ -866,7 +875,7 @@ impl TextField for DomainNameIfHomepageNoTokenizer {"domain_name_if_homepage_no_tokenizer"}- fn tokenizer(&self) -> Tokenizer {+ fn tokenizer(&self, _: Option<&whatlang::Lang>) -> Tokenizer {Tokenizer::Identity(Identity {})}@@ -913,7 +922,7 @@ impl TextField for DomainIfHomepageNoTokenizer {"domain_if_homepage_no_tokenizer"}- fn tokenizer(&self) -> Tokenizer {+ fn tokenizer(&self, _: Option<&whatlang::Lang>) -> Tokenizer {Tokenizer::Identity(Identity {})}@@ -1109,7 +1118,7 @@ impl TextField for SchemaOrgJson {"schema_org_json"}- fn tokenizer(&self) -> Tokenizer {+ fn tokenizer(&self, _: Option<&whatlang::Lang>) -> Tokenizer {Tokenizer::Identity(Identity {})}@@ -1145,7 +1154,7 @@ impl TextField for FlattenedSchemaOrgJson {true}- fn tokenizer(&self) -> Tokenizer {+ fn tokenizer(&self, _: Option<&whatlang::Lang>) -> Tokenizer {Tokenizer::Json(JsonField)}@@ -1181,11 +1190,11 @@ impl TextField for CleanBodyBigrams {CleanBody.into()}- fn tokenizer(&self) -> Tokenizer {+ fn tokenizer(&self, _: Option<&whatlang::Lang>) -> Tokenizer {Tokenizer::Bigram(BigramTokenizer::default())}- fn query_tokenizer(&self) -> Tokenizer {+ fn query_tokenizer(&self, _: Option<&whatlang::Lang>) -> Tokenizer {Tokenizer::default()}@@ -1225,11 +1234,11 @@ impl TextField for TitleBigrams {Title.into()}- fn tokenizer(&self) -> Tokenizer {+ fn tokenizer(&self, _: Option<&whatlang::Lang>) -> Tokenizer {Tokenizer::Bigram(BigramTokenizer::default())}- fn query_tokenizer(&self) -> Tokenizer {+ fn query_tokenizer(&self, _: Option<&whatlang::Lang>) -> Tokenizer {Tokenizer::default()}@@ -1275,11 +1284,11 @@ impl TextField for CleanBodyTrigrams {CleanBody.into()}- fn tokenizer(&self) -> Tokenizer {+ fn tokenizer(&self, _: Option<&whatlang::Lang>) -> Tokenizer {Tokenizer::Trigram(TrigramTokenizer::default())}- fn query_tokenizer(&self) -> Tokenizer {+ fn query_tokenizer(&self, _: Option<&whatlang::Lang>) -> Tokenizer {Tokenizer::default()}@@ -1319,11 +1328,11 @@ impl TextField for TitleTrigrams {Title.into()}- fn tokenizer(&self) -> Tokenizer {+ fn tokenizer(&self, _: Option<&whatlang::Lang>) -> Tokenizer {Tokenizer::Trigram(TrigramTokenizer::default())}- fn query_tokenizer(&self) -> Tokenizer {+ fn query_tokenizer(&self, _: Option<&whatlang::Lang>) -> Tokenizer {Tokenizer::default()}@@ -1389,7 +1398,7 @@ impl TextField for SafetyClassification {"safety_classification"}- fn tokenizer(&self) -> Tokenizer {+ fn tokenizer(&self, _: Option<&whatlang::Lang>) -> Tokenizer {Tokenizer::Identity(Identity {})}@@ -1431,7 +1440,7 @@ impl TextField for InsertionTimestamp {"insertion_timestamp"}- fn tokenizer(&self) -> Tokenizer {+ fn tokenizer(&self, _: Option<&whatlang::Lang>) -> Tokenizer {Tokenizer::Identity(Identity {})}@@ -1474,7 +1483,7 @@ impl TextField for RecipeFirstIngredientTagId {"recipe_first_ingredient_tag_id"}- fn tokenizer(&self) -> Tokenizer {+ fn tokenizer(&self, _: Option<&whatlang::Lang>) -> Tokenizer {Tokenizer::Identity(Identity {})}diff --git a/crates/core/src/tokenizer/mod.rs b/crates/core/src/tokenizer/mod.rsindex 8985fe7e..89a7b061 100644--- a/crates/core/src/tokenizer/mod.rs+++ b/crates/core/src/tokenizer/mod.rs@@ -67,10 +67,6 @@ pub enum Tokenizer {}impl Tokenizer {- pub fn new_stemmed() -> Self {- Self::Stemmed(Stemmed::default())- }-pub fn as_str(&self) -> &'static str {match self {Tokenizer::Normal(_) => Normal::as_str(),@@ -84,6 +80,12 @@ impl Tokenizer {}}+impl From<Stemmed> for Tokenizer {+ fn from(stemmed: Stemmed) -> Self {+ Self::Stemmed(stemmed)+ }+}+impl Default for Tokenizer {fn default() -> Self {Self::Normal(Normal::default())diff --git a/crates/core/src/webpage/html/into_tantivy.rs b/crates/core/src/webpage/html/into_tantivy.rsindex 6bd3dd24..9bd58d38 100644--- a/crates/core/src/webpage/html/into_tantivy.rs+++ b/crates/core/src/webpage/html/into_tantivy.rs@@ -108,7 +108,7 @@ impl Html {}fn pretokenize_string(&self, text: String, field: TextFieldEnum) -> PreTokenizedString {- self.pretokenize_string_with(text, field.tokenizer())+ self.pretokenize_string_with(text, field.tokenizer(self.lang()))}fn pretokenize_string_with(