web search done right
Fix bm25 scaling of terms based on their idf (#163)
Mikkel Denkerdc52379789b7
diff --git a/crates/core/src/query/pattern_query/mod.rs b/crates/core/src/query/pattern_query/mod.rsindex 5960c04a..0057690f 100644--- a/crates/core/src/query/pattern_query/mod.rs+++ b/crates/core/src/query/pattern_query/mod.rs@@ -23,7 +23,6 @@ use tantivy::tokenizer::Tokenizer;use crate::{fastfield_reader::FastFieldReader,- ranking::bm25::Bm25Weight,schema::{Field, TextField},};@@ -126,32 +125,16 @@ impl PatternQuery {impl tantivy::query::Query for PatternQuery {fn weight(&self,- scoring: tantivy::query::EnableScoring<'_>,+ _scoring: tantivy::query::EnableScoring<'_>,) -> tantivy::Result<Box<dyn tantivy::query::Weight>> {- let bm25_weight = match scoring {- tantivy::query::EnableScoring::Enabled {- searcher,- statistics_provider: _,- } => {- if self.raw_terms.is_empty() {- None- } else {- Some(Bm25Weight::for_terms(searcher, &self.raw_terms)?)- }- }- tantivy::query::EnableScoring::Disabled { .. } => None,- };-if self.can_optimize_site_domain {return Ok(Box::new(FastSiteDomainPatternWeight {term: self.raw_terms[0].clone(),field: self.field,- similarity_weight: bm25_weight,}));}Ok(Box::new(PatternWeight {- similarity_weight: bm25_weight,raw_terms: self.raw_terms.clone(),patterns: self.patterns.clone(),field: self.field,diff --git a/crates/core/src/query/pattern_query/scorer.rs b/crates/core/src/query/pattern_query/scorer.rsindex f11b76fd..19157ea3 100644--- a/crates/core/src/query/pattern_query/scorer.rs+++ b/crates/core/src/query/pattern_query/scorer.rs@@ -24,7 +24,6 @@ use tantivy::{use crate::{fastfield_reader::{self, FastFieldReader},query::intersection::Intersection,- ranking::bm25::Bm25Weight,schema::FastField,};@@ -86,17 +85,6 @@ impl DocSet for PatternScorer {}}-impl PatternScorer {- pub fn term_freq(&self) -> u32 {- match self {- PatternScorer::Normal(scorer) => scorer.phrase_count(),- PatternScorer::FastSiteDomain(scorer) => scorer.term_freq(),- PatternScorer::Everything(_) => 0,- PatternScorer::EmptyField(_) => 0,- }- }-}-pub struct AllScorer {pub doc: DocId,pub max_doc: DocId,@@ -190,27 +178,13 @@ impl Scorer for EmptyFieldScorer {}pub struct FastSiteDomainPatternScorer {- pub similarity_weight: Option<Bm25Weight>,pub posting: SegmentPostings,pub fieldnorm_reader: FieldNormReader,}-impl FastSiteDomainPatternScorer {- pub fn term_freq(&self) -> u32 {- self.posting.term_freq()- }-}impl Scorer for FastSiteDomainPatternScorer {fn score(&mut self) -> Score {- self.similarity_weight- .as_ref()- .map(|w| {- w.score(- self.fieldnorm_reader.fieldnorm_id(self.doc()),- self.posting.term_freq(),- )- })- .unwrap_or_default()+ 1.0}}@@ -230,8 +204,6 @@ impl DocSet for FastSiteDomainPatternScorer {pub struct NormalPatternScorer {pattern_all_simple: bool,- similarity_weight: Option<Bm25Weight>,- fieldnorm_reader: FieldNormReader,intersection_docset: Intersection<SegmentPostings>,pattern: Vec<SmallPatternPart>,num_query_terms: usize,@@ -244,9 +216,7 @@ pub struct NormalPatternScorer {impl NormalPatternScorer {pub fn new(- similarity_weight: Option<Bm25Weight>,term_postings_list: Vec<SegmentPostings>,- fieldnorm_reader: FieldNormReader,pattern: Vec<SmallPatternPart>,segment: tantivy::SegmentId,num_tokens_field: FastField,@@ -259,8 +229,6 @@ impl NormalPatternScorer {pattern_all_simple: pattern.iter().all(|p| matches!(p, SmallPatternPart::Term)),intersection_docset: Intersection::new(term_postings_list),num_query_terms,- similarity_weight,- fieldnorm_reader,pattern,left: Vec::with_capacity(100),right: Vec::with_capacity(100),@@ -275,9 +243,6 @@ impl NormalPatternScorer {s}- fn phrase_count(&self) -> u32 {- self.phrase_count- }fn pattern_match(&mut self) -> bool {if self.num_query_terms == 1 && self.pattern_all_simple {@@ -377,14 +342,7 @@ impl NormalPatternScorer {impl Scorer for NormalPatternScorer {fn score(&mut self) -> Score {- self.similarity_weight- .as_ref()- .map(|scorer| {- let doc = self.doc();- let fieldnorm_id = self.fieldnorm_reader.fieldnorm_id(doc);- scorer.score(fieldnorm_id, self.phrase_count())- })- .unwrap_or_default()+ 1.0}}diff --git a/crates/core/src/query/pattern_query/weight.rs b/crates/core/src/query/pattern_query/weight.rsindex b5d5b5ae..bf4d1488 100644--- a/crates/core/src/query/pattern_query/weight.rs+++ b/crates/core/src/query/pattern_query/weight.rs@@ -24,7 +24,6 @@ use tantivy::{use crate::{fastfield_reader::FastFieldReader,- ranking::bm25::Bm25Weight,schema::{FastField, Field, TextField},};@@ -36,7 +35,6 @@ use super::SmallPatternPart;pub struct FastSiteDomainPatternWeight {pub term: tantivy::Term,pub field: tantivy::schema::Field,- pub similarity_weight: Option<Bm25Weight>,}impl FastSiteDomainPatternWeight {@@ -44,24 +42,13 @@ impl FastSiteDomainPatternWeight {&self,reader: &tantivy::SegmentReader,) -> tantivy::Result<FieldNormReader> {- if self.similarity_weight.is_some() {- if let Some(fieldnorm_reader) = reader.fieldnorms_readers().get_field(self.field)? {- return Ok(fieldnorm_reader);- }- }Ok(FieldNormReader::constant(reader.max_doc(), 1))}fn pattern_scorer(&self,reader: &tantivy::SegmentReader,- boost: tantivy::Score,) -> tantivy::Result<Option<FastSiteDomainPatternScorer>> {- let similarity_weight = self- .similarity_weight- .as_ref()- .map(|weight| weight.boost_by(boost));-let fieldnorm_reader = self.fieldnorm_reader(reader)?;let field_no_tokenizer = match Field::get(self.field.field_id() as usize) {@@ -87,7 +74,6 @@ impl FastSiteDomainPatternWeight {.read_postings(&self.term, opt)?{Some(posting) => Ok(Some(FastSiteDomainPatternScorer {- similarity_weight,posting,fieldnorm_reader,})),@@ -100,9 +86,9 @@ impl tantivy::query::Weight for FastSiteDomainPatternWeight {fn scorer(&self,reader: &tantivy::SegmentReader,- boost: tantivy::Score,+ _boost: tantivy::Score,) -> tantivy::Result<Box<dyn tantivy::query::Scorer>> {- if let Some(scorer) = self.pattern_scorer(reader, boost)? {+ if let Some(scorer) = self.pattern_scorer(reader)? {Ok(Box::new(PatternScorer::FastSiteDomain(scorer)))} else {Ok(Box::new(EmptyScorer))@@ -114,7 +100,7 @@ impl tantivy::query::Weight for FastSiteDomainPatternWeight {reader: &tantivy::SegmentReader,doc: tantivy::DocId,) -> tantivy::Result<tantivy::query::Explanation> {- let scorer_opt = self.pattern_scorer(reader, 1.0)?;+ let scorer_opt = self.pattern_scorer(reader)?;if scorer_opt.is_none() {return Err(TantivyError::InvalidArgument(format!("Document #({doc}) does not match (empty scorer)"@@ -126,22 +112,12 @@ impl tantivy::query::Weight for FastSiteDomainPatternWeight {"Document #({doc}) does not match")));}- let fieldnorm_reader = self.fieldnorm_reader(reader)?;- let fieldnorm_id = fieldnorm_reader.fieldnorm_id(doc);- let term_freq = scorer.term_freq();- let mut explanation = Explanation::new("Pattern Scorer", scorer.score());- explanation.add_detail(- self.similarity_weight- .as_ref()- .unwrap()- .explain(fieldnorm_id, term_freq),- );+ let explanation = Explanation::new("Pattern Scorer", scorer.score());Ok(explanation)}}pub struct PatternWeight {- pub similarity_weight: Option<Bm25Weight>,pub patterns: Vec<PatternPart>,pub raw_terms: Vec<tantivy::Term>,pub field: tantivy::schema::Field,@@ -149,22 +125,9 @@ pub struct PatternWeight {}impl PatternWeight {- fn fieldnorm_reader(- &self,- reader: &tantivy::SegmentReader,- ) -> tantivy::Result<FieldNormReader> {- if self.similarity_weight.is_some() {- if let Some(fieldnorm_reader) = reader.fieldnorms_readers().get_field(self.field)? {- return Ok(fieldnorm_reader);- }- }- Ok(FieldNormReader::constant(reader.max_doc(), 1))- }-pub(crate) fn pattern_scorer(&self,reader: &tantivy::SegmentReader,- boost: tantivy::Score,) -> tantivy::Result<Option<PatternScorer>> {if self.patterns.is_empty() {return Ok(None);@@ -226,13 +189,6 @@ impl PatternWeight {})));}- let similarity_weight = self- .similarity_weight- .as_ref()- .map(|weight| weight.boost_by(boost));-- let fieldnorm_reader = self.fieldnorm_reader(reader)?;-let mut term_postings_list = Vec::with_capacity(self.raw_terms.len());for term in &self.raw_terms {if let Some(postings) = reader@@ -256,9 +212,7 @@ impl PatternWeight {.collect();Ok(Some(PatternScorer::Normal(NormalPatternScorer::new(- similarity_weight,term_postings_list,- fieldnorm_reader,small_patterns,reader.segment_id(),num_tokens_fastfield,@@ -271,9 +225,9 @@ impl tantivy::query::Weight for PatternWeight {fn scorer(&self,reader: &tantivy::SegmentReader,- boost: tantivy::Score,+ _boost: tantivy::Score,) -> tantivy::Result<Box<dyn tantivy::query::Scorer>> {- if let Some(scorer) = self.pattern_scorer(reader, boost)? {+ if let Some(scorer) = self.pattern_scorer(reader)? {Ok(Box::new(scorer))} else {Ok(Box::new(EmptyScorer))@@ -285,7 +239,7 @@ impl tantivy::query::Weight for PatternWeight {reader: &tantivy::SegmentReader,doc: tantivy::DocId,) -> tantivy::Result<tantivy::query::Explanation> {- let scorer_opt = self.pattern_scorer(reader, 1.0)?;+ let scorer_opt = self.pattern_scorer(reader)?;if scorer_opt.is_none() {return Err(TantivyError::InvalidArgument(format!("Document #({doc}) does not match (empty scorer)"@@ -297,16 +251,7 @@ impl tantivy::query::Weight for PatternWeight {"Document #({doc}) does not match")));}- let fieldnorm_reader = self.fieldnorm_reader(reader)?;- let fieldnorm_id = fieldnorm_reader.fieldnorm_id(doc);- let term_freq = scorer.term_freq();- let mut explanation = Explanation::new("Pattern Scorer", scorer.score());- explanation.add_detail(- self.similarity_weight- .as_ref()- .unwrap()- .explain(fieldnorm_id, term_freq),- );+ let explanation = Explanation::new("Pattern Scorer", scorer.score());Ok(explanation)}}diff --git a/crates/core/src/ranking/bm25.rs b/crates/core/src/ranking/bm25.rsindex e79de857..e9649da6 100644--- a/crates/core/src/ranking/bm25.rs+++ b/crates/core/src/ranking/bm25.rs@@ -1,5 +1,6 @@// source: https://github.com/quickwit-oss/tantivy/blob/main/src/query/bm25.rs+use itertools::Itertools;use serde::{Deserialize, Serialize};use tantivy::fieldnorm::FieldNormReader;@@ -35,27 +36,20 @@ pub struct Bm25Params {}#[derive(Clone)]-pub struct Bm25Weight {- idf_explain: Explanation,- weight: Score,- cache: [Score; 256],- average_fieldnorm: Score,+pub struct MultiBm25Weight {+ weights: Vec<Bm25Weight>,}-impl Bm25Weight {- pub fn boost_by(&self, boost: Score) -> Bm25Weight {- Bm25Weight {- idf_explain: self.idf_explain.clone(),- weight: self.weight * boost,- cache: self.cache,- average_fieldnorm: self.average_fieldnorm,+impl MultiBm25Weight {+ pub fn for_terms(searcher: &Searcher, terms: &[Term]) -> tantivy::Result<Self> {+ if terms.is_empty() {+ return Ok(Self {+ weights: Vec::new(),+ });}- }- pub fn for_terms(searcher: &Searcher, terms: &[Term]) -> tantivy::Result<Bm25Weight> {- assert!(!terms.is_empty(), "Bm25 requires at least one term");let field = terms[0].field();- for term in &terms[1..] {+ for term in terms.iter().skip(1) {assert_eq!(term.field(),field,@@ -72,21 +66,54 @@ impl Bm25Weight {}let average_fieldnorm = total_num_tokens as Score / total_num_docs as Score;- if terms.len() == 1 {- let term_doc_freq = searcher.doc_freq(&terms[0])?;- Ok(Bm25Weight::for_one_term(+ let mut weights = Vec::new();++ for term in terms {+ let term_doc_freq = searcher.doc_freq(term)?;+ weights.push(Bm25Weight::for_one_term(term_doc_freq,total_num_docs,average_fieldnorm,- ))- } else {- let mut idf_sum: Score = 0.0;- for term in terms {- let term_doc_freq = searcher.doc_freq(term)?;- idf_sum += idf(term_doc_freq, total_num_docs);- }- let idf_explain = Explanation::new("idf", idf_sum);- Ok(Bm25Weight::new(idf_explain, average_fieldnorm))+ ));+ }++ Ok(Self { weights })+ }++ #[inline]+ pub fn score(&self, stats: impl Iterator<Item = (u8, u32)>) -> Score {+ stats+ .zip_eq(self.weights.iter())+ .map(|((fieldnorm_id, term_freq), weight)| weight.score(fieldnorm_id, term_freq))+ .sum()+ }++ pub fn idf(&self) -> impl Iterator<Item = f32> + '_ {+ self.weights.iter().map(|w| w.weight)+ }++ pub fn boost_by(&self, boost: Score) -> Self {+ Self {+ weights: self.weights.iter().map(|w| w.boost_by(boost)).collect(),+ }+ }+}++#[derive(Clone)]+pub struct Bm25Weight {+ idf_explain: Explanation,+ weight: Score,+ cache: [Score; 256],+ average_fieldnorm: Score,+}++impl Bm25Weight {+ pub fn boost_by(&self, boost: Score) -> Bm25Weight {+ Bm25Weight {+ idf_explain: self.idf_explain.clone(),+ weight: self.weight * boost,+ cache: self.cache,+ average_fieldnorm: self.average_fieldnorm,}}@@ -158,3 +185,28 @@ impl Bm25Weight {explanation}}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn test_bm25_idf_scaling() {+ // assume the query is something like 'the end'+ // 'the' appears in almost all docs (98)+ // 'end' appears in a smalle subset (20)+ let weight = MultiBm25Weight {+ weights: vec![+ Bm25Weight::for_one_term(98, 100, 1.0),+ Bm25Weight::for_one_term(20, 100, 1.0),+ ],+ };++ // if a document has high frequency of 'end'+ // it should have a higher score than a document that+ // has an almost equally high frequency of 'the'+ let high_the = weight.score(vec![(0, 15), (0, 10)].into_iter());+ let high_end = weight.score(vec![(0, 8), (0, 13)].into_iter());+ assert!(high_end > high_the);+ }+}diff --git a/crates/core/src/ranking/models/lambdamart.rs b/crates/core/src/ranking/models/lambdamart.rsindex b62cf9fb..24737cfd 100644--- a/crates/core/src/ranking/models/lambdamart.rs+++ b/crates/core/src/ranking/models/lambdamart.rs@@ -310,25 +310,25 @@ mod tests {assert!(!model.trees.is_empty());let mut features = EnumMap::new();- features.insert(Signal::Bm25BacklinkText, 85.7750244140625);+ features.insert(Signal::IdfSumBacklinkText, 85.7750244140625);features.insert(Signal::Bm25CleanBody, 67.41311645507812);features.insert(Signal::Bm25CleanBodyBigrams, 0.0);features.insert(Signal::Bm25CleanBodyTrigrams, 0.0);- features.insert(Signal::Bm25Domain, 43.332096099853516);- features.insert(Signal::Bm25DomainIfHomepage, 0.0);- features.insert(Signal::Bm25DomainIfHomepageNoTokenizer, 0.0);- features.insert(Signal::Bm25DomainNameIfHomepageNoTokenizer, 0.0);- features.insert(Signal::Bm25DomainNameNoTokenizer, 0.0);- features.insert(Signal::Bm25DomainNoTokenizer, 0.0);- features.insert(Signal::Bm25Site, 61.47410202026367);- features.insert(Signal::Bm25SiteNoTokenizer, 0.0);+ features.insert(Signal::IdfSumDomain, 43.332096099853516);+ features.insert(Signal::IdfSumDomainIfHomepage, 0.0);+ features.insert(Signal::IdfSumDomainIfHomepageNoTokenizer, 0.0);+ features.insert(Signal::IdfSumDomainNameIfHomepageNoTokenizer, 0.0);+ features.insert(Signal::IdfSumDomainNameNoTokenizer, 0.0);+ features.insert(Signal::IdfSumDomainNoTokenizer, 0.0);+ features.insert(Signal::IdfSumSite, 61.47410202026367);+ features.insert(Signal::IdfSumSiteNoTokenizer, 0.0);features.insert(Signal::Bm25StemmedCleanBody, 65.94627380371094);features.insert(Signal::Bm25StemmedTitle, 0.0);features.insert(Signal::Bm25Title, 59.817813873291016);features.insert(Signal::Bm25TitleBigrams, 0.0);- features.insert(Signal::Bm25TitleIfHomepage, 0.0);+ features.insert(Signal::IdfSumTitleIfHomepage, 0.0);features.insert(Signal::Bm25TitleTrigrams, 0.0);- features.insert(Signal::Bm25Url, 57.07925033569336);+ features.insert(Signal::IdfSumUrl, 57.07925033569336);features.insert(Signal::FetchTimeMs, 0.023255813953488372);features.insert(Signal::HostCentrality, 0.017958538);features.insert(Signal::InboundSimilarity, 0.0);diff --git a/crates/core/src/ranking/signal.rs b/crates/core/src/ranking/signal.rsindex 35ab50a9..1422ae0e 100644--- a/crates/core/src/ranking/signal.rs+++ b/crates/core/src/ranking/signal.rs@@ -24,6 +24,7 @@ use crate::{webgraph::NodeID,webpage::Webpage,};+use itertools::Itertools;use optics::ast::RankingTarget;use optics::Optic;use serde::{Deserialize, Serialize};@@ -45,7 +46,7 @@ use crate::{webpage::region::{Region, RegionCount},};-use super::bm25::Bm25Weight;+use super::bm25::MultiBm25Weight;use super::models::linear::LinearRegression;use super::{inbound_similarity, query_centrality};@@ -75,28 +76,28 @@ pub enum Signal {Bm25StemmedCleanBody,#[serde(rename = "bm25_all_body")]Bm25AllBody,- #[serde(rename = "bm25_url")]- Bm25Url,- #[serde(rename = "bm25_site")]- Bm25Site,- #[serde(rename = "bm25_domain")]- Bm25Domain,- #[serde(rename = "bm25_site_no_tokenizer")]- Bm25SiteNoTokenizer,- #[serde(rename = "bm25_domain_no_tokenizer")]- Bm25DomainNoTokenizer,- #[serde(rename = "bm25_domain_name_no_tokenizer")]- Bm25DomainNameNoTokenizer,- #[serde(rename = "bm25_domain_if_homepage")]- Bm25DomainIfHomepage,- #[serde(rename = "bm25_domain_name_if_homepage_no_tokenizer")]- Bm25DomainNameIfHomepageNoTokenizer,- #[serde(rename = "bm25_domain_if_homepage_no_tokenizer")]- Bm25DomainIfHomepageNoTokenizer,- #[serde(rename = "bm25_title_if_homepage")]- Bm25TitleIfHomepage,- #[serde(rename = "bm25_backlink_text")]- Bm25BacklinkText,+ #[serde(rename = "idf_sum_url")]+ IdfSumUrl,+ #[serde(rename = "idf_sum_site")]+ IdfSumSite,+ #[serde(rename = "idf_sum_domain")]+ IdfSumDomain,+ #[serde(rename = "idf_sum_site_no_tokenizer")]+ IdfSumSiteNoTokenizer,+ #[serde(rename = "idf_sum_domain_no_tokenizer")]+ IdfSumDomainNoTokenizer,+ #[serde(rename = "idf_sum_domain_name_no_tokenizer")]+ IdfSumDomainNameNoTokenizer,+ #[serde(rename = "idf_sum_domain_if_homepage")]+ IdfSumDomainIfHomepage,+ #[serde(rename = "idf_sum_domain_name_if_homepage_no_tokenizer")]+ IdfSumDomainNameIfHomepageNoTokenizer,+ #[serde(rename = "idf_sum_domain_if_homepage_no_tokenizer")]+ IdfSumDomainIfHomepageNoTokenizer,+ #[serde(rename = "idf_sum_title_if_homepage")]+ IdfSumTitleIfHomepage,+ #[serde(rename = "idf_sum_backlink_text")]+ IdfSumBacklinkText,#[serde(rename = "cross_encoder_snippet")]CrossEncoderSnippet,#[serde(rename = "cross_encoder_title")]@@ -149,17 +150,17 @@ pub const ALL_SIGNALS: [Signal; 37] = [Signal::Bm25StemmedTitle,Signal::Bm25StemmedCleanBody,Signal::Bm25AllBody,- Signal::Bm25Url,- Signal::Bm25Site,- Signal::Bm25Domain,- Signal::Bm25SiteNoTokenizer,- Signal::Bm25DomainNoTokenizer,- Signal::Bm25DomainNameNoTokenizer,- Signal::Bm25DomainIfHomepage,- Signal::Bm25DomainNameIfHomepageNoTokenizer,- Signal::Bm25DomainIfHomepageNoTokenizer,- Signal::Bm25TitleIfHomepage,- Signal::Bm25BacklinkText,+ Signal::IdfSumUrl,+ Signal::IdfSumSite,+ Signal::IdfSumDomain,+ Signal::IdfSumSiteNoTokenizer,+ Signal::IdfSumDomainNoTokenizer,+ Signal::IdfSumDomainNameNoTokenizer,+ Signal::IdfSumDomainIfHomepage,+ Signal::IdfSumDomainNameIfHomepageNoTokenizer,+ Signal::IdfSumDomainIfHomepageNoTokenizer,+ Signal::IdfSumTitleIfHomepage,+ Signal::IdfSumBacklinkText,Signal::CrossEncoderSnippet,Signal::CrossEncoderTitle,Signal::HostCentrality,@@ -249,19 +250,36 @@ fn bm25(field: &mut TextFieldData, doc: DocId) -> f64 {return 0.0;}- let mut term_freq = 0;- for posting in &mut field.postings {- if posting.doc() == doc || (posting.doc() < doc && posting.seek(doc) == doc) {- term_freq += posting.term_freq();- }- }+ let fieldnorm_id = field.fieldnorm_reader.fieldnorm_id(doc);++ field+ .weight+ .score(field.postings.iter_mut().map(move |posting| {+ if posting.doc() == doc || (posting.doc() < doc && posting.seek(doc) == doc) {+ (fieldnorm_id, posting.term_freq())+ } else {+ (fieldnorm_id, 0)+ }+ })) as f64+}- if term_freq == 0 {+fn idf_sum(field: &mut TextFieldData, doc: DocId) -> f64 {+ if field.postings.is_empty() {return 0.0;}- let fieldnorm_id = field.fieldnorm_reader.fieldnorm_id(doc);- field.weight.score(fieldnorm_id, term_freq) as f64+ field+ .postings+ .iter_mut()+ .zip_eq(field.weight.idf())+ .filter_map(|(posting, idf)| {+ if posting.doc() == doc || (posting.doc() < doc && posting.seek(doc) == doc) {+ Some(idf)+ } else {+ None+ }+ })+ .sum::<f32>() as f64}impl Signal {@@ -280,17 +298,17 @@ impl Signal {Signal::Bm25StemmedTitle => 0.003,Signal::Bm25StemmedCleanBody => 0.001,Signal::Bm25AllBody => 0.0,- Signal::Bm25Url => 0.0003,- Signal::Bm25Site => 0.00015,- Signal::Bm25Domain => 0.0003,- Signal::Bm25SiteNoTokenizer => 0.00015,- Signal::Bm25DomainNoTokenizer => 0.0002,- Signal::Bm25DomainNameNoTokenizer => 0.0002,- Signal::Bm25DomainIfHomepage => 0.0004,- Signal::Bm25DomainNameIfHomepageNoTokenizer => 0.0036,- Signal::Bm25DomainIfHomepageNoTokenizer => 0.0036,- Signal::Bm25TitleIfHomepage => 0.00022,- Signal::Bm25BacklinkText => 0.003,+ Signal::IdfSumUrl => 0.0003,+ Signal::IdfSumSite => 0.00015,+ Signal::IdfSumDomain => 0.0003,+ Signal::IdfSumSiteNoTokenizer => 0.00015,+ Signal::IdfSumDomainNoTokenizer => 0.0002,+ Signal::IdfSumDomainNameNoTokenizer => 0.0002,+ Signal::IdfSumDomainIfHomepage => 0.0004,+ Signal::IdfSumDomainNameIfHomepageNoTokenizer => 0.0036,+ Signal::IdfSumDomainIfHomepageNoTokenizer => 0.0036,+ Signal::IdfSumTitleIfHomepage => 0.00022,+ Signal::IdfSumBacklinkText => 0.003,Signal::CrossEncoderSnippet => 0.17,Signal::CrossEncoderTitle => 0.17,Signal::HostCentrality => 0.5,@@ -393,22 +411,26 @@ impl Signal {| Signal::Bm25CleanBodyTrigrams| Signal::Bm25StemmedTitle| Signal::Bm25StemmedCleanBody- | Signal::Bm25AllBody- | Signal::Bm25Url- | Signal::Bm25Site- | Signal::Bm25Domain- | Signal::Bm25SiteNoTokenizer- | Signal::Bm25DomainNoTokenizer- | Signal::Bm25DomainNameNoTokenizer- | Signal::Bm25DomainIfHomepage- | Signal::Bm25DomainNameIfHomepageNoTokenizer- | Signal::Bm25DomainIfHomepageNoTokenizer- | Signal::Bm25TitleIfHomepage- | Signal::Bm25BacklinkText => seg_reader+ | Signal::Bm25AllBody => seg_reader.text_fields.get_mut(self.as_textfield().unwrap()).map(|field| bm25(field, doc)),+ Signal::IdfSumUrl+ | Signal::IdfSumSite+ | Signal::IdfSumDomain+ | Signal::IdfSumSiteNoTokenizer+ | Signal::IdfSumDomainNoTokenizer+ | Signal::IdfSumDomainNameNoTokenizer+ | Signal::IdfSumDomainIfHomepage+ | Signal::IdfSumDomainNameIfHomepageNoTokenizer+ | Signal::IdfSumDomainIfHomepageNoTokenizer+ | Signal::IdfSumTitleIfHomepage+ | Signal::IdfSumBacklinkText => seg_reader+ .text_fields+ .get_mut(self.as_textfield().unwrap())+ .map(|field| idf_sum(field, doc)),+Signal::CrossEncoderSnippet => None, // this is calculated in a later stepSignal::CrossEncoderTitle => None, // this is calculated in a later stepSignal::LambdaMART => None,@@ -501,17 +523,17 @@ impl Signal {| Signal::Bm25StemmedTitle| Signal::Bm25StemmedCleanBody| Signal::Bm25AllBody- | Signal::Bm25Url- | Signal::Bm25Site- | Signal::Bm25Domain- | Signal::Bm25SiteNoTokenizer- | Signal::Bm25DomainNoTokenizer- | Signal::Bm25DomainNameNoTokenizer- | Signal::Bm25DomainIfHomepage- | Signal::Bm25DomainNameIfHomepageNoTokenizer- | Signal::Bm25DomainIfHomepageNoTokenizer- | Signal::Bm25TitleIfHomepage- | Signal::Bm25BacklinkText+ | Signal::IdfSumUrl+ | Signal::IdfSumSite+ | Signal::IdfSumDomain+ | Signal::IdfSumSiteNoTokenizer+ | Signal::IdfSumDomainNoTokenizer+ | Signal::IdfSumDomainNameNoTokenizer+ | Signal::IdfSumDomainIfHomepage+ | Signal::IdfSumDomainNameIfHomepageNoTokenizer+ | Signal::IdfSumDomainIfHomepageNoTokenizer+ | Signal::IdfSumTitleIfHomepage+ | Signal::IdfSumBacklinkText| Signal::CrossEncoderSnippet| Signal::CrossEncoderTitle| Signal::InboundSimilarity@@ -560,19 +582,21 @@ impl Signal {Signal::Bm25StemmedTitle => Some(TextField::StemmedTitle),Signal::Bm25StemmedCleanBody => Some(TextField::StemmedCleanBody),Signal::Bm25AllBody => Some(TextField::AllBody),- Signal::Bm25Url => Some(TextField::Url),- Signal::Bm25Site => Some(TextField::SiteWithout),- Signal::Bm25Domain => Some(TextField::Domain),- Signal::Bm25SiteNoTokenizer => Some(TextField::SiteNoTokenizer),- Signal::Bm25DomainNoTokenizer => Some(TextField::DomainNoTokenizer),- Signal::Bm25DomainNameNoTokenizer => Some(TextField::DomainNameNoTokenizer),- Signal::Bm25DomainIfHomepage => Some(TextField::DomainIfHomepage),- Signal::Bm25DomainNameIfHomepageNoTokenizer => {+ Signal::IdfSumUrl => Some(TextField::Url),+ Signal::IdfSumSite => Some(TextField::SiteWithout),+ Signal::IdfSumDomain => Some(TextField::Domain),+ Signal::IdfSumSiteNoTokenizer => Some(TextField::SiteNoTokenizer),+ Signal::IdfSumDomainNoTokenizer => Some(TextField::DomainNoTokenizer),+ Signal::IdfSumDomainNameNoTokenizer => Some(TextField::DomainNameNoTokenizer),+ Signal::IdfSumDomainIfHomepage => Some(TextField::DomainIfHomepage),+ Signal::IdfSumDomainNameIfHomepageNoTokenizer => {Some(TextField::DomainNameIfHomepageNoTokenizer)}- Signal::Bm25TitleIfHomepage => Some(TextField::TitleIfHomepage),- Signal::Bm25BacklinkText => Some(TextField::BacklinkText),- Signal::Bm25DomainIfHomepageNoTokenizer => Some(TextField::DomainIfHomepageNoTokenizer),+ Signal::IdfSumTitleIfHomepage => Some(TextField::TitleIfHomepage),+ Signal::IdfSumBacklinkText => Some(TextField::BacklinkText),+ Signal::IdfSumDomainIfHomepageNoTokenizer => {+ Some(TextField::DomainIfHomepageNoTokenizer)+ }_ => None,}}@@ -635,7 +659,7 @@ impl SignalCoefficient {#[derive(Clone)]struct TextFieldData {postings: Vec<SegmentPostings>,- weight: Bm25Weight,+ weight: MultiBm25Weight,fieldnorm_reader: FieldNormReader,}@@ -797,19 +821,20 @@ impl SignalAggregator {continue;}- let weight = Bm25Weight::for_terms(tv_searcher, &terms)?;-let fieldnorm_reader = segment_reader.get_fieldnorms_reader(tv_field)?;let inverted_index = segment_reader.inverted_index(tv_field)?;+ let mut matching_terms = Vec::with_capacity(terms.len());let mut postings = Vec::with_capacity(terms.len());for term in &terms {if let Some(p) =inverted_index.read_postings(term, text_field.index_option())?{postings.push(p);+ matching_terms.push(term.clone());}}+ let weight = MultiBm25Weight::for_terms(tv_searcher, &matching_terms)?;text_fields.insert(text_field,diff --git a/crates/core/testcases/lambdamart.txt b/crates/core/testcases/lambdamart.txtindex 1b61722e..41415863 100644--- a/crates/core/testcases/lambdamart.txt+++ b/crates/core/testcases/lambdamart.txt@@ -5,7 +5,7 @@ num_tree_per_iteration=1label_index=0max_feature_idx=28objective=lambdarank-feature_names=bm25_backlink_text inbound_similarity bm25_clean_body_bigrams bm25_domain_if_homepage_no_tokenizer page_centrality is_homepage bm25_title_if_homepage url_slashes fetch_time_ms region tracker_score bm25_domain bm25_domain_no_tokenizer bm25_title bm25_stemmed_title bm25_domain_name_no_tokenizer host_centrality url_digits bm25_domain_name_if_homepage_no_tokenizer bm25_title_trigrams bm25_clean_body_trigrams bm25_title_bigrams bm25_stemmed_clean_body bm25_site bm25_site_no_tokenizer bm25_url bm25_domain_if_homepage update_timestamp bm25_clean_body+feature_names=idf_sum_backlink_text inbound_similarity bm25_clean_body_bigrams idf_sum_domain_if_homepage_no_tokenizer page_centrality is_homepage idf_sum_title_if_homepage url_slashes fetch_time_ms region tracker_score idf_sum_domain idf_sum_domain_no_tokenizer bm25_title bm25_stemmed_title idf_sum_domain_name_no_tokenizer host_centrality url_digits idf_sum_domain_name_if_homepage_no_tokenizer bm25_title_trigrams bm25_clean_body_trigrams bm25_title_bigrams bm25_stemmed_clean_body idf_sum_site idf_sum_site_no_tokenizer idf_sum_url idf_sum_domain_if_homepage update_timestamp bm25_clean_bodyfeature_infos=[0:71.599037170410156] none [0:59.439960479736328] none [0:0.041696587] [0:1] [0:24.348766326904297] [0.14285714285714285:0.5] [0:0.16666666666666666] [0:0.70508371084865562] [0.009433962264150943:1] none none [6.0304698944091797:45.648475646972656] [0:44.676078796386719] none [7.6000000000000006e-08:0.045836137999999998] [0.066666666666666666:1] none none none [0:48.516033172607422] [0:40.354255676269531] [0:44.526699066162109] none [0:44.291984558105469] none none [4.4346046447753906:41.890125274658203]tree_sizes=1051 753 851 859 958 856 759 960 858 651 862 971 753 861 861 973 859 761 865 869 982 763 986 763 984 759 764 765 861 762 767 765 869 869 874 872 873 879 871 883 773 774 877 768 765 777 988 878 767 879@@ -965,9 +965,9 @@ feature_importances:fetch_time_ms=40host_centrality=35bm25_title_bigrams=34-bm25_url=33+idf_sum_url=33bm25_clean_body_bigrams=26-bm25_backlink_text=21+idf_sum_backlink_text=21page_centrality=20bm25_stemmed_title=20bm25_title=18@@ -976,9 +976,9 @@ url_slashes=9bm25_stemmed_clean_body=6bm25_clean_body=5is_homepage=2-bm25_title_if_homepage=2+idf_sum_title_if_homepage=2region=2-bm25_site=1+idf_sum_site=1parameters:[boosting: gbdt]