web search done right
Add term cover percentage as ranking signals
Mikkel Denkerd608aa217f6c
diff --git a/crates/core/src/ranking/core/computer/mod.rs b/crates/core/src/ranking/core/computer/mod.rsindex 1939e50a..c244489c 100644--- a/crates/core/src/ranking/core/computer/mod.rs+++ b/crates/core/src/ranking/core/computer/mod.rs@@ -45,9 +45,14 @@ use super::{ComputedSignal, Signal, SignalCoefficient, SignalEnum};mod order;pub use order::SignalComputeOrder;+fn posting_contains(posting: &mut SegmentPostings, doc: DocId) -> bool {+ posting.doc() == doc || (posting.doc() < doc && posting.seek(doc) == doc)+}+#[derive(Clone)]pub struct TextFieldData {postings: Vec<SegmentPostings>,+ num_query_terms: usize,bm25: MultiBm25Weight,bm25f: MultiBm25FWeight,fieldnorm_reader: FieldNormReader,@@ -55,6 +60,24 @@ pub struct TextFieldData {}impl TextFieldData {+ pub fn coverage(&mut self, doc: DocId) -> f64 {+ if self.postings.is_empty() {+ return 0.0;+ }++ self.postings+ .iter_mut()+ .map(|posting| {+ if posting_contains(posting, doc) {+ 1.0+ } else {+ 0.0+ }+ })+ .sum::<f64>()+ / self.num_query_terms as f64+ }+pub fn bm25(&mut self, doc: DocId) -> f64 {if self.postings.is_empty() {return 0.0;@@ -64,7 +87,7 @@ impl TextFieldData {self.bm25.score(self.postings.iter_mut().map(move |posting| {- if posting.doc() == doc || (posting.doc() < doc && posting.seek(doc) == doc) {+ if posting_contains(posting, doc) {(fieldnorm_id, posting.term_freq())} else {(fieldnorm_id, 0)@@ -82,7 +105,7 @@ impl TextFieldData {.iter_mut().zip_eq(idf).filter_map(|(posting, idf)| {- if posting.doc() == doc || (posting.doc() < doc && posting.seek(doc) == doc) {+ if posting_contains(posting, doc) {Some(idf)} else {None@@ -313,6 +336,7 @@ impl SignalComputer {bm25f,fieldnorm_reader,signal_coefficient: self.coefficient(&signal),+ num_query_terms: terms.len(),},);}diff --git a/crates/core/src/ranking/core/computer/order.rs b/crates/core/src/ranking/core/computer/order.rsindex 217ac2ff..6b167257 100644--- a/crates/core/src/ranking/core/computer/order.rs+++ b/crates/core/src/ranking/core/computer/order.rs@@ -37,14 +37,18 @@ impl SignalComputeOrder {for signal in SignalEnum::all() {if let Some(text_field) = signal.as_textfield() {- let mono = text_field.monogram_field();+ if signal.has_sibling_ngrams() {+ let mono = text_field.monogram_field();- if !text_signals.contains_key(mono) {- text_signals.insert(mono, NGramComputeOrder::default());- }+ if !text_signals.contains_key(mono) {+ text_signals.insert(mono, NGramComputeOrder::default());+ }- let ngram = text_field.ngram_size();- text_signals.get_mut(mono).unwrap().push(signal, ngram);+ let ngram = text_field.ngram_size();+ text_signals.get_mut(mono).unwrap().push(signal, ngram);+ } else {+ other_signals.push(signal);+ }} else {other_signals.push(signal);}diff --git a/crates/core/src/ranking/core/prelude.rs b/crates/core/src/ranking/core/prelude.rsindex d41f0d38..825b538b 100644--- a/crates/core/src/ranking/core/prelude.rs+++ b/crates/core/src/ranking/core/prelude.rs@@ -48,6 +48,10 @@ pub trait Signal:self.as_field().and_then(|field| field.as_text())}+ fn has_sibling_ngrams(&self) -> bool {+ false+ }+fn as_numericalfield(&self) -> Option<NumericalFieldEnum> {self.as_field().and_then(|field| field.as_numerical())}@@ -82,9 +86,11 @@ pub trait Signal:pub enum SignalEnum {Bm25F,Bm25Title,+ TitleCoverage,Bm25TitleBigrams,Bm25TitleTrigrams,Bm25CleanBody,+ CleanBodyCoverage,Bm25CleanBodyBigrams,Bm25CleanBodyTrigrams,Bm25StemmedTitle,@@ -129,9 +135,11 @@ enum_dispatch_from_discriminant!(SignalEnumDiscriminants => SignalEnum,[Bm25F,Bm25Title,+ TitleCoverage,Bm25TitleBigrams,Bm25TitleTrigrams,Bm25CleanBody,+ CleanBodyCoverage,Bm25CleanBodyBigrams,Bm25CleanBodyTrigrams,Bm25StemmedTitle,diff --git a/crates/core/src/ranking/core/text.rs b/crates/core/src/ranking/core/text.rsindex 6ffb05bf..aa9754a4 100644--- a/crates/core/src/ranking/core/text.rs+++ b/crates/core/src/ranking/core/text.rs@@ -77,6 +77,10 @@ impl Signal for Bm25Title {Some(Field::Text(schema::text_field::Title.into()))}+ fn has_sibling_ngrams(&self) -> bool {+ true+ }+fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {let mut seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();@@ -87,6 +91,38 @@ impl Signal for Bm25Title {}}+#[derive(+ Debug,+ Clone,+ Copy,+ PartialEq,+ Eq,+ Hash,+ serde::Serialize,+ serde::Deserialize,+ bincode::Encode,+ bincode::Decode,+)]+pub struct TitleCoverage;+impl Signal for TitleCoverage {+ fn default_coefficient(&self) -> f64 {+ 0.01+ }++ fn as_field(&self) -> Option<Field> {+ Some(Field::Text(schema::text_field::Title.into()))+ }++ fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {+ let mut seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();++ seg_reader+ .text_fields_mut()+ .get_mut(self.as_textfield().unwrap())+ .map(|field| field.coverage(doc))+ }+}+#[derive(Debug,Clone,@@ -105,6 +141,10 @@ impl Signal for Bm25TitleBigrams {0.005}+ fn has_sibling_ngrams(&self) -> bool {+ true+ }+fn as_field(&self) -> Option<Field> {Some(Field::Text(schema::text_field::TitleBigrams.into()))}@@ -137,6 +177,10 @@ impl Signal for Bm25TitleTrigrams {0.005}+ fn has_sibling_ngrams(&self) -> bool {+ true+ }+fn as_field(&self) -> Option<Field> {Some(Field::Text(schema::text_field::TitleTrigrams.into()))}@@ -169,6 +213,10 @@ impl Signal for Bm25CleanBody {0.005}+ fn has_sibling_ngrams(&self) -> bool {+ true+ }+fn as_field(&self) -> Option<Field> {Some(Field::Text(schema::text_field::CleanBody.into()))}@@ -183,6 +231,38 @@ impl Signal for Bm25CleanBody {}}+#[derive(+ Debug,+ Clone,+ Copy,+ PartialEq,+ Eq,+ Hash,+ serde::Serialize,+ serde::Deserialize,+ bincode::Encode,+ bincode::Decode,+)]+pub struct CleanBodyCoverage;+impl Signal for CleanBodyCoverage {+ fn default_coefficient(&self) -> f64 {+ 0.01+ }++ fn as_field(&self) -> Option<Field> {+ Some(Field::Text(schema::text_field::CleanBody.into()))+ }++ fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {+ let mut seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();++ seg_reader+ .text_fields_mut()+ .get_mut(self.as_textfield().unwrap())+ .map(|field| field.coverage(doc))+ }+}+#[derive(Debug,Clone,@@ -201,6 +281,10 @@ impl Signal for Bm25CleanBodyBigrams {0.005}+ fn has_sibling_ngrams(&self) -> bool {+ true+ }+fn as_field(&self) -> Option<Field> {Some(Field::Text(schema::text_field::CleanBodyBigrams.into()))}@@ -233,6 +317,10 @@ impl Signal for Bm25CleanBodyTrigrams {0.005}+ fn has_sibling_ngrams(&self) -> bool {+ true+ }+fn as_field(&self) -> Option<Field> {Some(Field::Text(schema::text_field::CleanBodyTrigrams.into()))}diff --git a/crates/core/src/ranking/mod.rs b/crates/core/src/ranking/mod.rsindex 3bf70d88..1d126e37 100644--- a/crates/core/src/ranking/mod.rs+++ b/crates/core/src/ranking/mod.rs@@ -940,4 +940,156 @@ mod tests {assert_eq!(result.webpages.len(), 2);assert_eq!(result.webpages[0].url, "https://www.a.com/");}++ #[test]+ fn title_coverage() {+ let mut index = Index::temporary().expect("Unable to open index");++ index+ .insert(&Webpage {+ html: Html::parse(+ &format!(+ r#"+ <html>+ <head>+ <title>Test website</title>+ </head>+ <body>+ {CONTENT} {}+ </body>+ </html>+ "#,+ crate::rand_words(100)+ ),+ "https://www.first.com",+ )+ .unwrap(),+ host_centrality: 1.0,+ fetch_time_ms: 2,+ ..Default::default()+ })+ .expect("failed to insert webpage");+ index.commit().expect("failed to commit index");+ let searcher = LocalSearcher::new(index);++ let result = searcher+ .search(&SearchQuery {+ query: "test website".to_string(),+ return_ranking_signals: true,+ ..Default::default()+ })+ .expect("Search failed");++ assert_eq!(result.webpages.len(), 1);+ assert_eq!(+ result.webpages[0]+ .ranking_signals+ .as_ref()+ .unwrap()+ .get(&crate::ranking::SignalEnum::from(crate::ranking::core::TitleCoverage).into())+ .unwrap()+ .value,+ 1.0,+ );++ let result = searcher+ .search(&SearchQuery {+ query: "test example".to_string(),+ return_ranking_signals: true,+ ..Default::default()+ })+ .expect("Search failed");++ assert_eq!(result.webpages.len(), 1);+ assert_eq!(+ result.webpages[0]+ .ranking_signals+ .as_ref()+ .unwrap()+ .get(&crate::ranking::SignalEnum::from(crate::ranking::core::TitleCoverage).into())+ .unwrap()+ .value,+ 0.5,+ );+ }++ #[test]+ fn clean_body_coverage() {+ let mut index = Index::temporary().expect("Unable to open index");++ let mut page = Webpage {+ html: Html::parse(+ &format!(+ r#"+ <html>+ <head>+ <title>a b c</title>+ </head>+ <body>+ {CONTENT} {}+ </body>+ </html>+ "#,+ crate::rand_words(100)+ ),+ "https://www.first.com",+ )+ .unwrap(),+ host_centrality: 1.0,+ fetch_time_ms: 2,+ ..Default::default()+ };++ page.html.set_clean_text("test website".to_string());++ index.insert(&page).expect("failed to insert webpage");+ index.commit().expect("failed to commit index");+ let searcher = LocalSearcher::new(index);++ let result = searcher+ .search(&SearchQuery {+ query: "test website".to_string(),+ return_ranking_signals: true,+ ..Default::default()+ })+ .expect("Search failed");++ assert_eq!(result.webpages.len(), 1);+ assert_eq!(+ result.webpages[0]+ .ranking_signals+ .as_ref()+ .unwrap()+ .get(+ &crate::ranking::SignalEnum::from(crate::ranking::core::CleanBodyCoverage)+ .into()+ )+ .unwrap()+ .value,+ 1.0,+ );++ let result = searcher+ .search(&SearchQuery {+ query: "test b".to_string(),+ return_ranking_signals: true,+ ..Default::default()+ })+ .expect("Search failed");++ assert_eq!(result.webpages.len(), 1);+ assert_eq!(+ result.webpages[0]+ .ranking_signals+ .as_ref()+ .unwrap()+ .get(+ &crate::ranking::SignalEnum::from(crate::ranking::core::CleanBodyCoverage)+ .into()+ )+ .unwrap()+ .value,+ 0.5,+ );+ }}