Skip to content

web search done right

Add term cover percentage as ranking signals

Mikkel Denkerd608aa217f6c
diff --git a/crates/core/src/ranking/core/computer/mod.rs b/crates/core/src/ranking/core/computer/mod.rs
index 1939e50a..c244489c 100644
--- a/crates/core/src/ranking/core/computer/mod.rs
+++ b/crates/core/src/ranking/core/computer/mod.rs
@@ -45,9 +45,14 @@ use super::{ComputedSignal, Signal, SignalCoefficient, SignalEnum};
mod order;
pub use order::SignalComputeOrder;
+fn posting_contains(posting: &mut SegmentPostings, doc: DocId) -> bool {
+ posting.doc() == doc || (posting.doc() < doc && posting.seek(doc) == doc)
+}
+
#[derive(Clone)]
pub struct TextFieldData {
postings: Vec<SegmentPostings>,
+ num_query_terms: usize,
bm25: MultiBm25Weight,
bm25f: MultiBm25FWeight,
fieldnorm_reader: FieldNormReader,
@@ -55,6 +60,24 @@ pub struct TextFieldData {
}
impl TextFieldData {
+ pub fn coverage(&mut self, doc: DocId) -> f64 {
+ if self.postings.is_empty() {
+ return 0.0;
+ }
+
+ self.postings
+ .iter_mut()
+ .map(|posting| {
+ if posting_contains(posting, doc) {
+ 1.0
+ } else {
+ 0.0
+ }
+ })
+ .sum::<f64>()
+ / self.num_query_terms as f64
+ }
+
pub fn bm25(&mut self, doc: DocId) -> f64 {
if self.postings.is_empty() {
return 0.0;
@@ -64,7 +87,7 @@ impl TextFieldData {
self.bm25
.score(self.postings.iter_mut().map(move |posting| {
- if posting.doc() == doc || (posting.doc() < doc && posting.seek(doc) == doc) {
+ if posting_contains(posting, doc) {
(fieldnorm_id, posting.term_freq())
} else {
(fieldnorm_id, 0)
@@ -82,7 +105,7 @@ impl TextFieldData {
.iter_mut()
.zip_eq(idf)
.filter_map(|(posting, idf)| {
- if posting.doc() == doc || (posting.doc() < doc && posting.seek(doc) == doc) {
+ if posting_contains(posting, doc) {
Some(idf)
} else {
None
@@ -313,6 +336,7 @@ impl SignalComputer {
bm25f,
fieldnorm_reader,
signal_coefficient: self.coefficient(&signal),
+ num_query_terms: terms.len(),
},
);
}
diff --git a/crates/core/src/ranking/core/computer/order.rs b/crates/core/src/ranking/core/computer/order.rs
index 217ac2ff..6b167257 100644
--- a/crates/core/src/ranking/core/computer/order.rs
+++ b/crates/core/src/ranking/core/computer/order.rs
@@ -37,14 +37,18 @@ impl SignalComputeOrder {
for signal in SignalEnum::all() {
if let Some(text_field) = signal.as_textfield() {
- let mono = text_field.monogram_field();
+ if signal.has_sibling_ngrams() {
+ let mono = text_field.monogram_field();
- if !text_signals.contains_key(mono) {
- text_signals.insert(mono, NGramComputeOrder::default());
- }
+ if !text_signals.contains_key(mono) {
+ text_signals.insert(mono, NGramComputeOrder::default());
+ }
- let ngram = text_field.ngram_size();
- text_signals.get_mut(mono).unwrap().push(signal, ngram);
+ let ngram = text_field.ngram_size();
+ text_signals.get_mut(mono).unwrap().push(signal, ngram);
+ } else {
+ other_signals.push(signal);
+ }
} else {
other_signals.push(signal);
}
diff --git a/crates/core/src/ranking/core/prelude.rs b/crates/core/src/ranking/core/prelude.rs
index d41f0d38..825b538b 100644
--- a/crates/core/src/ranking/core/prelude.rs
+++ b/crates/core/src/ranking/core/prelude.rs
@@ -48,6 +48,10 @@ pub trait Signal:
self.as_field().and_then(|field| field.as_text())
}
+ fn has_sibling_ngrams(&self) -> bool {
+ false
+ }
+
fn as_numericalfield(&self) -> Option<NumericalFieldEnum> {
self.as_field().and_then(|field| field.as_numerical())
}
@@ -82,9 +86,11 @@ pub trait Signal:
pub enum SignalEnum {
Bm25F,
Bm25Title,
+ TitleCoverage,
Bm25TitleBigrams,
Bm25TitleTrigrams,
Bm25CleanBody,
+ CleanBodyCoverage,
Bm25CleanBodyBigrams,
Bm25CleanBodyTrigrams,
Bm25StemmedTitle,
@@ -129,9 +135,11 @@ enum_dispatch_from_discriminant!(SignalEnumDiscriminants => SignalEnum,
[
Bm25F,
Bm25Title,
+ TitleCoverage,
Bm25TitleBigrams,
Bm25TitleTrigrams,
Bm25CleanBody,
+ CleanBodyCoverage,
Bm25CleanBodyBigrams,
Bm25CleanBodyTrigrams,
Bm25StemmedTitle,
diff --git a/crates/core/src/ranking/core/text.rs b/crates/core/src/ranking/core/text.rs
index 6ffb05bf..aa9754a4 100644
--- a/crates/core/src/ranking/core/text.rs
+++ b/crates/core/src/ranking/core/text.rs
@@ -77,6 +77,10 @@ impl Signal for Bm25Title {
Some(Field::Text(schema::text_field::Title.into()))
}
+ fn has_sibling_ngrams(&self) -> bool {
+ true
+ }
+
fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {
let mut seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();
@@ -87,6 +91,38 @@ impl Signal for Bm25Title {
}
}
+#[derive(
+ Debug,
+ Clone,
+ Copy,
+ PartialEq,
+ Eq,
+ Hash,
+ serde::Serialize,
+ serde::Deserialize,
+ bincode::Encode,
+ bincode::Decode,
+)]
+pub struct TitleCoverage;
+impl Signal for TitleCoverage {
+ fn default_coefficient(&self) -> f64 {
+ 0.01
+ }
+
+ fn as_field(&self) -> Option<Field> {
+ Some(Field::Text(schema::text_field::Title.into()))
+ }
+
+ fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {
+ let mut seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();
+
+ seg_reader
+ .text_fields_mut()
+ .get_mut(self.as_textfield().unwrap())
+ .map(|field| field.coverage(doc))
+ }
+}
+
#[derive(
Debug,
Clone,
@@ -105,6 +141,10 @@ impl Signal for Bm25TitleBigrams {
0.005
}
+ fn has_sibling_ngrams(&self) -> bool {
+ true
+ }
+
fn as_field(&self) -> Option<Field> {
Some(Field::Text(schema::text_field::TitleBigrams.into()))
}
@@ -137,6 +177,10 @@ impl Signal for Bm25TitleTrigrams {
0.005
}
+ fn has_sibling_ngrams(&self) -> bool {
+ true
+ }
+
fn as_field(&self) -> Option<Field> {
Some(Field::Text(schema::text_field::TitleTrigrams.into()))
}
@@ -169,6 +213,10 @@ impl Signal for Bm25CleanBody {
0.005
}
+ fn has_sibling_ngrams(&self) -> bool {
+ true
+ }
+
fn as_field(&self) -> Option<Field> {
Some(Field::Text(schema::text_field::CleanBody.into()))
}
@@ -183,6 +231,38 @@ impl Signal for Bm25CleanBody {
}
}
+#[derive(
+ Debug,
+ Clone,
+ Copy,
+ PartialEq,
+ Eq,
+ Hash,
+ serde::Serialize,
+ serde::Deserialize,
+ bincode::Encode,
+ bincode::Decode,
+)]
+pub struct CleanBodyCoverage;
+impl Signal for CleanBodyCoverage {
+ fn default_coefficient(&self) -> f64 {
+ 0.01
+ }
+
+ fn as_field(&self) -> Option<Field> {
+ Some(Field::Text(schema::text_field::CleanBody.into()))
+ }
+
+ fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {
+ let mut seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();
+
+ seg_reader
+ .text_fields_mut()
+ .get_mut(self.as_textfield().unwrap())
+ .map(|field| field.coverage(doc))
+ }
+}
+
#[derive(
Debug,
Clone,
@@ -201,6 +281,10 @@ impl Signal for Bm25CleanBodyBigrams {
0.005
}
+ fn has_sibling_ngrams(&self) -> bool {
+ true
+ }
+
fn as_field(&self) -> Option<Field> {
Some(Field::Text(schema::text_field::CleanBodyBigrams.into()))
}
@@ -233,6 +317,10 @@ impl Signal for Bm25CleanBodyTrigrams {
0.005
}
+ fn has_sibling_ngrams(&self) -> bool {
+ true
+ }
+
fn as_field(&self) -> Option<Field> {
Some(Field::Text(schema::text_field::CleanBodyTrigrams.into()))
}
diff --git a/crates/core/src/ranking/mod.rs b/crates/core/src/ranking/mod.rs
index 3bf70d88..1d126e37 100644
--- a/crates/core/src/ranking/mod.rs
+++ b/crates/core/src/ranking/mod.rs
@@ -940,4 +940,156 @@ mod tests {
assert_eq!(result.webpages.len(), 2);
assert_eq!(result.webpages[0].url, "https://www.a.com/");
}
+
+ #[test]
+ fn title_coverage() {
+ let mut index = Index::temporary().expect("Unable to open index");
+
+ index
+ .insert(&Webpage {
+ html: Html::parse(
+ &format!(
+ r#"
+ <html>
+ <head>
+ <title>Test website</title>
+ </head>
+ <body>
+ {CONTENT} {}
+ </body>
+ </html>
+ "#,
+ crate::rand_words(100)
+ ),
+ "https://www.first.com",
+ )
+ .unwrap(),
+ host_centrality: 1.0,
+ fetch_time_ms: 2,
+ ..Default::default()
+ })
+ .expect("failed to insert webpage");
+ index.commit().expect("failed to commit index");
+ let searcher = LocalSearcher::new(index);
+
+ let result = searcher
+ .search(&SearchQuery {
+ query: "test website".to_string(),
+ return_ranking_signals: true,
+ ..Default::default()
+ })
+ .expect("Search failed");
+
+ assert_eq!(result.webpages.len(), 1);
+ assert_eq!(
+ result.webpages[0]
+ .ranking_signals
+ .as_ref()
+ .unwrap()
+ .get(&crate::ranking::SignalEnum::from(crate::ranking::core::TitleCoverage).into())
+ .unwrap()
+ .value,
+ 1.0,
+ );
+
+ let result = searcher
+ .search(&SearchQuery {
+ query: "test example".to_string(),
+ return_ranking_signals: true,
+ ..Default::default()
+ })
+ .expect("Search failed");
+
+ assert_eq!(result.webpages.len(), 1);
+ assert_eq!(
+ result.webpages[0]
+ .ranking_signals
+ .as_ref()
+ .unwrap()
+ .get(&crate::ranking::SignalEnum::from(crate::ranking::core::TitleCoverage).into())
+ .unwrap()
+ .value,
+ 0.5,
+ );
+ }
+
+ #[test]
+ fn clean_body_coverage() {
+ let mut index = Index::temporary().expect("Unable to open index");
+
+ let mut page = Webpage {
+ html: Html::parse(
+ &format!(
+ r#"
+ <html>
+ <head>
+ <title>a b c</title>
+ </head>
+ <body>
+ {CONTENT} {}
+ </body>
+ </html>
+ "#,
+ crate::rand_words(100)
+ ),
+ "https://www.first.com",
+ )
+ .unwrap(),
+ host_centrality: 1.0,
+ fetch_time_ms: 2,
+ ..Default::default()
+ };
+
+ page.html.set_clean_text("test website".to_string());
+
+ index.insert(&page).expect("failed to insert webpage");
+ index.commit().expect("failed to commit index");
+ let searcher = LocalSearcher::new(index);
+
+ let result = searcher
+ .search(&SearchQuery {
+ query: "test website".to_string(),
+ return_ranking_signals: true,
+ ..Default::default()
+ })
+ .expect("Search failed");
+
+ assert_eq!(result.webpages.len(), 1);
+ assert_eq!(
+ result.webpages[0]
+ .ranking_signals
+ .as_ref()
+ .unwrap()
+ .get(
+ &crate::ranking::SignalEnum::from(crate::ranking::core::CleanBodyCoverage)
+ .into()
+ )
+ .unwrap()
+ .value,
+ 1.0,
+ );
+
+ let result = searcher
+ .search(&SearchQuery {
+ query: "test b".to_string(),
+ return_ranking_signals: true,
+ ..Default::default()
+ })
+ .expect("Search failed");
+
+ assert_eq!(result.webpages.len(), 1);
+ assert_eq!(
+ result.webpages[0]
+ .ranking_signals
+ .as_ref()
+ .unwrap()
+ .get(
+ &crate::ranking::SignalEnum::from(crate::ranking::core::CleanBodyCoverage)
+ .into()
+ )
+ .unwrap()
+ .value,
+ 0.5,
+ );
+ }
}