Skip to content

web search done right

derank similar in ranking pipeline

Mikkel Denker9d3c2836afef
diff --git a/core/src/ranking/mod.rs b/core/src/ranking/mod.rs
index 0f221c76..fd0519a7 100644
--- a/core/src/ranking/mod.rs
+++ b/core/src/ranking/mod.rs
@@ -805,11 +805,10 @@ mod tests {
#[test]
fn term_proximity_ranking() {
let mut index = Index::temporary().expect("Unable to open index");
- index
- .insert(Webpage {
- html: Html::parse(
- &format!(
- r#"
+ let mut w = Webpage {
+ html: Html::parse(
+ &format!(
+ r#"
<html>
<head>
<title>Test website</title>
@@ -819,27 +818,31 @@ mod tests {
</body>
</html>
"#,
- crate::rand_words(100)
- ),
- "https://www.first.com",
+ crate::rand_words(100)
),
- backlinks: vec![],
- host_centrality: 1.0,
- pre_computed_score: 0.0,
- crawl_stability: 0.0,
- fetch_time_ms: 500,
- page_centrality: 0.0,
- primary_image: None,
- host_topic: None,
- node_id: None,
- dmoz_description: None,
- })
- .expect("failed to insert webpage");
- index
- .insert(Webpage {
- html: Html::parse(
- &format!(
- r#"
+ "https://www.first.com",
+ ),
+ backlinks: vec![],
+ host_centrality: 1.0,
+ pre_computed_score: 0.0,
+ crawl_stability: 0.0,
+ fetch_time_ms: 500,
+ page_centrality: 0.0,
+ primary_image: None,
+ host_topic: None,
+ node_id: None,
+ dmoz_description: None,
+ };
+ w.html.set_clean_text(format!(
+ "{CONTENT} termA termB d d d d d d d d d {}",
+ crate::rand_words(100)
+ ));
+
+ index.insert(w).expect("failed to insert webpage");
+ let mut w = Webpage {
+ html: Html::parse(
+ &format!(
+ r#"
<html>
<head>
<title>Test website</title>
@@ -849,27 +852,31 @@ mod tests {
</body>
</html>
"#,
- crate::rand_words(100)
- ),
- "https://www.third.com",
+ crate::rand_words(100)
),
- backlinks: vec![],
- host_centrality: 1.0,
- fetch_time_ms: 500,
- pre_computed_score: 0.0,
- crawl_stability: 0.0,
- page_centrality: 0.0,
- primary_image: None,
- host_topic: None,
- node_id: None,
- dmoz_description: None,
- })
- .expect("failed to insert webpage");
- index
- .insert(Webpage {
- html: Html::parse(
- &format!(
- r#"
+ "https://www.third.com",
+ ),
+ backlinks: vec![],
+ host_centrality: 1.0,
+ fetch_time_ms: 500,
+ pre_computed_score: 0.0,
+ crawl_stability: 0.0,
+ page_centrality: 0.0,
+ primary_image: None,
+ host_topic: None,
+ node_id: None,
+ dmoz_description: None,
+ };
+ w.html.set_clean_text(format!(
+ "{CONTENT} termA d d d d d d d d d termB {}",
+ crate::rand_words(100)
+ ));
+ index.insert(w).expect("failed to insert webpage");
+
+ let mut w = Webpage {
+ html: Html::parse(
+ &format!(
+ r#"
<html>
<head>
<title>Test website</title>
@@ -879,22 +886,27 @@ mod tests {
</body>
</html>
"#,
- crate::rand_words(100)
- ),
- "https://www.second.com",
+ crate::rand_words(100)
),
- backlinks: vec![],
- host_centrality: 1.0,
- fetch_time_ms: 500,
- pre_computed_score: 0.0,
- crawl_stability: 0.0,
- page_centrality: 0.0,
- primary_image: None,
- host_topic: None,
- node_id: None,
- dmoz_description: None,
- })
- .expect("failed to insert webpage");
+ "https://www.second.com",
+ ),
+ backlinks: vec![],
+ host_centrality: 1.0,
+ fetch_time_ms: 500,
+ pre_computed_score: 0.0,
+ crawl_stability: 0.0,
+ page_centrality: 0.0,
+ primary_image: None,
+ host_topic: None,
+ node_id: None,
+ dmoz_description: None,
+ };
+ w.html.set_clean_text(format!(
+ "{CONTENT} termA d d d d termB d d d d d {}",
+ crate::rand_words(100)
+ ));
+
+ index.insert(w).expect("failed to insert webpage");
index.commit().expect("failed to commit index");
let searcher = LocalSearcher::new(index);
diff --git a/core/src/ranking/pipeline.rs b/core/src/ranking/pipeline.rs
index 83917a9e..c1d474d8 100644
--- a/core/src/ranking/pipeline.rs
+++ b/core/src/ranking/pipeline.rs
@@ -14,11 +14,17 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
-use std::{cmp::Ordering, sync::Arc};
+use std::sync::Arc;
use serde::{Deserialize, Serialize};
-use crate::{enum_map::EnumMap, inverted_index::WebsitePointer, searcher::SearchQuery, Result};
+use crate::{
+ collector::{self, BucketCollector},
+ enum_map::EnumMap,
+ inverted_index::WebsitePointer,
+ searcher::SearchQuery,
+ Result,
+};
use super::{
models::{cross_encoder::CrossEncoder, lambdamart::LambdaMART},
@@ -30,6 +36,23 @@ pub trait AsRankingWebsite: Clone {
fn as_mut_ranking(&mut self) -> &mut RankingWebsite;
}
+impl<T> collector::Doc for T
+where
+ T: AsRankingWebsite,
+{
+ fn score(&self) -> &f64 {
+ &self.as_ranking().score
+ }
+
+ fn id(&self) -> &tantivy::DocId {
+ &self.as_ranking().pointer.address.doc_id
+ }
+
+ fn hashes(&self) -> collector::Hashes {
+ self.as_ranking().pointer.hashes
+ }
+}
+
impl AsRankingWebsite for RankingWebsite {
fn as_ranking(&self) -> &RankingWebsite {
self
@@ -247,6 +270,7 @@ struct RankingStage<T: AsRankingWebsite> {
prev: Prev<T>,
top_n: usize,
memory: Option<Vec<T>>,
+ derank_similar: bool,
}
impl<T: AsRankingWebsite> RankingStage<T> {
@@ -279,18 +303,20 @@ impl<T: AsRankingWebsite> RankingStage<T> {
for website in websites.iter_mut() {
let boost = website.as_ranking().optic_boost;
if let Some(boost) = boost {
- website.as_mut_ranking().score *= boost;
+ if boost != 0.0 {
+ website.as_mut_ranking().score *= boost;
+ }
}
}
- websites.sort_by(|a, b| {
- b.as_ranking()
- .score
- .partial_cmp(&a.as_ranking().score)
- .unwrap_or(Ordering::Equal)
- });
+ let mut collector = BucketCollector::new((page + 1) * top_n);
+
+ for website in websites {
+ collector.insert(website);
+ }
- websites
+ collector
+ .into_sorted_vec(self.derank_similar)
.into_iter()
.skip(page * top_n)
.take(top_n)
@@ -322,6 +348,7 @@ impl<T: AsRankingWebsite> RankingPipeline<T> {
prev: Prev::Initial,
memory: None,
top_n: 20,
+ derank_similar: false,
};
Ok(Self {
@@ -351,6 +378,7 @@ impl<T: AsRankingWebsite> RankingPipeline<T> {
prev: Prev::Initial,
memory: None,
top_n: 10_000,
+ derank_similar: true,
};
Self {
@@ -414,6 +442,8 @@ mod tests {
fn sample_websites(n: usize) -> Vec<RankingWebsite> {
(0..n)
.map(|i| -> RankingWebsite {
+ let mut signals = EnumMap::new();
+ signals.insert(Signal::LinearRegression, 1.0 / i as f64);
RankingWebsite {
pointer: WebsitePointer {
score: Score { total: 0.0 },
@@ -428,7 +458,7 @@ mod tests {
doc_id: i as u32,
},
},
- signals: EnumMap::new(),
+ signals,
optic_boost: None,
title: None,
clean_body: None,
diff --git a/core/src/searcher/distributed.rs b/core/src/searcher/distributed.rs
index 6cfd82e3..56cbd580 100644
--- a/core/src/searcher/distributed.rs
+++ b/core/src/searcher/distributed.rs
@@ -18,7 +18,7 @@ use crate::{
bangs::{Bang, BangHit, Bangs},
ceil_char_boundary,
cluster::{member::Service, Cluster},
- collector::{self, BucketCollector},
+ collector::BucketCollector,
exponential_backoff::ExponentialBackoff,
floor_char_boundary,
inverted_index::{self, RetrievedWebpage},
@@ -259,20 +259,6 @@ impl AsRankingWebsite for ScoredWebsitePointer {
}
}
-impl collector::Doc for ScoredWebsitePointer {
- fn score(&self) -> &f64 {
- &self.website.pointer.score.total
- }
-
- fn id(&self) -> &tantivy::DocId {
- &self.website.pointer.address.doc_id
- }
-
- fn hashes(&self) -> collector::Hashes {
- self.website.pointer.hashes
- }
-}
-
impl DistributedSearcher {
pub fn new(
cluster: Cluster,