web search done right
derank similar in ranking pipeline
Mikkel Denker9d3c2836afef
diff --git a/core/src/ranking/mod.rs b/core/src/ranking/mod.rsindex 0f221c76..fd0519a7 100644--- a/core/src/ranking/mod.rs+++ b/core/src/ranking/mod.rs@@ -805,11 +805,10 @@ mod tests {#[test]fn term_proximity_ranking() {let mut index = Index::temporary().expect("Unable to open index");- index- .insert(Webpage {- html: Html::parse(- &format!(- r#"+ let mut w = Webpage {+ html: Html::parse(+ &format!(+ r#"<html><head><title>Test website</title>@@ -819,27 +818,31 @@ mod tests {</body></html>"#,- crate::rand_words(100)- ),- "https://www.first.com",+ crate::rand_words(100)),- backlinks: vec![],- host_centrality: 1.0,- pre_computed_score: 0.0,- crawl_stability: 0.0,- fetch_time_ms: 500,- page_centrality: 0.0,- primary_image: None,- host_topic: None,- node_id: None,- dmoz_description: None,- })- .expect("failed to insert webpage");- index- .insert(Webpage {- html: Html::parse(- &format!(- r#"+ "https://www.first.com",+ ),+ backlinks: vec![],+ host_centrality: 1.0,+ pre_computed_score: 0.0,+ crawl_stability: 0.0,+ fetch_time_ms: 500,+ page_centrality: 0.0,+ primary_image: None,+ host_topic: None,+ node_id: None,+ dmoz_description: None,+ };+ w.html.set_clean_text(format!(+ "{CONTENT} termA termB d d d d d d d d d {}",+ crate::rand_words(100)+ ));++ index.insert(w).expect("failed to insert webpage");+ let mut w = Webpage {+ html: Html::parse(+ &format!(+ r#"<html><head><title>Test website</title>@@ -849,27 +852,31 @@ mod tests {</body></html>"#,- crate::rand_words(100)- ),- "https://www.third.com",+ crate::rand_words(100)),- backlinks: vec![],- host_centrality: 1.0,- fetch_time_ms: 500,- pre_computed_score: 0.0,- crawl_stability: 0.0,- page_centrality: 0.0,- primary_image: None,- host_topic: None,- node_id: None,- dmoz_description: None,- })- .expect("failed to insert webpage");- index- .insert(Webpage {- html: Html::parse(- &format!(- r#"+ "https://www.third.com",+ ),+ backlinks: vec![],+ host_centrality: 1.0,+ fetch_time_ms: 500,+ pre_computed_score: 0.0,+ crawl_stability: 0.0,+ page_centrality: 0.0,+ primary_image: None,+ host_topic: None,+ node_id: None,+ dmoz_description: None,+ };+ w.html.set_clean_text(format!(+ "{CONTENT} termA d d d d d d d d d termB {}",+ crate::rand_words(100)+ ));+ index.insert(w).expect("failed to insert webpage");++ let mut w = Webpage {+ html: Html::parse(+ &format!(+ r#"<html><head><title>Test website</title>@@ -879,22 +886,27 @@ mod tests {</body></html>"#,- crate::rand_words(100)- ),- "https://www.second.com",+ crate::rand_words(100)),- backlinks: vec![],- host_centrality: 1.0,- fetch_time_ms: 500,- pre_computed_score: 0.0,- crawl_stability: 0.0,- page_centrality: 0.0,- primary_image: None,- host_topic: None,- node_id: None,- dmoz_description: None,- })- .expect("failed to insert webpage");+ "https://www.second.com",+ ),+ backlinks: vec![],+ host_centrality: 1.0,+ fetch_time_ms: 500,+ pre_computed_score: 0.0,+ crawl_stability: 0.0,+ page_centrality: 0.0,+ primary_image: None,+ host_topic: None,+ node_id: None,+ dmoz_description: None,+ };+ w.html.set_clean_text(format!(+ "{CONTENT} termA d d d d termB d d d d d {}",+ crate::rand_words(100)+ ));++ index.insert(w).expect("failed to insert webpage");index.commit().expect("failed to commit index");let searcher = LocalSearcher::new(index);diff --git a/core/src/ranking/pipeline.rs b/core/src/ranking/pipeline.rsindex 83917a9e..c1d474d8 100644--- a/core/src/ranking/pipeline.rs+++ b/core/src/ranking/pipeline.rs@@ -14,11 +14,17 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.-use std::{cmp::Ordering, sync::Arc};+use std::sync::Arc;use serde::{Deserialize, Serialize};-use crate::{enum_map::EnumMap, inverted_index::WebsitePointer, searcher::SearchQuery, Result};+use crate::{+ collector::{self, BucketCollector},+ enum_map::EnumMap,+ inverted_index::WebsitePointer,+ searcher::SearchQuery,+ Result,+};use super::{models::{cross_encoder::CrossEncoder, lambdamart::LambdaMART},@@ -30,6 +36,23 @@ pub trait AsRankingWebsite: Clone {fn as_mut_ranking(&mut self) -> &mut RankingWebsite;}+impl<T> collector::Doc for T+where+ T: AsRankingWebsite,+{+ fn score(&self) -> &f64 {+ &self.as_ranking().score+ }++ fn id(&self) -> &tantivy::DocId {+ &self.as_ranking().pointer.address.doc_id+ }++ fn hashes(&self) -> collector::Hashes {+ self.as_ranking().pointer.hashes+ }+}+impl AsRankingWebsite for RankingWebsite {fn as_ranking(&self) -> &RankingWebsite {self@@ -247,6 +270,7 @@ struct RankingStage<T: AsRankingWebsite> {prev: Prev<T>,top_n: usize,memory: Option<Vec<T>>,+ derank_similar: bool,}impl<T: AsRankingWebsite> RankingStage<T> {@@ -279,18 +303,20 @@ impl<T: AsRankingWebsite> RankingStage<T> {for website in websites.iter_mut() {let boost = website.as_ranking().optic_boost;if let Some(boost) = boost {- website.as_mut_ranking().score *= boost;+ if boost != 0.0 {+ website.as_mut_ranking().score *= boost;+ }}}- websites.sort_by(|a, b| {- b.as_ranking()- .score- .partial_cmp(&a.as_ranking().score)- .unwrap_or(Ordering::Equal)- });+ let mut collector = BucketCollector::new((page + 1) * top_n);++ for website in websites {+ collector.insert(website);+ }- websites+ collector+ .into_sorted_vec(self.derank_similar).into_iter().skip(page * top_n).take(top_n)@@ -322,6 +348,7 @@ impl<T: AsRankingWebsite> RankingPipeline<T> {prev: Prev::Initial,memory: None,top_n: 20,+ derank_similar: false,};Ok(Self {@@ -351,6 +378,7 @@ impl<T: AsRankingWebsite> RankingPipeline<T> {prev: Prev::Initial,memory: None,top_n: 10_000,+ derank_similar: true,};Self {@@ -414,6 +442,8 @@ mod tests {fn sample_websites(n: usize) -> Vec<RankingWebsite> {(0..n).map(|i| -> RankingWebsite {+ let mut signals = EnumMap::new();+ signals.insert(Signal::LinearRegression, 1.0 / i as f64);RankingWebsite {pointer: WebsitePointer {score: Score { total: 0.0 },@@ -428,7 +458,7 @@ mod tests {doc_id: i as u32,},},- signals: EnumMap::new(),+ signals,optic_boost: None,title: None,clean_body: None,diff --git a/core/src/searcher/distributed.rs b/core/src/searcher/distributed.rsindex 6cfd82e3..56cbd580 100644--- a/core/src/searcher/distributed.rs+++ b/core/src/searcher/distributed.rs@@ -18,7 +18,7 @@ use crate::{bangs::{Bang, BangHit, Bangs},ceil_char_boundary,cluster::{member::Service, Cluster},- collector::{self, BucketCollector},+ collector::BucketCollector,exponential_backoff::ExponentialBackoff,floor_char_boundary,inverted_index::{self, RetrievedWebpage},@@ -259,20 +259,6 @@ impl AsRankingWebsite for ScoredWebsitePointer {}}-impl collector::Doc for ScoredWebsitePointer {- fn score(&self) -> &f64 {- &self.website.pointer.score.total- }-- fn id(&self) -> &tantivy::DocId {- &self.website.pointer.address.doc_id- }-- fn hashes(&self) -> collector::Hashes {- self.website.pointer.hashes- }-}-impl DistributedSearcher {pub fn new(cluster: Cluster,