web search done right
Use inbound edges similarity between liked sites and search results as a ranking metric.
Mikkel Denkerf0274317b48f
diff --git a/core/src/ranking/bitvec_similarity.rs b/core/src/ranking/bitvec_similarity.rsindex bf7467f7..ff3ab1d9 100644--- a/core/src/ranking/bitvec_similarity.rs+++ b/core/src/ranking/bitvec_similarity.rs@@ -17,13 +17,13 @@use itertools::Itertools;use serde::{Deserialize, Serialize};-#[derive(Serialize, Deserialize)]+#[derive(Serialize, Deserialize, Debug, Clone)]struct Block {data: u64,offset: usize,}-#[derive(Serialize, Deserialize)]+#[derive(Serialize, Deserialize, Debug, Clone)]pub struct BitVec {blocks: Vec<Block>,sqrt_len: f64,@@ -56,7 +56,7 @@ impl BitVec {}}- fn sim(&self, other: &Self) -> f64 {+ pub fn sim(&self, other: &Self) -> f64 {let mut i = 0;let mut j = 0;@@ -161,7 +161,6 @@ mod tests {let expected = naive_sim(&a, &b);- dbg!(expected);assert!(expected < 0.01);let a = BitVec::new(a);diff --git a/core/src/ranking/inbound_similarity.rs b/core/src/ranking/inbound_similarity.rsindex 8bd65a52..330a0769 100644--- a/core/src/ranking/inbound_similarity.rs+++ b/core/src/ranking/inbound_similarity.rs@@ -18,9 +18,9 @@ use std::{fs::File,io::{BufReader, BufWriter, Read},path::Path,+ sync::Arc,};-use optics::SiteRankings;use serde::{Deserialize, Serialize};use crate::{@@ -30,20 +30,45 @@ use crate::{};use super::bitvec_similarity;-const HARMONIC_CENTRALITY_THRESHOLD: f64 = 0.038;+const DEFAULT_HARMONIC_CENTRALITY_THRESHOLD: f64 = 0.038;+const SCORE_SCALE: f64 = 5.0;-pub struct Scorer<'a> {- liked: Vec<&'a bitvec_similarity::BitVec>,- disliked: Vec<&'a bitvec_similarity::BitVec>,+pub struct Scorer {+ liked: Vec<bitvec_similarity::BitVec>,+ disliked: Vec<bitvec_similarity::BitVec>,+ vectors: Arc<IntMap<bitvec_similarity::BitVec>>,+}++impl Scorer {+ pub fn score(&self, node: &NodeID) -> f64 {+ match self.vectors.get(&node.0) {+ Some(vec) => (SCORE_SCALE+ + (self.liked.iter().map(|liked| liked.sim(vec)).sum::<f64>()+ - self+ .disliked+ .iter()+ .map(|disliked| disliked.sim(vec))+ .sum::<f64>()))+ .max(0.0),+ None => 0.0,+ }+ }}#[derive(Serialize, Deserialize, Default)]pub struct InboundSimilarity {- vectors: IntMap<bitvec_similarity::BitVec>,+ vectors: Arc<IntMap<bitvec_similarity::BitVec>>,}impl InboundSimilarity {pub fn build(graph: &Webgraph, harmonic: &HarmonicCentrality) -> Self {+ Self::build_with_threshold(graph, harmonic, DEFAULT_HARMONIC_CENTRALITY_THRESHOLD)+ }+ fn build_with_threshold(+ graph: &Webgraph,+ harmonic: &HarmonicCentrality,+ harmonic_centrality_threshold: f64,+ ) -> Self {let mut vectors = IntMap::new();let nodes: Vec<_> = graph.nodes().collect();@@ -57,7 +82,7 @@ impl InboundSimilarity {.filter(|edge| match graph.id2node(&edge.from) {Some(node) => {let score = *harmonic.host.get(&node.into_host()).unwrap_or(&0.0);- score >= HARMONIC_CENTRALITY_THRESHOLD+ score >= harmonic_centrality_threshold}None => false,})@@ -69,21 +94,27 @@ impl InboundSimilarity {}}- Self { vectors }+ Self {+ vectors: Arc::new(vectors),+ }}- pub fn scorer<'a>(&'a self, liked_sites: &[NodeID], disliked_sites: &[NodeID]) -> Scorer<'a> {+ pub fn scorer(&self, liked_sites: &[NodeID], disliked_sites: &[NodeID]) -> Scorer {let liked: Vec<_> = liked_sites.iter()- .filter_map(|id| self.vectors.get(&id.0))+ .filter_map(|id| self.vectors.get(&id.0).cloned()).collect();let disliked: Vec<_> = disliked_sites.iter()- .filter_map(|id| self.vectors.get(&id.0))+ .filter_map(|id| self.vectors.get(&id.0).cloned()).collect();- Scorer { liked, disliked }+ Scorer {+ liked,+ disliked,+ vectors: self.vectors.clone(),+ }}pub fn save<P: AsRef<Path>>(&self, path: P) -> Result<()> {@@ -110,3 +141,142 @@ impl InboundSimilarity {Ok(bincode::deserialize(&buf)?)}}++#[cfg(test)]+mod tests {+ use optics::SiteRankings;++ use crate::{+ gen_temp_path,+ index::Index,+ rand_words,+ ranking::centrality_store::CentralityStore,+ searcher::{LocalSearcher, SearchQuery},+ webgraph::{Node, WebgraphBuilder},+ webpage::{Html, Webpage},+ };++ use super::*;++ #[test]+ fn it_favors_liked_sites() {+ let mut graph = WebgraphBuilder::new_memory().open();++ graph.insert(Node::from("a.com"), Node::from("b.com"), String::new());+ graph.insert(Node::from("c.com"), Node::from("d.com"), String::new());+ graph.insert(Node::from("a.com"), Node::from("e.com"), String::new());++ graph.commit();++ let harmonic = HarmonicCentrality::calculate(&graph);+ let inbound = InboundSimilarity::build_with_threshold(&graph, &harmonic, -1.0);++ let scorer = inbound.scorer(&[graph.node2id(&Node::from("b.com")).unwrap()], &[]);+ let e = graph.node2id(&Node::from("e.com")).unwrap();+ let d = graph.node2id(&Node::from("d.com")).unwrap();++ assert!(scorer.score(&e) > scorer.score(&d));+ }++ #[test]+ fn it_ranks_search_results() {+ let mut graph = WebgraphBuilder::new_memory().open();++ graph.insert(Node::from("a.com"), Node::from("b.com"), String::new());+ graph.insert(Node::from("c.com"), Node::from("d.com"), String::new());+ graph.insert(Node::from("a.com"), Node::from("e.com"), String::new());++ graph.commit();++ let harmonic = HarmonicCentrality::calculate(&graph);+ let inbound = InboundSimilarity::build_with_threshold(&graph, &harmonic, -1.0);++ let mut centrality_store = CentralityStore::build(&graph, gen_temp_path());+ centrality_store.inbound_similarity = inbound;++ let mut index = Index::temporary().expect("Unable to open index");++ index+ .insert(Webpage {+ html: Html::parse(+ &format!(+ r#"+ <html>+ <head>+ <title>Title</title>+ </head>+ <body>+ example {}+ </body>+ </html>+ "#,+ rand_words(1000)+ ),+ "e.com",+ ),+ backlinks: vec![],+ host_centrality: 0.0,+ fetch_time_ms: 500,+ page_centrality: 0.0,+ pre_computed_score: 0.0,+ crawl_stability: 0.0,+ primary_image: None,+ host_topic: None,+ node_id: Some(graph.node2id(&Node::from("e.com")).unwrap()),+ dmoz_description: None,+ })+ .expect("failed to insert webpage");+ index+ .insert(Webpage {+ html: Html::parse(+ &format!(+ r#"+ <html>+ <head>+ <title>Title</title>+ </head>+ <body>+ example {}+ </body>+ </html>+ "#,+ rand_words(1000)+ ),+ "d.com",+ ),+ backlinks: vec![],+ host_centrality: 0.01,+ fetch_time_ms: 500,+ page_centrality: 0.0,+ pre_computed_score: 0.0,+ crawl_stability: 0.0,+ primary_image: None,+ host_topic: None,+ node_id: Some(graph.node2id(&Node::from("d.com")).unwrap()),+ dmoz_description: None,+ })+ .expect("failed to insert webpage");++ index.commit().unwrap();++ let mut searcher = LocalSearcher::new(index);+ searcher.set_centrality_store(centrality_store);++ let res = searcher+ .search(&SearchQuery {+ query: "example".to_string(),+ site_rankings: Some(SiteRankings {+ liked: vec!["a.com".to_string()],+ disliked: vec![],+ blocked: vec![],+ }),+ ..Default::default()+ })+ .unwrap()+ .webpages;++ assert_eq!(res.len(), 2);+ assert_eq!(&res[0].url, "https://e.com");+ assert_eq!(&res[1].url, "https://d.com");+ }+}diff --git a/core/src/ranking/pipeline.rs b/core/src/ranking/pipeline.rsindex eed33d83..0afd24ef 100644--- a/core/src/ranking/pipeline.rs+++ b/core/src/ranking/pipeline.rs@@ -51,6 +51,7 @@ pub struct RankingWebsite {pub topic_centrality: f64,pub personal_centrality: f64,pub query_centrality: f64,+ pub inbound_similarity: f64,pub title: String,pub clean_body: String,pub score: f64,@@ -68,6 +69,7 @@ impl RankingWebsite {topic_centrality: 0.0,personal_centrality: 0.0,query_centrality: 0.0,+ inbound_similarity: 0.0,title: String::new(),clean_body: String::new(),score: pointer.score.total,@@ -88,6 +90,7 @@ impl RankingWebsite {let node = value.value.as_u64().unwrap().into();res.personal_centrality = aggregator.personal_centrality(node);+ res.inbound_similarity = aggregator.inbound_similarity(node);res.topic_centrality = aggregator.topic_centrality(node).unwrap_or_default();res.query_centrality = aggregator.query_centrality(node).unwrap_or_default();}@@ -399,6 +402,7 @@ mod tests {topic_centrality: 0.0,personal_centrality: 0.0,query_centrality: 0.0,+ inbound_similarity: 0.0,title: String::new(),clean_body: String::new(),score: 1.0 / i as f64,@@ -571,6 +575,7 @@ mod tests {topic_centrality: 0.0,personal_centrality: 0.0,query_centrality: 0.0,+ inbound_similarity: 0.0,title: String::new(),clean_body: String::new(),score: 1.0,diff --git a/core/src/ranking/signal.rs b/core/src/ranking/signal.rsindex 3ef45e9c..721a4dc2 100644--- a/core/src/ranking/signal.rs+++ b/core/src/ranking/signal.rs@@ -36,7 +36,7 @@ use crate::{webpage::region::{Region, RegionCount},};-use super::initial::Score;+use super::{inbound_similarity, initial::Score};#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]pub enum Signal {@@ -52,6 +52,7 @@ pub enum Signal {CrawlStability,TopicCentrality,QueryCentrality,+ InboundSimilarity,}pub const ALL_SIGNALS: [Signal; 12] = [@@ -77,6 +78,7 @@ struct SignalValue {personal_centrality: Option<f64>,topic_score: Option<f64>,query_centrality: Option<f64>,+ inbound_similarity: Option<f64>,}impl Signal {@@ -142,6 +144,7 @@ impl Signal {Signal::CrawlStability => value.fastfield_value.unwrap() as f64 / FLOAT_SCALING as f64,Signal::TopicCentrality => value.topic_score.unwrap_or_default(),Signal::QueryCentrality => value.query_centrality.unwrap_or_default(),+ Signal::InboundSimilarity => value.inbound_similarity.unwrap_or_default(),}}@@ -159,6 +162,7 @@ impl Signal {Signal::TrackerScore => 20.0,Signal::Region => 60.0,Signal::CrawlStability => 20.0,+ Signal::InboundSimilarity => 1000.0,}}@@ -175,6 +179,8 @@ impl Signal {"personal_centrality" => Some(Signal::PersonalCentrality),"topic_centrality" => Some(Signal::TopicCentrality),"query_centrality" => Some(Signal::QueryCentrality),+ "inbound_similarity" => Some(Signal::InboundSimilarity),+ "crawl_stability" => Some(Signal::CrawlStability),_ => None,}}@@ -193,6 +199,7 @@ impl Signal {Signal::PersonalCentrality => None,Signal::TopicCentrality => None,Signal::QueryCentrality => None,+ Signal::InboundSimilarity => None,}}}@@ -260,7 +267,8 @@ impl FieldBoost {pub struct SignalAggregator {fastfield_cache: Option<Arc<fastfield_cache::SegmentCache>>,signal_coefficients: SignalCoefficient,- personal_centrality: Vec<Arc<online_harmonic::Scorer>>,+ personal_centrality: Option<Arc<online_harmonic::Scorer>>,+ inbound_similariy: Option<Arc<inbound_similarity::Scorer>>,field_boost: FieldBoost,fetch_time_ms_cache: [f64; 1000],update_time_cache: Vec<f64>,@@ -299,7 +307,8 @@ impl SignalAggregator {Self {fastfield_cache: None,- personal_centrality: Vec::new(),+ personal_centrality: None,+ inbound_similariy: None,signal_coefficients,field_boost,fetch_time_ms_cache,@@ -321,8 +330,12 @@ impl SignalAggregator {self.query_centrality = Some(Arc::new(query_centrality));}- pub fn add_personal_harmonic(&mut self, personal_centrality: online_harmonic::Scorer) {- self.personal_centrality.push(Arc::new(personal_centrality))+ pub fn set_personal_harmonic(&mut self, personal_centrality: online_harmonic::Scorer) {+ self.personal_centrality = Some(Arc::new(personal_centrality));+ }++ pub fn set_inbound_similarity(&mut self, scorer: inbound_similarity::Scorer) {+ self.inbound_similariy = Some(Arc::new(scorer));}pub fn topic_centrality(&self, host_id: NodeID) -> Option<f64> {@@ -339,9 +352,16 @@ impl SignalAggregator {pub fn personal_centrality(&self, host_id: NodeID) -> f64 {self.personal_centrality- .iter()+ .as_ref().map(|scorer| scorer.score(host_id))- .sum()+ .unwrap_or_default()+ }++ pub fn inbound_similarity(&self, host_id: NodeID) -> f64 {+ self.inbound_similariy+ .as_ref()+ .map(|scorer| scorer.score(&host_id))+ .unwrap_or_default()}pub fn score(@@ -353,10 +373,16 @@ impl SignalAggregator {selected_region: Option<Region>,) -> Score {let host_id = self.fastfield_cache.as_ref().and_then(|cache| {- cache+ let node_id = cache.get_doc_cache(&FastField::HostNodeID).get_u64(&doc)- .map(NodeID::from)+ .unwrap();++ if node_id == u64::MAX {+ None+ } else {+ Some(NodeID::from(node_id))+ }});let topic_score = host_id.and_then(|host_id| self.topic_centrality(host_id));@@ -364,6 +390,7 @@ impl SignalAggregator {let query_centrality = host_id.and_then(|host_id| self.query_centrality(host_id));let personal_centrality = host_id.map(|host_id| self.personal_centrality(host_id));+ let inbound_similarity = host_id.map(|host_id| self.inbound_similarity(host_id));let score = ALL_SIGNALS.into_iter()@@ -384,6 +411,7 @@ impl SignalAggregator {current_timestamp,selected_region,personal_centrality,+ inbound_similarity,topic_score,query_centrality,},@@ -421,6 +449,7 @@ impl SignalAggregator {Signal::Bm25| Signal::PersonalCentrality| Signal::TopicCentrality+ | Signal::InboundSimilarity| Signal::QueryCentrality => {panic!("signal cannot be determined from webpage")}@@ -440,6 +469,7 @@ impl SignalAggregator {personal_centrality: None,topic_score: None,query_centrality: None,+ inbound_similarity: None,},)})diff --git a/core/src/searcher/local.rs b/core/src/searcher/local.rsindex 46973b68..338fa72b 100644--- a/core/src/searcher/local.rs+++ b/core/src/searcher/local.rs@@ -31,6 +31,7 @@ use crate::ranking::{online_centrality_scorer, Ranker, SignalAggregator};use crate::search_prettifier::{DisplayedEntity, DisplayedWebpage, HighlightedSpellCorrection};use crate::spell::Correction;use crate::webgraph::centrality::topic::TopicCentrality;+use crate::webgraph::Node;use crate::webpage::region::Region;use crate::webpage::Url;use crate::{inverted_index, Error, Result};@@ -79,7 +80,7 @@ impl LocalSearcher {None => None,};- let mut query_aggregator = optic+ let query_aggregator = optic.as_ref().and_then(|optic| {optic@@ -89,16 +90,6 @@ impl LocalSearcher {}).unwrap_or_default();- if let (Some(optic), Some(harmonic)) = (- optic.as_ref(),- self.centrality_store- .as_ref()- .map(|store| &store.online_harmonic),- ) {- query_aggregator- .add_personal_harmonic(online_centrality_scorer(&optic.site_rankings, harmonic));- }-let mut parsed_query = Query::parse(&query.query,self.index.schema(),@@ -195,13 +186,33 @@ impl LocalSearcher {.and_then(|pipeline| pipeline.stages.first().map(|stage| stage.aggregator())).unwrap_or_default();- if let Some(harmonic) = self- .centrality_store- .as_ref()- .map(|store| &store.online_harmonic)- {- aggregator- .add_personal_harmonic(online_centrality_scorer(&optic.site_rankings, harmonic));+ if let Some(store) = &self.centrality_store {+ aggregator.set_personal_harmonic(online_centrality_scorer(+ &optic.site_rankings,+ &store.online_harmonic,+ ));++ let liked_sites: Vec<_> = optic+ .site_rankings+ .liked+ .iter()+ .map(|site| Node::from(site.clone()).into_host())+ .filter_map(|node| store.node2id.get(&node).copied())+ .collect();++ let disliked_sites: Vec<_> = optic+ .site_rankings+ .disliked+ .iter()+ .map(|site| Node::from(site.clone()).into_host())+ .filter_map(|node| store.node2id.get(&node).copied())+ .collect();++ aggregator.set_inbound_similarity(+ store+ .inbound_similarity+ .scorer(&liked_sites, &disliked_sites),+ )}let ranker = self.ranker(&parsed_query, de_rank_similar, aggregator)?;