Skip to content

web search done right

Use inbound edges similarity between liked sites and search results as a ranking metric.

Mikkel Denkerf0274317b48f
diff --git a/core/src/ranking/bitvec_similarity.rs b/core/src/ranking/bitvec_similarity.rs
index bf7467f7..ff3ab1d9 100644
--- a/core/src/ranking/bitvec_similarity.rs
+++ b/core/src/ranking/bitvec_similarity.rs
@@ -17,13 +17,13 @@
use itertools::Itertools;
use serde::{Deserialize, Serialize};
-#[derive(Serialize, Deserialize)]
+#[derive(Serialize, Deserialize, Debug, Clone)]
struct Block {
data: u64,
offset: usize,
}
-#[derive(Serialize, Deserialize)]
+#[derive(Serialize, Deserialize, Debug, Clone)]
pub struct BitVec {
blocks: Vec<Block>,
sqrt_len: f64,
@@ -56,7 +56,7 @@ impl BitVec {
}
}
- fn sim(&self, other: &Self) -> f64 {
+ pub fn sim(&self, other: &Self) -> f64 {
let mut i = 0;
let mut j = 0;
@@ -161,7 +161,6 @@ mod tests {
let expected = naive_sim(&a, &b);
- dbg!(expected);
assert!(expected < 0.01);
let a = BitVec::new(a);
diff --git a/core/src/ranking/inbound_similarity.rs b/core/src/ranking/inbound_similarity.rs
index 8bd65a52..330a0769 100644
--- a/core/src/ranking/inbound_similarity.rs
+++ b/core/src/ranking/inbound_similarity.rs
@@ -18,9 +18,9 @@ use std::{
fs::File,
io::{BufReader, BufWriter, Read},
path::Path,
+ sync::Arc,
};
-use optics::SiteRankings;
use serde::{Deserialize, Serialize};
use crate::{
@@ -30,20 +30,45 @@ use crate::{
};
use super::bitvec_similarity;
-const HARMONIC_CENTRALITY_THRESHOLD: f64 = 0.038;
+const DEFAULT_HARMONIC_CENTRALITY_THRESHOLD: f64 = 0.038;
+const SCORE_SCALE: f64 = 5.0;
-pub struct Scorer<'a> {
- liked: Vec<&'a bitvec_similarity::BitVec>,
- disliked: Vec<&'a bitvec_similarity::BitVec>,
+pub struct Scorer {
+ liked: Vec<bitvec_similarity::BitVec>,
+ disliked: Vec<bitvec_similarity::BitVec>,
+ vectors: Arc<IntMap<bitvec_similarity::BitVec>>,
+}
+
+impl Scorer {
+ pub fn score(&self, node: &NodeID) -> f64 {
+ match self.vectors.get(&node.0) {
+ Some(vec) => (SCORE_SCALE
+ + (self.liked.iter().map(|liked| liked.sim(vec)).sum::<f64>()
+ - self
+ .disliked
+ .iter()
+ .map(|disliked| disliked.sim(vec))
+ .sum::<f64>()))
+ .max(0.0),
+ None => 0.0,
+ }
+ }
}
#[derive(Serialize, Deserialize, Default)]
pub struct InboundSimilarity {
- vectors: IntMap<bitvec_similarity::BitVec>,
+ vectors: Arc<IntMap<bitvec_similarity::BitVec>>,
}
impl InboundSimilarity {
pub fn build(graph: &Webgraph, harmonic: &HarmonicCentrality) -> Self {
+ Self::build_with_threshold(graph, harmonic, DEFAULT_HARMONIC_CENTRALITY_THRESHOLD)
+ }
+ fn build_with_threshold(
+ graph: &Webgraph,
+ harmonic: &HarmonicCentrality,
+ harmonic_centrality_threshold: f64,
+ ) -> Self {
let mut vectors = IntMap::new();
let nodes: Vec<_> = graph.nodes().collect();
@@ -57,7 +82,7 @@ impl InboundSimilarity {
.filter(|edge| match graph.id2node(&edge.from) {
Some(node) => {
let score = *harmonic.host.get(&node.into_host()).unwrap_or(&0.0);
- score >= HARMONIC_CENTRALITY_THRESHOLD
+ score >= harmonic_centrality_threshold
}
None => false,
})
@@ -69,21 +94,27 @@ impl InboundSimilarity {
}
}
- Self { vectors }
+ Self {
+ vectors: Arc::new(vectors),
+ }
}
- pub fn scorer<'a>(&'a self, liked_sites: &[NodeID], disliked_sites: &[NodeID]) -> Scorer<'a> {
+ pub fn scorer(&self, liked_sites: &[NodeID], disliked_sites: &[NodeID]) -> Scorer {
let liked: Vec<_> = liked_sites
.iter()
- .filter_map(|id| self.vectors.get(&id.0))
+ .filter_map(|id| self.vectors.get(&id.0).cloned())
.collect();
let disliked: Vec<_> = disliked_sites
.iter()
- .filter_map(|id| self.vectors.get(&id.0))
+ .filter_map(|id| self.vectors.get(&id.0).cloned())
.collect();
- Scorer { liked, disliked }
+ Scorer {
+ liked,
+ disliked,
+ vectors: self.vectors.clone(),
+ }
}
pub fn save<P: AsRef<Path>>(&self, path: P) -> Result<()> {
@@ -110,3 +141,142 @@ impl InboundSimilarity {
Ok(bincode::deserialize(&buf)?)
}
}
+
+#[cfg(test)]
+mod tests {
+ use optics::SiteRankings;
+
+ use crate::{
+ gen_temp_path,
+ index::Index,
+ rand_words,
+ ranking::centrality_store::CentralityStore,
+ searcher::{LocalSearcher, SearchQuery},
+ webgraph::{Node, WebgraphBuilder},
+ webpage::{Html, Webpage},
+ };
+
+ use super::*;
+
+ #[test]
+ fn it_favors_liked_sites() {
+ let mut graph = WebgraphBuilder::new_memory().open();
+
+ graph.insert(Node::from("a.com"), Node::from("b.com"), String::new());
+ graph.insert(Node::from("c.com"), Node::from("d.com"), String::new());
+ graph.insert(Node::from("a.com"), Node::from("e.com"), String::new());
+
+ graph.commit();
+
+ let harmonic = HarmonicCentrality::calculate(&graph);
+ let inbound = InboundSimilarity::build_with_threshold(&graph, &harmonic, -1.0);
+
+ let scorer = inbound.scorer(&[graph.node2id(&Node::from("b.com")).unwrap()], &[]);
+ let e = graph.node2id(&Node::from("e.com")).unwrap();
+ let d = graph.node2id(&Node::from("d.com")).unwrap();
+
+ assert!(scorer.score(&e) > scorer.score(&d));
+ }
+
+ #[test]
+ fn it_ranks_search_results() {
+ let mut graph = WebgraphBuilder::new_memory().open();
+
+ graph.insert(Node::from("a.com"), Node::from("b.com"), String::new());
+ graph.insert(Node::from("c.com"), Node::from("d.com"), String::new());
+ graph.insert(Node::from("a.com"), Node::from("e.com"), String::new());
+
+ graph.commit();
+
+ let harmonic = HarmonicCentrality::calculate(&graph);
+ let inbound = InboundSimilarity::build_with_threshold(&graph, &harmonic, -1.0);
+
+ let mut centrality_store = CentralityStore::build(&graph, gen_temp_path());
+ centrality_store.inbound_similarity = inbound;
+
+ let mut index = Index::temporary().expect("Unable to open index");
+
+ index
+ .insert(Webpage {
+ html: Html::parse(
+ &format!(
+ r#"
+ <html>
+ <head>
+ <title>Title</title>
+ </head>
+ <body>
+ example {}
+ </body>
+ </html>
+ "#,
+ rand_words(1000)
+ ),
+ "e.com",
+ ),
+ backlinks: vec![],
+ host_centrality: 0.0,
+ fetch_time_ms: 500,
+ page_centrality: 0.0,
+ pre_computed_score: 0.0,
+ crawl_stability: 0.0,
+ primary_image: None,
+ host_topic: None,
+ node_id: Some(graph.node2id(&Node::from("e.com")).unwrap()),
+ dmoz_description: None,
+ })
+ .expect("failed to insert webpage");
+ index
+ .insert(Webpage {
+ html: Html::parse(
+ &format!(
+ r#"
+ <html>
+ <head>
+ <title>Title</title>
+ </head>
+ <body>
+ example {}
+ </body>
+ </html>
+ "#,
+ rand_words(1000)
+ ),
+ "d.com",
+ ),
+ backlinks: vec![],
+ host_centrality: 0.01,
+ fetch_time_ms: 500,
+ page_centrality: 0.0,
+ pre_computed_score: 0.0,
+ crawl_stability: 0.0,
+ primary_image: None,
+ host_topic: None,
+ node_id: Some(graph.node2id(&Node::from("d.com")).unwrap()),
+ dmoz_description: None,
+ })
+ .expect("failed to insert webpage");
+
+ index.commit().unwrap();
+
+ let mut searcher = LocalSearcher::new(index);
+ searcher.set_centrality_store(centrality_store);
+
+ let res = searcher
+ .search(&SearchQuery {
+ query: "example".to_string(),
+ site_rankings: Some(SiteRankings {
+ liked: vec!["a.com".to_string()],
+ disliked: vec![],
+ blocked: vec![],
+ }),
+ ..Default::default()
+ })
+ .unwrap()
+ .webpages;
+
+ assert_eq!(res.len(), 2);
+ assert_eq!(&res[0].url, "https://e.com");
+ assert_eq!(&res[1].url, "https://d.com");
+ }
+}
diff --git a/core/src/ranking/pipeline.rs b/core/src/ranking/pipeline.rs
index eed33d83..0afd24ef 100644
--- a/core/src/ranking/pipeline.rs
+++ b/core/src/ranking/pipeline.rs
@@ -51,6 +51,7 @@ pub struct RankingWebsite {
pub topic_centrality: f64,
pub personal_centrality: f64,
pub query_centrality: f64,
+ pub inbound_similarity: f64,
pub title: String,
pub clean_body: String,
pub score: f64,
@@ -68,6 +69,7 @@ impl RankingWebsite {
topic_centrality: 0.0,
personal_centrality: 0.0,
query_centrality: 0.0,
+ inbound_similarity: 0.0,
title: String::new(),
clean_body: String::new(),
score: pointer.score.total,
@@ -88,6 +90,7 @@ impl RankingWebsite {
let node = value.value.as_u64().unwrap().into();
res.personal_centrality = aggregator.personal_centrality(node);
+ res.inbound_similarity = aggregator.inbound_similarity(node);
res.topic_centrality = aggregator.topic_centrality(node).unwrap_or_default();
res.query_centrality = aggregator.query_centrality(node).unwrap_or_default();
}
@@ -399,6 +402,7 @@ mod tests {
topic_centrality: 0.0,
personal_centrality: 0.0,
query_centrality: 0.0,
+ inbound_similarity: 0.0,
title: String::new(),
clean_body: String::new(),
score: 1.0 / i as f64,
@@ -571,6 +575,7 @@ mod tests {
topic_centrality: 0.0,
personal_centrality: 0.0,
query_centrality: 0.0,
+ inbound_similarity: 0.0,
title: String::new(),
clean_body: String::new(),
score: 1.0,
diff --git a/core/src/ranking/signal.rs b/core/src/ranking/signal.rs
index 3ef45e9c..721a4dc2 100644
--- a/core/src/ranking/signal.rs
+++ b/core/src/ranking/signal.rs
@@ -36,7 +36,7 @@ use crate::{
webpage::region::{Region, RegionCount},
};
-use super::initial::Score;
+use super::{inbound_similarity, initial::Score};
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum Signal {
@@ -52,6 +52,7 @@ pub enum Signal {
CrawlStability,
TopicCentrality,
QueryCentrality,
+ InboundSimilarity,
}
pub const ALL_SIGNALS: [Signal; 12] = [
@@ -77,6 +78,7 @@ struct SignalValue {
personal_centrality: Option<f64>,
topic_score: Option<f64>,
query_centrality: Option<f64>,
+ inbound_similarity: Option<f64>,
}
impl Signal {
@@ -142,6 +144,7 @@ impl Signal {
Signal::CrawlStability => value.fastfield_value.unwrap() as f64 / FLOAT_SCALING as f64,
Signal::TopicCentrality => value.topic_score.unwrap_or_default(),
Signal::QueryCentrality => value.query_centrality.unwrap_or_default(),
+ Signal::InboundSimilarity => value.inbound_similarity.unwrap_or_default(),
}
}
@@ -159,6 +162,7 @@ impl Signal {
Signal::TrackerScore => 20.0,
Signal::Region => 60.0,
Signal::CrawlStability => 20.0,
+ Signal::InboundSimilarity => 1000.0,
}
}
@@ -175,6 +179,8 @@ impl Signal {
"personal_centrality" => Some(Signal::PersonalCentrality),
"topic_centrality" => Some(Signal::TopicCentrality),
"query_centrality" => Some(Signal::QueryCentrality),
+ "inbound_similarity" => Some(Signal::InboundSimilarity),
+ "crawl_stability" => Some(Signal::CrawlStability),
_ => None,
}
}
@@ -193,6 +199,7 @@ impl Signal {
Signal::PersonalCentrality => None,
Signal::TopicCentrality => None,
Signal::QueryCentrality => None,
+ Signal::InboundSimilarity => None,
}
}
}
@@ -260,7 +267,8 @@ impl FieldBoost {
pub struct SignalAggregator {
fastfield_cache: Option<Arc<fastfield_cache::SegmentCache>>,
signal_coefficients: SignalCoefficient,
- personal_centrality: Vec<Arc<online_harmonic::Scorer>>,
+ personal_centrality: Option<Arc<online_harmonic::Scorer>>,
+ inbound_similariy: Option<Arc<inbound_similarity::Scorer>>,
field_boost: FieldBoost,
fetch_time_ms_cache: [f64; 1000],
update_time_cache: Vec<f64>,
@@ -299,7 +307,8 @@ impl SignalAggregator {
Self {
fastfield_cache: None,
- personal_centrality: Vec::new(),
+ personal_centrality: None,
+ inbound_similariy: None,
signal_coefficients,
field_boost,
fetch_time_ms_cache,
@@ -321,8 +330,12 @@ impl SignalAggregator {
self.query_centrality = Some(Arc::new(query_centrality));
}
- pub fn add_personal_harmonic(&mut self, personal_centrality: online_harmonic::Scorer) {
- self.personal_centrality.push(Arc::new(personal_centrality))
+ pub fn set_personal_harmonic(&mut self, personal_centrality: online_harmonic::Scorer) {
+ self.personal_centrality = Some(Arc::new(personal_centrality));
+ }
+
+ pub fn set_inbound_similarity(&mut self, scorer: inbound_similarity::Scorer) {
+ self.inbound_similariy = Some(Arc::new(scorer));
}
pub fn topic_centrality(&self, host_id: NodeID) -> Option<f64> {
@@ -339,9 +352,16 @@ impl SignalAggregator {
pub fn personal_centrality(&self, host_id: NodeID) -> f64 {
self.personal_centrality
- .iter()
+ .as_ref()
.map(|scorer| scorer.score(host_id))
- .sum()
+ .unwrap_or_default()
+ }
+
+ pub fn inbound_similarity(&self, host_id: NodeID) -> f64 {
+ self.inbound_similariy
+ .as_ref()
+ .map(|scorer| scorer.score(&host_id))
+ .unwrap_or_default()
}
pub fn score(
@@ -353,10 +373,16 @@ impl SignalAggregator {
selected_region: Option<Region>,
) -> Score {
let host_id = self.fastfield_cache.as_ref().and_then(|cache| {
- cache
+ let node_id = cache
.get_doc_cache(&FastField::HostNodeID)
.get_u64(&doc)
- .map(NodeID::from)
+ .unwrap();
+
+ if node_id == u64::MAX {
+ None
+ } else {
+ Some(NodeID::from(node_id))
+ }
});
let topic_score = host_id.and_then(|host_id| self.topic_centrality(host_id));
@@ -364,6 +390,7 @@ impl SignalAggregator {
let query_centrality = host_id.and_then(|host_id| self.query_centrality(host_id));
let personal_centrality = host_id.map(|host_id| self.personal_centrality(host_id));
+ let inbound_similarity = host_id.map(|host_id| self.inbound_similarity(host_id));
let score = ALL_SIGNALS
.into_iter()
@@ -384,6 +411,7 @@ impl SignalAggregator {
current_timestamp,
selected_region,
personal_centrality,
+ inbound_similarity,
topic_score,
query_centrality,
},
@@ -421,6 +449,7 @@ impl SignalAggregator {
Signal::Bm25
| Signal::PersonalCentrality
| Signal::TopicCentrality
+ | Signal::InboundSimilarity
| Signal::QueryCentrality => {
panic!("signal cannot be determined from webpage")
}
@@ -440,6 +469,7 @@ impl SignalAggregator {
personal_centrality: None,
topic_score: None,
query_centrality: None,
+ inbound_similarity: None,
},
)
})
diff --git a/core/src/searcher/local.rs b/core/src/searcher/local.rs
index 46973b68..338fa72b 100644
--- a/core/src/searcher/local.rs
+++ b/core/src/searcher/local.rs
@@ -31,6 +31,7 @@ use crate::ranking::{online_centrality_scorer, Ranker, SignalAggregator};
use crate::search_prettifier::{DisplayedEntity, DisplayedWebpage, HighlightedSpellCorrection};
use crate::spell::Correction;
use crate::webgraph::centrality::topic::TopicCentrality;
+use crate::webgraph::Node;
use crate::webpage::region::Region;
use crate::webpage::Url;
use crate::{inverted_index, Error, Result};
@@ -79,7 +80,7 @@ impl LocalSearcher {
None => None,
};
- let mut query_aggregator = optic
+ let query_aggregator = optic
.as_ref()
.and_then(|optic| {
optic
@@ -89,16 +90,6 @@ impl LocalSearcher {
})
.unwrap_or_default();
- if let (Some(optic), Some(harmonic)) = (
- optic.as_ref(),
- self.centrality_store
- .as_ref()
- .map(|store| &store.online_harmonic),
- ) {
- query_aggregator
- .add_personal_harmonic(online_centrality_scorer(&optic.site_rankings, harmonic));
- }
-
let mut parsed_query = Query::parse(
&query.query,
self.index.schema(),
@@ -195,13 +186,33 @@ impl LocalSearcher {
.and_then(|pipeline| pipeline.stages.first().map(|stage| stage.aggregator()))
.unwrap_or_default();
- if let Some(harmonic) = self
- .centrality_store
- .as_ref()
- .map(|store| &store.online_harmonic)
- {
- aggregator
- .add_personal_harmonic(online_centrality_scorer(&optic.site_rankings, harmonic));
+ if let Some(store) = &self.centrality_store {
+ aggregator.set_personal_harmonic(online_centrality_scorer(
+ &optic.site_rankings,
+ &store.online_harmonic,
+ ));
+
+ let liked_sites: Vec<_> = optic
+ .site_rankings
+ .liked
+ .iter()
+ .map(|site| Node::from(site.clone()).into_host())
+ .filter_map(|node| store.node2id.get(&node).copied())
+ .collect();
+
+ let disliked_sites: Vec<_> = optic
+ .site_rankings
+ .disliked
+ .iter()
+ .map(|site| Node::from(site.clone()).into_host())
+ .filter_map(|node| store.node2id.get(&node).copied())
+ .collect();
+
+ aggregator.set_inbound_similarity(
+ store
+ .inbound_similarity
+ .scorer(&liked_sites, &disliked_sites),
+ )
}
let ranker = self.ranker(&parsed_query, de_rank_similar, aggregator)?;