Skip to content

web search done right

Improve speed of online similarity index build by only considering top-n nodes by harmonic centrality

Mikkel Denkerf7ad311a0f48
diff --git a/core/src/ranking/centrality_store.rs b/core/src/ranking/centrality_store.rs
index c3e58427..81af4842 100644
--- a/core/src/ranking/centrality_store.rs
+++ b/core/src/ranking/centrality_store.rs
@@ -17,6 +17,9 @@
use std::io::{BufReader, BufWriter, Read};
use std::{collections::BTreeMap, fs::File, path::Path};
+use tracing::debug;
+use tracing::log::trace;
+
use crate::Result;
use crate::{
kv::{rocksdb_store::RocksDbStore, Kv},
@@ -166,7 +169,10 @@ impl CentralityStore {
Self::store_host(&output_path, &mut store, harmonic_centrality);
+ debug!("Begin online harmonic");
store.online_harmonic = OnlineHarmonicCentrality::new(graph, &store.harmonic);
+
+ debug!("Begin inbound similarity index construction");
store.inbound_similarity = InboundSimilarity::build(graph, &store.harmonic);
store.flush();
@@ -175,16 +181,20 @@ impl CentralityStore {
}
pub fn flush(&self) {
+ trace!("flushing");
self.harmonic.flush();
+ trace!("saving online harmonic");
self.online_harmonic
.save(Path::new(&self.base_path).join("online_harmonic"))
.unwrap();
+ trace!("saving inbound similarity");
self.inbound_similarity
.save(Path::new(&self.base_path).join("inbound_similarity"))
.unwrap();
+ trace!("saving node2id");
let mut file = BufWriter::new(
File::options()
.create(true)
diff --git a/core/src/ranking/inbound_similarity.rs b/core/src/ranking/inbound_similarity.rs
index b362a352..2fcb0611 100644
--- a/core/src/ranking/inbound_similarity.rs
+++ b/core/src/ranking/inbound_similarity.rs
@@ -15,6 +15,8 @@
// along with this program. If not, see <https://www.gnu.org/licenses/>.
use std::{
+ cmp::{Ordering, Reverse},
+ collections::{BinaryHeap, HashSet},
fs::File,
io::{BufReader, BufWriter, Read},
path::Path,
@@ -30,8 +32,8 @@ use crate::{
};
use super::{bitvec_similarity, centrality_store::HarmonicCentralityStore};
-const DEFAULT_HARMONIC_CENTRALITY_THRESHOLD_FOR_INBOUND: f64 = 0.038;
-const DEFAULT_HARMONIC_CENTRALITY_THRESHOLD_FOR_NODES: f64 = 0.038;
+const DEFAULT_NUM_TOP_HARMONIC_CENTRALITY_FOR_INBOUND: usize = 1000;
+const DEFAULT_NUM_TOP_HARMONIC_CENTRALITY_FOR_NODES: usize = 1000;
const SCORE_SCALE: f64 = 5.0;
pub struct Scorer {
@@ -56,6 +58,32 @@ impl Scorer {
}
}
+#[derive(Clone, Debug)]
+struct ScoredNode {
+ node: NodeID,
+ score: f64,
+}
+
+impl PartialOrd for ScoredNode {
+ fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
+ self.score.partial_cmp(&other.score)
+ }
+}
+
+impl PartialEq for ScoredNode {
+ fn eq(&self, other: &Self) -> bool {
+ self.score == other.score
+ }
+}
+
+impl Ord for ScoredNode {
+ fn cmp(&self, other: &Self) -> std::cmp::Ordering {
+ self.partial_cmp(other).unwrap_or(Ordering::Equal)
+ }
+}
+
+impl Eq for ScoredNode {}
+
#[derive(Serialize, Deserialize, Default)]
pub struct InboundSimilarity {
vectors: Arc<IntMap<bitvec_similarity::BitVec>>,
@@ -66,36 +94,65 @@ impl InboundSimilarity {
Self::build_with_threshold(
graph,
harmonic,
- DEFAULT_HARMONIC_CENTRALITY_THRESHOLD_FOR_NODES,
- DEFAULT_HARMONIC_CENTRALITY_THRESHOLD_FOR_INBOUND,
+ DEFAULT_NUM_TOP_HARMONIC_CENTRALITY_FOR_NODES,
+ DEFAULT_NUM_TOP_HARMONIC_CENTRALITY_FOR_INBOUND,
)
}
fn build_with_threshold(
graph: &Webgraph,
harmonic: &HarmonicCentralityStore,
- harmonic_centrality_threshold_nodes: f64,
- harmonic_centrality_threshold_inbound: f64,
+ num_top_nodes: usize,
+ num_top_nodes_inbound: usize,
) -> Self {
let mut vectors = IntMap::new();
let nodes: Vec<_> = graph.nodes().collect();
+ let mut top_nodes: BinaryHeap<Reverse<ScoredNode>> =
+ BinaryHeap::with_capacity(num_top_nodes);
+ let mut top_nodes_inbound: BinaryHeap<Reverse<ScoredNode>> =
+ BinaryHeap::with_capacity(num_top_nodes_inbound);
+
+ for (node, centrality) in harmonic.host.iter() {
+ let scored_node = Reverse(ScoredNode {
+ node,
+ score: centrality,
+ });
+
+ if top_nodes.len() >= num_top_nodes {
+ if let Some(mut worst) = top_nodes.peek_mut() {
+ if worst.0.score < scored_node.0.score {
+ *worst = scored_node.clone();
+ }
+ }
+ } else {
+ top_nodes.push(scored_node.clone());
+ }
+
+ if top_nodes_inbound.len() >= num_top_nodes_inbound {
+ if let Some(mut worst) = top_nodes_inbound.peek_mut() {
+ if worst.0.score < scored_node.0.score {
+ *worst = scored_node;
+ }
+ }
+ } else {
+ top_nodes_inbound.push(scored_node);
+ }
+ }
+
+ let inbound_nodes: HashSet<NodeID> =
+ top_nodes_inbound.into_iter().map(|n| n.0.node).collect();
+
if let Some(max_node) = nodes.iter().max().copied() {
let mut buf = vec![false; max_node.0 as usize];
- for node_id in nodes.into_iter().filter(|node_id| {
- let score = harmonic.host.get(node_id).unwrap_or(0.0);
- score >= harmonic_centrality_threshold_nodes
- }) {
+ for node_id in top_nodes.into_iter().map(|n| n.0.node) {
buf.clear();
buf.resize(max_node.0 as usize, false);
for edge in graph
.raw_ingoing_edges(&node_id)
.into_iter()
- .filter(|edge| {
- let score = harmonic.host.get(&edge.from).unwrap_or(0.0);
- score >= harmonic_centrality_threshold_inbound
- })
+ .filter(|edge| inbound_nodes.contains(&edge.from))
{
buf[edge.from.0 as usize] = true;
}
@@ -176,6 +233,11 @@ mod tests {
graph.insert(Node::from("c.com"), Node::from("d.com"), String::new());
graph.insert(Node::from("a.com"), Node::from("e.com"), String::new());
+ graph.insert(Node::from("z.com"), Node::from("a.com"), String::new());
+ graph.insert(Node::from("z.com"), Node::from("b.com"), String::new());
+ graph.insert(Node::from("z.com"), Node::from("c.com"), String::new());
+ graph.insert(Node::from("z.com"), Node::from("d.com"), String::new());
+
graph.commit();
let harmonic = HarmonicCentrality::calculate(&graph);
@@ -189,7 +251,7 @@ mod tests {
harmonic_centrality_store.host.flush();
let inbound =
- InboundSimilarity::build_with_threshold(&graph, &harmonic_centrality_store, -1.0, -1.0);
+ InboundSimilarity::build_with_threshold(&graph, &harmonic_centrality_store, 1000, 1000);
let scorer = inbound.scorer(&[graph.node2id(&Node::from("b.com")).unwrap()], &[]);
let e = graph.node2id(&Node::from("e.com")).unwrap();
@@ -219,7 +281,7 @@ mod tests {
harmonic_centrality_store.host.flush();
let inbound =
- InboundSimilarity::build_with_threshold(&graph, &harmonic_centrality_store, -1.0, -1.0);
+ InboundSimilarity::build_with_threshold(&graph, &harmonic_centrality_store, 1000, 1000);
let mut centrality_store = CentralityStore::build(&graph, gen_temp_path());
centrality_store.inbound_similarity = inbound;
diff --git a/core/src/ranking/signal.rs b/core/src/ranking/signal.rs
index 721a4dc2..5d414cb8 100644
--- a/core/src/ranking/signal.rs
+++ b/core/src/ranking/signal.rs
@@ -19,7 +19,7 @@ use crate::{
schema::{FastField, TextField},
webgraph::{
centrality::{
- online_harmonic::{self, SHIFT},
+ online_harmonic::{self, SCORE_OFFSET},
topic,
},
NodeID,
@@ -347,7 +347,7 @@ impl SignalAggregator {
pub fn query_centrality(&self, host_id: NodeID) -> Option<f64> {
self.query_centrality
.as_ref()
- .map(|scorer| scorer.score(host_id) - SHIFT)
+ .map(|scorer| scorer.score(host_id) - SCORE_OFFSET)
}
pub fn personal_centrality(&self, host_id: NodeID) -> f64 {
diff --git a/core/src/webgraph/centrality/online_harmonic.rs b/core/src/webgraph/centrality/online_harmonic.rs
index 6103397d..96d9f81e 100644
--- a/core/src/webgraph/centrality/online_harmonic.rs
+++ b/core/src/webgraph/centrality/online_harmonic.rs
@@ -38,12 +38,12 @@ use crate::webgraph::{Edge, Node};
use crate::webgraph::{NodeID, Webgraph};
use crate::Result;
-const NUM_PROXY_NODES: usize = 500;
+const NUM_PROXY_NODES: usize = 50;
const BEST_PROXY_NODES_PER_USER_NODE: usize = 3;
const USER_NODES_LIMIT: usize = 100; // if the user specifies more than this number of nodes, the remaining nodes will be merged into existing
const MAX_DIST_PROXY: u8 = 3;
-const MAX_NUM_DISTANCE_NODES: usize = 10_000;
-pub const SHIFT: f64 = 1.0;
+const MAX_NUM_OUT_EDGES: usize = 200;
+pub const SCORE_OFFSET: f64 = 1.0;
#[derive(Serialize, Deserialize)]
pub struct ProxyNode {
@@ -144,7 +144,7 @@ impl Scorer {
return *cached;
}
- let res = (SHIFT
+ let res = (SCORE_OFFSET
+ (self
.liked_nodes
.iter()
@@ -183,7 +183,9 @@ struct UserNode {
impl UserNode {
fn new(id: NodeID, proxy_nodes: &[Arc<ProxyNode>]) -> Self {
- let mut heap = BinaryHeap::with_capacity(BEST_PROXY_NODES_PER_USER_NODE);
+ let mut heap: BinaryHeap<WeightedProxyNode> =
+ BinaryHeap::with_capacity(BEST_PROXY_NODES_PER_USER_NODE);
+
for proxy in proxy_nodes {
if let Some(dist_to_proxy) = proxy.dist_from_node.get(&id.0) {
let weighted_node = WeightedProxyNode {
@@ -191,9 +193,11 @@ impl UserNode {
dist: *dist_to_proxy,
};
- if heap.len() == BEST_PROXY_NODES_PER_USER_NODE {
+ if heap.len() >= BEST_PROXY_NODES_PER_USER_NODE {
if let Some(mut worst) = heap.peek_mut() {
- *worst = weighted_node;
+ if worst.dist > weighted_node.dist {
+ *worst = weighted_node;
+ }
}
} else {
heap.push(weighted_node);
@@ -266,6 +270,7 @@ where
F2: Fn(&Edge) -> NodeID,
{
let source_id = graph.node2id(&source);
+
if source_id.is_none() {
return BTreeMap::new();
}
@@ -291,7 +296,12 @@ where
continue;
}
- for edge in node_edges(v) {
+ let mut out_edges_taken = 0;
+ for edge in node_edges(v).into_iter() {
+ if out_edges_taken >= MAX_NUM_OUT_EDGES {
+ break;
+ }
+
if cost + 1 < *distances.get(&edge_node(&edge)).unwrap_or(&u8::MAX) {
let d = cost + 1;
@@ -299,15 +309,12 @@ where
continue;
}
+ out_edges_taken += 1;
let next = cmp::Reverse((d, edge_node(&edge)));
queue.push(next);
distances.insert(edge_node(&edge), d);
}
}
-
- if distances.len() > MAX_NUM_DISTANCE_NODES {
- break;
- }
}
distances
diff --git a/core/src/webgraph/centrality/topic.rs b/core/src/webgraph/centrality/topic.rs
index ee3836c7..0071dc49 100644
--- a/core/src/webgraph/centrality/topic.rs
+++ b/core/src/webgraph/centrality/topic.rs
@@ -37,7 +37,7 @@ use crate::{
index::Index,
ranking::{initial::InitialScoreTweaker, SignalAggregator},
schema::{FastField, Field, TextField},
- webgraph::{centrality::online_harmonic::SHIFT, NodeID, Webgraph},
+ webgraph::{centrality::online_harmonic::SCORE_OFFSET, NodeID, Webgraph},
};
use crate::{query::Query, Result};
@@ -184,7 +184,7 @@ impl TopicCentrality {
let scorer = harmonic.scorer(&top_sites, &[]);
for node in &nodes {
- node_scores[node.0 as usize][i] = scorer.score(*node) - SHIFT;
+ node_scores[node.0 as usize][i] = scorer.score(*node) - SCORE_OFFSET;
}
}