web search done right
Improve speed of online similarity index build by only considering top-n nodes by harmonic centrality
Mikkel Denkerf7ad311a0f48
diff --git a/core/src/ranking/centrality_store.rs b/core/src/ranking/centrality_store.rsindex c3e58427..81af4842 100644--- a/core/src/ranking/centrality_store.rs+++ b/core/src/ranking/centrality_store.rs@@ -17,6 +17,9 @@use std::io::{BufReader, BufWriter, Read};use std::{collections::BTreeMap, fs::File, path::Path};+use tracing::debug;+use tracing::log::trace;+use crate::Result;use crate::{kv::{rocksdb_store::RocksDbStore, Kv},@@ -166,7 +169,10 @@ impl CentralityStore {Self::store_host(&output_path, &mut store, harmonic_centrality);+ debug!("Begin online harmonic");store.online_harmonic = OnlineHarmonicCentrality::new(graph, &store.harmonic);++ debug!("Begin inbound similarity index construction");store.inbound_similarity = InboundSimilarity::build(graph, &store.harmonic);store.flush();@@ -175,16 +181,20 @@ impl CentralityStore {}pub fn flush(&self) {+ trace!("flushing");self.harmonic.flush();+ trace!("saving online harmonic");self.online_harmonic.save(Path::new(&self.base_path).join("online_harmonic")).unwrap();+ trace!("saving inbound similarity");self.inbound_similarity.save(Path::new(&self.base_path).join("inbound_similarity")).unwrap();+ trace!("saving node2id");let mut file = BufWriter::new(File::options().create(true)diff --git a/core/src/ranking/inbound_similarity.rs b/core/src/ranking/inbound_similarity.rsindex b362a352..2fcb0611 100644--- a/core/src/ranking/inbound_similarity.rs+++ b/core/src/ranking/inbound_similarity.rs@@ -15,6 +15,8 @@// along with this program. If not, see <https://www.gnu.org/licenses/>.use std::{+ cmp::{Ordering, Reverse},+ collections::{BinaryHeap, HashSet},fs::File,io::{BufReader, BufWriter, Read},path::Path,@@ -30,8 +32,8 @@ use crate::{};use super::{bitvec_similarity, centrality_store::HarmonicCentralityStore};-const DEFAULT_HARMONIC_CENTRALITY_THRESHOLD_FOR_INBOUND: f64 = 0.038;-const DEFAULT_HARMONIC_CENTRALITY_THRESHOLD_FOR_NODES: f64 = 0.038;+const DEFAULT_NUM_TOP_HARMONIC_CENTRALITY_FOR_INBOUND: usize = 1000;+const DEFAULT_NUM_TOP_HARMONIC_CENTRALITY_FOR_NODES: usize = 1000;const SCORE_SCALE: f64 = 5.0;pub struct Scorer {@@ -56,6 +58,32 @@ impl Scorer {}}+#[derive(Clone, Debug)]+struct ScoredNode {+ node: NodeID,+ score: f64,+}++impl PartialOrd for ScoredNode {+ fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {+ self.score.partial_cmp(&other.score)+ }+}++impl PartialEq for ScoredNode {+ fn eq(&self, other: &Self) -> bool {+ self.score == other.score+ }+}++impl Ord for ScoredNode {+ fn cmp(&self, other: &Self) -> std::cmp::Ordering {+ self.partial_cmp(other).unwrap_or(Ordering::Equal)+ }+}++impl Eq for ScoredNode {}+#[derive(Serialize, Deserialize, Default)]pub struct InboundSimilarity {vectors: Arc<IntMap<bitvec_similarity::BitVec>>,@@ -66,36 +94,65 @@ impl InboundSimilarity {Self::build_with_threshold(graph,harmonic,- DEFAULT_HARMONIC_CENTRALITY_THRESHOLD_FOR_NODES,- DEFAULT_HARMONIC_CENTRALITY_THRESHOLD_FOR_INBOUND,+ DEFAULT_NUM_TOP_HARMONIC_CENTRALITY_FOR_NODES,+ DEFAULT_NUM_TOP_HARMONIC_CENTRALITY_FOR_INBOUND,)}fn build_with_threshold(graph: &Webgraph,harmonic: &HarmonicCentralityStore,- harmonic_centrality_threshold_nodes: f64,- harmonic_centrality_threshold_inbound: f64,+ num_top_nodes: usize,+ num_top_nodes_inbound: usize,) -> Self {let mut vectors = IntMap::new();let nodes: Vec<_> = graph.nodes().collect();+ let mut top_nodes: BinaryHeap<Reverse<ScoredNode>> =+ BinaryHeap::with_capacity(num_top_nodes);+ let mut top_nodes_inbound: BinaryHeap<Reverse<ScoredNode>> =+ BinaryHeap::with_capacity(num_top_nodes_inbound);++ for (node, centrality) in harmonic.host.iter() {+ let scored_node = Reverse(ScoredNode {+ node,+ score: centrality,+ });++ if top_nodes.len() >= num_top_nodes {+ if let Some(mut worst) = top_nodes.peek_mut() {+ if worst.0.score < scored_node.0.score {+ *worst = scored_node.clone();+ }+ }+ } else {+ top_nodes.push(scored_node.clone());+ }++ if top_nodes_inbound.len() >= num_top_nodes_inbound {+ if let Some(mut worst) = top_nodes_inbound.peek_mut() {+ if worst.0.score < scored_node.0.score {+ *worst = scored_node;+ }+ }+ } else {+ top_nodes_inbound.push(scored_node);+ }+ }++ let inbound_nodes: HashSet<NodeID> =+ top_nodes_inbound.into_iter().map(|n| n.0.node).collect();+if let Some(max_node) = nodes.iter().max().copied() {let mut buf = vec![false; max_node.0 as usize];- for node_id in nodes.into_iter().filter(|node_id| {- let score = harmonic.host.get(node_id).unwrap_or(0.0);- score >= harmonic_centrality_threshold_nodes- }) {+ for node_id in top_nodes.into_iter().map(|n| n.0.node) {buf.clear();buf.resize(max_node.0 as usize, false);for edge in graph.raw_ingoing_edges(&node_id).into_iter()- .filter(|edge| {- let score = harmonic.host.get(&edge.from).unwrap_or(0.0);- score >= harmonic_centrality_threshold_inbound- })+ .filter(|edge| inbound_nodes.contains(&edge.from)){buf[edge.from.0 as usize] = true;}@@ -176,6 +233,11 @@ mod tests {graph.insert(Node::from("c.com"), Node::from("d.com"), String::new());graph.insert(Node::from("a.com"), Node::from("e.com"), String::new());+ graph.insert(Node::from("z.com"), Node::from("a.com"), String::new());+ graph.insert(Node::from("z.com"), Node::from("b.com"), String::new());+ graph.insert(Node::from("z.com"), Node::from("c.com"), String::new());+ graph.insert(Node::from("z.com"), Node::from("d.com"), String::new());+graph.commit();let harmonic = HarmonicCentrality::calculate(&graph);@@ -189,7 +251,7 @@ mod tests {harmonic_centrality_store.host.flush();let inbound =- InboundSimilarity::build_with_threshold(&graph, &harmonic_centrality_store, -1.0, -1.0);+ InboundSimilarity::build_with_threshold(&graph, &harmonic_centrality_store, 1000, 1000);let scorer = inbound.scorer(&[graph.node2id(&Node::from("b.com")).unwrap()], &[]);let e = graph.node2id(&Node::from("e.com")).unwrap();@@ -219,7 +281,7 @@ mod tests {harmonic_centrality_store.host.flush();let inbound =- InboundSimilarity::build_with_threshold(&graph, &harmonic_centrality_store, -1.0, -1.0);+ InboundSimilarity::build_with_threshold(&graph, &harmonic_centrality_store, 1000, 1000);let mut centrality_store = CentralityStore::build(&graph, gen_temp_path());centrality_store.inbound_similarity = inbound;diff --git a/core/src/ranking/signal.rs b/core/src/ranking/signal.rsindex 721a4dc2..5d414cb8 100644--- a/core/src/ranking/signal.rs+++ b/core/src/ranking/signal.rs@@ -19,7 +19,7 @@ use crate::{schema::{FastField, TextField},webgraph::{centrality::{- online_harmonic::{self, SHIFT},+ online_harmonic::{self, SCORE_OFFSET},topic,},NodeID,@@ -347,7 +347,7 @@ impl SignalAggregator {pub fn query_centrality(&self, host_id: NodeID) -> Option<f64> {self.query_centrality.as_ref()- .map(|scorer| scorer.score(host_id) - SHIFT)+ .map(|scorer| scorer.score(host_id) - SCORE_OFFSET)}pub fn personal_centrality(&self, host_id: NodeID) -> f64 {diff --git a/core/src/webgraph/centrality/online_harmonic.rs b/core/src/webgraph/centrality/online_harmonic.rsindex 6103397d..96d9f81e 100644--- a/core/src/webgraph/centrality/online_harmonic.rs+++ b/core/src/webgraph/centrality/online_harmonic.rs@@ -38,12 +38,12 @@ use crate::webgraph::{Edge, Node};use crate::webgraph::{NodeID, Webgraph};use crate::Result;-const NUM_PROXY_NODES: usize = 500;+const NUM_PROXY_NODES: usize = 50;const BEST_PROXY_NODES_PER_USER_NODE: usize = 3;const USER_NODES_LIMIT: usize = 100; // if the user specifies more than this number of nodes, the remaining nodes will be merged into existingconst MAX_DIST_PROXY: u8 = 3;-const MAX_NUM_DISTANCE_NODES: usize = 10_000;-pub const SHIFT: f64 = 1.0;+const MAX_NUM_OUT_EDGES: usize = 200;+pub const SCORE_OFFSET: f64 = 1.0;#[derive(Serialize, Deserialize)]pub struct ProxyNode {@@ -144,7 +144,7 @@ impl Scorer {return *cached;}- let res = (SHIFT+ let res = (SCORE_OFFSET+ (self.liked_nodes.iter()@@ -183,7 +183,9 @@ struct UserNode {impl UserNode {fn new(id: NodeID, proxy_nodes: &[Arc<ProxyNode>]) -> Self {- let mut heap = BinaryHeap::with_capacity(BEST_PROXY_NODES_PER_USER_NODE);+ let mut heap: BinaryHeap<WeightedProxyNode> =+ BinaryHeap::with_capacity(BEST_PROXY_NODES_PER_USER_NODE);+for proxy in proxy_nodes {if let Some(dist_to_proxy) = proxy.dist_from_node.get(&id.0) {let weighted_node = WeightedProxyNode {@@ -191,9 +193,11 @@ impl UserNode {dist: *dist_to_proxy,};- if heap.len() == BEST_PROXY_NODES_PER_USER_NODE {+ if heap.len() >= BEST_PROXY_NODES_PER_USER_NODE {if let Some(mut worst) = heap.peek_mut() {- *worst = weighted_node;+ if worst.dist > weighted_node.dist {+ *worst = weighted_node;+ }}} else {heap.push(weighted_node);@@ -266,6 +270,7 @@ whereF2: Fn(&Edge) -> NodeID,{let source_id = graph.node2id(&source);+if source_id.is_none() {return BTreeMap::new();}@@ -291,7 +296,12 @@ wherecontinue;}- for edge in node_edges(v) {+ let mut out_edges_taken = 0;+ for edge in node_edges(v).into_iter() {+ if out_edges_taken >= MAX_NUM_OUT_EDGES {+ break;+ }+if cost + 1 < *distances.get(&edge_node(&edge)).unwrap_or(&u8::MAX) {let d = cost + 1;@@ -299,15 +309,12 @@ wherecontinue;}+ out_edges_taken += 1;let next = cmp::Reverse((d, edge_node(&edge)));queue.push(next);distances.insert(edge_node(&edge), d);}}-- if distances.len() > MAX_NUM_DISTANCE_NODES {- break;- }}distancesdiff --git a/core/src/webgraph/centrality/topic.rs b/core/src/webgraph/centrality/topic.rsindex ee3836c7..0071dc49 100644--- a/core/src/webgraph/centrality/topic.rs+++ b/core/src/webgraph/centrality/topic.rs@@ -37,7 +37,7 @@ use crate::{index::Index,ranking::{initial::InitialScoreTweaker, SignalAggregator},schema::{FastField, Field, TextField},- webgraph::{centrality::online_harmonic::SHIFT, NodeID, Webgraph},+ webgraph::{centrality::online_harmonic::SCORE_OFFSET, NodeID, Webgraph},};use crate::{query::Query, Result};@@ -184,7 +184,7 @@ impl TopicCentrality {let scorer = harmonic.scorer(&top_sites, &[]);for node in &nodes {- node_scores[node.0 as usize][i] = scorer.score(*node) - SHIFT;+ node_scores[node.0 as usize][i] = scorer.score(*node) - SCORE_OFFSET;}}