web search done right
use in_degree + out_degree to choose proxy nodes for online centrality. This will hopefully make online centrality more accurate by having better proxy nodes
Mikkel Denkera75b1b62c454
diff --git a/Dockerfile b/Dockerfileindex f83ae99b..f0a5cbe9 100644--- a/Dockerfile+++ b/Dockerfile@@ -40,6 +40,7 @@ WORKDIR /cuelyCOPY --from=builder /cuely/bin/cuely /usr/local/bin/cuelyCOPY --from=builder /cuely/frontend/dist frontend/dist-COPY --from=builder /usr/lib/libonnxruntime.so* /usr/lib+COPY --from=builder /usr/lib/libonnxruntime.so.1.13.1 /usr/lib+COPY --from=builder /usr/lib/libonnxruntime.so /usr/libENTRYPOINT ["/usr/local/bin/cuely"]diff --git a/core/src/webgraph/centrality/online_harmonic.rs b/core/src/webgraph/centrality/online_harmonic.rsindex 93e8a58b..48a15059 100644--- a/core/src/webgraph/centrality/online_harmonic.rs+++ b/core/src/webgraph/centrality/online_harmonic.rs@@ -37,7 +37,7 @@ use crate::webgraph::{Node, ShortestPaths};use crate::webgraph::{NodeID, Webgraph};use crate::Result;-const NUM_PROXY_NODES: usize = 100_000;+const NUM_PROXY_NODES: usize = 10_000;const BEST_PROXY_NODES_PER_USER_NODE: usize = 3;const USER_NODES_LIMIT: usize = 100; // if the user specifies more than this number of nodes, the remaining nodes will be merged into existingpub const SHIFT: f64 = 1.0;@@ -227,6 +227,31 @@ impl UserNode {}}+struct ProxyNodeCandidate {+ node: Node,+ score: usize,+}++impl PartialOrd for ProxyNodeCandidate {+ fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {+ self.score.partial_cmp(&other.score)+ }+}++impl PartialEq for ProxyNodeCandidate {+ fn eq(&self, other: &Self) -> bool {+ self.score == other.score+ }+}++impl Ord for ProxyNodeCandidate {+ fn cmp(&self, other: &Self) -> std::cmp::Ordering {+ self.partial_cmp(other).unwrap_or(std::cmp::Ordering::Equal)+ }+}++impl Eq for ProxyNodeCandidate {}+#[derive(Serialize, Deserialize, Default)]pub struct OnlineHarmonicCentrality {pub node2id: HashMap<Node, NodeID>,@@ -243,21 +268,39 @@ impl OnlineHarmonicCentrality {// we should probably choose the proxy nodes based on their// betweenness centrality, but I don't know how we can approximate betweenness- // on a graph that cannot be in memory- let mut nodes = Vec::new();+ // on a graph that cannot be in memory.+ // For now, we will choose the nodes with highes in_degree + out_degree+ // as an estimate++ let mut nodes: BinaryHeap<ProxyNodeCandidate> = BinaryHeap::new();for id in graph.nodes() {if let Some(node) = graph.id2node(&id) {- if nodes.len() < num_proxy_nodes {- nodes.push(node.clone());- }+ node2id.insert(node.clone(), id);++ let degree_sum = graph.outgoing_edges(node.clone()).len()+ + graph.ingoing_edges(node.clone()).len();- node2id.insert(node, id);+ let candidate = ProxyNodeCandidate {+ node,+ score: degree_sum,+ };++ if nodes.len() >= num_proxy_nodes {+ if let Some(mut existing_node) = nodes.peek_mut() {+ if candidate.score > existing_node.score {+ *existing_node = candidate;+ }+ }+ } else {+ nodes.push(candidate);+ }}}let proxy_nodes: Vec<_> = nodes.into_iter().take(num_proxy_nodes)+ .map(|candidate| candidate.node).filter(|node| node2id.contains_key(node)).map(|node| {let dist_to_node = graph@@ -265,6 +308,7 @@ impl OnlineHarmonicCentrality {.into_iter().map(|(n, v)| (n.0, v)).collect();+let dist_from_node = graph.raw_reversed_distances(node.clone()).into_iter()diff --git a/core/src/webgraph/mod.rs b/core/src/webgraph/mod.rsindex b6160680..9b5f16e8 100644--- a/core/src/webgraph/mod.rs+++ b/core/src/webgraph/mod.rs@@ -184,7 +184,7 @@ whereF1: Fn(NodeID) -> Vec<Edge>,F2: Fn(&Edge) -> NodeID,{- const MAX_DIST: usize = 5;+ const MAX_DIST: usize = 3;let source_id = graph.node2id(&source);if source_id.is_none() {return HashMap::new();