web search done right
More memory efficient shortest path
Mikkel Denker14172e6d8aee
diff --git a/core/src/webgraph/centrality/online_harmonic.rs b/core/src/webgraph/centrality/online_harmonic.rsindex 48a15059..420d0791 100644--- a/core/src/webgraph/centrality/online_harmonic.rs+++ b/core/src/webgraph/centrality/online_harmonic.rs@@ -37,7 +37,7 @@ use crate::webgraph::{Node, ShortestPaths};use crate::webgraph::{NodeID, Webgraph};use crate::Result;-const NUM_PROXY_NODES: usize = 10_000;+const NUM_PROXY_NODES: usize = 100;const BEST_PROXY_NODES_PER_USER_NODE: usize = 3;const USER_NODES_LIMIT: usize = 100; // if the user specifies more than this number of nodes, the remaining nodes will be merged into existingpub const SHIFT: f64 = 1.0;@@ -45,12 +45,12 @@ pub const SHIFT: f64 = 1.0;#[derive(Serialize, Deserialize)]pub struct ProxyNode {pub id: NodeID,- dist_from_node: IntMap<usize>, // from node to proxy node- dist_to_node: IntMap<usize>, // from proxy node to other node+ dist_from_node: IntMap<u8>, // from node to proxy node+ dist_to_node: IntMap<u8>, // from proxy node to other node}impl ProxyNode {- fn dist(&self, from: &NodeID, to: &NodeID) -> Option<usize> {+ fn dist(&self, from: &NodeID, to: &NodeID) -> Option<u8> {if let Some(from_node_to_proxy) = self.dist_from_node.get(&from.0) {if let Some(from_proxy_to_node) = self.dist_to_node.get(&to.0) {return Some(from_node_to_proxy + from_proxy_to_node);@@ -63,7 +63,7 @@ impl ProxyNode {struct WeightedProxyNode {node: Arc<ProxyNode>,- dist: usize,+ dist: u8,}impl PartialOrd for WeightedProxyNode {@@ -105,7 +105,7 @@ fn create_user_nodes(nodes: &[NodeID], proxy_nodes: &[Arc<ProxyNode>]) -> Vec<Us} else {let mut best = res.iter_mut()- .min_by_key(|curr| user_node.best_dist(&curr.id).unwrap_or(1_000_000))+ .min_by_key(|curr| user_node.best_dist(&curr.id).unwrap_or(255)).unwrap();best.weight += 1;@@ -205,7 +205,7 @@ impl UserNode {}}- fn best_dist(&self, node: &NodeID) -> Option<usize> {+ fn best_dist(&self, node: &NodeID) -> Option<u8> {let mut best = None;for proxy in &self.proxy_nodes {@@ -299,9 +299,9 @@ impl OnlineHarmonicCentrality {let proxy_nodes: Vec<_> = nodes.into_iter()- .take(num_proxy_nodes).map(|candidate| candidate.node).filter(|node| node2id.contains_key(node))+ .take(num_proxy_nodes).map(|node| {let dist_to_node = graph.raw_distances(node.clone())diff --git a/core/src/webgraph/mod.rs b/core/src/webgraph/mod.rsindex bde2b18f..b4af1070 100644--- a/core/src/webgraph/mod.rs+++ b/core/src/webgraph/mod.rs@@ -170,10 +170,10 @@ where}pub trait ShortestPaths {- fn distances(&self, source: Node) -> HashMap<Node, usize>;- fn raw_distances(&self, source: Node) -> HashMap<NodeID, usize>;- fn raw_reversed_distances(&self, source: Node) -> HashMap<NodeID, usize>;- fn reversed_distances(&self, source: Node) -> HashMap<Node, usize>;+ fn distances(&self, source: Node) -> HashMap<Node, u8>;+ fn raw_distances(&self, source: Node) -> HashMap<NodeID, u8>;+ fn raw_reversed_distances(&self, source: Node) -> HashMap<NodeID, u8>;+ fn reversed_distances(&self, source: Node) -> HashMap<Node, u8>;}fn dijkstra<F1, F2>(@@ -181,23 +181,23 @@ fn dijkstra<F1, F2>(node_edges: F1,edge_node: F2,graph: &Webgraph,-) -> HashMap<NodeID, usize>+) -> HashMap<NodeID, u8>whereF1: Fn(NodeID) -> Vec<Edge>,F2: Fn(&Edge) -> NodeID,{- const MAX_DIST: usize = 3;+ const MAX_DIST: u8 = 3;let source_id = graph.node2id(&source);if source_id.is_none() {return HashMap::new();}let source_id = source_id.unwrap();- let mut distances: HashMap<NodeID, usize> = HashMap::default();+ let mut distances: HashMap<NodeID, u8> = HashMap::default();let mut queue = BinaryHeap::new();- queue.push(cmp::Reverse((0_usize, source_id)));+ queue.push(cmp::Reverse((0, source_id)));distances.insert(source_id, 0);while let Some(state) = queue.pop() {@@ -207,14 +207,14 @@ wherecontinue;}- let current_dist = distances.get(&v).unwrap_or(&usize::MAX);+ let current_dist = distances.get(&v).unwrap_or(&u8::MAX);if cost > *current_dist {continue;}for edge in node_edges(v) {- if cost + 1 < *distances.get(&edge_node(&edge)).unwrap_or(&usize::MAX) {+ if cost + 1 < *distances.get(&edge_node(&edge)).unwrap_or(&u8::MAX) {let d = cost + 1;if d > MAX_DIST {@@ -232,14 +232,14 @@ where}impl ShortestPaths for Webgraph {- fn distances(&self, source: Node) -> HashMap<Node, usize> {+ fn distances(&self, source: Node) -> HashMap<Node, u8> {self.raw_distances(source).into_iter().map(|(id, dist)| (self.id2node(&id).expect("unknown node"), dist)).collect()}- fn raw_distances(&self, source: Node) -> HashMap<NodeID, usize> {+ fn raw_distances(&self, source: Node) -> HashMap<NodeID, u8> {dijkstra(source,|node| self.raw_outgoing_edges(&node),@@ -248,7 +248,7 @@ impl ShortestPaths for Webgraph {)}- fn raw_reversed_distances(&self, source: Node) -> HashMap<NodeID, usize> {+ fn raw_reversed_distances(&self, source: Node) -> HashMap<NodeID, u8> {dijkstra(source,|node| self.raw_ingoing_edges(&node),@@ -257,7 +257,7 @@ impl ShortestPaths for Webgraph {)}- fn reversed_distances(&self, source: Node) -> HashMap<Node, usize> {+ fn reversed_distances(&self, source: Node) -> HashMap<Node, u8> {self.raw_reversed_distances(source).into_iter().map(|(id, dist)| (self.id2node(&id).expect("unknown node"), dist))diff --git a/core/src/webgraph/segment.rs b/core/src/webgraph/segment.rsindex aa20d7a0..5fd5250f 100644--- a/core/src/webgraph/segment.rs+++ b/core/src/webgraph/segment.rs@@ -326,13 +326,13 @@ impl Store for RocksDbStore {Segment {adjacency: Adjacency {- tree: CachedStore::new(adjacency, 50_000),+ tree: CachedStore::new(adjacency, 100),},reversed_adjacency: Adjacency {- tree: CachedStore::new(reversed_adjacency, 50_000),+ tree: CachedStore::new(reversed_adjacency, 100),},- id_mapping: CachedStore::new(id_mapping, 100_000),- rev_id_mapping: CachedStore::new(rev_id_mapping, 100_000),+ id_mapping: CachedStore::new(id_mapping, 100),+ rev_id_mapping: CachedStore::new(rev_id_mapping, 100),meta: CachedStore::new(meta, 1_000),store: Default::default(),path: path.as_ref().as_os_str().to_str().unwrap().to_string(),@@ -350,13 +350,13 @@ impl Store for RocksDbStore {Segment {adjacency: Adjacency {- tree: CachedStore::new(adjacency, 50_000),+ tree: CachedStore::new(adjacency, 100),},reversed_adjacency: Adjacency {- tree: CachedStore::new(reversed_adjacency, 50_000),+ tree: CachedStore::new(reversed_adjacency, 100),},- id_mapping: CachedStore::new(id_mapping, 100_000),- rev_id_mapping: CachedStore::new(rev_id_mapping, 100_000),+ id_mapping: CachedStore::new(id_mapping, 100),+ rev_id_mapping: CachedStore::new(rev_id_mapping, 100),meta: CachedStore::new(meta, 1_000),store: Default::default(),path: path.as_ref().as_os_str().to_str().unwrap().to_string(),