Skip to content

web search done right

More memory efficient shortest path

Mikkel Denker14172e6d8aee
diff --git a/core/src/webgraph/centrality/online_harmonic.rs b/core/src/webgraph/centrality/online_harmonic.rs
index 48a15059..420d0791 100644
--- a/core/src/webgraph/centrality/online_harmonic.rs
+++ b/core/src/webgraph/centrality/online_harmonic.rs
@@ -37,7 +37,7 @@ use crate::webgraph::{Node, ShortestPaths};
use crate::webgraph::{NodeID, Webgraph};
use crate::Result;
-const NUM_PROXY_NODES: usize = 10_000;
+const NUM_PROXY_NODES: usize = 100;
const BEST_PROXY_NODES_PER_USER_NODE: usize = 3;
const USER_NODES_LIMIT: usize = 100; // if the user specifies more than this number of nodes, the remaining nodes will be merged into existing
pub const SHIFT: f64 = 1.0;
@@ -45,12 +45,12 @@ pub const SHIFT: f64 = 1.0;
#[derive(Serialize, Deserialize)]
pub struct ProxyNode {
pub id: NodeID,
- dist_from_node: IntMap<usize>, // from node to proxy node
- dist_to_node: IntMap<usize>, // from proxy node to other node
+ dist_from_node: IntMap<u8>, // from node to proxy node
+ dist_to_node: IntMap<u8>, // from proxy node to other node
}
impl ProxyNode {
- fn dist(&self, from: &NodeID, to: &NodeID) -> Option<usize> {
+ fn dist(&self, from: &NodeID, to: &NodeID) -> Option<u8> {
if let Some(from_node_to_proxy) = self.dist_from_node.get(&from.0) {
if let Some(from_proxy_to_node) = self.dist_to_node.get(&to.0) {
return Some(from_node_to_proxy + from_proxy_to_node);
@@ -63,7 +63,7 @@ impl ProxyNode {
struct WeightedProxyNode {
node: Arc<ProxyNode>,
- dist: usize,
+ dist: u8,
}
impl PartialOrd for WeightedProxyNode {
@@ -105,7 +105,7 @@ fn create_user_nodes(nodes: &[NodeID], proxy_nodes: &[Arc<ProxyNode>]) -> Vec<Us
} else {
let mut best = res
.iter_mut()
- .min_by_key(|curr| user_node.best_dist(&curr.id).unwrap_or(1_000_000))
+ .min_by_key(|curr| user_node.best_dist(&curr.id).unwrap_or(255))
.unwrap();
best.weight += 1;
@@ -205,7 +205,7 @@ impl UserNode {
}
}
- fn best_dist(&self, node: &NodeID) -> Option<usize> {
+ fn best_dist(&self, node: &NodeID) -> Option<u8> {
let mut best = None;
for proxy in &self.proxy_nodes {
@@ -299,9 +299,9 @@ impl OnlineHarmonicCentrality {
let proxy_nodes: Vec<_> = nodes
.into_iter()
- .take(num_proxy_nodes)
.map(|candidate| candidate.node)
.filter(|node| node2id.contains_key(node))
+ .take(num_proxy_nodes)
.map(|node| {
let dist_to_node = graph
.raw_distances(node.clone())
diff --git a/core/src/webgraph/mod.rs b/core/src/webgraph/mod.rs
index bde2b18f..b4af1070 100644
--- a/core/src/webgraph/mod.rs
+++ b/core/src/webgraph/mod.rs
@@ -170,10 +170,10 @@ where
}
pub trait ShortestPaths {
- fn distances(&self, source: Node) -> HashMap<Node, usize>;
- fn raw_distances(&self, source: Node) -> HashMap<NodeID, usize>;
- fn raw_reversed_distances(&self, source: Node) -> HashMap<NodeID, usize>;
- fn reversed_distances(&self, source: Node) -> HashMap<Node, usize>;
+ fn distances(&self, source: Node) -> HashMap<Node, u8>;
+ fn raw_distances(&self, source: Node) -> HashMap<NodeID, u8>;
+ fn raw_reversed_distances(&self, source: Node) -> HashMap<NodeID, u8>;
+ fn reversed_distances(&self, source: Node) -> HashMap<Node, u8>;
}
fn dijkstra<F1, F2>(
@@ -181,23 +181,23 @@ fn dijkstra<F1, F2>(
node_edges: F1,
edge_node: F2,
graph: &Webgraph,
-) -> HashMap<NodeID, usize>
+) -> HashMap<NodeID, u8>
where
F1: Fn(NodeID) -> Vec<Edge>,
F2: Fn(&Edge) -> NodeID,
{
- const MAX_DIST: usize = 3;
+ const MAX_DIST: u8 = 3;
let source_id = graph.node2id(&source);
if source_id.is_none() {
return HashMap::new();
}
let source_id = source_id.unwrap();
- let mut distances: HashMap<NodeID, usize> = HashMap::default();
+ let mut distances: HashMap<NodeID, u8> = HashMap::default();
let mut queue = BinaryHeap::new();
- queue.push(cmp::Reverse((0_usize, source_id)));
+ queue.push(cmp::Reverse((0, source_id)));
distances.insert(source_id, 0);
while let Some(state) = queue.pop() {
@@ -207,14 +207,14 @@ where
continue;
}
- let current_dist = distances.get(&v).unwrap_or(&usize::MAX);
+ let current_dist = distances.get(&v).unwrap_or(&u8::MAX);
if cost > *current_dist {
continue;
}
for edge in node_edges(v) {
- if cost + 1 < *distances.get(&edge_node(&edge)).unwrap_or(&usize::MAX) {
+ if cost + 1 < *distances.get(&edge_node(&edge)).unwrap_or(&u8::MAX) {
let d = cost + 1;
if d > MAX_DIST {
@@ -232,14 +232,14 @@ where
}
impl ShortestPaths for Webgraph {
- fn distances(&self, source: Node) -> HashMap<Node, usize> {
+ fn distances(&self, source: Node) -> HashMap<Node, u8> {
self.raw_distances(source)
.into_iter()
.map(|(id, dist)| (self.id2node(&id).expect("unknown node"), dist))
.collect()
}
- fn raw_distances(&self, source: Node) -> HashMap<NodeID, usize> {
+ fn raw_distances(&self, source: Node) -> HashMap<NodeID, u8> {
dijkstra(
source,
|node| self.raw_outgoing_edges(&node),
@@ -248,7 +248,7 @@ impl ShortestPaths for Webgraph {
)
}
- fn raw_reversed_distances(&self, source: Node) -> HashMap<NodeID, usize> {
+ fn raw_reversed_distances(&self, source: Node) -> HashMap<NodeID, u8> {
dijkstra(
source,
|node| self.raw_ingoing_edges(&node),
@@ -257,7 +257,7 @@ impl ShortestPaths for Webgraph {
)
}
- fn reversed_distances(&self, source: Node) -> HashMap<Node, usize> {
+ fn reversed_distances(&self, source: Node) -> HashMap<Node, u8> {
self.raw_reversed_distances(source)
.into_iter()
.map(|(id, dist)| (self.id2node(&id).expect("unknown node"), dist))
diff --git a/core/src/webgraph/segment.rs b/core/src/webgraph/segment.rs
index aa20d7a0..5fd5250f 100644
--- a/core/src/webgraph/segment.rs
+++ b/core/src/webgraph/segment.rs
@@ -326,13 +326,13 @@ impl Store for RocksDbStore {
Segment {
adjacency: Adjacency {
- tree: CachedStore::new(adjacency, 50_000),
+ tree: CachedStore::new(adjacency, 100),
},
reversed_adjacency: Adjacency {
- tree: CachedStore::new(reversed_adjacency, 50_000),
+ tree: CachedStore::new(reversed_adjacency, 100),
},
- id_mapping: CachedStore::new(id_mapping, 100_000),
- rev_id_mapping: CachedStore::new(rev_id_mapping, 100_000),
+ id_mapping: CachedStore::new(id_mapping, 100),
+ rev_id_mapping: CachedStore::new(rev_id_mapping, 100),
meta: CachedStore::new(meta, 1_000),
store: Default::default(),
path: path.as_ref().as_os_str().to_str().unwrap().to_string(),
@@ -350,13 +350,13 @@ impl Store for RocksDbStore {
Segment {
adjacency: Adjacency {
- tree: CachedStore::new(adjacency, 50_000),
+ tree: CachedStore::new(adjacency, 100),
},
reversed_adjacency: Adjacency {
- tree: CachedStore::new(reversed_adjacency, 50_000),
+ tree: CachedStore::new(reversed_adjacency, 100),
},
- id_mapping: CachedStore::new(id_mapping, 100_000),
- rev_id_mapping: CachedStore::new(rev_id_mapping, 100_000),
+ id_mapping: CachedStore::new(id_mapping, 100),
+ rev_id_mapping: CachedStore::new(rev_id_mapping, 100),
meta: CachedStore::new(meta, 1_000),
store: Default::default(),
path: path.as_ref().as_os_str().to_str().unwrap().to_string(),