Skip to content

web search done right

Deduplicate webgraph edges during lookup. Some edges may be duplicate if they are present in multiple segments.

Mikkel Denker445043b1d25d
diff --git a/core/src/webgraph/centrality/harmonic.rs b/core/src/webgraph/centrality/harmonic.rs
index b9cc6d5e..3c961830 100644
--- a/core/src/webgraph/centrality/harmonic.rs
+++ b/core/src/webgraph/centrality/harmonic.rs
@@ -372,13 +372,18 @@ mod tests {
let centrality = HarmonicCentrality::calculate(&graph);
graph.insert(Node::from("A"), Node::from("B"), "1".to_string());
+ graph.commit();
graph.insert(Node::from("A"), Node::from("B"), "2".to_string());
+ graph.commit();
graph.insert(Node::from("A"), Node::from("B"), "3".to_string());
+ graph.commit();
graph.insert(Node::from("A"), Node::from("B"), "4".to_string());
+ graph.commit();
graph.insert(Node::from("A"), Node::from("B"), "5".to_string());
+ graph.commit();
graph.insert(Node::from("A"), Node::from("B"), "6".to_string());
+ graph.commit();
graph.insert(Node::from("A"), Node::from("B"), "7".to_string());
-
graph.commit();
let centrality_extra = HarmonicCentrality::calculate(&graph);
diff --git a/core/src/webgraph/mod.rs b/core/src/webgraph/mod.rs
index 2e0688f9..fb6b02c5 100644
--- a/core/src/webgraph/mod.rs
+++ b/core/src/webgraph/mod.rs
@@ -510,7 +510,8 @@ impl Webgraph {
}
fn inner_ingoing_edges(&self, node: &NodeID, load_labels: bool) -> Vec<Edge> {
- self.executor
+ let mut edges: Vec<_> = self
+ .executor
.map(
|segment| segment.ingoing_edges(node, load_labels),
self.segments.iter(),
@@ -518,7 +519,12 @@ impl Webgraph {
.unwrap()
.into_iter()
.flatten()
- .collect()
+ .collect();
+
+ edges.sort_unstable_by_key(|edge| edge.from);
+ edges.dedup_by_key(|edge| edge.from);
+
+ edges
}
pub fn raw_outgoing_edges(&self, node: &NodeID) -> Vec<Edge> {
@@ -526,7 +532,8 @@ impl Webgraph {
}
fn inner_outgoing_edges(&self, node: &NodeID, load_labels: bool) -> Vec<Edge> {
- self.executor
+ let mut edges: Vec<_> = self
+ .executor
.map(
|segment| segment.outgoing_edges(node, load_labels),
self.segments.iter(),
@@ -534,7 +541,12 @@ impl Webgraph {
.unwrap()
.into_iter()
.flatten()
- .collect()
+ .collect();
+
+ edges.sort_unstable_by_key(|edge| edge.to);
+ edges.dedup_by_key(|edge| edge.to);
+
+ edges
}
pub fn outgoing_edges(&self, node: Node) -> Vec<FullEdge> {
@@ -560,6 +572,9 @@ impl Webgraph {
self.id2node.iter().map(|(id, node)| (node, id))
}
+ /// Iterate all edges in the graph at least once.
+ /// Some edges may be returned multiple times.
+ /// This happens if they are present in more than one segment.
pub fn edges(&self) -> impl Iterator<Item = Edge> + '_ {
self.segments.iter().flat_map(|segment| segment.edges())
}