web search done right
Deduplicate webgraph edges during lookup. Some edges may be duplicate if they are present in multiple segments.
Mikkel Denker445043b1d25d
diff --git a/core/src/webgraph/centrality/harmonic.rs b/core/src/webgraph/centrality/harmonic.rsindex b9cc6d5e..3c961830 100644--- a/core/src/webgraph/centrality/harmonic.rs+++ b/core/src/webgraph/centrality/harmonic.rs@@ -372,13 +372,18 @@ mod tests {let centrality = HarmonicCentrality::calculate(&graph);graph.insert(Node::from("A"), Node::from("B"), "1".to_string());+ graph.commit();graph.insert(Node::from("A"), Node::from("B"), "2".to_string());+ graph.commit();graph.insert(Node::from("A"), Node::from("B"), "3".to_string());+ graph.commit();graph.insert(Node::from("A"), Node::from("B"), "4".to_string());+ graph.commit();graph.insert(Node::from("A"), Node::from("B"), "5".to_string());+ graph.commit();graph.insert(Node::from("A"), Node::from("B"), "6".to_string());+ graph.commit();graph.insert(Node::from("A"), Node::from("B"), "7".to_string());-graph.commit();let centrality_extra = HarmonicCentrality::calculate(&graph);diff --git a/core/src/webgraph/mod.rs b/core/src/webgraph/mod.rsindex 2e0688f9..fb6b02c5 100644--- a/core/src/webgraph/mod.rs+++ b/core/src/webgraph/mod.rs@@ -510,7 +510,8 @@ impl Webgraph {}fn inner_ingoing_edges(&self, node: &NodeID, load_labels: bool) -> Vec<Edge> {- self.executor+ let mut edges: Vec<_> = self+ .executor.map(|segment| segment.ingoing_edges(node, load_labels),self.segments.iter(),@@ -518,7 +519,12 @@ impl Webgraph {.unwrap().into_iter().flatten()- .collect()+ .collect();++ edges.sort_unstable_by_key(|edge| edge.from);+ edges.dedup_by_key(|edge| edge.from);++ edges}pub fn raw_outgoing_edges(&self, node: &NodeID) -> Vec<Edge> {@@ -526,7 +532,8 @@ impl Webgraph {}fn inner_outgoing_edges(&self, node: &NodeID, load_labels: bool) -> Vec<Edge> {- self.executor+ let mut edges: Vec<_> = self+ .executor.map(|segment| segment.outgoing_edges(node, load_labels),self.segments.iter(),@@ -534,7 +541,12 @@ impl Webgraph {.unwrap().into_iter().flatten()- .collect()+ .collect();++ edges.sort_unstable_by_key(|edge| edge.to);+ edges.dedup_by_key(|edge| edge.to);++ edges}pub fn outgoing_edges(&self, node: Node) -> Vec<FullEdge> {@@ -560,6 +572,9 @@ impl Webgraph {self.id2node.iter().map(|(id, node)| (node, id))}+ /// Iterate all edges in the graph at least once.+ /// Some edges may be returned multiple times.+ /// This happens if they are present in more than one segment.pub fn edges(&self) -> impl Iterator<Item = Edge> + '_ {self.segments.iter().flat_map(|segment| segment.edges())}