Skip to content

web search done right

fix webgraph merge bug of missing edges

Mikkel Denkerfeafa7507a13
diff --git a/crates/core/src/webgraph/edge.rs b/crates/core/src/webgraph/edge.rs
index f1ef91af..4cfcf67f 100644
--- a/crates/core/src/webgraph/edge.rs
+++ b/crates/core/src/webgraph/edge.rs
@@ -170,6 +170,7 @@ where
}
}
+#[derive(Clone)]
pub struct StoredEdge<L = ()> {
pub other: NodeDatum,
pub rel: RelFlags,
diff --git a/crates/core/src/webgraph/merge.rs b/crates/core/src/webgraph/merge.rs
index ef1cc91e..6dad73e5 100644
--- a/crates/core/src/webgraph/merge.rs
+++ b/crates/core/src/webgraph/merge.rs
@@ -17,6 +17,7 @@
use std::{cmp::Reverse, collections::BinaryHeap};
use super::{store::EdgeRange, NodeID, StoredEdge};
+use std::hash::Hash;
pub struct MergeNode<O = ()> {
node: NodeID,
@@ -178,12 +179,14 @@ impl<'a, L> Iterator for EdgeMerger<'a, L> {
if let Some(edge) = &res {
while let Some(mut peek) = self.iters.peek_mut() {
- if peek.0.peek().map(|x| x.other.sort_key) == Some(edge.other.sort_key) {
+ if peek.0.peek().map(|x| x.other.id) == Some(edge.other.id) {
peek.0.next().unwrap();
} else {
break;
}
}
+ } else {
+ debug_assert!(self.iters.iter().all(|p| p.0.peek().is_none()));
}
res
@@ -218,6 +221,13 @@ impl PartialEq for NodeDatum {
impl Eq for NodeDatum {}
+impl Hash for NodeDatum {
+ fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
+ self.id.hash(state);
+ self.sort_key.hash(state);
+ }
+}
+
impl NodeDatum {
pub fn new(node: NodeID, sort_key: u64) -> Self {
Self { id: node, sort_key }
@@ -256,6 +266,12 @@ impl<L> PartialEq for StoredEdge<L> {
impl<L> Eq for StoredEdge<L> {}
+impl<L> Hash for StoredEdge<L> {
+ fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
+ self.other.hash(state);
+ }
+}
+
#[cfg(test)]
mod tests {
use crate::webpage::html::links::RelFlags;
@@ -303,6 +319,38 @@ mod tests {
assert!(!merger.advance(&mut buf));
}
+ #[test]
+ fn test_merge_nodes_unequal_len() {
+ let a = vec![
+ MergeNode::new(1u64.into(), EdgeRange::new(0..10, 1), 0..10),
+ MergeNode::new(4u64.into(), EdgeRange::new(0..10, 2), 0..10),
+ MergeNode::new(5u64.into(), EdgeRange::new(0..10, 3), 0..10),
+ ];
+
+ let b = vec![MergeNode::new(2u64.into(), EdgeRange::new(0..10, 4), 0..10)];
+
+ let mut merger = MergeIter::new(vec![a.into_iter(), b.into_iter()]);
+ let mut buf = Vec::new();
+
+ assert!(merger.advance(&mut buf));
+ assert_eq!(buf.len(), 1);
+ assert_eq!(buf[0].id(), 1u64.into());
+
+ assert!(merger.advance(&mut buf));
+ assert_eq!(buf.len(), 1);
+ assert_eq!(buf[0].id(), 2u64.into());
+
+ assert!(merger.advance(&mut buf));
+ assert_eq!(buf.len(), 1);
+ assert_eq!(buf[0].id(), 4u64.into());
+
+ assert!(merger.advance(&mut buf));
+ assert_eq!(buf.len(), 1);
+ assert_eq!(buf[0].id(), 5u64.into());
+
+ assert!(!merger.advance(&mut buf));
+ }
+
#[test]
fn test_datum_merge() {
let a = vec![
@@ -326,4 +374,26 @@ mod tests {
assert_eq!(merger.next().unwrap().other.sort_key(), 5);
assert!(merger.next().is_none());
}
+
+ #[test]
+ fn test_datum_merge_unequal_len() {
+ let a = vec![
+ StoredEdge::new(NodeDatum::new(1u64.into(), 1), RelFlags::default()),
+ StoredEdge::new(NodeDatum::new(2u64.into(), 4), RelFlags::default()),
+ StoredEdge::new(NodeDatum::new(3u64.into(), 5), RelFlags::default()),
+ ];
+
+ let b = vec![StoredEdge::new(
+ NodeDatum::new(4u64.into(), 2),
+ RelFlags::default(),
+ )];
+
+ let mut merger = EdgeMerger::new(vec![a.into_iter(), b.into_iter()]);
+
+ assert_eq!(merger.next().unwrap().other.sort_key(), 1);
+ assert_eq!(merger.next().unwrap().other.sort_key(), 2);
+ assert_eq!(merger.next().unwrap().other.sort_key(), 4);
+ assert_eq!(merger.next().unwrap().other.sort_key(), 5);
+ assert!(merger.next().is_none());
+ }
}
diff --git a/crates/core/src/webgraph/mod.rs b/crates/core/src/webgraph/mod.rs
index 912685d8..120f5a0e 100644
--- a/crates/core/src/webgraph/mod.rs
+++ b/crates/core/src/webgraph/mod.rs
@@ -442,12 +442,12 @@ pub mod tests {
}
pub fn test_graph() -> Webgraph {
- // ┌------┐
- // │ │
+ // ┌-----┐
+ // │ │
// ┌───A◄─┐ │
- // │ │ │
+ // │ │ │
// ▼ │ │
- // B─────►C◄┘
+ // B─────►C◄-┘
// ▲
// │
// │
@@ -505,7 +505,7 @@ pub mod tests {
}
#[test]
- fn merge() {
+ fn merge_path() {
let mut graphs = Vec::new();
for (from, to, label) in &[
(Node::from("A"), Node::from("B"), String::new()),
@@ -532,6 +532,7 @@ pub mod tests {
graph.merge(other).unwrap();
}
+ graph.merge_all_segments(Compression::default()).unwrap();
graph.optimize_read();
assert_eq!(
@@ -540,6 +541,75 @@ pub mod tests {
);
}
+ #[test]
+ fn merge_simple() {
+ let mut graphs = Vec::new();
+ for (from, to, label) in test_edges() {
+ let mut wrt = WebgraphWriter::new(
+ crate::gen_temp_path(),
+ Executor::single_thread(),
+ Compression::default(),
+ None,
+ );
+ wrt.insert(from.clone(), to.clone(), label.clone(), RelFlags::default());
+ graphs.push(wrt.finalize());
+ }
+
+ let mut graph = graphs.pop().unwrap();
+
+ for other in graphs {
+ graph.merge(other).unwrap();
+ }
+
+ graph.merge_all_segments(Compression::default()).unwrap();
+ graph.optimize_read();
+
+ let mut res = graph.outgoing_edges(Node::from("A"), EdgeLimit::Unlimited);
+ res.sort_by(|a, b| a.to.cmp(&b.to));
+ assert_eq!(res.len(), 2);
+
+ assert_eq!(res[0].to, Node::from("B"));
+ assert_eq!(res[1].to, Node::from("C"));
+
+ let mut res = graph.outgoing_edges(Node::from("B"), EdgeLimit::Unlimited);
+ res.sort_by(|a, b| a.to.cmp(&b.to));
+
+ assert_eq!(res.len(), 1);
+ assert_eq!(res[0].to, Node::from("C"));
+
+ let mut res = graph.outgoing_edges(Node::from("C"), EdgeLimit::Unlimited);
+ res.sort_by(|a, b| a.to.cmp(&b.to));
+
+ assert_eq!(res.len(), 1);
+ assert_eq!(res[0].to, Node::from("A"));
+
+ let mut res = graph.outgoing_edges(Node::from("D"), EdgeLimit::Unlimited);
+ res.sort_by(|a, b| a.to.cmp(&b.to));
+
+ assert_eq!(res.len(), 1);
+ assert_eq!(res[0].to, Node::from("C"));
+
+ let mut res = graph.ingoing_edges(Node::from("A"), EdgeLimit::Unlimited);
+ res.sort_by(|a, b| a.from.cmp(&b.from));
+
+ assert_eq!(res.len(), 1);
+ assert_eq!(res[0].from, Node::from("C"));
+
+ let mut res = graph.ingoing_edges(Node::from("B"), EdgeLimit::Unlimited);
+ res.sort_by(|a, b| a.from.cmp(&b.from));
+
+ assert_eq!(res.len(), 1);
+ assert_eq!(res[0].from, Node::from("A"));
+
+ let mut res = graph.ingoing_edges(Node::from("C"), EdgeLimit::Unlimited);
+ res.sort_by(|a, b| a.from.cmp(&b.from));
+
+ assert_eq!(res.len(), 3);
+ assert_eq!(res[0].from, Node::from("A"));
+ assert_eq!(res[1].from, Node::from("B"));
+ assert_eq!(res[2].from, Node::from("D"));
+ }
+
#[test]
fn merge_cycle() {
let mut graphs = Vec::new();
@@ -576,6 +646,97 @@ pub mod tests {
graph.distances(Node::from("A")).get(&Node::from("C")),
Some(&2)
);
+
+ assert_eq!(
+ graph
+ .outgoing_edges(Node::from("A"), EdgeLimit::Unlimited)
+ .len(),
+ 1
+ );
+ assert_eq!(
+ graph
+ .outgoing_edges(Node::from("B"), EdgeLimit::Unlimited)
+ .len(),
+ 1
+ );
+ assert_eq!(
+ graph
+ .outgoing_edges(Node::from("C"), EdgeLimit::Unlimited)
+ .len(),
+ 1
+ );
+ }
+
+ #[test]
+ fn merge_star() {
+ let mut graphs = Vec::new();
+ for (from, to, label) in &[
+ (Node::from("A"), Node::from("B"), String::new()),
+ (Node::from("A"), Node::from("C"), String::new()),
+ (Node::from("A"), Node::from("D"), String::new()),
+ (Node::from("A"), Node::from("E"), String::new()),
+ ] {
+ let mut wrt = WebgraphWriter::new(
+ crate::gen_temp_path(),
+ Executor::single_thread(),
+ Compression::default(),
+ None,
+ );
+ wrt.insert(from.clone(), to.clone(), label.clone(), RelFlags::default());
+ graphs.push(wrt.finalize());
+ }
+
+ let mut graph = graphs.pop().unwrap();
+
+ for other in graphs {
+ graph.merge(other).unwrap();
+ }
+
+ graph.merge_all_segments(Compression::default()).unwrap();
+ graph.optimize_read();
+
+ assert_eq!(
+ graph
+ .outgoing_edges(Node::from("A"), EdgeLimit::Unlimited)
+ .len(),
+ 4
+ );
+ }
+
+ #[test]
+ fn merge_reverse_star() {
+ let mut graphs = Vec::new();
+ for (from, to, label) in &[
+ (Node::from("B"), Node::from("A"), String::new()),
+ (Node::from("C"), Node::from("A"), String::new()),
+ (Node::from("D"), Node::from("A"), String::new()),
+ (Node::from("E"), Node::from("A"), String::new()),
+ ] {
+ let mut wrt = WebgraphWriter::new(
+ crate::gen_temp_path(),
+ Executor::single_thread(),
+ Compression::default(),
+ None,
+ );
+ wrt.insert(from.clone(), to.clone(), label.clone(), RelFlags::default());
+ graphs.push(wrt.finalize());
+ }
+
+ let mut graph = graphs.pop().unwrap();
+
+ for other in graphs {
+ graph.merge(other).unwrap();
+ }
+
+ graph.merge_all_segments(Compression::default()).unwrap();
+ graph.optimize_read();
+
+ assert_eq!(
+ graph
+ .ingoing_edges(Node::from("A"), EdgeLimit::Unlimited)
+ .len(),
+ 4
+ );
}
#[test]
diff --git a/crates/core/src/webgraph/store.rs b/crates/core/src/webgraph/store.rs
index 8f3148da..95ec3af5 100644
--- a/crates/core/src/webgraph/store.rs
+++ b/crates/core/src/webgraph/store.rs
@@ -340,16 +340,16 @@ impl EdgeStore {
for node in buf {
let store = &stores[node.ord().as_usize()];
- let edge_nodes = store.edges.slice(usize_range(node.range().range.clone()));
- let edge_labels = store
+ let stored_edges = store.edges.slice(usize_range(node.range().range.clone()));
+ let labels = store
.edge_labels
.slice(usize_range(node.labels()))
.map(|r| r.decompress())
.flat_map(|block| block.labels.into_iter());
edges.push(
- edge_nodes
- .zip_eq(edge_labels)
+ stored_edges
+ .zip_eq(labels)
.map(|(edge, label)| edge.with_label(label)),
);
}
diff --git a/crates/core/src/webgraph/store_writer.rs b/crates/core/src/webgraph/store_writer.rs
index 116381a5..cb8bdd6b 100644
--- a/crates/core/src/webgraph/store_writer.rs
+++ b/crates/core/src/webgraph/store_writer.rs
@@ -222,8 +222,8 @@ struct FinalEdgeStoreWriter {
ranges: RangesDb,
hosts: HostDb,
- edge_labels: IterableStoreWriter<CompressedLabelBlock, File>,
- edge_nodes: ConstIterableStoreWriter<StoredEdge, File>,
+ labels: IterableStoreWriter<CompressedLabelBlock, File>,
+ edges: ConstIterableStoreWriter<StoredEdge, File>,
host_centrality_rank_store: Option<Arc<speedy_kv::Db<NodeID, u64>>>,
@@ -242,29 +242,29 @@ impl FinalEdgeStoreWriter {
) -> Self {
let ranges = RangesDb::open(path.as_ref().join("ranges"));
- let edge_labels_file = File::options()
+ let labels_file = File::options()
.read(true)
.create(true)
.truncate(false)
.write(true)
.open(path.as_ref().join("labels"))
.unwrap();
- let edge_labels = IterableStoreWriter::new(edge_labels_file);
+ let labels = IterableStoreWriter::new(labels_file);
- let edge_nodes_file = File::options()
+ let edges_file = File::options()
.read(true)
.create(true)
.truncate(false)
.write(true)
.open(path.as_ref().join("edges"))
.unwrap();
- let edge_nodes = ConstIterableStoreWriter::new(edge_nodes_file);
+ let edges = ConstIterableStoreWriter::new(edges_file);
Self {
ranges,
hosts: HostDb::open(path.as_ref().join("hosts")),
- edge_labels,
- edge_nodes,
+ labels,
+ edges,
reversed,
compression,
path: path.as_ref().to_path_buf(),
@@ -304,7 +304,7 @@ impl FinalEdgeStoreWriter {
debug_assert!(self.ranges.labels_get_raw(&node_bytes).is_none());
let mut edge_labels = Vec::new();
- let mut edge_nodes: Vec<StoredEdge> = Vec::new();
+ let mut stored_edges: Vec<StoredEdge> = Vec::new();
for edge in edges {
edge_labels.push(edge.label.clone());
@@ -323,7 +323,7 @@ impl FinalEdgeStoreWriter {
let datum = NodeDatum::new(node, sort_key);
let rel = edge.rel;
- edge_nodes.push(StoredEdge::new(datum, rel));
+ stored_edges.push(StoredEdge::new(datum, rel));
}
let edge_labels: Vec<_> = edge_labels
@@ -339,7 +339,7 @@ impl FinalEdgeStoreWriter {
let mut last_node_offset = None;
for block in &edge_labels {
- let offset = self.edge_labels.write(block).unwrap();
+ let offset = self.labels.write(block).unwrap();
if first_label_offset.is_none() {
first_label_offset = Some(offset);
@@ -348,8 +348,8 @@ impl FinalEdgeStoreWriter {
last_label_offset = Some(offset);
}
- for node in &edge_nodes {
- let offset = self.edge_nodes.write(node).unwrap();
+ for node in &stored_edges {
+ let offset = self.edges.write(node).unwrap();
if first_node_offset.is_none() {
first_node_offset = Some(offset);
@@ -446,7 +446,7 @@ impl FinalEdgeStoreWriter {
self.ranges.commit();
- self.edge_nodes.flush().unwrap();
- self.edge_labels.flush().unwrap();
+ self.edges.flush().unwrap();
+ self.labels.flush().unwrap();
}
}