web search done right
fix webgraph merge bug of missing edges
Mikkel Denkerfeafa7507a13
diff --git a/crates/core/src/webgraph/edge.rs b/crates/core/src/webgraph/edge.rsindex f1ef91af..4cfcf67f 100644--- a/crates/core/src/webgraph/edge.rs+++ b/crates/core/src/webgraph/edge.rs@@ -170,6 +170,7 @@ where}}+#[derive(Clone)]pub struct StoredEdge<L = ()> {pub other: NodeDatum,pub rel: RelFlags,diff --git a/crates/core/src/webgraph/merge.rs b/crates/core/src/webgraph/merge.rsindex ef1cc91e..6dad73e5 100644--- a/crates/core/src/webgraph/merge.rs+++ b/crates/core/src/webgraph/merge.rs@@ -17,6 +17,7 @@use std::{cmp::Reverse, collections::BinaryHeap};use super::{store::EdgeRange, NodeID, StoredEdge};+use std::hash::Hash;pub struct MergeNode<O = ()> {node: NodeID,@@ -178,12 +179,14 @@ impl<'a, L> Iterator for EdgeMerger<'a, L> {if let Some(edge) = &res {while let Some(mut peek) = self.iters.peek_mut() {- if peek.0.peek().map(|x| x.other.sort_key) == Some(edge.other.sort_key) {+ if peek.0.peek().map(|x| x.other.id) == Some(edge.other.id) {peek.0.next().unwrap();} else {break;}}+ } else {+ debug_assert!(self.iters.iter().all(|p| p.0.peek().is_none()));}res@@ -218,6 +221,13 @@ impl PartialEq for NodeDatum {impl Eq for NodeDatum {}+impl Hash for NodeDatum {+ fn hash<H: std::hash::Hasher>(&self, state: &mut H) {+ self.id.hash(state);+ self.sort_key.hash(state);+ }+}+impl NodeDatum {pub fn new(node: NodeID, sort_key: u64) -> Self {Self { id: node, sort_key }@@ -256,6 +266,12 @@ impl<L> PartialEq for StoredEdge<L> {impl<L> Eq for StoredEdge<L> {}+impl<L> Hash for StoredEdge<L> {+ fn hash<H: std::hash::Hasher>(&self, state: &mut H) {+ self.other.hash(state);+ }+}+#[cfg(test)]mod tests {use crate::webpage::html::links::RelFlags;@@ -303,6 +319,38 @@ mod tests {assert!(!merger.advance(&mut buf));}+ #[test]+ fn test_merge_nodes_unequal_len() {+ let a = vec![+ MergeNode::new(1u64.into(), EdgeRange::new(0..10, 1), 0..10),+ MergeNode::new(4u64.into(), EdgeRange::new(0..10, 2), 0..10),+ MergeNode::new(5u64.into(), EdgeRange::new(0..10, 3), 0..10),+ ];++ let b = vec![MergeNode::new(2u64.into(), EdgeRange::new(0..10, 4), 0..10)];++ let mut merger = MergeIter::new(vec![a.into_iter(), b.into_iter()]);+ let mut buf = Vec::new();++ assert!(merger.advance(&mut buf));+ assert_eq!(buf.len(), 1);+ assert_eq!(buf[0].id(), 1u64.into());++ assert!(merger.advance(&mut buf));+ assert_eq!(buf.len(), 1);+ assert_eq!(buf[0].id(), 2u64.into());++ assert!(merger.advance(&mut buf));+ assert_eq!(buf.len(), 1);+ assert_eq!(buf[0].id(), 4u64.into());++ assert!(merger.advance(&mut buf));+ assert_eq!(buf.len(), 1);+ assert_eq!(buf[0].id(), 5u64.into());++ assert!(!merger.advance(&mut buf));+ }+#[test]fn test_datum_merge() {let a = vec![@@ -326,4 +374,26 @@ mod tests {assert_eq!(merger.next().unwrap().other.sort_key(), 5);assert!(merger.next().is_none());}++ #[test]+ fn test_datum_merge_unequal_len() {+ let a = vec![+ StoredEdge::new(NodeDatum::new(1u64.into(), 1), RelFlags::default()),+ StoredEdge::new(NodeDatum::new(2u64.into(), 4), RelFlags::default()),+ StoredEdge::new(NodeDatum::new(3u64.into(), 5), RelFlags::default()),+ ];++ let b = vec![StoredEdge::new(+ NodeDatum::new(4u64.into(), 2),+ RelFlags::default(),+ )];++ let mut merger = EdgeMerger::new(vec![a.into_iter(), b.into_iter()]);++ assert_eq!(merger.next().unwrap().other.sort_key(), 1);+ assert_eq!(merger.next().unwrap().other.sort_key(), 2);+ assert_eq!(merger.next().unwrap().other.sort_key(), 4);+ assert_eq!(merger.next().unwrap().other.sort_key(), 5);+ assert!(merger.next().is_none());+ }}diff --git a/crates/core/src/webgraph/mod.rs b/crates/core/src/webgraph/mod.rsindex 912685d8..120f5a0e 100644--- a/crates/core/src/webgraph/mod.rs+++ b/crates/core/src/webgraph/mod.rs@@ -442,12 +442,12 @@ pub mod tests {}pub fn test_graph() -> Webgraph {- // ┌------┐- // │ │+ // ┌-----┐+ // │ │// ┌───A◄─┐ │- // │ │ │+ // │ │ │// ▼ │ │- // B─────►C◄┘+ // B─────►C◄-┘// ▲// │// │@@ -505,7 +505,7 @@ pub mod tests {}#[test]- fn merge() {+ fn merge_path() {let mut graphs = Vec::new();for (from, to, label) in &[(Node::from("A"), Node::from("B"), String::new()),@@ -532,6 +532,7 @@ pub mod tests {graph.merge(other).unwrap();}+ graph.merge_all_segments(Compression::default()).unwrap();graph.optimize_read();assert_eq!(@@ -540,6 +541,75 @@ pub mod tests {);}+ #[test]+ fn merge_simple() {+ let mut graphs = Vec::new();+ for (from, to, label) in test_edges() {+ let mut wrt = WebgraphWriter::new(+ crate::gen_temp_path(),+ Executor::single_thread(),+ Compression::default(),+ None,+ );+ wrt.insert(from.clone(), to.clone(), label.clone(), RelFlags::default());+ graphs.push(wrt.finalize());+ }++ let mut graph = graphs.pop().unwrap();++ for other in graphs {+ graph.merge(other).unwrap();+ }++ graph.merge_all_segments(Compression::default()).unwrap();+ graph.optimize_read();++ let mut res = graph.outgoing_edges(Node::from("A"), EdgeLimit::Unlimited);+ res.sort_by(|a, b| a.to.cmp(&b.to));+ assert_eq!(res.len(), 2);++ assert_eq!(res[0].to, Node::from("B"));+ assert_eq!(res[1].to, Node::from("C"));++ let mut res = graph.outgoing_edges(Node::from("B"), EdgeLimit::Unlimited);+ res.sort_by(|a, b| a.to.cmp(&b.to));++ assert_eq!(res.len(), 1);+ assert_eq!(res[0].to, Node::from("C"));++ let mut res = graph.outgoing_edges(Node::from("C"), EdgeLimit::Unlimited);+ res.sort_by(|a, b| a.to.cmp(&b.to));++ assert_eq!(res.len(), 1);+ assert_eq!(res[0].to, Node::from("A"));++ let mut res = graph.outgoing_edges(Node::from("D"), EdgeLimit::Unlimited);+ res.sort_by(|a, b| a.to.cmp(&b.to));++ assert_eq!(res.len(), 1);+ assert_eq!(res[0].to, Node::from("C"));++ let mut res = graph.ingoing_edges(Node::from("A"), EdgeLimit::Unlimited);+ res.sort_by(|a, b| a.from.cmp(&b.from));++ assert_eq!(res.len(), 1);+ assert_eq!(res[0].from, Node::from("C"));++ let mut res = graph.ingoing_edges(Node::from("B"), EdgeLimit::Unlimited);+ res.sort_by(|a, b| a.from.cmp(&b.from));++ assert_eq!(res.len(), 1);+ assert_eq!(res[0].from, Node::from("A"));++ let mut res = graph.ingoing_edges(Node::from("C"), EdgeLimit::Unlimited);+ res.sort_by(|a, b| a.from.cmp(&b.from));++ assert_eq!(res.len(), 3);+ assert_eq!(res[0].from, Node::from("A"));+ assert_eq!(res[1].from, Node::from("B"));+ assert_eq!(res[2].from, Node::from("D"));+ }+#[test]fn merge_cycle() {let mut graphs = Vec::new();@@ -576,6 +646,97 @@ pub mod tests {graph.distances(Node::from("A")).get(&Node::from("C")),Some(&2));++ assert_eq!(+ graph+ .outgoing_edges(Node::from("A"), EdgeLimit::Unlimited)+ .len(),+ 1+ );+ assert_eq!(+ graph+ .outgoing_edges(Node::from("B"), EdgeLimit::Unlimited)+ .len(),+ 1+ );+ assert_eq!(+ graph+ .outgoing_edges(Node::from("C"), EdgeLimit::Unlimited)+ .len(),+ 1+ );+ }++ #[test]+ fn merge_star() {+ let mut graphs = Vec::new();+ for (from, to, label) in &[+ (Node::from("A"), Node::from("B"), String::new()),+ (Node::from("A"), Node::from("C"), String::new()),+ (Node::from("A"), Node::from("D"), String::new()),+ (Node::from("A"), Node::from("E"), String::new()),+ ] {+ let mut wrt = WebgraphWriter::new(+ crate::gen_temp_path(),+ Executor::single_thread(),+ Compression::default(),+ None,+ );+ wrt.insert(from.clone(), to.clone(), label.clone(), RelFlags::default());+ graphs.push(wrt.finalize());+ }++ let mut graph = graphs.pop().unwrap();++ for other in graphs {+ graph.merge(other).unwrap();+ }++ graph.merge_all_segments(Compression::default()).unwrap();+ graph.optimize_read();++ assert_eq!(+ graph+ .outgoing_edges(Node::from("A"), EdgeLimit::Unlimited)+ .len(),+ 4+ );+ }++ #[test]+ fn merge_reverse_star() {+ let mut graphs = Vec::new();+ for (from, to, label) in &[+ (Node::from("B"), Node::from("A"), String::new()),+ (Node::from("C"), Node::from("A"), String::new()),+ (Node::from("D"), Node::from("A"), String::new()),+ (Node::from("E"), Node::from("A"), String::new()),+ ] {+ let mut wrt = WebgraphWriter::new(+ crate::gen_temp_path(),+ Executor::single_thread(),+ Compression::default(),+ None,+ );+ wrt.insert(from.clone(), to.clone(), label.clone(), RelFlags::default());+ graphs.push(wrt.finalize());+ }++ let mut graph = graphs.pop().unwrap();++ for other in graphs {+ graph.merge(other).unwrap();+ }++ graph.merge_all_segments(Compression::default()).unwrap();+ graph.optimize_read();++ assert_eq!(+ graph+ .ingoing_edges(Node::from("A"), EdgeLimit::Unlimited)+ .len(),+ 4+ );}#[test]diff --git a/crates/core/src/webgraph/store.rs b/crates/core/src/webgraph/store.rsindex 8f3148da..95ec3af5 100644--- a/crates/core/src/webgraph/store.rs+++ b/crates/core/src/webgraph/store.rs@@ -340,16 +340,16 @@ impl EdgeStore {for node in buf {let store = &stores[node.ord().as_usize()];- let edge_nodes = store.edges.slice(usize_range(node.range().range.clone()));- let edge_labels = store+ let stored_edges = store.edges.slice(usize_range(node.range().range.clone()));+ let labels = store.edge_labels.slice(usize_range(node.labels())).map(|r| r.decompress()).flat_map(|block| block.labels.into_iter());edges.push(- edge_nodes- .zip_eq(edge_labels)+ stored_edges+ .zip_eq(labels).map(|(edge, label)| edge.with_label(label)),);}diff --git a/crates/core/src/webgraph/store_writer.rs b/crates/core/src/webgraph/store_writer.rsindex 116381a5..cb8bdd6b 100644--- a/crates/core/src/webgraph/store_writer.rs+++ b/crates/core/src/webgraph/store_writer.rs@@ -222,8 +222,8 @@ struct FinalEdgeStoreWriter {ranges: RangesDb,hosts: HostDb,- edge_labels: IterableStoreWriter<CompressedLabelBlock, File>,- edge_nodes: ConstIterableStoreWriter<StoredEdge, File>,+ labels: IterableStoreWriter<CompressedLabelBlock, File>,+ edges: ConstIterableStoreWriter<StoredEdge, File>,host_centrality_rank_store: Option<Arc<speedy_kv::Db<NodeID, u64>>>,@@ -242,29 +242,29 @@ impl FinalEdgeStoreWriter {) -> Self {let ranges = RangesDb::open(path.as_ref().join("ranges"));- let edge_labels_file = File::options()+ let labels_file = File::options().read(true).create(true).truncate(false).write(true).open(path.as_ref().join("labels")).unwrap();- let edge_labels = IterableStoreWriter::new(edge_labels_file);+ let labels = IterableStoreWriter::new(labels_file);- let edge_nodes_file = File::options()+ let edges_file = File::options().read(true).create(true).truncate(false).write(true).open(path.as_ref().join("edges")).unwrap();- let edge_nodes = ConstIterableStoreWriter::new(edge_nodes_file);+ let edges = ConstIterableStoreWriter::new(edges_file);Self {ranges,hosts: HostDb::open(path.as_ref().join("hosts")),- edge_labels,- edge_nodes,+ labels,+ edges,reversed,compression,path: path.as_ref().to_path_buf(),@@ -304,7 +304,7 @@ impl FinalEdgeStoreWriter {debug_assert!(self.ranges.labels_get_raw(&node_bytes).is_none());let mut edge_labels = Vec::new();- let mut edge_nodes: Vec<StoredEdge> = Vec::new();+ let mut stored_edges: Vec<StoredEdge> = Vec::new();for edge in edges {edge_labels.push(edge.label.clone());@@ -323,7 +323,7 @@ impl FinalEdgeStoreWriter {let datum = NodeDatum::new(node, sort_key);let rel = edge.rel;- edge_nodes.push(StoredEdge::new(datum, rel));+ stored_edges.push(StoredEdge::new(datum, rel));}let edge_labels: Vec<_> = edge_labels@@ -339,7 +339,7 @@ impl FinalEdgeStoreWriter {let mut last_node_offset = None;for block in &edge_labels {- let offset = self.edge_labels.write(block).unwrap();+ let offset = self.labels.write(block).unwrap();if first_label_offset.is_none() {first_label_offset = Some(offset);@@ -348,8 +348,8 @@ impl FinalEdgeStoreWriter {last_label_offset = Some(offset);}- for node in &edge_nodes {- let offset = self.edge_nodes.write(node).unwrap();+ for node in &stored_edges {+ let offset = self.edges.write(node).unwrap();if first_node_offset.is_none() {first_node_offset = Some(offset);@@ -446,7 +446,7 @@ impl FinalEdgeStoreWriter {self.ranges.commit();- self.edge_nodes.flush().unwrap();- self.edge_labels.flush().unwrap();+ self.edges.flush().unwrap();+ self.labels.flush().unwrap();}}