Skip to content

web search done right

upper bounds for in degree and out degree in webgraph

Mikkel Denkerb16043629781
diff --git a/crates/core/src/entrypoint/webgraph_server.rs b/crates/core/src/entrypoint/webgraph_server.rs
index fb374436..ae21fb04 100644
--- a/crates/core/src/entrypoint/webgraph_server.rs
+++ b/crates/core/src/entrypoint/webgraph_server.rs
@@ -180,6 +180,32 @@ impl Message<WebGraphService> for PagesByHosts {
}
}
+#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]
+pub struct InDegreeUpperBound {
+ pub node: NodeID,
+}
+
+impl Message<WebGraphService> for InDegreeUpperBound {
+ type Response = u64;
+
+ async fn handle(self, server: &WebGraphService) -> Self::Response {
+ server.graph.in_degree_upper_bound(&self.node)
+ }
+}
+
+#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]
+pub struct OutDegreeUpperBound {
+ pub node: NodeID,
+}
+
+impl Message<WebGraphService> for OutDegreeUpperBound {
+ type Response = u64;
+
+ async fn handle(self, server: &WebGraphService) -> Self::Response {
+ server.graph.out_degree_upper_bound(&self.node)
+ }
+}
+
pub async fn run(config: config::WebgraphServerConfig) -> Result<()> {
let addr: SocketAddr = config.host;
diff --git a/crates/core/src/webgraph/mod.rs b/crates/core/src/webgraph/mod.rs
index b56bcb1e..70a8695a 100644
--- a/crates/core/src/webgraph/mod.rs
+++ b/crates/core/src/webgraph/mod.rs
@@ -325,6 +325,20 @@ impl Webgraph {
.collect()
}
+ pub fn out_degree_upper_bound(&self, node: &NodeID) -> u64 {
+ self.segments
+ .iter()
+ .map(|segment| segment.out_degree(node))
+ .sum()
+ }
+
+ pub fn in_degree_upper_bound(&self, node: &NodeID) -> u64 {
+ self.segments
+ .iter()
+ .map(|segment| segment.in_degree(node))
+ .sum()
+ }
+
pub fn outgoing_edges(&self, node: Node, limit: EdgeLimit) -> Vec<FullEdge> {
let dedup = |edges: &mut Vec<SegmentEdge<String>>| {
edges.sort_by_key(|e| e.to.node());
diff --git a/crates/core/src/webgraph/segment.rs b/crates/core/src/webgraph/segment.rs
index d65f7ba7..614b526f 100644
--- a/crates/core/src/webgraph/segment.rs
+++ b/crates/core/src/webgraph/segment.rs
@@ -172,6 +172,10 @@ impl Segment {
self.adjacency.get_without_label(node, limit)
}
+ pub fn out_degree(&self, node: &NodeID) -> u64 {
+ self.adjacency.degree(node)
+ }
+
pub fn ingoing_edges_with_label(
&self,
node: &NodeID,
@@ -184,6 +188,10 @@ impl Segment {
self.reversed_adjacency.get_without_label(node, limit)
}
+ pub fn in_degree(&self, node: &NodeID) -> u64 {
+ self.reversed_adjacency.degree(node)
+ }
+
pub fn pages_by_host(&self, host_node: &NodeID) -> Vec<NodeID> {
self.reversed_adjacency.nodes_by_host(host_node)
}
diff --git a/crates/core/src/webgraph/store.rs b/crates/core/src/webgraph/store.rs
index 21ecd574..e594b600 100644
--- a/crates/core/src/webgraph/store.rs
+++ b/crates/core/src/webgraph/store.rs
@@ -486,6 +486,18 @@ impl EdgeStore {
Ok(())
}
+ pub fn degree(&self, node: &NodeID) -> u64 {
+ let node_bytes = node.as_u64().to_be_bytes();
+
+ match self.ranges.edges.get_raw(&node_bytes) {
+ Some(node_range_bytes) => {
+ let range = EdgeRange::deserialize(node_range_bytes.as_bytes());
+ self.edges.slice(usize_range(range.range)).len() as u64
+ }
+ None => 0,
+ }
+ }
+
pub fn get_with_label(&self, node: &NodeID, limit: &EdgeLimit) -> Vec<SegmentEdge<String>> {
let node_bytes = node.as_u64().to_be_bytes();
@@ -792,4 +804,31 @@ mod tests {
Edge::from(e1.clone()).from.node()
);
}
+
+ #[test]
+ fn test_degree() {
+ let temp_dir = crate::gen_temp_dir().unwrap();
+ let mut kv: EdgeStoreWriter =
+ EdgeStoreWriter::new(&temp_dir, Compression::default(), false, None);
+
+ let e1 = InsertableEdge {
+ from: FullNodeID {
+ id: NodeID::from(1_u64),
+ host: NodeID::from(1_u64),
+ },
+ to: FullNodeID {
+ id: NodeID::from(2_u64),
+ host: NodeID::from(2_u64),
+ },
+ label: "1".to_string(),
+ rel: RelFlags::default(),
+ };
+
+ kv.put(e1.clone());
+
+ let store = kv.finalize();
+
+ assert_eq!(store.degree(&NodeID::from(1_u64)), 1);
+ assert_eq!(store.degree(&NodeID::from(2_u64)), 0);
+ }
}
diff --git a/crates/file-store/src/iterable.rs b/crates/file-store/src/iterable.rs
index 54abca10..9900355a 100644
--- a/crates/file-store/src/iterable.rs
+++ b/crates/file-store/src/iterable.rs
@@ -339,6 +339,19 @@ impl<T> ConstIterableStoreReader<T> {
}
}
+impl<T> ConstIterableStoreReader<T>
+where
+ T: ConstSerializable,
+{
+ pub fn len(&self) -> usize {
+ self.data.len() / T::BYTES
+ }
+
+ pub fn is_empty(&self) -> bool {
+ self.len() == 0
+ }
+}
+
impl<T> Iterator for ConstIterableStoreReader<T>
where
T: ConstSerializable,
@@ -431,4 +444,17 @@ mod tests {
let items: Vec<i32> = reader.collect();
assert_eq!(items, vec![1, 2, 3]);
}
+
+ #[test]
+ fn test_const_iterable_store_len() {
+ let mut writer = ConstIterableStoreWriter::new(Vec::new());
+ writer.write(&1).unwrap();
+ writer.write(&2).unwrap();
+ writer.write(&3).unwrap();
+ let writer = writer.finalize().unwrap();
+
+ let reader = ConstIterableStoreReader::<i32>::from_bytes(writer);
+
+ assert_eq!(reader.len(), 3);
+ }
}