web search done right
upper bounds for in degree and out degree in webgraph
Mikkel Denkerb16043629781
diff --git a/crates/core/src/entrypoint/webgraph_server.rs b/crates/core/src/entrypoint/webgraph_server.rsindex fb374436..ae21fb04 100644--- a/crates/core/src/entrypoint/webgraph_server.rs+++ b/crates/core/src/entrypoint/webgraph_server.rs@@ -180,6 +180,32 @@ impl Message<WebGraphService> for PagesByHosts {}}+#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]+pub struct InDegreeUpperBound {+ pub node: NodeID,+}++impl Message<WebGraphService> for InDegreeUpperBound {+ type Response = u64;++ async fn handle(self, server: &WebGraphService) -> Self::Response {+ server.graph.in_degree_upper_bound(&self.node)+ }+}++#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]+pub struct OutDegreeUpperBound {+ pub node: NodeID,+}++impl Message<WebGraphService> for OutDegreeUpperBound {+ type Response = u64;++ async fn handle(self, server: &WebGraphService) -> Self::Response {+ server.graph.out_degree_upper_bound(&self.node)+ }+}+pub async fn run(config: config::WebgraphServerConfig) -> Result<()> {let addr: SocketAddr = config.host;diff --git a/crates/core/src/webgraph/mod.rs b/crates/core/src/webgraph/mod.rsindex b56bcb1e..70a8695a 100644--- a/crates/core/src/webgraph/mod.rs+++ b/crates/core/src/webgraph/mod.rs@@ -325,6 +325,20 @@ impl Webgraph {.collect()}+ pub fn out_degree_upper_bound(&self, node: &NodeID) -> u64 {+ self.segments+ .iter()+ .map(|segment| segment.out_degree(node))+ .sum()+ }++ pub fn in_degree_upper_bound(&self, node: &NodeID) -> u64 {+ self.segments+ .iter()+ .map(|segment| segment.in_degree(node))+ .sum()+ }+pub fn outgoing_edges(&self, node: Node, limit: EdgeLimit) -> Vec<FullEdge> {let dedup = |edges: &mut Vec<SegmentEdge<String>>| {edges.sort_by_key(|e| e.to.node());diff --git a/crates/core/src/webgraph/segment.rs b/crates/core/src/webgraph/segment.rsindex d65f7ba7..614b526f 100644--- a/crates/core/src/webgraph/segment.rs+++ b/crates/core/src/webgraph/segment.rs@@ -172,6 +172,10 @@ impl Segment {self.adjacency.get_without_label(node, limit)}+ pub fn out_degree(&self, node: &NodeID) -> u64 {+ self.adjacency.degree(node)+ }+pub fn ingoing_edges_with_label(&self,node: &NodeID,@@ -184,6 +188,10 @@ impl Segment {self.reversed_adjacency.get_without_label(node, limit)}+ pub fn in_degree(&self, node: &NodeID) -> u64 {+ self.reversed_adjacency.degree(node)+ }+pub fn pages_by_host(&self, host_node: &NodeID) -> Vec<NodeID> {self.reversed_adjacency.nodes_by_host(host_node)}diff --git a/crates/core/src/webgraph/store.rs b/crates/core/src/webgraph/store.rsindex 21ecd574..e594b600 100644--- a/crates/core/src/webgraph/store.rs+++ b/crates/core/src/webgraph/store.rs@@ -486,6 +486,18 @@ impl EdgeStore {Ok(())}+ pub fn degree(&self, node: &NodeID) -> u64 {+ let node_bytes = node.as_u64().to_be_bytes();++ match self.ranges.edges.get_raw(&node_bytes) {+ Some(node_range_bytes) => {+ let range = EdgeRange::deserialize(node_range_bytes.as_bytes());+ self.edges.slice(usize_range(range.range)).len() as u64+ }+ None => 0,+ }+ }+pub fn get_with_label(&self, node: &NodeID, limit: &EdgeLimit) -> Vec<SegmentEdge<String>> {let node_bytes = node.as_u64().to_be_bytes();@@ -792,4 +804,31 @@ mod tests {Edge::from(e1.clone()).from.node());}++ #[test]+ fn test_degree() {+ let temp_dir = crate::gen_temp_dir().unwrap();+ let mut kv: EdgeStoreWriter =+ EdgeStoreWriter::new(&temp_dir, Compression::default(), false, None);++ let e1 = InsertableEdge {+ from: FullNodeID {+ id: NodeID::from(1_u64),+ host: NodeID::from(1_u64),+ },+ to: FullNodeID {+ id: NodeID::from(2_u64),+ host: NodeID::from(2_u64),+ },+ label: "1".to_string(),+ rel: RelFlags::default(),+ };++ kv.put(e1.clone());++ let store = kv.finalize();++ assert_eq!(store.degree(&NodeID::from(1_u64)), 1);+ assert_eq!(store.degree(&NodeID::from(2_u64)), 0);+ }}diff --git a/crates/file-store/src/iterable.rs b/crates/file-store/src/iterable.rsindex 54abca10..9900355a 100644--- a/crates/file-store/src/iterable.rs+++ b/crates/file-store/src/iterable.rs@@ -339,6 +339,19 @@ impl<T> ConstIterableStoreReader<T> {}}+impl<T> ConstIterableStoreReader<T>+where+ T: ConstSerializable,+{+ pub fn len(&self) -> usize {+ self.data.len() / T::BYTES+ }++ pub fn is_empty(&self) -> bool {+ self.len() == 0+ }+}+impl<T> Iterator for ConstIterableStoreReader<T>whereT: ConstSerializable,@@ -431,4 +444,17 @@ mod tests {let items: Vec<i32> = reader.collect();assert_eq!(items, vec![1, 2, 3]);}++ #[test]+ fn test_const_iterable_store_len() {+ let mut writer = ConstIterableStoreWriter::new(Vec::new());+ writer.write(&1).unwrap();+ writer.write(&2).unwrap();+ writer.write(&3).unwrap();+ let writer = writer.finalize().unwrap();++ let reader = ConstIterableStoreReader::<i32>::from_bytes(writer);++ assert_eq!(reader.len(), 3);+ }}