web search done right
store sort-key in webgraph edges to properly apply limits with multiple segments
Mikkel Denker295444de4ef0
diff --git a/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/worker.rs b/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/worker.rsindex c2244109..ee67f684 100644--- a/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/worker.rs+++ b/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/worker.rs@@ -123,7 +123,9 @@ impl RemoteWorker for RemoteApproxCentralityWorker {pub fn run(config: ApproxHarmonicWorkerConfig) -> Result<()> {let tokio_conf = config.clone();- let graph = Webgraph::builder(config.graph_path).open();+ let graph = Webgraph::builder(config.graph_path)+ .single_threaded()+ .open();let worker = ApproxCentralityWorker::new(graph, config.shard);let service = Service::ApproxHarmonicWorker {host: tokio_conf.host,diff --git a/crates/core/src/webgraph/edge.rs b/crates/core/src/webgraph/edge.rsindex 4af8d194..d415a8bc 100644--- a/crates/core/src/webgraph/edge.rs+++ b/crates/core/src/webgraph/edge.rs@@ -16,7 +16,7 @@use utoipa::ToSchema;-use super::{FullNodeID, Node, NodeID};+use super::{store::NodeDatum, FullNodeID, Node, NodeID};pub const MAX_LABEL_LENGTH: usize = 1024;@@ -59,7 +59,7 @@ where}#[derive(Debug, Clone, PartialEq, Eq, Hash, bincode::Encode, bincode::Decode)]-pub struct InnerEdge<L>+pub struct InsertableEdge<L>whereL: EdgeLabel,{@@ -68,11 +68,12 @@ wherepub label: L,}-impl<L> From<InnerEdge<L>> for Edge<L>+#[cfg(test)]+impl<L> From<InsertableEdge<L>> for Edge<L>whereL: EdgeLabel,{- fn from(edge: InnerEdge<L>) -> Self {+ fn from(edge: InsertableEdge<L>) -> Self {Edge {from: edge.from.id,to: edge.to.id,@@ -99,3 +100,54 @@ pub struct FullEdge {pub to: Node,pub label: String,}++#[derive(Debug, Clone, PartialEq)]+pub struct SegmentEdge<L>+where+ L: EdgeLabel,+{+ pub from: NodeDatum,+ pub to: NodeDatum,+ pub label: L,+}++impl<L> From<SegmentEdge<L>> for Edge<L>+where+ L: EdgeLabel,+{+ fn from(edge: SegmentEdge<L>) -> Self {+ Edge {+ from: edge.from.node(),+ to: edge.to.node(),+ label: edge.label,+ }+ }+}++#[cfg(test)]+impl<L> From<Edge<L>> for SegmentEdge<L>+where+ L: EdgeLabel,+{+ fn from(edge: Edge<L>) -> Self {+ SegmentEdge {+ from: NodeDatum::new(edge.from, 0),+ to: NodeDatum::new(edge.to, 0),+ label: edge.label,+ }+ }+}++#[cfg(test)]+impl<L> From<InsertableEdge<L>> for SegmentEdge<L>+where+ L: EdgeLabel,+{+ fn from(edge: InsertableEdge<L>) -> Self {+ SegmentEdge {+ from: NodeDatum::new(edge.from.id, 0),+ to: NodeDatum::new(edge.to.id, 0),+ label: edge.label,+ }+ }+}diff --git a/crates/core/src/webgraph/mod.rs b/crates/core/src/webgraph/mod.rsindex c73ff160..52f97384 100644--- a/crates/core/src/webgraph/mod.rs+++ b/crates/core/src/webgraph/mod.rs@@ -94,6 +94,18 @@ pub enum EdgeLimit {Limit(usize),}+impl EdgeLimit {+ pub fn apply<'a, T>(+ &self,+ it: impl Iterator<Item = T> + 'a,+ ) -> Box<dyn Iterator<Item = T> + 'a> {+ match self {+ EdgeLimit::Unlimited => Box::new(it),+ EdgeLimit::Limit(limit) => Box::new(it.take(*limit)),+ }+ }+}+pub struct Webgraph {path: String,segments: Vec<Segment>,@@ -171,22 +183,25 @@ impl Webgraph {}pub fn ingoing_edges(&self, node: Node, limit: EdgeLimit) -> Vec<FullEdge> {- let dedup = |edges: &mut Vec<Edge<String>>| {- edges.sort_by_key(|e| e.from);- edges.dedup_by_key(|e| e.from);+ let dedup = |edges: &mut Vec<SegmentEdge<String>>| {+ edges.sort_by_key(|e| e.from.node());+ edges.dedup_by_key(|e| e.from.node());};- self.inner_edges(+ let mut edges = self.inner_edges(|segment| segment.ingoing_edges_with_label(&node.id(), &limit),dedup,- )- .into_iter()- .map(|e| FullEdge {- from: self.id2node(&e.from).unwrap(),- to: self.id2node(&e.to).unwrap(),- label: e.label,- })- .collect()+ );+ edges.sort_by(|a, b| a.from.sort_key().cmp(&b.from.sort_key()));++ limit+ .apply(edges.into_iter())+ .map(|e| FullEdge {+ from: self.id2node(&e.from.node()).unwrap(),+ to: self.id2node(&e.to.node()).unwrap(),+ label: e.label,+ })+ .collect()}pub fn pages_by_host(&self, host_node: &NodeID) -> Vec<NodeID> {@@ -208,12 +223,22 @@ impl Webgraph {}pub fn raw_ingoing_edges(&self, node: &NodeID, limit: EdgeLimit) -> Vec<Edge<()>> {- let dedup = |edges: &mut Vec<Edge<()>>| {- edges.sort_by_key(|e| e.from);- edges.dedup_by_key(|e| e.from);+ let dedup = |edges: &mut Vec<SegmentEdge<()>>| {+ edges.sort_by_key(|e| e.from.node());+ edges.dedup_by_key(|e| e.from.node());};- self.inner_edges(|segment| segment.ingoing_edges(node, &limit), dedup)+ let mut edges = self.inner_edges(|segment| segment.ingoing_edges(node, &limit), dedup);+ edges.sort_by(|a, b| a.from.sort_key().cmp(&b.from.sort_key()));++ limit+ .apply(edges.into_iter())+ .map(|e| Edge {+ from: e.from.node(),+ to: e.to.node(),+ label: e.label,+ })+ .collect()}pub fn raw_ingoing_edges_with_labels(@@ -221,15 +246,25 @@ impl Webgraph {node: &NodeID,limit: EdgeLimit,) -> Vec<Edge<String>> {- let dedup = |edges: &mut Vec<Edge<String>>| {- edges.sort_by_key(|e| e.from);- edges.dedup_by_key(|e| e.from);+ let dedup = |edges: &mut Vec<SegmentEdge<String>>| {+ edges.sort_by_key(|e| e.from.node());+ edges.dedup_by_key(|e| e.from.node());};- self.inner_edges(+ let mut edges = self.inner_edges(|segment| segment.ingoing_edges_with_label(node, &limit),dedup,- )+ );+ edges.sort_by(|a, b| a.from.sort_key().cmp(&b.from.sort_key()));++ limit+ .apply(edges.into_iter())+ .map(|e| Edge {+ from: e.from.node(),+ to: e.to.node(),+ label: e.label,+ })+ .collect()}pub fn raw_outgoing_edges_with_labels(@@ -237,50 +272,74 @@ impl Webgraph {node: &NodeID,limit: EdgeLimit,) -> Vec<Edge<String>> {- let dedup = |edges: &mut Vec<Edge<String>>| {- edges.sort_by_key(|e| e.from);- edges.dedup_by_key(|e| e.from);+ let dedup = |edges: &mut Vec<SegmentEdge<String>>| {+ edges.sort_by_key(|e| e.to.node());+ edges.dedup_by_key(|e| e.to.node());};- self.inner_edges(+ let mut edges = self.inner_edges(|segment| segment.outgoing_edges_with_label(node, &limit),dedup,- )+ );++ edges.sort_by(|a, b| a.to.sort_key().cmp(&b.to.sort_key()));++ limit+ .apply(edges.into_iter())+ .map(|e| Edge {+ from: e.from.node(),+ to: e.to.node(),+ label: e.label,+ })+ .collect()}pub fn outgoing_edges(&self, node: Node, limit: EdgeLimit) -> Vec<FullEdge> {- let dedup = |edges: &mut Vec<Edge<String>>| {- edges.sort_by_key(|e| e.to);- edges.dedup_by_key(|e| e.to);+ let dedup = |edges: &mut Vec<SegmentEdge<String>>| {+ edges.sort_by_key(|e| e.to.node());+ edges.dedup_by_key(|e| e.to.node());};- self.inner_edges(+ let mut edges = self.inner_edges(|segment| segment.outgoing_edges_with_label(&node.id(), &limit),dedup,- )- .into_iter()- .map(|e| FullEdge {- from: self.id2node(&e.from).unwrap(),- to: self.id2node(&e.to).unwrap(),- label: e.label,- })- .collect()+ );+ edges.sort_by(|a, b| a.to.sort_key().cmp(&b.to.sort_key()));++ limit+ .apply(edges.into_iter())+ .map(|e| FullEdge {+ from: self.id2node(&e.from.node()).unwrap(),+ to: self.id2node(&e.to.node()).unwrap(),+ label: e.label,+ })+ .collect()}pub fn raw_outgoing_edges(&self, node: &NodeID, limit: EdgeLimit) -> Vec<Edge<()>> {- let dedup = |edges: &mut Vec<Edge<()>>| {- edges.sort_by_key(|e| e.to);- edges.dedup_by_key(|e| e.to);+ let dedup = |edges: &mut Vec<SegmentEdge<()>>| {+ edges.sort_by_key(|e| e.to.node());+ edges.dedup_by_key(|e| e.to.node());};- self.inner_edges(|segment| segment.outgoing_edges(node, &limit), dedup)+ let mut edges = self.inner_edges(|segment| segment.outgoing_edges(node, &limit), dedup);+ edges.sort_by(|a, b| a.to.sort_key().cmp(&b.to.sort_key()));++ limit+ .apply(edges.into_iter())+ .map(|e| Edge {+ from: e.from.node(),+ to: e.to.node(),+ label: e.label,+ })+ .collect()}- fn inner_edges<F1, F2, L>(&self, loader: F1, dedup: F2) -> Vec<Edge<L>>+ fn inner_edges<F1, F2, L>(&self, loader: F1, dedup: F2) -> Vec<SegmentEdge<L>>whereL: EdgeLabel,- F1: Sized + Sync + Fn(&Segment) -> Vec<Edge<L>>,- F2: Fn(&mut Vec<Edge<L>>),+ F1: Sized + Sync + Fn(&Segment) -> Vec<SegmentEdge<L>>,+ F2: Fn(&mut Vec<SegmentEdge<L>>),{let mut edges: Vec<_> = self.executor@@ -330,13 +389,15 @@ impl Webgraph {/// Some edges may be returned multiple times./// This happens if they are present in more than one segment.pub fn edges(&self) -> impl Iterator<Item = Edge<()>> + '_ {- self.segments.iter().flat_map(|segment| segment.edges())+ self.segments+ .iter()+ .flat_map(|segment| segment.edges().map(|e| e.into()))}pub fn par_edges(&self) -> impl ParallelIterator<Item = Edge<()>> + '_ {self.segments.par_iter()- .flat_map(|segment| segment.edges().par_bridge())+ .flat_map(|segment| segment.edges().par_bridge().map(|e| e.into()))}}@@ -544,6 +605,57 @@ pub mod tests {.len(),1);++ let mut graphs = Vec::new();+ for (from, to, label) in &[+ (Node::from("A"), Node::from("B"), String::new()),+ (Node::from("A"), Node::from("C"), String::new()),+ ] {+ let mut wrt = WebgraphWriter::new(+ crate::gen_temp_path(),+ Executor::single_thread(),+ Compression::default(),+ None,+ );+ wrt.insert(from.clone(), to.clone(), label.clone());+ graphs.push(wrt.finalize());+ }++ let mut graph = graphs.pop().unwrap();++ for other in graphs {+ graph.merge(other).unwrap();+ }++ assert_eq!(+ graph+ .outgoing_edges(Node::from("A"), EdgeLimit::Unlimited)+ .len(),+ 2+ );++ assert_eq!(+ graph+ .outgoing_edges(Node::from("A"), EdgeLimit::Limit(1))+ .len(),+ 1+ );++ graph.optimize_read();++ assert_eq!(+ graph+ .outgoing_edges(Node::from("A"), EdgeLimit::Unlimited)+ .len(),+ 2+ );++ assert_eq!(+ graph+ .outgoing_edges(Node::from("A"), EdgeLimit::Limit(1))+ .len(),+ 1+ );}#[test]diff --git a/crates/core/src/webgraph/segment.rs b/crates/core/src/webgraph/segment.rsindex 0291a391..40af0b5e 100644--- a/crates/core/src/webgraph/segment.rs+++ b/crates/core/src/webgraph/segment.rs@@ -20,8 +20,8 @@ use std::{};use super::{- store::EdgeStore, store_writer::EdgeStoreWriter, Compression, Edge, EdgeLimit, InnerEdge,- NodeID,+ store::EdgeStore, store_writer::EdgeStoreWriter, Compression, EdgeLimit, InsertableEdge,+ NodeID, SegmentEdge,};const ADJACENCY_STORE: &str = "adjacency";@@ -76,7 +76,7 @@ impl SegmentWriter {}}- pub fn insert(&mut self, edge: InnerEdge<String>) {+ pub fn insert(&mut self, edge: InsertableEdge<String>) {self.adjacency.put(edge.clone());self.reversed_adjacency.put(edge);}@@ -110,19 +110,27 @@ impl Segment {}}- pub fn outgoing_edges_with_label(&self, node: &NodeID, limit: &EdgeLimit) -> Vec<Edge<String>> {+ pub fn outgoing_edges_with_label(+ &self,+ node: &NodeID,+ limit: &EdgeLimit,+ ) -> Vec<SegmentEdge<String>> {self.adjacency.get_with_label(node, limit)}- pub fn outgoing_edges(&self, node: &NodeID, limit: &EdgeLimit) -> Vec<Edge<()>> {+ pub fn outgoing_edges(&self, node: &NodeID, limit: &EdgeLimit) -> Vec<SegmentEdge<()>> {self.adjacency.get_without_label(node, limit)}- pub fn ingoing_edges_with_label(&self, node: &NodeID, limit: &EdgeLimit) -> Vec<Edge<String>> {+ pub fn ingoing_edges_with_label(+ &self,+ node: &NodeID,+ limit: &EdgeLimit,+ ) -> Vec<SegmentEdge<String>> {self.reversed_adjacency.get_with_label(node, limit)}- pub fn ingoing_edges(&self, node: &NodeID, limit: &EdgeLimit) -> Vec<Edge<()>> {+ pub fn ingoing_edges(&self, node: &NodeID, limit: &EdgeLimit) -> Vec<SegmentEdge<()>> {self.reversed_adjacency.get_without_label(node, limit)}@@ -138,7 +146,7 @@ impl Segment {Path::new(&self.folder_path).join(&self.id)}- pub fn edges(&self) -> impl Iterator<Item = Edge<()>> + '_ + Send + Sync {+ pub fn edges(&self) -> impl Iterator<Item = SegmentEdge<()>> + '_ + Send + Sync {self.adjacency.iter_without_label()}@@ -150,7 +158,7 @@ impl Segment {#[cfg(test)]mod test {- use crate::webgraph::FullNodeID;+ use crate::webgraph::{Edge, FullNodeID};use super::*;@@ -186,22 +194,22 @@ mod test {host: NodeID::from(0_u64),};- edges.push(InnerEdge {+ edges.push(InsertableEdge {from: a.clone(),to: b.clone(),label: String::new(),});- edges.push(InnerEdge {+ edges.push(InsertableEdge {from: b.clone(),to: c.clone(),label: String::new(),});- edges.push(InnerEdge {+ edges.push(InsertableEdge {from: c.clone(),to: a.clone(),label: String::new(),});- edges.push(InnerEdge {+ edges.push(InsertableEdge {from: a.clone(),to: c.clone(),label: String::new(),@@ -214,7 +222,7 @@ mod test {let mut out: Vec<_> = segment.outgoing_edges(&a.id, &EdgeLimit::Unlimited);- out.sort_by(|a, b| a.to.cmp(&b.to));+ out.sort_by(|a, b| a.to.node().cmp(&b.to.node()));assert_eq!(out,@@ -223,35 +231,39 @@ mod test {from: a.id,to: b.id,label: ()- },+ }+ .into(),Edge {from: a.id,to: c.id,label: ()- },+ }+ .into(),]);let mut out: Vec<_> = segment.outgoing_edges(&b.id, &EdgeLimit::Unlimited);- out.sort_by(|a, b| a.to.cmp(&b.to));+ out.sort_by(|a, b| a.to.node().cmp(&b.to.node()));assert_eq!(out,vec![Edge {from: b.id,to: c.id,label: ()- },]+ }+ .into(),]);let mut out: Vec<_> = segment.outgoing_edges(&c.id, &EdgeLimit::Unlimited);- out.sort_by(|a, b| a.to.cmp(&b.to));+ out.sort_by(|a, b| a.to.node().cmp(&b.to.node()));assert_eq!(out,vec![Edge {from: c.id,to: a.id,label: ()- },]+ }+ .into(),]);let out: Vec<_> = segment.ingoing_edges(&a.id, &EdgeLimit::Unlimited);@@ -261,7 +273,8 @@ mod test {from: c.id,to: a.id,label: ()- },]+ }+ .into(),]);let out: Vec<_> = segment.ingoing_edges(&b.id, &EdgeLimit::Unlimited);@@ -271,11 +284,12 @@ mod test {from: a.id,to: b.id,label: ()- },]+ }+ .into(),]);let mut out: Vec<_> = segment.ingoing_edges(&c.id, &EdgeLimit::Unlimited);- out.sort_by(|a, b| a.from.cmp(&b.from));+ out.sort_by(|a, b| a.from.node().cmp(&b.from.node()));assert_eq!(out,vec![@@ -283,12 +297,14 @@ mod test {from: a.id,to: c.id,label: ()- },+ }+ .into(),Edge {from: b.id,to: c.id,label: ()- },+ }+ .into(),]);}diff --git a/crates/core/src/webgraph/store.rs b/crates/core/src/webgraph/store.rsindex 023a590d..50129ec3 100644--- a/crates/core/src/webgraph/store.rs+++ b/crates/core/src/webgraph/store.rs@@ -23,7 +23,7 @@ use file_store::{use fst::Automaton;use itertools::Itertools;-use super::{Compression, Edge, EdgeLimit, FullNodeID, NodeID};+use super::{Compression, EdgeLimit, FullNodeID, NodeID, SegmentEdge};#[derive(Debug,@@ -88,8 +88,25 @@ impl HostDb {}}+#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]+pub struct NodeRange {+ range: std::ops::Range<u64>,+ sort_key: u64,+}++impl NodeRange {+ pub fn new(range: std::ops::Range<u64>, sort_key: u64) -> Self {+ Self { range, sort_key }+ }+}++#[inline]+fn usize_range(range: std::ops::Range<u64>) -> std::ops::Range<usize> {+ range.start as usize..range.end as usize+}+pub struct RangesDb {- nodes: speedy_kv::Db<NodeID, std::ops::Range<u64>>,+ nodes: speedy_kv::Db<NodeID, NodeRange>,labels: speedy_kv::Db<NodeID, std::ops::Range<u64>>,}@@ -114,7 +131,7 @@ impl RangesDb {pub fn nodes_get_raw<'a>(&'a self,key: &'a [u8],- ) -> Option<speedy_kv::SerializedRef<'a, Range<u64>>> {+ ) -> Option<speedy_kv::SerializedRef<'a, NodeRange>> {self.nodes.get_raw(key)}@@ -134,6 +151,27 @@ impl RangesDb {}}+#[derive(Debug, Clone, PartialEq)]+pub struct NodeDatum {+ id: NodeID,+ sort_key: u64,+}++impl NodeDatum {+ pub fn new(node: NodeID, sort_key: u64) -> Self {+ Self { id: node, sort_key }+ }++ #[inline]+ pub fn node(&self) -> NodeID {+ self.id+ }++ #[inline]+ pub fn sort_key(&self) -> u64 {+ self.sort_key+ }+}impl ConstSerializable for NodeID {const BYTES: usize = std::mem::size_of::<NodeID>();@@ -147,6 +185,22 @@ impl ConstSerializable for NodeID {}}+impl ConstSerializable for NodeDatum {+ const BYTES: usize = std::mem::size_of::<NodeDatum>();++ fn serialize(&self, buf: &mut [u8]) {+ self.id.serialize(&mut buf[..NodeID::BYTES]);+ self.sort_key.serialize(&mut buf[NodeID::BYTES..]);+ }++ fn deserialize(buf: &[u8]) -> Self {+ let node = NodeID::deserialize(&buf[..NodeID::BYTES]);+ let sort_key = u64::deserialize(&buf[NodeID::BYTES..]);++ Self { id: node, sort_key }+ }+}+pub const NUM_LABELS_PER_BLOCK: usize = 128;#[derive(bincode::Encode, bincode::Decode)]@@ -190,7 +244,7 @@ pub struct EdgeStore {hosts: HostDb,edge_labels: IterableStoreReader<CompressedLabelBlock>,- edge_nodes: ConstIterableStoreReader<NodeID>,+ edge_nodes: ConstIterableStoreReader<NodeDatum>,}impl EdgeStore {@@ -215,7 +269,7 @@ impl EdgeStore {self.hosts.optimize_read();}- pub fn get_with_label(&self, node: &NodeID, limit: &EdgeLimit) -> Vec<Edge<String>> {+ pub fn get_with_label(&self, node: &NodeID, limit: &EdgeLimit) -> Vec<SegmentEdge<String>> {let node_bytes = node.as_u64().to_le_bytes();match (@@ -223,12 +277,12 @@ impl EdgeStore {self.ranges.labels.get_raw(&node_bytes),) {(Some(node_range_bytes), Some(edge_range_bytes)) => {- let (node_range, _) = bincode::decode_from_slice::<Range<usize>, _>(+ let (node_range, _) = bincode::decode_from_slice::<NodeRange, _>(node_range_bytes.as_bytes(),bincode::config::standard(),).unwrap();- let (edge_range, _) = bincode::decode_from_slice::<Range<usize>, _>(+ let (edge_range, _) = bincode::decode_from_slice::<Range<u64>, _>(edge_range_bytes.as_bytes(),bincode::config::standard(),)@@ -236,35 +290,27 @@ impl EdgeStore {let edge_labels = self.edge_labels- .slice(edge_range)+ .slice(usize_range(edge_range)).map(|r| r.unwrap().decompress()).flat_map(|block| block.labels.into_iter());- let edge_labels: Vec<_> = match limit {- EdgeLimit::Unlimited => edge_labels.collect(),- EdgeLimit::Limit(limit) => edge_labels.take(*limit).collect(),- };-- let edge_nodes = self.edge_nodes.slice(node_range);+ let edge_labels = limit.apply(edge_labels);- let edge_nodes: Vec<_> = match limit {- EdgeLimit::Unlimited => edge_nodes.collect(),- EdgeLimit::Limit(limit) => edge_nodes.take(*limit).collect(),- };+ let edge_nodes = self.edge_nodes.slice(usize_range(node_range.range));+ let edge_nodes = limit.apply(edge_nodes);edge_labels- .into_iter().zip_eq(edge_nodes).map(|(label, other)| {if self.reversed {- Edge {+ SegmentEdge {from: other,- to: *node,+ to: NodeDatum::new(*node, node_range.sort_key),label,}} else {- Edge {- from: *node,+ SegmentEdge {+ from: NodeDatum::new(*node, node_range.sort_key),to: other,label,}@@ -276,36 +322,33 @@ impl EdgeStore {}}- pub fn get_without_label(&self, node: &NodeID, limit: &EdgeLimit) -> Vec<Edge<()>> {+ pub fn get_without_label(&self, node: &NodeID, limit: &EdgeLimit) -> Vec<SegmentEdge<()>> {let node_bytes = node.as_u64().to_le_bytes();match self.ranges.nodes.get_raw(&node_bytes) {Some(node_range_bytes) => {- let (node_range, _) = bincode::decode_from_slice::<Range<usize>, _>(+ let (node_range, _) = bincode::decode_from_slice::<NodeRange, _>(node_range_bytes.as_bytes(),bincode::config::standard(),).unwrap();- let edge_nodes = self.edge_nodes.slice(node_range);+ let edge_nodes = self.edge_nodes.slice(usize_range(node_range.range));- let edge_nodes: Vec<_> = match limit {- EdgeLimit::Unlimited => edge_nodes.collect(),- EdgeLimit::Limit(limit) => edge_nodes.take(*limit).collect(),- };+ let edge_nodes = limit.apply(edge_nodes);edge_nodes.into_iter().map(|other| {if self.reversed {- Edge {+ SegmentEdge {from: other,- to: *node,+ to: NodeDatum::new(*node, node_range.sort_key),label: (),}} else {- Edge {- from: *node,+ SegmentEdge {+ from: NodeDatum::new(*node, node_range.sort_key),to: other,label: (),}@@ -321,29 +364,32 @@ impl EdgeStore {self.hosts.get(host)}- pub fn iter_without_label(&self) -> impl Iterator<Item = Edge<()>> + '_ + Send + Sync {+ pub fn iter_without_label(&self) -> impl Iterator<Item = SegmentEdge<()>> + '_ + Send + Sync {self.ranges.nodes.iter_raw().flat_map(move |(key, val)| {let node = u64::from_le_bytes((key.as_bytes()).try_into().unwrap());let node = NodeID::from(node);- let (node_range, _) = bincode::decode_from_slice::<Range<usize>, _>(+ let (node_range, _) = bincode::decode_from_slice::<NodeRange, _>(val.as_bytes(),bincode::config::standard(),).unwrap();- let edge_nodes = self.edge_nodes.slice(node_range).collect::<Vec<_>>();+ let edge_nodes = self+ .edge_nodes+ .slice(usize_range(node_range.range))+ .collect::<Vec<_>>();edge_nodes.into_iter().map(move |other| {if self.reversed {- Edge {+ SegmentEdge {from: other,- to: node,+ to: NodeDatum::new(node, node_range.sort_key),label: (),}} else {- Edge {- from: node,+ SegmentEdge {+ from: NodeDatum::new(node, node_range.sort_key),to: other,label: (),}@@ -357,7 +403,7 @@ impl EdgeStore {mod tests {use std::sync::Arc;- use crate::webgraph::{store_writer::EdgeStoreWriter, InnerEdge};+ use crate::webgraph::{store_writer::EdgeStoreWriter, Edge, InsertableEdge};use super::*;@@ -370,7 +416,7 @@ mod tests {None,);- let e = InnerEdge {+ let e = InsertableEdge {from: FullNodeID {id: NodeID::from(0_u64),host: NodeID::from(0_u64),@@ -389,7 +435,7 @@ mod tests {let edges: Vec<_> = store.get_with_label(&NodeID::from(0_u64), &EdgeLimit::Unlimited);assert_eq!(edges.len(), 1);- assert_eq!(&edges[0], &Edge::from(e.clone()));+ assert_eq!(&edges[0], &SegmentEdge::from(e.clone()));let edges: Vec<_> = store.get_with_label(&NodeID::from(1_u64), &EdgeLimit::Unlimited);@@ -409,7 +455,7 @@ mod tests {None,);- let e = InnerEdge {+ let e = InsertableEdge {from: FullNodeID {id: NodeID::from(0_u64),host: NodeID::from(0_u64),@@ -430,7 +476,7 @@ mod tests {let edges: Vec<_> = store.get_with_label(&NodeID::from(1_u64), &EdgeLimit::Unlimited);assert_eq!(edges.len(), 1);- assert_eq!(&edges[0], &Edge::from(e.clone()));+ assert_eq!(&edges[0], &SegmentEdge::from(e.clone()));}#[test]@@ -443,7 +489,7 @@ mod tests {);for i in 0..10 {- let e = InnerEdge {+ let e = InsertableEdge {from: FullNodeID {id: NodeID::from(i as u64),host: NodeID::from(0_u64),@@ -482,7 +528,7 @@ mod tests {Some(Arc::new(rank_store)),);- let e1 = InnerEdge {+ let e1 = InsertableEdge {from: FullNodeID {id: NodeID::from(1_u64),host: NodeID::from(1_u64),@@ -494,7 +540,7 @@ mod tests {label: "1".to_string(),};- let e2 = InnerEdge {+ let e2 = InsertableEdge {from: FullNodeID {id: NodeID::from(2_u64),host: NodeID::from(2_u64),@@ -506,7 +552,7 @@ mod tests {label: "2".to_string(),};- let e3 = InnerEdge {+ let e3 = InsertableEdge {from: FullNodeID {id: NodeID::from(3_u64),host: NodeID::from(3_u64),@@ -528,8 +574,8 @@ mod tests {assert_eq!(edges.len(), 3);- assert_eq!(&edges[0], &Edge::from(e2.clone()));- assert_eq!(&edges[1], &Edge::from(e3.clone()));- assert_eq!(&edges[2], &Edge::from(e1.clone()));+ assert_eq!(Edge::from(edges[0].clone()), Edge::from(e2.clone()));+ assert_eq!(Edge::from(edges[1].clone()), Edge::from(e3.clone()));+ assert_eq!(Edge::from(edges[2].clone()), Edge::from(e1.clone()));}}diff --git a/crates/core/src/webgraph/store_writer.rs b/crates/core/src/webgraph/store_writer.rsindex 32095524..d3854c64 100644--- a/crates/core/src/webgraph/store_writer.rs+++ b/crates/core/src/webgraph/store_writer.rs@@ -26,21 +26,24 @@ use std::{use itertools::Itertools;-use crate::{webgraph::store::NUM_LABELS_PER_BLOCK, Result};+use crate::{+ webgraph::store::{NodeDatum, NodeRange, NUM_LABELS_PER_BLOCK},+ Result,+};use file_store::iterable::{ConstIterableStoreWriter, IterableStoreReader, IterableStoreWriter, SortedIterableStoreReader,};use super::{store::{CompressedLabelBlock, EdgeStore, HostDb, LabelBlock, RangesDb},- Compression, EdgeLabel, InnerEdge, NodeID,+ Compression, EdgeLabel, InsertableEdge, NodeID,};#[derive(bincode::Encode, bincode::Decode)]struct SortableEdge<L: EdgeLabel> {sort_node: NodeID,secondary_node: NodeID,- edge: InnerEdge<L>,+ edge: InsertableEdge<L>,}impl<L: EdgeLabel> PartialOrd for SortableEdge<L> {@@ -151,7 +154,7 @@ impl EdgeStoreWriter {Ok(())}- pub fn put(&mut self, edge: InnerEdge<String>) {+ pub fn put(&mut self, edge: InsertableEdge<String>) {let (sort_node, secondary_node) = if self.reversed {(edge.to.id, edge.from.id)} else {@@ -213,7 +216,7 @@ struct FinalEdgeStoreWriter {hosts: HostDb,edge_labels: IterableStoreWriter<CompressedLabelBlock, File>,- edge_nodes: ConstIterableStoreWriter<NodeID, File>,+ edge_nodes: ConstIterableStoreWriter<NodeDatum, File>,host_centrality_rank_store: Option<Arc<speedy_kv::Db<NodeID, u64>>>,@@ -265,7 +268,7 @@ impl FinalEdgeStoreWriter {/// The edges *must* have been de-duplicated by their from/to node./// I.e. if the store is not reversed, there should only ever be a single/// put for each from node, and vice versa.- fn put_store(&mut self, edges: &mut [InnerEdge<String>]) {+ fn put_store(&mut self, edges: &mut [InsertableEdge<String>]) {if edges.is_empty() {return;}@@ -294,15 +297,25 @@ impl FinalEdgeStoreWriter {debug_assert!(self.ranges.labels_get_raw(&node_bytes).is_none());let mut edge_labels = Vec::new();- let mut edge_nodes = Vec::new();+ let mut edge_nodes: Vec<NodeDatum> = Vec::new();for edge in edges {edge_labels.push(edge.label.clone());- edge_nodes.push(if self.reversed {- edge.from.id++ let (node, host) = if self.reversed {+ (edge.from.id, edge.from.host)} else {- edge.to.id- });+ (edge.to.id, edge.to.host)+ };++ let sort_key = self+ .host_centrality_rank_store+ .as_ref()+ .map(|store| store.get(&host).unwrap().unwrap_or(u64::MAX))+ .unwrap_or(0);++ let datum = NodeDatum::new(node, sort_key);+ edge_nodes.push(datum);}let edge_labels: Vec<_> = edge_labels@@ -342,10 +355,19 @@ impl FinalEdgeStoreWriter {end: last_label_offset.unwrap().start + last_label_offset.unwrap().num_bytes,};- let node_range = Range {- start: first_node_offset.unwrap().start,- end: last_node_offset.unwrap().start + last_node_offset.unwrap().num_bytes,- };+ let sort_key = self+ .host_centrality_rank_store+ .as_ref()+ .map(|store| store.get(&node.host).unwrap().unwrap_or(u64::MAX))+ .unwrap_or(0);++ let node_range: NodeRange = NodeRange::new(+ Range {+ start: first_node_offset.unwrap().start,+ end: last_node_offset.unwrap().start + last_node_offset.unwrap().num_bytes,+ },+ sort_key,+ );self.ranges.insert_raw_node(node_bytes.to_vec(),@@ -362,7 +384,10 @@ impl FinalEdgeStoreWriter {////// **IMPORTANT** The edges must be sorted by/// either the from or to node, depending on the value of `reversed`.- pub fn build_store(&mut self, edges: impl Iterator<Item = InnerEdge<String>>) -> EdgeStore {+ pub fn build_store(+ &mut self,+ edges: impl Iterator<Item = InsertableEdge<String>>,+ ) -> EdgeStore {let mut inserts_since_last_flush = 0;// create batches of consecutive edges with the same from/to node@@ -374,7 +399,7 @@ impl FinalEdgeStoreWriter {|| (!self.reversed && edge.from.id != last_node){batch.sort_unstable_by_key(- |e: &InnerEdge<_>| if self.reversed { e.from.id } else { e.to.id },+ |e: &InsertableEdge<_>| if self.reversed { e.from.id } else { e.to.id },);batch.dedup_by_key(|e| if self.reversed { e.from.id } else { e.to.id });let batch_len = batch.len();@@ -399,7 +424,7 @@ impl FinalEdgeStoreWriter {if !batch.is_empty() {batch.sort_unstable_by_key(- |e: &InnerEdge<_>| if self.reversed { e.from.id } else { e.to.id },+ |e: &InsertableEdge<_>| if self.reversed { e.from.id } else { e.to.id },);batch.dedup_by_key(|e| if self.reversed { e.from.id } else { e.to.id });self.put_store(&mut batch);diff --git a/crates/core/src/webgraph/writer.rs b/crates/core/src/webgraph/writer.rsindex ba8d2b5b..943ca6c5 100644--- a/crates/core/src/webgraph/writer.rs+++ b/crates/core/src/webgraph/writer.rs@@ -19,8 +19,8 @@ use std::{fs, path::Path, sync::Arc};use crate::executor::Executor;use super::{- id_node_db::Id2NodeDb, segment::SegmentWriter, Compression, FullNodeID, InnerEdge, Meta, Node,- NodeID, Webgraph, MAX_LABEL_LENGTH,+ id_node_db::Id2NodeDb, segment::SegmentWriter, Compression, FullNodeID, InsertableEdge, Meta,+ Node, NodeID, Webgraph, MAX_LABEL_LENGTH,};pub struct WebgraphWriter {@@ -95,7 +95,7 @@ impl WebgraphWriter {self.id_or_assign(to.clone()),);- let edge = InnerEdge {+ let edge = InsertableEdge {from: from_id,to: to_id,label: label.chars().take(MAX_LABEL_LENGTH).collect(),