Skip to content

web search done right

store sort-key in webgraph edges to properly apply limits with multiple segments

Mikkel Denker295444de4ef0
diff --git a/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/worker.rs b/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/worker.rs
index c2244109..ee67f684 100644
--- a/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/worker.rs
+++ b/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/worker.rs
@@ -123,7 +123,9 @@ impl RemoteWorker for RemoteApproxCentralityWorker {
pub fn run(config: ApproxHarmonicWorkerConfig) -> Result<()> {
let tokio_conf = config.clone();
- let graph = Webgraph::builder(config.graph_path).open();
+ let graph = Webgraph::builder(config.graph_path)
+ .single_threaded()
+ .open();
let worker = ApproxCentralityWorker::new(graph, config.shard);
let service = Service::ApproxHarmonicWorker {
host: tokio_conf.host,
diff --git a/crates/core/src/webgraph/edge.rs b/crates/core/src/webgraph/edge.rs
index 4af8d194..d415a8bc 100644
--- a/crates/core/src/webgraph/edge.rs
+++ b/crates/core/src/webgraph/edge.rs
@@ -16,7 +16,7 @@
use utoipa::ToSchema;
-use super::{FullNodeID, Node, NodeID};
+use super::{store::NodeDatum, FullNodeID, Node, NodeID};
pub const MAX_LABEL_LENGTH: usize = 1024;
@@ -59,7 +59,7 @@ where
}
#[derive(Debug, Clone, PartialEq, Eq, Hash, bincode::Encode, bincode::Decode)]
-pub struct InnerEdge<L>
+pub struct InsertableEdge<L>
where
L: EdgeLabel,
{
@@ -68,11 +68,12 @@ where
pub label: L,
}
-impl<L> From<InnerEdge<L>> for Edge<L>
+#[cfg(test)]
+impl<L> From<InsertableEdge<L>> for Edge<L>
where
L: EdgeLabel,
{
- fn from(edge: InnerEdge<L>) -> Self {
+ fn from(edge: InsertableEdge<L>) -> Self {
Edge {
from: edge.from.id,
to: edge.to.id,
@@ -99,3 +100,54 @@ pub struct FullEdge {
pub to: Node,
pub label: String,
}
+
+#[derive(Debug, Clone, PartialEq)]
+pub struct SegmentEdge<L>
+where
+ L: EdgeLabel,
+{
+ pub from: NodeDatum,
+ pub to: NodeDatum,
+ pub label: L,
+}
+
+impl<L> From<SegmentEdge<L>> for Edge<L>
+where
+ L: EdgeLabel,
+{
+ fn from(edge: SegmentEdge<L>) -> Self {
+ Edge {
+ from: edge.from.node(),
+ to: edge.to.node(),
+ label: edge.label,
+ }
+ }
+}
+
+#[cfg(test)]
+impl<L> From<Edge<L>> for SegmentEdge<L>
+where
+ L: EdgeLabel,
+{
+ fn from(edge: Edge<L>) -> Self {
+ SegmentEdge {
+ from: NodeDatum::new(edge.from, 0),
+ to: NodeDatum::new(edge.to, 0),
+ label: edge.label,
+ }
+ }
+}
+
+#[cfg(test)]
+impl<L> From<InsertableEdge<L>> for SegmentEdge<L>
+where
+ L: EdgeLabel,
+{
+ fn from(edge: InsertableEdge<L>) -> Self {
+ SegmentEdge {
+ from: NodeDatum::new(edge.from.id, 0),
+ to: NodeDatum::new(edge.to.id, 0),
+ label: edge.label,
+ }
+ }
+}
diff --git a/crates/core/src/webgraph/mod.rs b/crates/core/src/webgraph/mod.rs
index c73ff160..52f97384 100644
--- a/crates/core/src/webgraph/mod.rs
+++ b/crates/core/src/webgraph/mod.rs
@@ -94,6 +94,18 @@ pub enum EdgeLimit {
Limit(usize),
}
+impl EdgeLimit {
+ pub fn apply<'a, T>(
+ &self,
+ it: impl Iterator<Item = T> + 'a,
+ ) -> Box<dyn Iterator<Item = T> + 'a> {
+ match self {
+ EdgeLimit::Unlimited => Box::new(it),
+ EdgeLimit::Limit(limit) => Box::new(it.take(*limit)),
+ }
+ }
+}
+
pub struct Webgraph {
path: String,
segments: Vec<Segment>,
@@ -171,22 +183,25 @@ impl Webgraph {
}
pub fn ingoing_edges(&self, node: Node, limit: EdgeLimit) -> Vec<FullEdge> {
- let dedup = |edges: &mut Vec<Edge<String>>| {
- edges.sort_by_key(|e| e.from);
- edges.dedup_by_key(|e| e.from);
+ let dedup = |edges: &mut Vec<SegmentEdge<String>>| {
+ edges.sort_by_key(|e| e.from.node());
+ edges.dedup_by_key(|e| e.from.node());
};
- self.inner_edges(
+ let mut edges = self.inner_edges(
|segment| segment.ingoing_edges_with_label(&node.id(), &limit),
dedup,
- )
- .into_iter()
- .map(|e| FullEdge {
- from: self.id2node(&e.from).unwrap(),
- to: self.id2node(&e.to).unwrap(),
- label: e.label,
- })
- .collect()
+ );
+ edges.sort_by(|a, b| a.from.sort_key().cmp(&b.from.sort_key()));
+
+ limit
+ .apply(edges.into_iter())
+ .map(|e| FullEdge {
+ from: self.id2node(&e.from.node()).unwrap(),
+ to: self.id2node(&e.to.node()).unwrap(),
+ label: e.label,
+ })
+ .collect()
}
pub fn pages_by_host(&self, host_node: &NodeID) -> Vec<NodeID> {
@@ -208,12 +223,22 @@ impl Webgraph {
}
pub fn raw_ingoing_edges(&self, node: &NodeID, limit: EdgeLimit) -> Vec<Edge<()>> {
- let dedup = |edges: &mut Vec<Edge<()>>| {
- edges.sort_by_key(|e| e.from);
- edges.dedup_by_key(|e| e.from);
+ let dedup = |edges: &mut Vec<SegmentEdge<()>>| {
+ edges.sort_by_key(|e| e.from.node());
+ edges.dedup_by_key(|e| e.from.node());
};
- self.inner_edges(|segment| segment.ingoing_edges(node, &limit), dedup)
+ let mut edges = self.inner_edges(|segment| segment.ingoing_edges(node, &limit), dedup);
+ edges.sort_by(|a, b| a.from.sort_key().cmp(&b.from.sort_key()));
+
+ limit
+ .apply(edges.into_iter())
+ .map(|e| Edge {
+ from: e.from.node(),
+ to: e.to.node(),
+ label: e.label,
+ })
+ .collect()
}
pub fn raw_ingoing_edges_with_labels(
@@ -221,15 +246,25 @@ impl Webgraph {
node: &NodeID,
limit: EdgeLimit,
) -> Vec<Edge<String>> {
- let dedup = |edges: &mut Vec<Edge<String>>| {
- edges.sort_by_key(|e| e.from);
- edges.dedup_by_key(|e| e.from);
+ let dedup = |edges: &mut Vec<SegmentEdge<String>>| {
+ edges.sort_by_key(|e| e.from.node());
+ edges.dedup_by_key(|e| e.from.node());
};
- self.inner_edges(
+ let mut edges = self.inner_edges(
|segment| segment.ingoing_edges_with_label(node, &limit),
dedup,
- )
+ );
+ edges.sort_by(|a, b| a.from.sort_key().cmp(&b.from.sort_key()));
+
+ limit
+ .apply(edges.into_iter())
+ .map(|e| Edge {
+ from: e.from.node(),
+ to: e.to.node(),
+ label: e.label,
+ })
+ .collect()
}
pub fn raw_outgoing_edges_with_labels(
@@ -237,50 +272,74 @@ impl Webgraph {
node: &NodeID,
limit: EdgeLimit,
) -> Vec<Edge<String>> {
- let dedup = |edges: &mut Vec<Edge<String>>| {
- edges.sort_by_key(|e| e.from);
- edges.dedup_by_key(|e| e.from);
+ let dedup = |edges: &mut Vec<SegmentEdge<String>>| {
+ edges.sort_by_key(|e| e.to.node());
+ edges.dedup_by_key(|e| e.to.node());
};
- self.inner_edges(
+ let mut edges = self.inner_edges(
|segment| segment.outgoing_edges_with_label(node, &limit),
dedup,
- )
+ );
+
+ edges.sort_by(|a, b| a.to.sort_key().cmp(&b.to.sort_key()));
+
+ limit
+ .apply(edges.into_iter())
+ .map(|e| Edge {
+ from: e.from.node(),
+ to: e.to.node(),
+ label: e.label,
+ })
+ .collect()
}
pub fn outgoing_edges(&self, node: Node, limit: EdgeLimit) -> Vec<FullEdge> {
- let dedup = |edges: &mut Vec<Edge<String>>| {
- edges.sort_by_key(|e| e.to);
- edges.dedup_by_key(|e| e.to);
+ let dedup = |edges: &mut Vec<SegmentEdge<String>>| {
+ edges.sort_by_key(|e| e.to.node());
+ edges.dedup_by_key(|e| e.to.node());
};
- self.inner_edges(
+ let mut edges = self.inner_edges(
|segment| segment.outgoing_edges_with_label(&node.id(), &limit),
dedup,
- )
- .into_iter()
- .map(|e| FullEdge {
- from: self.id2node(&e.from).unwrap(),
- to: self.id2node(&e.to).unwrap(),
- label: e.label,
- })
- .collect()
+ );
+ edges.sort_by(|a, b| a.to.sort_key().cmp(&b.to.sort_key()));
+
+ limit
+ .apply(edges.into_iter())
+ .map(|e| FullEdge {
+ from: self.id2node(&e.from.node()).unwrap(),
+ to: self.id2node(&e.to.node()).unwrap(),
+ label: e.label,
+ })
+ .collect()
}
pub fn raw_outgoing_edges(&self, node: &NodeID, limit: EdgeLimit) -> Vec<Edge<()>> {
- let dedup = |edges: &mut Vec<Edge<()>>| {
- edges.sort_by_key(|e| e.to);
- edges.dedup_by_key(|e| e.to);
+ let dedup = |edges: &mut Vec<SegmentEdge<()>>| {
+ edges.sort_by_key(|e| e.to.node());
+ edges.dedup_by_key(|e| e.to.node());
};
- self.inner_edges(|segment| segment.outgoing_edges(node, &limit), dedup)
+ let mut edges = self.inner_edges(|segment| segment.outgoing_edges(node, &limit), dedup);
+ edges.sort_by(|a, b| a.to.sort_key().cmp(&b.to.sort_key()));
+
+ limit
+ .apply(edges.into_iter())
+ .map(|e| Edge {
+ from: e.from.node(),
+ to: e.to.node(),
+ label: e.label,
+ })
+ .collect()
}
- fn inner_edges<F1, F2, L>(&self, loader: F1, dedup: F2) -> Vec<Edge<L>>
+ fn inner_edges<F1, F2, L>(&self, loader: F1, dedup: F2) -> Vec<SegmentEdge<L>>
where
L: EdgeLabel,
- F1: Sized + Sync + Fn(&Segment) -> Vec<Edge<L>>,
- F2: Fn(&mut Vec<Edge<L>>),
+ F1: Sized + Sync + Fn(&Segment) -> Vec<SegmentEdge<L>>,
+ F2: Fn(&mut Vec<SegmentEdge<L>>),
{
let mut edges: Vec<_> = self
.executor
@@ -330,13 +389,15 @@ impl Webgraph {
/// Some edges may be returned multiple times.
/// This happens if they are present in more than one segment.
pub fn edges(&self) -> impl Iterator<Item = Edge<()>> + '_ {
- self.segments.iter().flat_map(|segment| segment.edges())
+ self.segments
+ .iter()
+ .flat_map(|segment| segment.edges().map(|e| e.into()))
}
pub fn par_edges(&self) -> impl ParallelIterator<Item = Edge<()>> + '_ {
self.segments
.par_iter()
- .flat_map(|segment| segment.edges().par_bridge())
+ .flat_map(|segment| segment.edges().par_bridge().map(|e| e.into()))
}
}
@@ -544,6 +605,57 @@ pub mod tests {
.len(),
1
);
+
+ let mut graphs = Vec::new();
+ for (from, to, label) in &[
+ (Node::from("A"), Node::from("B"), String::new()),
+ (Node::from("A"), Node::from("C"), String::new()),
+ ] {
+ let mut wrt = WebgraphWriter::new(
+ crate::gen_temp_path(),
+ Executor::single_thread(),
+ Compression::default(),
+ None,
+ );
+ wrt.insert(from.clone(), to.clone(), label.clone());
+ graphs.push(wrt.finalize());
+ }
+
+ let mut graph = graphs.pop().unwrap();
+
+ for other in graphs {
+ graph.merge(other).unwrap();
+ }
+
+ assert_eq!(
+ graph
+ .outgoing_edges(Node::from("A"), EdgeLimit::Unlimited)
+ .len(),
+ 2
+ );
+
+ assert_eq!(
+ graph
+ .outgoing_edges(Node::from("A"), EdgeLimit::Limit(1))
+ .len(),
+ 1
+ );
+
+ graph.optimize_read();
+
+ assert_eq!(
+ graph
+ .outgoing_edges(Node::from("A"), EdgeLimit::Unlimited)
+ .len(),
+ 2
+ );
+
+ assert_eq!(
+ graph
+ .outgoing_edges(Node::from("A"), EdgeLimit::Limit(1))
+ .len(),
+ 1
+ );
}
#[test]
diff --git a/crates/core/src/webgraph/segment.rs b/crates/core/src/webgraph/segment.rs
index 0291a391..40af0b5e 100644
--- a/crates/core/src/webgraph/segment.rs
+++ b/crates/core/src/webgraph/segment.rs
@@ -20,8 +20,8 @@ use std::{
};
use super::{
- store::EdgeStore, store_writer::EdgeStoreWriter, Compression, Edge, EdgeLimit, InnerEdge,
- NodeID,
+ store::EdgeStore, store_writer::EdgeStoreWriter, Compression, EdgeLimit, InsertableEdge,
+ NodeID, SegmentEdge,
};
const ADJACENCY_STORE: &str = "adjacency";
@@ -76,7 +76,7 @@ impl SegmentWriter {
}
}
- pub fn insert(&mut self, edge: InnerEdge<String>) {
+ pub fn insert(&mut self, edge: InsertableEdge<String>) {
self.adjacency.put(edge.clone());
self.reversed_adjacency.put(edge);
}
@@ -110,19 +110,27 @@ impl Segment {
}
}
- pub fn outgoing_edges_with_label(&self, node: &NodeID, limit: &EdgeLimit) -> Vec<Edge<String>> {
+ pub fn outgoing_edges_with_label(
+ &self,
+ node: &NodeID,
+ limit: &EdgeLimit,
+ ) -> Vec<SegmentEdge<String>> {
self.adjacency.get_with_label(node, limit)
}
- pub fn outgoing_edges(&self, node: &NodeID, limit: &EdgeLimit) -> Vec<Edge<()>> {
+ pub fn outgoing_edges(&self, node: &NodeID, limit: &EdgeLimit) -> Vec<SegmentEdge<()>> {
self.adjacency.get_without_label(node, limit)
}
- pub fn ingoing_edges_with_label(&self, node: &NodeID, limit: &EdgeLimit) -> Vec<Edge<String>> {
+ pub fn ingoing_edges_with_label(
+ &self,
+ node: &NodeID,
+ limit: &EdgeLimit,
+ ) -> Vec<SegmentEdge<String>> {
self.reversed_adjacency.get_with_label(node, limit)
}
- pub fn ingoing_edges(&self, node: &NodeID, limit: &EdgeLimit) -> Vec<Edge<()>> {
+ pub fn ingoing_edges(&self, node: &NodeID, limit: &EdgeLimit) -> Vec<SegmentEdge<()>> {
self.reversed_adjacency.get_without_label(node, limit)
}
@@ -138,7 +146,7 @@ impl Segment {
Path::new(&self.folder_path).join(&self.id)
}
- pub fn edges(&self) -> impl Iterator<Item = Edge<()>> + '_ + Send + Sync {
+ pub fn edges(&self) -> impl Iterator<Item = SegmentEdge<()>> + '_ + Send + Sync {
self.adjacency.iter_without_label()
}
@@ -150,7 +158,7 @@ impl Segment {
#[cfg(test)]
mod test {
- use crate::webgraph::FullNodeID;
+ use crate::webgraph::{Edge, FullNodeID};
use super::*;
@@ -186,22 +194,22 @@ mod test {
host: NodeID::from(0_u64),
};
- edges.push(InnerEdge {
+ edges.push(InsertableEdge {
from: a.clone(),
to: b.clone(),
label: String::new(),
});
- edges.push(InnerEdge {
+ edges.push(InsertableEdge {
from: b.clone(),
to: c.clone(),
label: String::new(),
});
- edges.push(InnerEdge {
+ edges.push(InsertableEdge {
from: c.clone(),
to: a.clone(),
label: String::new(),
});
- edges.push(InnerEdge {
+ edges.push(InsertableEdge {
from: a.clone(),
to: c.clone(),
label: String::new(),
@@ -214,7 +222,7 @@ mod test {
let mut out: Vec<_> = segment.outgoing_edges(&a.id, &EdgeLimit::Unlimited);
- out.sort_by(|a, b| a.to.cmp(&b.to));
+ out.sort_by(|a, b| a.to.node().cmp(&b.to.node()));
assert_eq!(
out,
@@ -223,35 +231,39 @@ mod test {
from: a.id,
to: b.id,
label: ()
- },
+ }
+ .into(),
Edge {
from: a.id,
to: c.id,
label: ()
- },
+ }
+ .into(),
]
);
let mut out: Vec<_> = segment.outgoing_edges(&b.id, &EdgeLimit::Unlimited);
- out.sort_by(|a, b| a.to.cmp(&b.to));
+ out.sort_by(|a, b| a.to.node().cmp(&b.to.node()));
assert_eq!(
out,
vec![Edge {
from: b.id,
to: c.id,
label: ()
- },]
+ }
+ .into(),]
);
let mut out: Vec<_> = segment.outgoing_edges(&c.id, &EdgeLimit::Unlimited);
- out.sort_by(|a, b| a.to.cmp(&b.to));
+ out.sort_by(|a, b| a.to.node().cmp(&b.to.node()));
assert_eq!(
out,
vec![Edge {
from: c.id,
to: a.id,
label: ()
- },]
+ }
+ .into(),]
);
let out: Vec<_> = segment.ingoing_edges(&a.id, &EdgeLimit::Unlimited);
@@ -261,7 +273,8 @@ mod test {
from: c.id,
to: a.id,
label: ()
- },]
+ }
+ .into(),]
);
let out: Vec<_> = segment.ingoing_edges(&b.id, &EdgeLimit::Unlimited);
@@ -271,11 +284,12 @@ mod test {
from: a.id,
to: b.id,
label: ()
- },]
+ }
+ .into(),]
);
let mut out: Vec<_> = segment.ingoing_edges(&c.id, &EdgeLimit::Unlimited);
- out.sort_by(|a, b| a.from.cmp(&b.from));
+ out.sort_by(|a, b| a.from.node().cmp(&b.from.node()));
assert_eq!(
out,
vec![
@@ -283,12 +297,14 @@ mod test {
from: a.id,
to: c.id,
label: ()
- },
+ }
+ .into(),
Edge {
from: b.id,
to: c.id,
label: ()
- },
+ }
+ .into(),
]
);
}
diff --git a/crates/core/src/webgraph/store.rs b/crates/core/src/webgraph/store.rs
index 023a590d..50129ec3 100644
--- a/crates/core/src/webgraph/store.rs
+++ b/crates/core/src/webgraph/store.rs
@@ -23,7 +23,7 @@ use file_store::{
use fst::Automaton;
use itertools::Itertools;
-use super::{Compression, Edge, EdgeLimit, FullNodeID, NodeID};
+use super::{Compression, EdgeLimit, FullNodeID, NodeID, SegmentEdge};
#[derive(
Debug,
@@ -88,8 +88,25 @@ impl HostDb {
}
}
+#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]
+pub struct NodeRange {
+ range: std::ops::Range<u64>,
+ sort_key: u64,
+}
+
+impl NodeRange {
+ pub fn new(range: std::ops::Range<u64>, sort_key: u64) -> Self {
+ Self { range, sort_key }
+ }
+}
+
+#[inline]
+fn usize_range(range: std::ops::Range<u64>) -> std::ops::Range<usize> {
+ range.start as usize..range.end as usize
+}
+
pub struct RangesDb {
- nodes: speedy_kv::Db<NodeID, std::ops::Range<u64>>,
+ nodes: speedy_kv::Db<NodeID, NodeRange>,
labels: speedy_kv::Db<NodeID, std::ops::Range<u64>>,
}
@@ -114,7 +131,7 @@ impl RangesDb {
pub fn nodes_get_raw<'a>(
&'a self,
key: &'a [u8],
- ) -> Option<speedy_kv::SerializedRef<'a, Range<u64>>> {
+ ) -> Option<speedy_kv::SerializedRef<'a, NodeRange>> {
self.nodes.get_raw(key)
}
@@ -134,6 +151,27 @@ impl RangesDb {
}
}
+#[derive(Debug, Clone, PartialEq)]
+pub struct NodeDatum {
+ id: NodeID,
+ sort_key: u64,
+}
+
+impl NodeDatum {
+ pub fn new(node: NodeID, sort_key: u64) -> Self {
+ Self { id: node, sort_key }
+ }
+
+ #[inline]
+ pub fn node(&self) -> NodeID {
+ self.id
+ }
+
+ #[inline]
+ pub fn sort_key(&self) -> u64 {
+ self.sort_key
+ }
+}
impl ConstSerializable for NodeID {
const BYTES: usize = std::mem::size_of::<NodeID>();
@@ -147,6 +185,22 @@ impl ConstSerializable for NodeID {
}
}
+impl ConstSerializable for NodeDatum {
+ const BYTES: usize = std::mem::size_of::<NodeDatum>();
+
+ fn serialize(&self, buf: &mut [u8]) {
+ self.id.serialize(&mut buf[..NodeID::BYTES]);
+ self.sort_key.serialize(&mut buf[NodeID::BYTES..]);
+ }
+
+ fn deserialize(buf: &[u8]) -> Self {
+ let node = NodeID::deserialize(&buf[..NodeID::BYTES]);
+ let sort_key = u64::deserialize(&buf[NodeID::BYTES..]);
+
+ Self { id: node, sort_key }
+ }
+}
+
pub const NUM_LABELS_PER_BLOCK: usize = 128;
#[derive(bincode::Encode, bincode::Decode)]
@@ -190,7 +244,7 @@ pub struct EdgeStore {
hosts: HostDb,
edge_labels: IterableStoreReader<CompressedLabelBlock>,
- edge_nodes: ConstIterableStoreReader<NodeID>,
+ edge_nodes: ConstIterableStoreReader<NodeDatum>,
}
impl EdgeStore {
@@ -215,7 +269,7 @@ impl EdgeStore {
self.hosts.optimize_read();
}
- pub fn get_with_label(&self, node: &NodeID, limit: &EdgeLimit) -> Vec<Edge<String>> {
+ pub fn get_with_label(&self, node: &NodeID, limit: &EdgeLimit) -> Vec<SegmentEdge<String>> {
let node_bytes = node.as_u64().to_le_bytes();
match (
@@ -223,12 +277,12 @@ impl EdgeStore {
self.ranges.labels.get_raw(&node_bytes),
) {
(Some(node_range_bytes), Some(edge_range_bytes)) => {
- let (node_range, _) = bincode::decode_from_slice::<Range<usize>, _>(
+ let (node_range, _) = bincode::decode_from_slice::<NodeRange, _>(
node_range_bytes.as_bytes(),
bincode::config::standard(),
)
.unwrap();
- let (edge_range, _) = bincode::decode_from_slice::<Range<usize>, _>(
+ let (edge_range, _) = bincode::decode_from_slice::<Range<u64>, _>(
edge_range_bytes.as_bytes(),
bincode::config::standard(),
)
@@ -236,35 +290,27 @@ impl EdgeStore {
let edge_labels = self
.edge_labels
- .slice(edge_range)
+ .slice(usize_range(edge_range))
.map(|r| r.unwrap().decompress())
.flat_map(|block| block.labels.into_iter());
- let edge_labels: Vec<_> = match limit {
- EdgeLimit::Unlimited => edge_labels.collect(),
- EdgeLimit::Limit(limit) => edge_labels.take(*limit).collect(),
- };
-
- let edge_nodes = self.edge_nodes.slice(node_range);
+ let edge_labels = limit.apply(edge_labels);
- let edge_nodes: Vec<_> = match limit {
- EdgeLimit::Unlimited => edge_nodes.collect(),
- EdgeLimit::Limit(limit) => edge_nodes.take(*limit).collect(),
- };
+ let edge_nodes = self.edge_nodes.slice(usize_range(node_range.range));
+ let edge_nodes = limit.apply(edge_nodes);
edge_labels
- .into_iter()
.zip_eq(edge_nodes)
.map(|(label, other)| {
if self.reversed {
- Edge {
+ SegmentEdge {
from: other,
- to: *node,
+ to: NodeDatum::new(*node, node_range.sort_key),
label,
}
} else {
- Edge {
- from: *node,
+ SegmentEdge {
+ from: NodeDatum::new(*node, node_range.sort_key),
to: other,
label,
}
@@ -276,36 +322,33 @@ impl EdgeStore {
}
}
- pub fn get_without_label(&self, node: &NodeID, limit: &EdgeLimit) -> Vec<Edge<()>> {
+ pub fn get_without_label(&self, node: &NodeID, limit: &EdgeLimit) -> Vec<SegmentEdge<()>> {
let node_bytes = node.as_u64().to_le_bytes();
match self.ranges.nodes.get_raw(&node_bytes) {
Some(node_range_bytes) => {
- let (node_range, _) = bincode::decode_from_slice::<Range<usize>, _>(
+ let (node_range, _) = bincode::decode_from_slice::<NodeRange, _>(
node_range_bytes.as_bytes(),
bincode::config::standard(),
)
.unwrap();
- let edge_nodes = self.edge_nodes.slice(node_range);
+ let edge_nodes = self.edge_nodes.slice(usize_range(node_range.range));
- let edge_nodes: Vec<_> = match limit {
- EdgeLimit::Unlimited => edge_nodes.collect(),
- EdgeLimit::Limit(limit) => edge_nodes.take(*limit).collect(),
- };
+ let edge_nodes = limit.apply(edge_nodes);
edge_nodes
.into_iter()
.map(|other| {
if self.reversed {
- Edge {
+ SegmentEdge {
from: other,
- to: *node,
+ to: NodeDatum::new(*node, node_range.sort_key),
label: (),
}
} else {
- Edge {
- from: *node,
+ SegmentEdge {
+ from: NodeDatum::new(*node, node_range.sort_key),
to: other,
label: (),
}
@@ -321,29 +364,32 @@ impl EdgeStore {
self.hosts.get(host)
}
- pub fn iter_without_label(&self) -> impl Iterator<Item = Edge<()>> + '_ + Send + Sync {
+ pub fn iter_without_label(&self) -> impl Iterator<Item = SegmentEdge<()>> + '_ + Send + Sync {
self.ranges.nodes.iter_raw().flat_map(move |(key, val)| {
let node = u64::from_le_bytes((key.as_bytes()).try_into().unwrap());
let node = NodeID::from(node);
- let (node_range, _) = bincode::decode_from_slice::<Range<usize>, _>(
+ let (node_range, _) = bincode::decode_from_slice::<NodeRange, _>(
val.as_bytes(),
bincode::config::standard(),
)
.unwrap();
- let edge_nodes = self.edge_nodes.slice(node_range).collect::<Vec<_>>();
+ let edge_nodes = self
+ .edge_nodes
+ .slice(usize_range(node_range.range))
+ .collect::<Vec<_>>();
edge_nodes.into_iter().map(move |other| {
if self.reversed {
- Edge {
+ SegmentEdge {
from: other,
- to: node,
+ to: NodeDatum::new(node, node_range.sort_key),
label: (),
}
} else {
- Edge {
- from: node,
+ SegmentEdge {
+ from: NodeDatum::new(node, node_range.sort_key),
to: other,
label: (),
}
@@ -357,7 +403,7 @@ impl EdgeStore {
mod tests {
use std::sync::Arc;
- use crate::webgraph::{store_writer::EdgeStoreWriter, InnerEdge};
+ use crate::webgraph::{store_writer::EdgeStoreWriter, Edge, InsertableEdge};
use super::*;
@@ -370,7 +416,7 @@ mod tests {
None,
);
- let e = InnerEdge {
+ let e = InsertableEdge {
from: FullNodeID {
id: NodeID::from(0_u64),
host: NodeID::from(0_u64),
@@ -389,7 +435,7 @@ mod tests {
let edges: Vec<_> = store.get_with_label(&NodeID::from(0_u64), &EdgeLimit::Unlimited);
assert_eq!(edges.len(), 1);
- assert_eq!(&edges[0], &Edge::from(e.clone()));
+ assert_eq!(&edges[0], &SegmentEdge::from(e.clone()));
let edges: Vec<_> = store.get_with_label(&NodeID::from(1_u64), &EdgeLimit::Unlimited);
@@ -409,7 +455,7 @@ mod tests {
None,
);
- let e = InnerEdge {
+ let e = InsertableEdge {
from: FullNodeID {
id: NodeID::from(0_u64),
host: NodeID::from(0_u64),
@@ -430,7 +476,7 @@ mod tests {
let edges: Vec<_> = store.get_with_label(&NodeID::from(1_u64), &EdgeLimit::Unlimited);
assert_eq!(edges.len(), 1);
- assert_eq!(&edges[0], &Edge::from(e.clone()));
+ assert_eq!(&edges[0], &SegmentEdge::from(e.clone()));
}
#[test]
@@ -443,7 +489,7 @@ mod tests {
);
for i in 0..10 {
- let e = InnerEdge {
+ let e = InsertableEdge {
from: FullNodeID {
id: NodeID::from(i as u64),
host: NodeID::from(0_u64),
@@ -482,7 +528,7 @@ mod tests {
Some(Arc::new(rank_store)),
);
- let e1 = InnerEdge {
+ let e1 = InsertableEdge {
from: FullNodeID {
id: NodeID::from(1_u64),
host: NodeID::from(1_u64),
@@ -494,7 +540,7 @@ mod tests {
label: "1".to_string(),
};
- let e2 = InnerEdge {
+ let e2 = InsertableEdge {
from: FullNodeID {
id: NodeID::from(2_u64),
host: NodeID::from(2_u64),
@@ -506,7 +552,7 @@ mod tests {
label: "2".to_string(),
};
- let e3 = InnerEdge {
+ let e3 = InsertableEdge {
from: FullNodeID {
id: NodeID::from(3_u64),
host: NodeID::from(3_u64),
@@ -528,8 +574,8 @@ mod tests {
assert_eq!(edges.len(), 3);
- assert_eq!(&edges[0], &Edge::from(e2.clone()));
- assert_eq!(&edges[1], &Edge::from(e3.clone()));
- assert_eq!(&edges[2], &Edge::from(e1.clone()));
+ assert_eq!(Edge::from(edges[0].clone()), Edge::from(e2.clone()));
+ assert_eq!(Edge::from(edges[1].clone()), Edge::from(e3.clone()));
+ assert_eq!(Edge::from(edges[2].clone()), Edge::from(e1.clone()));
}
}
diff --git a/crates/core/src/webgraph/store_writer.rs b/crates/core/src/webgraph/store_writer.rs
index 32095524..d3854c64 100644
--- a/crates/core/src/webgraph/store_writer.rs
+++ b/crates/core/src/webgraph/store_writer.rs
@@ -26,21 +26,24 @@ use std::{
use itertools::Itertools;
-use crate::{webgraph::store::NUM_LABELS_PER_BLOCK, Result};
+use crate::{
+ webgraph::store::{NodeDatum, NodeRange, NUM_LABELS_PER_BLOCK},
+ Result,
+};
use file_store::iterable::{
ConstIterableStoreWriter, IterableStoreReader, IterableStoreWriter, SortedIterableStoreReader,
};
use super::{
store::{CompressedLabelBlock, EdgeStore, HostDb, LabelBlock, RangesDb},
- Compression, EdgeLabel, InnerEdge, NodeID,
+ Compression, EdgeLabel, InsertableEdge, NodeID,
};
#[derive(bincode::Encode, bincode::Decode)]
struct SortableEdge<L: EdgeLabel> {
sort_node: NodeID,
secondary_node: NodeID,
- edge: InnerEdge<L>,
+ edge: InsertableEdge<L>,
}
impl<L: EdgeLabel> PartialOrd for SortableEdge<L> {
@@ -151,7 +154,7 @@ impl EdgeStoreWriter {
Ok(())
}
- pub fn put(&mut self, edge: InnerEdge<String>) {
+ pub fn put(&mut self, edge: InsertableEdge<String>) {
let (sort_node, secondary_node) = if self.reversed {
(edge.to.id, edge.from.id)
} else {
@@ -213,7 +216,7 @@ struct FinalEdgeStoreWriter {
hosts: HostDb,
edge_labels: IterableStoreWriter<CompressedLabelBlock, File>,
- edge_nodes: ConstIterableStoreWriter<NodeID, File>,
+ edge_nodes: ConstIterableStoreWriter<NodeDatum, File>,
host_centrality_rank_store: Option<Arc<speedy_kv::Db<NodeID, u64>>>,
@@ -265,7 +268,7 @@ impl FinalEdgeStoreWriter {
/// The edges *must* have been de-duplicated by their from/to node.
/// I.e. if the store is not reversed, there should only ever be a single
/// put for each from node, and vice versa.
- fn put_store(&mut self, edges: &mut [InnerEdge<String>]) {
+ fn put_store(&mut self, edges: &mut [InsertableEdge<String>]) {
if edges.is_empty() {
return;
}
@@ -294,15 +297,25 @@ impl FinalEdgeStoreWriter {
debug_assert!(self.ranges.labels_get_raw(&node_bytes).is_none());
let mut edge_labels = Vec::new();
- let mut edge_nodes = Vec::new();
+ let mut edge_nodes: Vec<NodeDatum> = Vec::new();
for edge in edges {
edge_labels.push(edge.label.clone());
- edge_nodes.push(if self.reversed {
- edge.from.id
+
+ let (node, host) = if self.reversed {
+ (edge.from.id, edge.from.host)
} else {
- edge.to.id
- });
+ (edge.to.id, edge.to.host)
+ };
+
+ let sort_key = self
+ .host_centrality_rank_store
+ .as_ref()
+ .map(|store| store.get(&host).unwrap().unwrap_or(u64::MAX))
+ .unwrap_or(0);
+
+ let datum = NodeDatum::new(node, sort_key);
+ edge_nodes.push(datum);
}
let edge_labels: Vec<_> = edge_labels
@@ -342,10 +355,19 @@ impl FinalEdgeStoreWriter {
end: last_label_offset.unwrap().start + last_label_offset.unwrap().num_bytes,
};
- let node_range = Range {
- start: first_node_offset.unwrap().start,
- end: last_node_offset.unwrap().start + last_node_offset.unwrap().num_bytes,
- };
+ let sort_key = self
+ .host_centrality_rank_store
+ .as_ref()
+ .map(|store| store.get(&node.host).unwrap().unwrap_or(u64::MAX))
+ .unwrap_or(0);
+
+ let node_range: NodeRange = NodeRange::new(
+ Range {
+ start: first_node_offset.unwrap().start,
+ end: last_node_offset.unwrap().start + last_node_offset.unwrap().num_bytes,
+ },
+ sort_key,
+ );
self.ranges.insert_raw_node(
node_bytes.to_vec(),
@@ -362,7 +384,10 @@ impl FinalEdgeStoreWriter {
///
/// **IMPORTANT** The edges must be sorted by
/// either the from or to node, depending on the value of `reversed`.
- pub fn build_store(&mut self, edges: impl Iterator<Item = InnerEdge<String>>) -> EdgeStore {
+ pub fn build_store(
+ &mut self,
+ edges: impl Iterator<Item = InsertableEdge<String>>,
+ ) -> EdgeStore {
let mut inserts_since_last_flush = 0;
// create batches of consecutive edges with the same from/to node
@@ -374,7 +399,7 @@ impl FinalEdgeStoreWriter {
|| (!self.reversed && edge.from.id != last_node)
{
batch.sort_unstable_by_key(
- |e: &InnerEdge<_>| if self.reversed { e.from.id } else { e.to.id },
+ |e: &InsertableEdge<_>| if self.reversed { e.from.id } else { e.to.id },
);
batch.dedup_by_key(|e| if self.reversed { e.from.id } else { e.to.id });
let batch_len = batch.len();
@@ -399,7 +424,7 @@ impl FinalEdgeStoreWriter {
if !batch.is_empty() {
batch.sort_unstable_by_key(
- |e: &InnerEdge<_>| if self.reversed { e.from.id } else { e.to.id },
+ |e: &InsertableEdge<_>| if self.reversed { e.from.id } else { e.to.id },
);
batch.dedup_by_key(|e| if self.reversed { e.from.id } else { e.to.id });
self.put_store(&mut batch);
diff --git a/crates/core/src/webgraph/writer.rs b/crates/core/src/webgraph/writer.rs
index ba8d2b5b..943ca6c5 100644
--- a/crates/core/src/webgraph/writer.rs
+++ b/crates/core/src/webgraph/writer.rs
@@ -19,8 +19,8 @@ use std::{fs, path::Path, sync::Arc};
use crate::executor::Executor;
use super::{
- id_node_db::Id2NodeDb, segment::SegmentWriter, Compression, FullNodeID, InnerEdge, Meta, Node,
- NodeID, Webgraph, MAX_LABEL_LENGTH,
+ id_node_db::Id2NodeDb, segment::SegmentWriter, Compression, FullNodeID, InsertableEdge, Meta,
+ Node, NodeID, Webgraph, MAX_LABEL_LENGTH,
};
pub struct WebgraphWriter {
@@ -95,7 +95,7 @@ impl WebgraphWriter {
self.id_or_assign(to.clone()),
);
- let edge = InnerEdge {
+ let edge = InsertableEdge {
from: from_id,
to: to_id,
label: label.chars().take(MAX_LABEL_LENGTH).collect(),