Skip to content

web search done right

optionally merge all webgraph segments into a single segment for improved read performance

Mikkel Denker817bda9738bd
diff --git a/crates/core/src/config/defaults.rs b/crates/core/src/config/defaults.rs
index 803d8575..fb1853e7 100644
--- a/crates/core/src/config/defaults.rs
+++ b/crates/core/src/config/defaults.rs
@@ -200,3 +200,10 @@ impl ApproxHarmonic {
false
}
}
+
+pub struct Webgraph;
+impl Webgraph {
+ pub fn merge_all_segments() -> bool {
+ true
+ }
+}
diff --git a/crates/core/src/config/mod.rs b/crates/core/src/config/mod.rs
index 274bedd2..39c005d1 100644
--- a/crates/core/src/config/mod.rs
+++ b/crates/core/src/config/mod.rs
@@ -74,6 +74,9 @@ pub struct WebgraphConstructConfig {
pub batch_size: Option<usize>,
pub canonical_index_path: Option<String>,
pub host_centrality_rank_store_path: Option<String>,
+
+ #[serde(default = "defaults::Webgraph::merge_all_segments")]
+ pub merge_all_segments: bool,
}
#[derive(Debug, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, Clone)]
diff --git a/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/mapper.rs b/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/mapper.rs
index 6e677885..ce3fe0c3 100644
--- a/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/mapper.rs
+++ b/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/mapper.rs
@@ -90,7 +90,7 @@ impl Mapper for ApproxCentralityMapper {
) {
let pool = rayon::ThreadPoolBuilder::new()
.num_threads(usize::from(std::thread::available_parallelism().unwrap()))
- .stack_size(800_000_000)
+ .stack_size(80_000_000)
.thread_name(move |num| format!("approx-harmonic-mapper-{num}"))
.build()
.unwrap();
@@ -115,6 +115,8 @@ impl Mapper for ApproxCentralityMapper {
let workers = Workers::new(worker.clone());
let num_samples = dht.next().meta.get(()).unwrap().num_samples_per_worker;
+ tracing::info!("Sampling {} nodes", num_samples);
+
let sampled = worker
.graph()
.random_nodes_with_outgoing(num_samples as usize);
diff --git a/crates/core/src/entrypoint/webgraph.rs b/crates/core/src/entrypoint/webgraph.rs
index 6d6bd3e6..54f0467d 100644
--- a/crates/core/src/entrypoint/webgraph.rs
+++ b/crates/core/src/entrypoint/webgraph.rs
@@ -261,6 +261,11 @@ impl Webgraph {
page_graph.merge(other_page)?;
}
+ if config.merge_all_segments {
+ host_graph.merge_all_segments(Default::default())?;
+ page_graph.merge_all_segments(Default::default())?;
+ }
+
host_graph.optimize_read();
page_graph.optimize_read();
diff --git a/crates/core/src/main.rs b/crates/core/src/main.rs
index 60afd253..36c3d0d9 100644
--- a/crates/core/src/main.rs
+++ b/crates/core/src/main.rs
@@ -218,6 +218,9 @@ enum WebgraphOptions {
Merge {
#[clap(required = true)]
paths: Vec<String>,
+
+ #[clap(default_value_t = stract::config::defaults::Webgraph::merge_all_segments())]
+ merge_all_segments: bool,
},
/// Deploy the webgraph server. The webgraph server is responsible for serving the webgraph to the search servers.
@@ -326,7 +329,10 @@ fn main() -> Result<()> {
let config = load_toml_config(config_path);
entrypoint::Webgraph::run(&config)?;
}
- WebgraphOptions::Merge { mut paths } => {
+ WebgraphOptions::Merge {
+ mut paths,
+ merge_all_segments,
+ } => {
let mut webgraph = WebgraphBuilder::new(paths.remove(0))
.single_threaded()
.open();
@@ -336,6 +342,10 @@ fn main() -> Result<()> {
webgraph.merge(other)?;
}
+ if merge_all_segments {
+ webgraph.merge_all_segments(Default::default())?;
+ }
+
webgraph.optimize_read();
}
WebgraphOptions::Server { config_path } => {
diff --git a/crates/core/src/webgraph/edge.rs b/crates/core/src/webgraph/edge.rs
index d415a8bc..b477585f 100644
--- a/crates/core/src/webgraph/edge.rs
+++ b/crates/core/src/webgraph/edge.rs
@@ -16,7 +16,7 @@
use utoipa::ToSchema;
-use super::{store::NodeDatum, FullNodeID, Node, NodeID};
+use super::{merge::NodeDatum, FullNodeID, Node, NodeID};
pub const MAX_LABEL_LENGTH: usize = 1024;
@@ -106,8 +106,8 @@ pub struct SegmentEdge<L>
where
L: EdgeLabel,
{
- pub from: NodeDatum,
- pub to: NodeDatum,
+ pub from: NodeDatum<()>,
+ pub to: NodeDatum<()>,
pub label: L,
}
diff --git a/crates/core/src/webgraph/merge.rs b/crates/core/src/webgraph/merge.rs
new file mode 100644
index 00000000..8d06b3e5
--- /dev/null
+++ b/crates/core/src/webgraph/merge.rs
@@ -0,0 +1,325 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2024 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/license
+
+use std::{cmp::Reverse, collections::BinaryHeap};
+
+use super::{store::NodeRange, NodeID};
+
+pub struct MergeNode<O = ()> {
+ node: NodeID,
+ range: NodeRange,
+ labels: std::ops::Range<u64>,
+ ord: O,
+}
+
+impl MergeNode {
+ pub fn new(node: NodeID, range: NodeRange, labels: std::ops::Range<u64>) -> Self {
+ Self {
+ node,
+ range,
+ labels,
+ ord: (),
+ }
+ }
+}
+
+impl<O> MergeNode<O> {
+ pub fn with_ord<O2>(self, ord: O2) -> MergeNode<O2> {
+ MergeNode {
+ node: self.node,
+ range: self.range,
+ labels: self.labels,
+ ord,
+ }
+ }
+
+ pub fn id(&self) -> NodeID {
+ self.node
+ }
+
+ pub fn range(&self) -> &NodeRange {
+ &self.range
+ }
+
+ pub fn labels(&self) -> std::ops::Range<u64> {
+ self.labels.clone()
+ }
+
+ pub fn ord(&self) -> &O {
+ &self.ord
+ }
+}
+
+impl<O> Ord for MergeNode<O> {
+ fn cmp(&self, other: &Self) -> std::cmp::Ordering {
+ // nodes in the fst are sorted by their id,
+ // not by their sort key
+ self.node.cmp(&other.node)
+ }
+}
+impl<O> PartialOrd for MergeNode<O> {
+ fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
+ Some(self.cmp(other))
+ }
+}
+impl<O> PartialEq for MergeNode<O> {
+ fn eq(&self, other: &Self) -> bool {
+ self.node == other.node
+ }
+}
+impl<O> Eq for MergeNode<O> {}
+
+#[derive(Clone, Copy)]
+pub struct MergeSegmentOrd(usize);
+impl MergeSegmentOrd {
+ pub fn new(ord: usize) -> Self {
+ Self(ord)
+ }
+
+ pub fn as_usize(&self) -> usize {
+ self.0
+ }
+}
+
+type MinHeap<T> = BinaryHeap<Reverse<T>>;
+
+/// Merge multiple iterators of MergeNodes into a single iterator based on the node id
+pub struct MergeIter<'a> {
+ iters: MinHeap<file_store::Peekable<Box<dyn Iterator<Item = MergeNode<MergeSegmentOrd>> + 'a>>>,
+}
+impl<'a> MergeIter<'a> {
+ pub fn new(iters: Vec<impl Iterator<Item = MergeNode> + 'a>) -> Self {
+ let iters = iters
+ .into_iter()
+ .enumerate()
+ .map(|(ord, iter)| {
+ let it = Box::new(iter.map(move |node| node.with_ord(MergeSegmentOrd::new(ord))))
+ as Box<dyn Iterator<Item = _>>;
+
+ Reverse(file_store::Peekable::new(it))
+ })
+ .collect();
+
+ Self { iters }
+ }
+
+ pub fn advance(&mut self, buf: &mut Vec<MergeNode<MergeSegmentOrd>>) -> bool {
+ buf.clear();
+
+ let next = {
+ let item = self.iters.peek_mut();
+
+ if item.is_none() {
+ return false;
+ }
+
+ let mut item = item.unwrap();
+
+ if item.0.peek().is_none() {
+ return false;
+ }
+
+ item.0.next().unwrap()
+ };
+
+ let node = next.node;
+ buf.push(next);
+
+ // advance all iterators that have the same node
+ while let Some(mut peek) = self.iters.peek_mut() {
+ if peek.0.peek().map(|x| x.node) == Some(node) {
+ buf.push(peek.0.next().unwrap());
+ } else {
+ break;
+ }
+ }
+
+ true
+ }
+}
+
+/// Merge multiple iterators of NodeDatum into a single iterator based on the sort key
+pub struct NodeDatumMerger<'a, L = String> {
+ iters: MinHeap<file_store::Peekable<Box<dyn Iterator<Item = NodeDatum<L>> + 'a>>>,
+}
+
+impl<'a, L> NodeDatumMerger<'a, L> {
+ pub fn new(iters: Vec<impl Iterator<Item = NodeDatum<L>> + 'a>) -> Self {
+ let iters = iters
+ .into_iter()
+ .map(|iter| {
+ let it = Box::new(iter) as Box<dyn Iterator<Item = _>>;
+ Reverse(file_store::Peekable::new(it))
+ })
+ .collect();
+
+ Self { iters }
+ }
+}
+
+impl<'a, L> Iterator for NodeDatumMerger<'a, L> {
+ type Item = NodeDatum<L>;
+
+ fn next(&mut self) -> Option<Self::Item> {
+ let res = self.iters.peek_mut().and_then(|mut item| item.0.next());
+
+ if let Some(datum) = &res {
+ while let Some(mut peek) = self.iters.peek_mut() {
+ if peek.0.peek().map(|x| x.sort_key) == Some(datum.sort_key) {
+ peek.0.next().unwrap();
+ } else {
+ break;
+ }
+ }
+ }
+
+ res
+ }
+}
+
+#[derive(Debug, Clone)]
+pub struct NodeDatum<L = ()> {
+ id: NodeID,
+ sort_key: u64,
+ label: L,
+}
+
+impl<L> PartialOrd for NodeDatum<L> {
+ fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
+ Some(self.cmp(other))
+ }
+}
+
+impl<L> Ord for NodeDatum<L> {
+ fn cmp(&self, other: &Self) -> std::cmp::Ordering {
+ self.sort_key
+ .cmp(&other.sort_key)
+ .then_with(|| self.id.cmp(&other.id))
+ }
+}
+
+impl<L> PartialEq for NodeDatum<L> {
+ fn eq(&self, other: &Self) -> bool {
+ self.id == other.id && self.sort_key == other.sort_key
+ }
+}
+
+impl<L> Eq for NodeDatum<L> {}
+
+impl NodeDatum {
+ pub fn new(node: NodeID, sort_key: u64) -> Self {
+ Self {
+ id: node,
+ sort_key,
+ label: (),
+ }
+ }
+}
+
+impl<L> NodeDatum<L> {
+ #[inline]
+ pub fn node(&self) -> NodeID {
+ self.id
+ }
+
+ #[inline]
+ pub fn sort_key(&self) -> u64 {
+ self.sort_key
+ }
+
+ pub fn with_label<L2>(self, label: L2) -> NodeDatum<L2> {
+ NodeDatum {
+ id: self.id,
+ sort_key: self.sort_key,
+ label,
+ }
+ }
+
+ #[inline]
+ pub fn label(&self) -> &L {
+ &self.label
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn test_merge_nodes() {
+ let a = vec![
+ MergeNode::new(1u64.into(), NodeRange::new(0..10, 1), 0..10),
+ MergeNode::new(4u64.into(), NodeRange::new(0..10, 2), 0..10),
+ MergeNode::new(5u64.into(), NodeRange::new(0..10, 3), 0..10),
+ ];
+
+ let b = vec![
+ MergeNode::new(2u64.into(), NodeRange::new(0..10, 4), 0..10),
+ MergeNode::new(3u64.into(), NodeRange::new(0..10, 5), 0..10),
+ MergeNode::new(5u64.into(), NodeRange::new(0..10, 3), 0..10),
+ ];
+
+ let mut merger = MergeIter::new(vec![a.into_iter(), b.into_iter()]);
+ let mut buf = Vec::new();
+
+ assert!(merger.advance(&mut buf));
+ assert_eq!(buf.len(), 1);
+ assert_eq!(buf[0].id(), 1u64.into());
+
+ assert!(merger.advance(&mut buf));
+ assert_eq!(buf.len(), 1);
+ assert_eq!(buf[0].id(), 2u64.into());
+
+ assert!(merger.advance(&mut buf));
+ assert_eq!(buf.len(), 1);
+ assert_eq!(buf[0].id(), 3u64.into());
+
+ assert!(merger.advance(&mut buf));
+ assert_eq!(buf.len(), 1);
+ assert_eq!(buf[0].id(), 4u64.into());
+
+ assert!(merger.advance(&mut buf));
+ assert_eq!(buf.len(), 2);
+ assert_eq!(buf[0].id(), 5u64.into());
+ assert_eq!(buf[1].id(), 5u64.into());
+
+ assert!(!merger.advance(&mut buf));
+ }
+
+ #[test]
+ fn test_datum_merge() {
+ let a = vec![
+ NodeDatum::new(1u64.into(), 1),
+ NodeDatum::new(2u64.into(), 4),
+ NodeDatum::new(3u64.into(), 5),
+ ];
+
+ let b = vec![
+ NodeDatum::new(4u64.into(), 2),
+ NodeDatum::new(5u64.into(), 3),
+ NodeDatum::new(3u64.into(), 5),
+ ];
+
+ let mut merger = NodeDatumMerger::new(vec![a.into_iter(), b.into_iter()]);
+
+ assert_eq!(merger.next().unwrap().sort_key(), 1);
+ assert_eq!(merger.next().unwrap().sort_key(), 2);
+ assert_eq!(merger.next().unwrap().sort_key(), 3);
+ assert_eq!(merger.next().unwrap().sort_key(), 4);
+ assert_eq!(merger.next().unwrap().sort_key(), 5);
+ assert!(merger.next().is_none());
+ }
+}
diff --git a/crates/core/src/webgraph/mod.rs b/crates/core/src/webgraph/mod.rs
index 52f97384..67af0386 100644
--- a/crates/core/src/webgraph/mod.rs
+++ b/crates/core/src/webgraph/mod.rs
@@ -22,11 +22,13 @@ use std::{fs, io};
use itertools::Itertools;
use rand::seq::{IteratorRandom, SliceRandom};
use rayon::prelude::*;
+use uuid::Uuid;
use self::id_node_db::Id2NodeDb;
use self::segment::Segment;
use crate::executor::Executor;
+use crate::Result;
pub use builder::WebgraphBuilder;
pub use compression::Compression;
pub use edge::*;
@@ -39,6 +41,7 @@ pub mod centrality;
mod compression;
mod edge;
mod id_node_db;
+mod merge;
mod node;
pub mod remote;
mod segment;
@@ -174,6 +177,23 @@ impl Webgraph {
Ok(())
}
+ pub fn merge_all_segments(&mut self, compression: Compression) -> Result<()> {
+ let segments = std::mem::take(&mut self.segments);
+
+ let id = Uuid::new_v4().to_string();
+ let path = Path::new(&self.path).join("segments");
+
+ Segment::merge(segments, compression, &path, id.clone())?;
+ let new_segment = Segment::open(path, id.clone());
+
+ self.segments.push(new_segment);
+ self.meta.comitted_segments = vec![id];
+
+ self.save_metadata();
+
+ Ok(())
+ }
+
pub fn optimize_read(&mut self) {
self.executor
.map(|s| s.optimize_read(), self.segments.iter_mut())
@@ -543,6 +563,13 @@ pub mod tests {
graph.distances(Node::from("A")).get(&Node::from("C")),
Some(&2)
);
+
+ graph.merge_all_segments(Compression::default()).unwrap();
+
+ assert_eq!(
+ graph.distances(Node::from("A")).get(&Node::from("C")),
+ Some(&2)
+ );
}
#[test]
diff --git a/crates/core/src/webgraph/segment.rs b/crates/core/src/webgraph/segment.rs
index 40af0b5e..d4bd308a 100644
--- a/crates/core/src/webgraph/segment.rs
+++ b/crates/core/src/webgraph/segment.rs
@@ -17,12 +17,14 @@
use std::{
path::{Path, PathBuf},
sync::Arc,
+ thread,
};
use super::{
store::EdgeStore, store_writer::EdgeStoreWriter, Compression, EdgeLimit, InsertableEdge,
NodeID, SegmentEdge,
};
+use crate::Result;
const ADJACENCY_STORE: &str = "adjacency";
const REVERSED_ADJACENCY_STORE: &str = "reversed_adjacency";
@@ -110,6 +112,44 @@ impl Segment {
}
}
+ pub fn merge<P: AsRef<Path>>(
+ segments: Vec<Self>,
+ label_compression: Compression,
+ folder: P,
+ id: String,
+ ) -> Result<()> {
+ let old_paths = segments.iter().map(|s| s.path()).collect::<Vec<_>>();
+
+ let (adjacency, reversed_adjacency) = segments
+ .into_iter()
+ .map(|s| (s.adjacency, s.reversed_adjacency))
+ .unzip();
+
+ let adjacency_path = folder.as_ref().join(&id).join(ADJACENCY_STORE);
+ let adjacency =
+ thread::spawn(move || EdgeStore::merge(adjacency, label_compression, adjacency_path));
+
+ let reversed_adjacency_path = folder.as_ref().join(&id).join(REVERSED_ADJACENCY_STORE);
+ let reversed_adjacency = thread::spawn(move || {
+ EdgeStore::merge(
+ reversed_adjacency,
+ label_compression,
+ reversed_adjacency_path,
+ )
+ });
+
+ adjacency.join().unwrap()?;
+ reversed_adjacency.join().unwrap()?;
+
+ for path in old_paths {
+ if Path::new(&path).exists() {
+ std::fs::remove_dir_all(path)?;
+ }
+ }
+
+ Ok(())
+ }
+
pub fn outgoing_edges_with_label(
&self,
node: &NodeID,
diff --git a/crates/core/src/webgraph/store.rs b/crates/core/src/webgraph/store.rs
index 50129ec3..5bb44797 100644
--- a/crates/core/src/webgraph/store.rs
+++ b/crates/core/src/webgraph/store.rs
@@ -14,16 +14,27 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
-use std::{ops::Range, path::Path};
+use std::{fs::File, ops::Range, path::Path};
+use crate::{
+ webgraph::merge::{MergeIter, NodeDatumMerger},
+ Result,
+};
+use anyhow::bail;
use file_store::{
- iterable::{ConstIterableStoreReader, IterableStoreReader},
+ iterable::{
+ ConstIterableStoreReader, ConstIterableStoreWriter, IterableStoreReader,
+ IterableStoreWriter,
+ },
ConstSerializable,
};
use fst::Automaton;
use itertools::Itertools;
-use super::{Compression, EdgeLimit, FullNodeID, NodeID, SegmentEdge};
+use super::{
+ merge::{MergeNode, MergeSegmentOrd, NodeDatum},
+ Compression, EdgeLimit, FullNodeID, NodeID, SegmentEdge,
+};
#[derive(
Debug,
@@ -86,6 +97,10 @@ impl HostDb {
pub fn flush(&mut self) {
self.db.commit().unwrap();
}
+
+ fn merge(&mut self, other: HostDb) {
+ self.db.merge(other.db).unwrap();
+ }
}
#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]
@@ -123,6 +138,10 @@ impl RangesDb {
self.labels.merge_all_segments().unwrap();
}
+ fn uncommitted_node_inserts(&self) -> usize {
+ self.nodes.uncommitted_inserts()
+ }
+
pub fn commit(&mut self) {
self.nodes.commit().unwrap();
self.labels.commit().unwrap();
@@ -149,29 +168,26 @@ impl RangesDb {
pub fn insert_raw_label(&mut self, node: Vec<u8>, range: Vec<u8>) {
self.labels.insert_raw(node, range);
}
-}
-#[derive(Debug, Clone, PartialEq)]
-pub struct NodeDatum {
- id: NodeID,
- sort_key: u64,
-}
+ fn merge_nodes(&self) -> impl Iterator<Item = MergeNode> + '_ {
+ self.nodes.iter_raw().zip_eq(self.labels.iter_raw()).map(
+ move |((key_node, val), (key_label, labels))| {
+ debug_assert_eq!(key_node, key_label);
-impl NodeDatum {
- pub fn new(node: NodeID, sort_key: u64) -> Self {
- Self { id: node, sort_key }
- }
+ let node = NodeID::deserialize(key_node.as_bytes());
+ let range = NodeRange::deserialize(val.as_bytes());
- #[inline]
- pub fn node(&self) -> NodeID {
- self.id
- }
+ let labels = {
+ let range = Range::deserialize(labels.as_bytes());
+ range.start..range.end
+ };
- #[inline]
- pub fn sort_key(&self) -> u64 {
- self.sort_key
+ MergeNode::new(node, range, labels)
+ },
+ )
}
}
+
impl ConstSerializable for NodeID {
const BYTES: usize = std::mem::size_of::<NodeID>();
@@ -189,15 +205,33 @@ impl ConstSerializable for NodeDatum {
const BYTES: usize = std::mem::size_of::<NodeDatum>();
fn serialize(&self, buf: &mut [u8]) {
- self.id.serialize(&mut buf[..NodeID::BYTES]);
- self.sort_key.serialize(&mut buf[NodeID::BYTES..]);
+ self.node().serialize(&mut buf[..NodeID::BYTES]);
+ self.sort_key().serialize(&mut buf[NodeID::BYTES..]);
}
fn deserialize(buf: &[u8]) -> Self {
let node = NodeID::deserialize(&buf[..NodeID::BYTES]);
let sort_key = u64::deserialize(&buf[NodeID::BYTES..]);
- Self { id: node, sort_key }
+ Self::new(node, sort_key)
+ }
+}
+
+impl ConstSerializable for NodeRange {
+ const BYTES: usize = std::mem::size_of::<NodeRange>();
+
+ fn serialize(&self, buf: &mut [u8]) {
+ const RANGE_BYTES: usize = std::mem::size_of::<u64>() * 2;
+ self.range.serialize(&mut buf[..RANGE_BYTES]);
+ self.sort_key.serialize(&mut buf[RANGE_BYTES..]);
+ }
+
+ fn deserialize(buf: &[u8]) -> Self {
+ const RANGE_BYTES: usize = std::mem::size_of::<u64>() * 2;
+ let range: Range<u64> = Range::deserialize(&buf[..RANGE_BYTES]);
+ let sort_key = u64::deserialize(&buf[RANGE_BYTES..]);
+
+ Self { range, sort_key }
}
}
@@ -269,6 +303,175 @@ impl EdgeStore {
self.hosts.optimize_read();
}
+ fn merge_postings_for_node<'a>(
+ buf: &[MergeNode<MergeSegmentOrd>],
+ stores: &'a [EdgeStore],
+ ) -> NodeDatumMerger<'a> {
+ let mut edges = Vec::new();
+
+ for node in buf {
+ let store = &stores[node.ord().as_usize()];
+ let edge_nodes = store
+ .edge_nodes
+ .slice(usize_range(node.range().range.clone()));
+ let edge_labels = store
+ .edge_labels
+ .slice(usize_range(node.labels()))
+ .map(|r| r.unwrap().decompress())
+ .flat_map(|block| block.labels.into_iter());
+
+ edges.push(
+ edge_nodes
+ .zip_eq(edge_labels)
+ .map(|(node, label)| node.with_label(label)),
+ );
+ }
+
+ NodeDatumMerger::new(edges)
+ }
+
+ fn merge_postings<P: AsRef<Path>>(
+ stores: &[EdgeStore],
+ label_compression: Compression,
+ path: P,
+ ) -> Result<Self> {
+ let reversed = stores[0].reversed;
+ let mut ranges = RangesDb::open(path.as_ref().join("ranges"));
+
+ let edge_labels_file = File::options()
+ .read(true)
+ .create(true)
+ .truncate(false)
+ .write(true)
+ .open(path.as_ref().join("labels"))
+ .unwrap();
+ let mut edge_labels = IterableStoreWriter::new(edge_labels_file);
+
+ let edge_nodes_file = File::options()
+ .read(true)
+ .create(true)
+ .truncate(false)
+ .write(true)
+ .open(path.as_ref().join("nodes"))
+ .unwrap();
+ let mut edge_nodes = ConstIterableStoreWriter::new(edge_nodes_file);
+
+ let mut merge_iter = MergeIter::new(
+ stores
+ .iter()
+ .map(|store| store.ranges.merge_nodes())
+ .collect(),
+ );
+
+ let mut buf = Vec::new();
+
+ while merge_iter.advance(&mut buf) {
+ if buf.is_empty() {
+ continue;
+ }
+
+ let edges = Self::merge_postings_for_node(&buf, stores);
+
+ // write postings
+ let node_sort_key = buf[0].range().sort_key;
+ let node_id = buf[0].id();
+ let mut first_label_offset = None;
+ let mut last_label_offset = None;
+ let mut first_node_offset = None;
+ let mut last_node_offset = None;
+
+ for chunk in edges.chunks(NUM_LABELS_PER_BLOCK).into_iter() {
+ let (labels, nodes): (Vec<_>, Vec<_>) = chunk
+ .map(|edge| (edge.label().clone(), edge.with_label(())))
+ .unzip();
+
+ let label_block = LabelBlock::new(labels).compress(label_compression);
+
+ let label_offset = edge_labels.write(&label_block).unwrap();
+
+ if first_label_offset.is_none() {
+ first_label_offset = Some(label_offset);
+ }
+
+ last_label_offset = Some(label_offset);
+
+ for node in nodes {
+ let offset = edge_nodes.write(&node).unwrap();
+
+ if first_node_offset.is_none() {
+ first_node_offset = Some(offset);
+ }
+
+ last_node_offset = Some(offset);
+ }
+ }
+
+ let label_range = Range {
+ start: first_label_offset.unwrap().start,
+ end: last_label_offset.unwrap().start + last_label_offset.unwrap().num_bytes,
+ };
+
+ let node_range: NodeRange = NodeRange::new(
+ Range {
+ start: first_node_offset.unwrap().start,
+ end: last_node_offset.unwrap().start + last_node_offset.unwrap().num_bytes,
+ },
+ node_sort_key,
+ );
+ let node_range_bytes = node_range.serialize_to_vec();
+
+ let node_bytes = node_id.serialize_to_vec();
+ ranges.insert_raw_node(node_bytes.clone(), node_range_bytes);
+
+ let label_range_bytes = label_range.serialize_to_vec();
+
+ ranges.insert_raw_label(node_bytes, label_range_bytes);
+
+ if ranges.uncommitted_node_inserts() > 100_000_000 {
+ ranges.commit();
+ edge_nodes.flush().unwrap();
+ edge_labels.flush().unwrap();
+ }
+ }
+
+ if ranges.uncommitted_node_inserts() > 0 {
+ ranges.commit();
+ edge_nodes.flush().unwrap();
+ edge_labels.flush().unwrap();
+ }
+
+ Ok(Self::open(path, reversed))
+ }
+
+ pub fn merge<P: AsRef<Path>>(
+ stores: Vec<EdgeStore>,
+ label_compression: Compression,
+ path: P,
+ ) -> Result<()> {
+ if stores.is_empty() {
+ return Ok(());
+ }
+
+ if !path.as_ref().exists() {
+ std::fs::create_dir_all(&path)?;
+ }
+
+ let reversed = stores[0].reversed;
+ if !stores.iter().all(|store| store.reversed == reversed) {
+ bail!("Cannot merge stores with different reversed flags");
+ }
+
+ let mut res = Self::merge_postings(&stores, label_compression, path)?;
+
+ for store in stores {
+ res.hosts.merge(store.hosts);
+ }
+
+ res.optimize_read();
+
+ Ok(())
+ }
+
pub fn get_with_label(&self, node: &NodeID, limit: &EdgeLimit) -> Vec<SegmentEdge<String>> {
let node_bytes = node.as_u64().to_le_bytes();
@@ -277,16 +480,8 @@ impl EdgeStore {
self.ranges.labels.get_raw(&node_bytes),
) {
(Some(node_range_bytes), Some(edge_range_bytes)) => {
- let (node_range, _) = bincode::decode_from_slice::<NodeRange, _>(
- node_range_bytes.as_bytes(),
- bincode::config::standard(),
- )
- .unwrap();
- let (edge_range, _) = bincode::decode_from_slice::<Range<u64>, _>(
- edge_range_bytes.as_bytes(),
- bincode::config::standard(),
- )
- .unwrap();
+ let node_range = NodeRange::deserialize(node_range_bytes.as_bytes());
+ let edge_range: Range<u64> = Range::deserialize(edge_range_bytes.as_bytes());
let edge_labels = self
.edge_labels
@@ -327,11 +522,7 @@ impl EdgeStore {
match self.ranges.nodes.get_raw(&node_bytes) {
Some(node_range_bytes) => {
- let (node_range, _) = bincode::decode_from_slice::<NodeRange, _>(
- node_range_bytes.as_bytes(),
- bincode::config::standard(),
- )
- .unwrap();
+ let node_range = NodeRange::deserialize(node_range_bytes.as_bytes());
let edge_nodes = self.edge_nodes.slice(usize_range(node_range.range));
@@ -369,11 +560,7 @@ impl EdgeStore {
let node = u64::from_le_bytes((key.as_bytes()).try_into().unwrap());
let node = NodeID::from(node);
- let (node_range, _) = bincode::decode_from_slice::<NodeRange, _>(
- val.as_bytes(),
- bincode::config::standard(),
- )
- .unwrap();
+ let node_range = NodeRange::deserialize(val.as_bytes());
let edge_nodes = self
.edge_nodes
diff --git a/crates/core/src/webgraph/store_writer.rs b/crates/core/src/webgraph/store_writer.rs
index d3854c64..84f3d0d7 100644
--- a/crates/core/src/webgraph/store_writer.rs
+++ b/crates/core/src/webgraph/store_writer.rs
@@ -27,14 +27,19 @@ use std::{
use itertools::Itertools;
use crate::{
- webgraph::store::{NodeDatum, NodeRange, NUM_LABELS_PER_BLOCK},
+ webgraph::store::{NodeRange, NUM_LABELS_PER_BLOCK},
Result,
};
-use file_store::iterable::{
- ConstIterableStoreWriter, IterableStoreReader, IterableStoreWriter, SortedIterableStoreReader,
+use file_store::{
+ iterable::{
+ ConstIterableStoreWriter, IterableStoreReader, IterableStoreWriter,
+ SortedIterableStoreReader,
+ },
+ ConstSerializable,
};
use super::{
+ merge::NodeDatum,
store::{CompressedLabelBlock, EdgeStore, HostDb, LabelBlock, RangesDb},
Compression, EdgeLabel, InsertableEdge, NodeID,
};
@@ -291,7 +296,7 @@ impl FinalEdgeStoreWriter {
}
self.hosts.insert(&node);
- let node_bytes = node.id.as_u64().to_le_bytes();
+ let node_bytes = node.id.serialize_to_vec();
debug_assert!(self.ranges.nodes_get_raw(&node_bytes).is_none());
debug_assert!(self.ranges.labels_get_raw(&node_bytes).is_none());
@@ -368,16 +373,14 @@ impl FinalEdgeStoreWriter {
},
sort_key,
);
+ let node_range_bytes = node_range.serialize_to_vec();
- self.ranges.insert_raw_node(
- node_bytes.to_vec(),
- bincode::encode_to_vec(node_range, bincode::config::standard()).unwrap(),
- );
+ self.ranges
+ .insert_raw_node(node_bytes.clone(), node_range_bytes);
- self.ranges.insert_raw_label(
- node_bytes.to_vec(),
- bincode::encode_to_vec(label_range, bincode::config::standard()).unwrap(),
- );
+ let label_range_bytes = label_range.serialize_to_vec();
+
+ self.ranges.insert_raw_label(node_bytes, label_range_bytes);
}
/// Build a new edge store from a set of edges.
diff --git a/crates/file-store/src/const_serializable.rs b/crates/file-store/src/const_serializable.rs
index 1029a946..2fe980de 100644
--- a/crates/file-store/src/const_serializable.rs
+++ b/crates/file-store/src/const_serializable.rs
@@ -23,11 +23,34 @@
// fn deserialize(bytes: [u8; Self::BYTES]) -> Self;
// }
+use std::ops::Range;
+
pub trait ConstSerializable {
const BYTES: usize;
fn serialize(&self, buf: &mut [u8]);
fn deserialize(buf: &[u8]) -> Self;
+
+ fn serialize_to_vec(&self) -> Vec<u8> {
+ let mut buf = vec![0; Self::BYTES];
+ self.serialize(&mut buf);
+ buf
+ }
+}
+
+impl ConstSerializable for Range<u64> {
+ const BYTES: usize = std::mem::size_of::<u64>() * 2;
+
+ fn serialize(&self, buf: &mut [u8]) {
+ self.start.serialize(&mut buf[..std::mem::size_of::<u64>()]);
+ self.end.serialize(&mut buf[std::mem::size_of::<u64>()..]);
+ }
+
+ fn deserialize(buf: &[u8]) -> Self {
+ let start = u64::deserialize(&buf[..std::mem::size_of::<u64>()]);
+ let end = u64::deserialize(&buf[std::mem::size_of::<u64>()..]);
+ start..end
+ }
}
macro_rules! impl_const_serializable_num {
diff --git a/crates/file-store/src/peekable.rs b/crates/file-store/src/peekable.rs
index d0757769..65aab11c 100644
--- a/crates/file-store/src/peekable.rs
+++ b/crates/file-store/src/peekable.rs
@@ -57,3 +57,54 @@ where
peeked
}
}
+
+impl<I, T> Ord for Peekable<I>
+where
+ I: Iterator<Item = T>,
+ T: Ord,
+{
+ fn cmp(&self, other: &Self) -> std::cmp::Ordering {
+ match (self.peek(), other.peek()) {
+ (Some(a), Some(b)) => a.cmp(b),
+ (Some(_), None) => std::cmp::Ordering::Less,
+ (None, Some(_)) => std::cmp::Ordering::Greater,
+ (None, None) => std::cmp::Ordering::Equal,
+ }
+ }
+}
+
+impl<I, T> PartialOrd for Peekable<I>
+where
+ I: Iterator<Item = T>,
+ T: PartialOrd,
+{
+ fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
+ match (self.peek(), other.peek()) {
+ (Some(a), Some(b)) => a.partial_cmp(b),
+ (Some(_), None) => Some(std::cmp::Ordering::Less),
+ (None, Some(_)) => Some(std::cmp::Ordering::Greater),
+ (None, None) => Some(std::cmp::Ordering::Equal),
+ }
+ }
+}
+
+impl<I, T> PartialEq for Peekable<I>
+where
+ I: Iterator<Item = T>,
+ T: PartialEq,
+{
+ fn eq(&self, other: &Self) -> bool {
+ match (self.peek(), other.peek()) {
+ (Some(a), Some(b)) => a == b,
+ (None, None) => true,
+ _ => false,
+ }
+ }
+}
+
+impl<I, T> Eq for Peekable<I>
+where
+ I: Iterator<Item = T>,
+ T: Eq,
+{
+}
diff --git a/crates/speedy-kv/src/segment.rs b/crates/speedy-kv/src/segment.rs
index 7bc7ede1..458a55d2 100644
--- a/crates/speedy-kv/src/segment.rs
+++ b/crates/speedy-kv/src/segment.rs
@@ -342,11 +342,7 @@ where
I: Iterator<Item = (SerializedRef<'a, K>, SerializedRef<'a, V>)>,
{
fn eq(&self, other: &Self) -> bool {
- match (self.iter.peek(), other.iter.peek()) {
- (Some((a, _)), Some((b, _))) => a == b && self.segment_ord == other.segment_ord,
- (None, None) => true,
- _ => false,
- }
+ self.iter == other.iter && self.segment_ord == other.segment_ord
}
}
diff --git a/scripts/requirements.txt b/scripts/requirements.txt
index c0d5dfc8..92441620 100644
--- a/scripts/requirements.txt
+++ b/scripts/requirements.txt
@@ -4,6 +4,3 @@ datasets
gensim
safetensors
pandas
-llama-cpp-python[server]
-llama-cpp-python
-huggingface-hub>=0.20.3