web search done right
optionally merge all webgraph segments into a single segment for improved read performance
Mikkel Denker817bda9738bd
diff --git a/crates/core/src/config/defaults.rs b/crates/core/src/config/defaults.rsindex 803d8575..fb1853e7 100644--- a/crates/core/src/config/defaults.rs+++ b/crates/core/src/config/defaults.rs@@ -200,3 +200,10 @@ impl ApproxHarmonic {false}}++pub struct Webgraph;+impl Webgraph {+ pub fn merge_all_segments() -> bool {+ true+ }+}diff --git a/crates/core/src/config/mod.rs b/crates/core/src/config/mod.rsindex 274bedd2..39c005d1 100644--- a/crates/core/src/config/mod.rs+++ b/crates/core/src/config/mod.rs@@ -74,6 +74,9 @@ pub struct WebgraphConstructConfig {pub batch_size: Option<usize>,pub canonical_index_path: Option<String>,pub host_centrality_rank_store_path: Option<String>,++ #[serde(default = "defaults::Webgraph::merge_all_segments")]+ pub merge_all_segments: bool,}#[derive(Debug, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, Clone)]diff --git a/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/mapper.rs b/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/mapper.rsindex 6e677885..ce3fe0c3 100644--- a/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/mapper.rs+++ b/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/mapper.rs@@ -90,7 +90,7 @@ impl Mapper for ApproxCentralityMapper {) {let pool = rayon::ThreadPoolBuilder::new().num_threads(usize::from(std::thread::available_parallelism().unwrap()))- .stack_size(800_000_000)+ .stack_size(80_000_000).thread_name(move |num| format!("approx-harmonic-mapper-{num}")).build().unwrap();@@ -115,6 +115,8 @@ impl Mapper for ApproxCentralityMapper {let workers = Workers::new(worker.clone());let num_samples = dht.next().meta.get(()).unwrap().num_samples_per_worker;+ tracing::info!("Sampling {} nodes", num_samples);+let sampled = worker.graph().random_nodes_with_outgoing(num_samples as usize);diff --git a/crates/core/src/entrypoint/webgraph.rs b/crates/core/src/entrypoint/webgraph.rsindex 6d6bd3e6..54f0467d 100644--- a/crates/core/src/entrypoint/webgraph.rs+++ b/crates/core/src/entrypoint/webgraph.rs@@ -261,6 +261,11 @@ impl Webgraph {page_graph.merge(other_page)?;}+ if config.merge_all_segments {+ host_graph.merge_all_segments(Default::default())?;+ page_graph.merge_all_segments(Default::default())?;+ }+host_graph.optimize_read();page_graph.optimize_read();diff --git a/crates/core/src/main.rs b/crates/core/src/main.rsindex 60afd253..36c3d0d9 100644--- a/crates/core/src/main.rs+++ b/crates/core/src/main.rs@@ -218,6 +218,9 @@ enum WebgraphOptions {Merge {#[clap(required = true)]paths: Vec<String>,++ #[clap(default_value_t = stract::config::defaults::Webgraph::merge_all_segments())]+ merge_all_segments: bool,},/// Deploy the webgraph server. The webgraph server is responsible for serving the webgraph to the search servers.@@ -326,7 +329,10 @@ fn main() -> Result<()> {let config = load_toml_config(config_path);entrypoint::Webgraph::run(&config)?;}- WebgraphOptions::Merge { mut paths } => {+ WebgraphOptions::Merge {+ mut paths,+ merge_all_segments,+ } => {let mut webgraph = WebgraphBuilder::new(paths.remove(0)).single_threaded().open();@@ -336,6 +342,10 @@ fn main() -> Result<()> {webgraph.merge(other)?;}+ if merge_all_segments {+ webgraph.merge_all_segments(Default::default())?;+ }+webgraph.optimize_read();}WebgraphOptions::Server { config_path } => {diff --git a/crates/core/src/webgraph/edge.rs b/crates/core/src/webgraph/edge.rsindex d415a8bc..b477585f 100644--- a/crates/core/src/webgraph/edge.rs+++ b/crates/core/src/webgraph/edge.rs@@ -16,7 +16,7 @@use utoipa::ToSchema;-use super::{store::NodeDatum, FullNodeID, Node, NodeID};+use super::{merge::NodeDatum, FullNodeID, Node, NodeID};pub const MAX_LABEL_LENGTH: usize = 1024;@@ -106,8 +106,8 @@ pub struct SegmentEdge<L>whereL: EdgeLabel,{- pub from: NodeDatum,- pub to: NodeDatum,+ pub from: NodeDatum<()>,+ pub to: NodeDatum<()>,pub label: L,}diff --git a/crates/core/src/webgraph/merge.rs b/crates/core/src/webgraph/merge.rsnew file mode 100644index 00000000..8d06b3e5--- /dev/null+++ b/crates/core/src/webgraph/merge.rs@@ -0,0 +1,325 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/license++use std::{cmp::Reverse, collections::BinaryHeap};++use super::{store::NodeRange, NodeID};++pub struct MergeNode<O = ()> {+ node: NodeID,+ range: NodeRange,+ labels: std::ops::Range<u64>,+ ord: O,+}++impl MergeNode {+ pub fn new(node: NodeID, range: NodeRange, labels: std::ops::Range<u64>) -> Self {+ Self {+ node,+ range,+ labels,+ ord: (),+ }+ }+}++impl<O> MergeNode<O> {+ pub fn with_ord<O2>(self, ord: O2) -> MergeNode<O2> {+ MergeNode {+ node: self.node,+ range: self.range,+ labels: self.labels,+ ord,+ }+ }++ pub fn id(&self) -> NodeID {+ self.node+ }++ pub fn range(&self) -> &NodeRange {+ &self.range+ }++ pub fn labels(&self) -> std::ops::Range<u64> {+ self.labels.clone()+ }++ pub fn ord(&self) -> &O {+ &self.ord+ }+}++impl<O> Ord for MergeNode<O> {+ fn cmp(&self, other: &Self) -> std::cmp::Ordering {+ // nodes in the fst are sorted by their id,+ // not by their sort key+ self.node.cmp(&other.node)+ }+}+impl<O> PartialOrd for MergeNode<O> {+ fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {+ Some(self.cmp(other))+ }+}+impl<O> PartialEq for MergeNode<O> {+ fn eq(&self, other: &Self) -> bool {+ self.node == other.node+ }+}+impl<O> Eq for MergeNode<O> {}++#[derive(Clone, Copy)]+pub struct MergeSegmentOrd(usize);+impl MergeSegmentOrd {+ pub fn new(ord: usize) -> Self {+ Self(ord)+ }++ pub fn as_usize(&self) -> usize {+ self.0+ }+}++type MinHeap<T> = BinaryHeap<Reverse<T>>;++/// Merge multiple iterators of MergeNodes into a single iterator based on the node id+pub struct MergeIter<'a> {+ iters: MinHeap<file_store::Peekable<Box<dyn Iterator<Item = MergeNode<MergeSegmentOrd>> + 'a>>>,+}+impl<'a> MergeIter<'a> {+ pub fn new(iters: Vec<impl Iterator<Item = MergeNode> + 'a>) -> Self {+ let iters = iters+ .into_iter()+ .enumerate()+ .map(|(ord, iter)| {+ let it = Box::new(iter.map(move |node| node.with_ord(MergeSegmentOrd::new(ord))))+ as Box<dyn Iterator<Item = _>>;++ Reverse(file_store::Peekable::new(it))+ })+ .collect();++ Self { iters }+ }++ pub fn advance(&mut self, buf: &mut Vec<MergeNode<MergeSegmentOrd>>) -> bool {+ buf.clear();++ let next = {+ let item = self.iters.peek_mut();++ if item.is_none() {+ return false;+ }++ let mut item = item.unwrap();++ if item.0.peek().is_none() {+ return false;+ }++ item.0.next().unwrap()+ };++ let node = next.node;+ buf.push(next);++ // advance all iterators that have the same node+ while let Some(mut peek) = self.iters.peek_mut() {+ if peek.0.peek().map(|x| x.node) == Some(node) {+ buf.push(peek.0.next().unwrap());+ } else {+ break;+ }+ }++ true+ }+}++/// Merge multiple iterators of NodeDatum into a single iterator based on the sort key+pub struct NodeDatumMerger<'a, L = String> {+ iters: MinHeap<file_store::Peekable<Box<dyn Iterator<Item = NodeDatum<L>> + 'a>>>,+}++impl<'a, L> NodeDatumMerger<'a, L> {+ pub fn new(iters: Vec<impl Iterator<Item = NodeDatum<L>> + 'a>) -> Self {+ let iters = iters+ .into_iter()+ .map(|iter| {+ let it = Box::new(iter) as Box<dyn Iterator<Item = _>>;+ Reverse(file_store::Peekable::new(it))+ })+ .collect();++ Self { iters }+ }+}++impl<'a, L> Iterator for NodeDatumMerger<'a, L> {+ type Item = NodeDatum<L>;++ fn next(&mut self) -> Option<Self::Item> {+ let res = self.iters.peek_mut().and_then(|mut item| item.0.next());++ if let Some(datum) = &res {+ while let Some(mut peek) = self.iters.peek_mut() {+ if peek.0.peek().map(|x| x.sort_key) == Some(datum.sort_key) {+ peek.0.next().unwrap();+ } else {+ break;+ }+ }+ }++ res+ }+}++#[derive(Debug, Clone)]+pub struct NodeDatum<L = ()> {+ id: NodeID,+ sort_key: u64,+ label: L,+}++impl<L> PartialOrd for NodeDatum<L> {+ fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {+ Some(self.cmp(other))+ }+}++impl<L> Ord for NodeDatum<L> {+ fn cmp(&self, other: &Self) -> std::cmp::Ordering {+ self.sort_key+ .cmp(&other.sort_key)+ .then_with(|| self.id.cmp(&other.id))+ }+}++impl<L> PartialEq for NodeDatum<L> {+ fn eq(&self, other: &Self) -> bool {+ self.id == other.id && self.sort_key == other.sort_key+ }+}++impl<L> Eq for NodeDatum<L> {}++impl NodeDatum {+ pub fn new(node: NodeID, sort_key: u64) -> Self {+ Self {+ id: node,+ sort_key,+ label: (),+ }+ }+}++impl<L> NodeDatum<L> {+ #[inline]+ pub fn node(&self) -> NodeID {+ self.id+ }++ #[inline]+ pub fn sort_key(&self) -> u64 {+ self.sort_key+ }++ pub fn with_label<L2>(self, label: L2) -> NodeDatum<L2> {+ NodeDatum {+ id: self.id,+ sort_key: self.sort_key,+ label,+ }+ }++ #[inline]+ pub fn label(&self) -> &L {+ &self.label+ }+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn test_merge_nodes() {+ let a = vec![+ MergeNode::new(1u64.into(), NodeRange::new(0..10, 1), 0..10),+ MergeNode::new(4u64.into(), NodeRange::new(0..10, 2), 0..10),+ MergeNode::new(5u64.into(), NodeRange::new(0..10, 3), 0..10),+ ];++ let b = vec![+ MergeNode::new(2u64.into(), NodeRange::new(0..10, 4), 0..10),+ MergeNode::new(3u64.into(), NodeRange::new(0..10, 5), 0..10),+ MergeNode::new(5u64.into(), NodeRange::new(0..10, 3), 0..10),+ ];++ let mut merger = MergeIter::new(vec![a.into_iter(), b.into_iter()]);+ let mut buf = Vec::new();++ assert!(merger.advance(&mut buf));+ assert_eq!(buf.len(), 1);+ assert_eq!(buf[0].id(), 1u64.into());++ assert!(merger.advance(&mut buf));+ assert_eq!(buf.len(), 1);+ assert_eq!(buf[0].id(), 2u64.into());++ assert!(merger.advance(&mut buf));+ assert_eq!(buf.len(), 1);+ assert_eq!(buf[0].id(), 3u64.into());++ assert!(merger.advance(&mut buf));+ assert_eq!(buf.len(), 1);+ assert_eq!(buf[0].id(), 4u64.into());++ assert!(merger.advance(&mut buf));+ assert_eq!(buf.len(), 2);+ assert_eq!(buf[0].id(), 5u64.into());+ assert_eq!(buf[1].id(), 5u64.into());++ assert!(!merger.advance(&mut buf));+ }++ #[test]+ fn test_datum_merge() {+ let a = vec![+ NodeDatum::new(1u64.into(), 1),+ NodeDatum::new(2u64.into(), 4),+ NodeDatum::new(3u64.into(), 5),+ ];++ let b = vec![+ NodeDatum::new(4u64.into(), 2),+ NodeDatum::new(5u64.into(), 3),+ NodeDatum::new(3u64.into(), 5),+ ];++ let mut merger = NodeDatumMerger::new(vec![a.into_iter(), b.into_iter()]);++ assert_eq!(merger.next().unwrap().sort_key(), 1);+ assert_eq!(merger.next().unwrap().sort_key(), 2);+ assert_eq!(merger.next().unwrap().sort_key(), 3);+ assert_eq!(merger.next().unwrap().sort_key(), 4);+ assert_eq!(merger.next().unwrap().sort_key(), 5);+ assert!(merger.next().is_none());+ }+}diff --git a/crates/core/src/webgraph/mod.rs b/crates/core/src/webgraph/mod.rsindex 52f97384..67af0386 100644--- a/crates/core/src/webgraph/mod.rs+++ b/crates/core/src/webgraph/mod.rs@@ -22,11 +22,13 @@ use std::{fs, io};use itertools::Itertools;use rand::seq::{IteratorRandom, SliceRandom};use rayon::prelude::*;+use uuid::Uuid;use self::id_node_db::Id2NodeDb;use self::segment::Segment;use crate::executor::Executor;+use crate::Result;pub use builder::WebgraphBuilder;pub use compression::Compression;pub use edge::*;@@ -39,6 +41,7 @@ pub mod centrality;mod compression;mod edge;mod id_node_db;+mod merge;mod node;pub mod remote;mod segment;@@ -174,6 +177,23 @@ impl Webgraph {Ok(())}+ pub fn merge_all_segments(&mut self, compression: Compression) -> Result<()> {+ let segments = std::mem::take(&mut self.segments);++ let id = Uuid::new_v4().to_string();+ let path = Path::new(&self.path).join("segments");++ Segment::merge(segments, compression, &path, id.clone())?;+ let new_segment = Segment::open(path, id.clone());++ self.segments.push(new_segment);+ self.meta.comitted_segments = vec![id];++ self.save_metadata();++ Ok(())+ }+pub fn optimize_read(&mut self) {self.executor.map(|s| s.optimize_read(), self.segments.iter_mut())@@ -543,6 +563,13 @@ pub mod tests {graph.distances(Node::from("A")).get(&Node::from("C")),Some(&2));++ graph.merge_all_segments(Compression::default()).unwrap();++ assert_eq!(+ graph.distances(Node::from("A")).get(&Node::from("C")),+ Some(&2)+ );}#[test]diff --git a/crates/core/src/webgraph/segment.rs b/crates/core/src/webgraph/segment.rsindex 40af0b5e..d4bd308a 100644--- a/crates/core/src/webgraph/segment.rs+++ b/crates/core/src/webgraph/segment.rs@@ -17,12 +17,14 @@use std::{path::{Path, PathBuf},sync::Arc,+ thread,};use super::{store::EdgeStore, store_writer::EdgeStoreWriter, Compression, EdgeLimit, InsertableEdge,NodeID, SegmentEdge,};+use crate::Result;const ADJACENCY_STORE: &str = "adjacency";const REVERSED_ADJACENCY_STORE: &str = "reversed_adjacency";@@ -110,6 +112,44 @@ impl Segment {}}+ pub fn merge<P: AsRef<Path>>(+ segments: Vec<Self>,+ label_compression: Compression,+ folder: P,+ id: String,+ ) -> Result<()> {+ let old_paths = segments.iter().map(|s| s.path()).collect::<Vec<_>>();++ let (adjacency, reversed_adjacency) = segments+ .into_iter()+ .map(|s| (s.adjacency, s.reversed_adjacency))+ .unzip();++ let adjacency_path = folder.as_ref().join(&id).join(ADJACENCY_STORE);+ let adjacency =+ thread::spawn(move || EdgeStore::merge(adjacency, label_compression, adjacency_path));++ let reversed_adjacency_path = folder.as_ref().join(&id).join(REVERSED_ADJACENCY_STORE);+ let reversed_adjacency = thread::spawn(move || {+ EdgeStore::merge(+ reversed_adjacency,+ label_compression,+ reversed_adjacency_path,+ )+ });++ adjacency.join().unwrap()?;+ reversed_adjacency.join().unwrap()?;++ for path in old_paths {+ if Path::new(&path).exists() {+ std::fs::remove_dir_all(path)?;+ }+ }++ Ok(())+ }+pub fn outgoing_edges_with_label(&self,node: &NodeID,diff --git a/crates/core/src/webgraph/store.rs b/crates/core/src/webgraph/store.rsindex 50129ec3..5bb44797 100644--- a/crates/core/src/webgraph/store.rs+++ b/crates/core/src/webgraph/store.rs@@ -14,16 +14,27 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.-use std::{ops::Range, path::Path};+use std::{fs::File, ops::Range, path::Path};+use crate::{+ webgraph::merge::{MergeIter, NodeDatumMerger},+ Result,+};+use anyhow::bail;use file_store::{- iterable::{ConstIterableStoreReader, IterableStoreReader},+ iterable::{+ ConstIterableStoreReader, ConstIterableStoreWriter, IterableStoreReader,+ IterableStoreWriter,+ },ConstSerializable,};use fst::Automaton;use itertools::Itertools;-use super::{Compression, EdgeLimit, FullNodeID, NodeID, SegmentEdge};+use super::{+ merge::{MergeNode, MergeSegmentOrd, NodeDatum},+ Compression, EdgeLimit, FullNodeID, NodeID, SegmentEdge,+};#[derive(Debug,@@ -86,6 +97,10 @@ impl HostDb {pub fn flush(&mut self) {self.db.commit().unwrap();}++ fn merge(&mut self, other: HostDb) {+ self.db.merge(other.db).unwrap();+ }}#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]@@ -123,6 +138,10 @@ impl RangesDb {self.labels.merge_all_segments().unwrap();}+ fn uncommitted_node_inserts(&self) -> usize {+ self.nodes.uncommitted_inserts()+ }+pub fn commit(&mut self) {self.nodes.commit().unwrap();self.labels.commit().unwrap();@@ -149,29 +168,26 @@ impl RangesDb {pub fn insert_raw_label(&mut self, node: Vec<u8>, range: Vec<u8>) {self.labels.insert_raw(node, range);}-}-#[derive(Debug, Clone, PartialEq)]-pub struct NodeDatum {- id: NodeID,- sort_key: u64,-}+ fn merge_nodes(&self) -> impl Iterator<Item = MergeNode> + '_ {+ self.nodes.iter_raw().zip_eq(self.labels.iter_raw()).map(+ move |((key_node, val), (key_label, labels))| {+ debug_assert_eq!(key_node, key_label);-impl NodeDatum {- pub fn new(node: NodeID, sort_key: u64) -> Self {- Self { id: node, sort_key }- }+ let node = NodeID::deserialize(key_node.as_bytes());+ let range = NodeRange::deserialize(val.as_bytes());- #[inline]- pub fn node(&self) -> NodeID {- self.id- }+ let labels = {+ let range = Range::deserialize(labels.as_bytes());+ range.start..range.end+ };- #[inline]- pub fn sort_key(&self) -> u64 {- self.sort_key+ MergeNode::new(node, range, labels)+ },+ )}}+impl ConstSerializable for NodeID {const BYTES: usize = std::mem::size_of::<NodeID>();@@ -189,15 +205,33 @@ impl ConstSerializable for NodeDatum {const BYTES: usize = std::mem::size_of::<NodeDatum>();fn serialize(&self, buf: &mut [u8]) {- self.id.serialize(&mut buf[..NodeID::BYTES]);- self.sort_key.serialize(&mut buf[NodeID::BYTES..]);+ self.node().serialize(&mut buf[..NodeID::BYTES]);+ self.sort_key().serialize(&mut buf[NodeID::BYTES..]);}fn deserialize(buf: &[u8]) -> Self {let node = NodeID::deserialize(&buf[..NodeID::BYTES]);let sort_key = u64::deserialize(&buf[NodeID::BYTES..]);- Self { id: node, sort_key }+ Self::new(node, sort_key)+ }+}++impl ConstSerializable for NodeRange {+ const BYTES: usize = std::mem::size_of::<NodeRange>();++ fn serialize(&self, buf: &mut [u8]) {+ const RANGE_BYTES: usize = std::mem::size_of::<u64>() * 2;+ self.range.serialize(&mut buf[..RANGE_BYTES]);+ self.sort_key.serialize(&mut buf[RANGE_BYTES..]);+ }++ fn deserialize(buf: &[u8]) -> Self {+ const RANGE_BYTES: usize = std::mem::size_of::<u64>() * 2;+ let range: Range<u64> = Range::deserialize(&buf[..RANGE_BYTES]);+ let sort_key = u64::deserialize(&buf[RANGE_BYTES..]);++ Self { range, sort_key }}}@@ -269,6 +303,175 @@ impl EdgeStore {self.hosts.optimize_read();}+ fn merge_postings_for_node<'a>(+ buf: &[MergeNode<MergeSegmentOrd>],+ stores: &'a [EdgeStore],+ ) -> NodeDatumMerger<'a> {+ let mut edges = Vec::new();++ for node in buf {+ let store = &stores[node.ord().as_usize()];+ let edge_nodes = store+ .edge_nodes+ .slice(usize_range(node.range().range.clone()));+ let edge_labels = store+ .edge_labels+ .slice(usize_range(node.labels()))+ .map(|r| r.unwrap().decompress())+ .flat_map(|block| block.labels.into_iter());++ edges.push(+ edge_nodes+ .zip_eq(edge_labels)+ .map(|(node, label)| node.with_label(label)),+ );+ }++ NodeDatumMerger::new(edges)+ }++ fn merge_postings<P: AsRef<Path>>(+ stores: &[EdgeStore],+ label_compression: Compression,+ path: P,+ ) -> Result<Self> {+ let reversed = stores[0].reversed;+ let mut ranges = RangesDb::open(path.as_ref().join("ranges"));++ let edge_labels_file = File::options()+ .read(true)+ .create(true)+ .truncate(false)+ .write(true)+ .open(path.as_ref().join("labels"))+ .unwrap();+ let mut edge_labels = IterableStoreWriter::new(edge_labels_file);++ let edge_nodes_file = File::options()+ .read(true)+ .create(true)+ .truncate(false)+ .write(true)+ .open(path.as_ref().join("nodes"))+ .unwrap();+ let mut edge_nodes = ConstIterableStoreWriter::new(edge_nodes_file);++ let mut merge_iter = MergeIter::new(+ stores+ .iter()+ .map(|store| store.ranges.merge_nodes())+ .collect(),+ );++ let mut buf = Vec::new();++ while merge_iter.advance(&mut buf) {+ if buf.is_empty() {+ continue;+ }++ let edges = Self::merge_postings_for_node(&buf, stores);++ // write postings+ let node_sort_key = buf[0].range().sort_key;+ let node_id = buf[0].id();+ let mut first_label_offset = None;+ let mut last_label_offset = None;+ let mut first_node_offset = None;+ let mut last_node_offset = None;++ for chunk in edges.chunks(NUM_LABELS_PER_BLOCK).into_iter() {+ let (labels, nodes): (Vec<_>, Vec<_>) = chunk+ .map(|edge| (edge.label().clone(), edge.with_label(())))+ .unzip();++ let label_block = LabelBlock::new(labels).compress(label_compression);++ let label_offset = edge_labels.write(&label_block).unwrap();++ if first_label_offset.is_none() {+ first_label_offset = Some(label_offset);+ }++ last_label_offset = Some(label_offset);++ for node in nodes {+ let offset = edge_nodes.write(&node).unwrap();++ if first_node_offset.is_none() {+ first_node_offset = Some(offset);+ }++ last_node_offset = Some(offset);+ }+ }++ let label_range = Range {+ start: first_label_offset.unwrap().start,+ end: last_label_offset.unwrap().start + last_label_offset.unwrap().num_bytes,+ };++ let node_range: NodeRange = NodeRange::new(+ Range {+ start: first_node_offset.unwrap().start,+ end: last_node_offset.unwrap().start + last_node_offset.unwrap().num_bytes,+ },+ node_sort_key,+ );+ let node_range_bytes = node_range.serialize_to_vec();++ let node_bytes = node_id.serialize_to_vec();+ ranges.insert_raw_node(node_bytes.clone(), node_range_bytes);++ let label_range_bytes = label_range.serialize_to_vec();++ ranges.insert_raw_label(node_bytes, label_range_bytes);++ if ranges.uncommitted_node_inserts() > 100_000_000 {+ ranges.commit();+ edge_nodes.flush().unwrap();+ edge_labels.flush().unwrap();+ }+ }++ if ranges.uncommitted_node_inserts() > 0 {+ ranges.commit();+ edge_nodes.flush().unwrap();+ edge_labels.flush().unwrap();+ }++ Ok(Self::open(path, reversed))+ }++ pub fn merge<P: AsRef<Path>>(+ stores: Vec<EdgeStore>,+ label_compression: Compression,+ path: P,+ ) -> Result<()> {+ if stores.is_empty() {+ return Ok(());+ }++ if !path.as_ref().exists() {+ std::fs::create_dir_all(&path)?;+ }++ let reversed = stores[0].reversed;+ if !stores.iter().all(|store| store.reversed == reversed) {+ bail!("Cannot merge stores with different reversed flags");+ }++ let mut res = Self::merge_postings(&stores, label_compression, path)?;++ for store in stores {+ res.hosts.merge(store.hosts);+ }++ res.optimize_read();++ Ok(())+ }+pub fn get_with_label(&self, node: &NodeID, limit: &EdgeLimit) -> Vec<SegmentEdge<String>> {let node_bytes = node.as_u64().to_le_bytes();@@ -277,16 +480,8 @@ impl EdgeStore {self.ranges.labels.get_raw(&node_bytes),) {(Some(node_range_bytes), Some(edge_range_bytes)) => {- let (node_range, _) = bincode::decode_from_slice::<NodeRange, _>(- node_range_bytes.as_bytes(),- bincode::config::standard(),- )- .unwrap();- let (edge_range, _) = bincode::decode_from_slice::<Range<u64>, _>(- edge_range_bytes.as_bytes(),- bincode::config::standard(),- )- .unwrap();+ let node_range = NodeRange::deserialize(node_range_bytes.as_bytes());+ let edge_range: Range<u64> = Range::deserialize(edge_range_bytes.as_bytes());let edge_labels = self.edge_labels@@ -327,11 +522,7 @@ impl EdgeStore {match self.ranges.nodes.get_raw(&node_bytes) {Some(node_range_bytes) => {- let (node_range, _) = bincode::decode_from_slice::<NodeRange, _>(- node_range_bytes.as_bytes(),- bincode::config::standard(),- )- .unwrap();+ let node_range = NodeRange::deserialize(node_range_bytes.as_bytes());let edge_nodes = self.edge_nodes.slice(usize_range(node_range.range));@@ -369,11 +560,7 @@ impl EdgeStore {let node = u64::from_le_bytes((key.as_bytes()).try_into().unwrap());let node = NodeID::from(node);- let (node_range, _) = bincode::decode_from_slice::<NodeRange, _>(- val.as_bytes(),- bincode::config::standard(),- )- .unwrap();+ let node_range = NodeRange::deserialize(val.as_bytes());let edge_nodes = self.edge_nodesdiff --git a/crates/core/src/webgraph/store_writer.rs b/crates/core/src/webgraph/store_writer.rsindex d3854c64..84f3d0d7 100644--- a/crates/core/src/webgraph/store_writer.rs+++ b/crates/core/src/webgraph/store_writer.rs@@ -27,14 +27,19 @@ use std::{use itertools::Itertools;use crate::{- webgraph::store::{NodeDatum, NodeRange, NUM_LABELS_PER_BLOCK},+ webgraph::store::{NodeRange, NUM_LABELS_PER_BLOCK},Result,};-use file_store::iterable::{- ConstIterableStoreWriter, IterableStoreReader, IterableStoreWriter, SortedIterableStoreReader,+use file_store::{+ iterable::{+ ConstIterableStoreWriter, IterableStoreReader, IterableStoreWriter,+ SortedIterableStoreReader,+ },+ ConstSerializable,};use super::{+ merge::NodeDatum,store::{CompressedLabelBlock, EdgeStore, HostDb, LabelBlock, RangesDb},Compression, EdgeLabel, InsertableEdge, NodeID,};@@ -291,7 +296,7 @@ impl FinalEdgeStoreWriter {}self.hosts.insert(&node);- let node_bytes = node.id.as_u64().to_le_bytes();+ let node_bytes = node.id.serialize_to_vec();debug_assert!(self.ranges.nodes_get_raw(&node_bytes).is_none());debug_assert!(self.ranges.labels_get_raw(&node_bytes).is_none());@@ -368,16 +373,14 @@ impl FinalEdgeStoreWriter {},sort_key,);+ let node_range_bytes = node_range.serialize_to_vec();- self.ranges.insert_raw_node(- node_bytes.to_vec(),- bincode::encode_to_vec(node_range, bincode::config::standard()).unwrap(),- );+ self.ranges+ .insert_raw_node(node_bytes.clone(), node_range_bytes);- self.ranges.insert_raw_label(- node_bytes.to_vec(),- bincode::encode_to_vec(label_range, bincode::config::standard()).unwrap(),- );+ let label_range_bytes = label_range.serialize_to_vec();++ self.ranges.insert_raw_label(node_bytes, label_range_bytes);}/// Build a new edge store from a set of edges.diff --git a/crates/file-store/src/const_serializable.rs b/crates/file-store/src/const_serializable.rsindex 1029a946..2fe980de 100644--- a/crates/file-store/src/const_serializable.rs+++ b/crates/file-store/src/const_serializable.rs@@ -23,11 +23,34 @@// fn deserialize(bytes: [u8; Self::BYTES]) -> Self;// }+use std::ops::Range;+pub trait ConstSerializable {const BYTES: usize;fn serialize(&self, buf: &mut [u8]);fn deserialize(buf: &[u8]) -> Self;++ fn serialize_to_vec(&self) -> Vec<u8> {+ let mut buf = vec![0; Self::BYTES];+ self.serialize(&mut buf);+ buf+ }+}++impl ConstSerializable for Range<u64> {+ const BYTES: usize = std::mem::size_of::<u64>() * 2;++ fn serialize(&self, buf: &mut [u8]) {+ self.start.serialize(&mut buf[..std::mem::size_of::<u64>()]);+ self.end.serialize(&mut buf[std::mem::size_of::<u64>()..]);+ }++ fn deserialize(buf: &[u8]) -> Self {+ let start = u64::deserialize(&buf[..std::mem::size_of::<u64>()]);+ let end = u64::deserialize(&buf[std::mem::size_of::<u64>()..]);+ start..end+ }}macro_rules! impl_const_serializable_num {diff --git a/crates/file-store/src/peekable.rs b/crates/file-store/src/peekable.rsindex d0757769..65aab11c 100644--- a/crates/file-store/src/peekable.rs+++ b/crates/file-store/src/peekable.rs@@ -57,3 +57,54 @@ wherepeeked}}++impl<I, T> Ord for Peekable<I>+where+ I: Iterator<Item = T>,+ T: Ord,+{+ fn cmp(&self, other: &Self) -> std::cmp::Ordering {+ match (self.peek(), other.peek()) {+ (Some(a), Some(b)) => a.cmp(b),+ (Some(_), None) => std::cmp::Ordering::Less,+ (None, Some(_)) => std::cmp::Ordering::Greater,+ (None, None) => std::cmp::Ordering::Equal,+ }+ }+}++impl<I, T> PartialOrd for Peekable<I>+where+ I: Iterator<Item = T>,+ T: PartialOrd,+{+ fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {+ match (self.peek(), other.peek()) {+ (Some(a), Some(b)) => a.partial_cmp(b),+ (Some(_), None) => Some(std::cmp::Ordering::Less),+ (None, Some(_)) => Some(std::cmp::Ordering::Greater),+ (None, None) => Some(std::cmp::Ordering::Equal),+ }+ }+}++impl<I, T> PartialEq for Peekable<I>+where+ I: Iterator<Item = T>,+ T: PartialEq,+{+ fn eq(&self, other: &Self) -> bool {+ match (self.peek(), other.peek()) {+ (Some(a), Some(b)) => a == b,+ (None, None) => true,+ _ => false,+ }+ }+}++impl<I, T> Eq for Peekable<I>+where+ I: Iterator<Item = T>,+ T: Eq,+{+}diff --git a/crates/speedy-kv/src/segment.rs b/crates/speedy-kv/src/segment.rsindex 7bc7ede1..458a55d2 100644--- a/crates/speedy-kv/src/segment.rs+++ b/crates/speedy-kv/src/segment.rs@@ -342,11 +342,7 @@ whereI: Iterator<Item = (SerializedRef<'a, K>, SerializedRef<'a, V>)>,{fn eq(&self, other: &Self) -> bool {- match (self.iter.peek(), other.iter.peek()) {- (Some((a, _)), Some((b, _))) => a == b && self.segment_ord == other.segment_ord,- (None, None) => true,- _ => false,- }+ self.iter == other.iter && self.segment_ord == other.segment_ord}}diff --git a/scripts/requirements.txt b/scripts/requirements.txtindex c0d5dfc8..92441620 100644--- a/scripts/requirements.txt+++ b/scripts/requirements.txt@@ -4,6 +4,3 @@ datasetsgensimsafetensorspandas-llama-cpp-python[server]-llama-cpp-python-huggingface-hub>=0.20.3