web search done right
Re-write webgraph storage backend. The webgraph storage is now essentially a '(from, to) -> label' map stored in rocksdb databases. This heavily simplifies inserts and merges, since we can now insert new edges directly into the db without having to read the existing edges.
Mikkel Denker8fd2b2a29205
diff --git a/core/src/entrypoint/configure.rs b/core/src/entrypoint/configure.rsindex 4ce6c1ee..c177cf0e 100644--- a/core/src/entrypoint/configure.rs+++ b/core/src/entrypoint/configure.rs@@ -85,8 +85,6 @@ fn create_webgraph() -> Result<()> {};worker.process_job(&job);- let graph = worker.graph;- std::fs::rename(graph.path, out_path)?;Ok(())}diff --git a/core/src/entrypoint/indexer.rs b/core/src/entrypoint/indexer.rsindex 9e53855d..846f6b77 100644--- a/core/src/entrypoint/indexer.rs+++ b/core/src/entrypoint/indexer.rs@@ -183,7 +183,7 @@ pub fn process_job(job: &Job, worker: &IndexingWorker) -> Index {webgraph.raw_ingoing_edges_with_labels(&Node::from(html.url()).id()).into_iter()- .map(|edge| edge.label.loaded().unwrap())+ .map(|edge| edge.label).collect()}).unwrap_or_default();diff --git a/core/src/entrypoint/webgraph.rs b/core/src/entrypoint/webgraph.rsindex a4da11ae..302ace83 100644--- a/core/src/entrypoint/webgraph.rs+++ b/core/src/entrypoint/webgraph.rs@@ -71,9 +71,7 @@ pub struct Job {}pub fn open_graph<P: AsRef<Path>>(path: P) -> webgraph::Webgraph {- WebgraphBuilder::new(path)- .commit_mode(webgraph::CommitMode::SingleSegment)- .open()+ WebgraphBuilder::new(path).open()}pub struct WebgraphWorker {diff --git a/core/src/webgraph/mod.rs b/core/src/webgraph/mod.rsindex 16709ceb..e491a428 100644--- a/core/src/webgraph/mod.rs+++ b/core/src/webgraph/mod.rs@@ -22,20 +22,20 @@ use std::path::Path;use std::sync::Arc;use std::{cmp, fs};+use itertools::Itertools;use rayon::prelude::*;use serde::{Deserialize, Serialize};use url::Url;-use crate::directory::{self, DirEntry};use crate::executor::Executor;use crate::intmap;pub mod centrality;mod store;-use self::segment::{LiveSegment, StoredSegment};-use self::store::Store;+use self::segment::StoredSegment;pub const MAX_LABEL_LENGTH: usize = 1024;+const MAX_BATCH_SIZE: usize = 1_000;#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord, Hash)]pub struct NodeID(u128);@@ -77,14 +77,20 @@ impl intmap::Key for NodeID {}#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq, Eq, Hash)]-pub(crate) struct FullStoredEdge {- other: NodeID,- label: String,+pub struct Edge<L>+where+ L: Send + Sync,+{+ pub from: NodeID,+ pub to: NodeID,+ pub label: L,}#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq, Eq, Hash)]-pub(crate) struct SmallStoredEdge {- other: NodeID,+pub struct FullEdge {+ pub from: Node,+ pub to: Node,+ pub label: String,}#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]@@ -194,57 +200,7 @@ pub fn normalize_url(url: &Url) -> String {normalized}-#[derive(Debug, PartialEq, Eq)]-pub struct Edge {- pub from: NodeID,- pub to: NodeID,- pub label: Loaded<String>,-}--#[derive(Debug, PartialEq, Eq)]-pub enum Loaded<T> {- Some(T),- NotYet,-}-impl<T> Loaded<T> {- pub fn loaded(self) -> Option<T> {- match self {- Loaded::Some(t) => Some(t),- Loaded::NotYet => None,- }- }-}--impl<T> std::fmt::Display for Loaded<T>-where- T: std::fmt::Display,-{- fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {- match self {- Loaded::Some(val) => {- write!(f, "{}", val)- }- Loaded::NotYet => Ok(()),- }- }-}--#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]-pub struct FullEdge {- pub from: Node,- pub to: Node,- pub label: String,-}--#[derive(Default, Clone, Copy)]-pub enum CommitMode {- #[default]- NewSegment,- SingleSegment,-}-pub struct WebgraphBuilder {- commit_mode: CommitMode,path: Box<Path>,executor: Executor,}@@ -260,24 +216,18 @@ impl WebgraphBuilder {pub fn new<P: AsRef<Path>>(path: P) -> Self {Self {- commit_mode: CommitMode::default(),path: path.as_ref().into(),executor: Executor::multi_thread("webgraph").unwrap(),}}- pub fn commit_mode(mut self, mode: CommitMode) -> Self {- self.commit_mode = mode;- self- }-pub fn single_threaded(mut self) -> Self {self.executor = Executor::single_thread();self}pub fn open(self) -> Webgraph {- Webgraph::open(self.path, self.commit_mode, self.executor)+ Webgraph::open(self.path, self.executor)}}@@ -288,10 +238,11 @@ pub trait ShortestPaths {fn reversed_distances(&self, source: Node) -> BTreeMap<Node, u8>;}-fn dijkstra<F1, F2>(source: Node, node_edges: F1, edge_node: F2) -> BTreeMap<NodeID, u8>+fn dijkstra<F1, F2, L>(source: Node, node_edges: F1, edge_node: F2) -> BTreeMap<NodeID, u8>where- F1: Fn(NodeID) -> Vec<Edge>,- F2: Fn(&Edge) -> NodeID,+ L: Send + Sync,+ F1: Fn(NodeID) -> Vec<Edge<L>>,+ F2: Fn(&Edge<L>) -> NodeID,{let source_id = source.id();let mut distances: BTreeMap<NodeID, u8> = BTreeMap::default();@@ -368,13 +319,92 @@ struct SegmentMergeCandidate {merges: Vec<StoredSegment>,}+struct Id2NodeDb {+ db: rocksdb::DB,+}++impl Id2NodeDb {+ fn open<P: AsRef<Path>>(path: P) -> Self {+ let mut opts = rocksdb::Options::default();+ opts.create_if_missing(true);+ opts.optimize_for_point_lookup(512);++ let mut block_opts = rocksdb::BlockBasedOptions::default();+ block_opts.set_ribbon_filter(5.0);++ opts.set_block_based_table_factory(&block_opts);++ let db = rocksdb::DB::open(&opts, path).unwrap();++ Self { db }+ }++ fn put(&mut self, id: &NodeID, node: &Node) {+ let mut opts = rocksdb::WriteOptions::default();+ opts.disable_wal(true);++ self.db+ .put_opt(+ id.bit_128().to_be_bytes(),+ bincode::serialize(node).unwrap(),+ &opts,+ )+ .unwrap();+ }++ fn get(&self, id: &NodeID) -> Option<Node> {+ self.db+ .get(id.bit_128().to_be_bytes())+ .unwrap()+ .map(|bytes| bincode::deserialize(&bytes).unwrap())+ }++ fn keys(&self) -> impl Iterator<Item = NodeID> + '_ {+ self.db+ .iterator(rocksdb::IteratorMode::Start)+ .filter_map(|r| {+ let (key, _) = r.ok()?;+ Some(NodeID(u128::from_be_bytes((*key).try_into().unwrap())))+ })+ }++ fn iter(&self) -> impl Iterator<Item = (NodeID, Node)> + '_ {+ self.db+ .iterator(rocksdb::IteratorMode::Start)+ .filter_map(|r| {+ let (key, value) = r.ok()?;++ Some((+ NodeID(u128::from_be_bytes((*key).try_into().unwrap())),+ bincode::deserialize(&value).unwrap(),+ ))+ })+ }++ fn batch_put(&mut self, iter: impl Iterator<Item = (NodeID, Node)>) {+ let mut batch = rocksdb::WriteBatch::default();++ for (id, node) in iter {+ batch.put(+ id.bit_128().to_be_bytes(),+ bincode::serialize(&node).unwrap(),+ );+ }++ self.db.write(batch).unwrap();+ }++ fn flush(&self) {+ self.db.flush().unwrap();+ }+}+pub struct Webgraph {pub path: String,- live_segment: LiveSegment,segments: Vec<StoredSegment>,executor: Arc<Executor>,- id2node: Store<NodeID, Node>,- commit_mode: CommitMode,+ insert_batch: Vec<Edge<String>>,+ id2node: Id2NodeDb,meta: Meta,}@@ -410,26 +440,34 @@ impl Webgraph {writer.write_all(json.as_bytes()).unwrap();}- fn open<P: AsRef<Path>>(path: P, commit_mode: CommitMode, executor: Executor) -> Self {+ fn open<P: AsRef<Path>>(path: P, executor: Executor) -> Self {fs::create_dir_all(&path).unwrap();- let meta = Self::meta(&path);+ let mut meta = Self::meta(&path);fs::create_dir_all(path.as_ref().join("segments")).unwrap();let mut segments = Vec::new();for segment in &meta.comitted_segments {segments.push(StoredSegment::open(- path.as_ref().join("segments").join(segment),+ path.as_ref().join("segments"),segment.clone(),));}+ if segments.is_empty() {+ segments.push(StoredSegment::open(+ path.as_ref().join("segments"),+ uuid::Uuid::new_v4().to_string(),+ ));++ meta.comitted_segments.push(segments[0].id());+ }+Self {path: path.as_ref().as_os_str().to_str().unwrap().to_string(),- live_segment: LiveSegment::default(),segments,- commit_mode,executor: Arc::new(executor),- id2node: Store::open(path.as_ref().join("id2node")),+ id2node: Id2NodeDb::open(path.as_ref().join("id2node")),+ insert_batch: Vec::with_capacity(MAX_BATCH_SIZE),meta,}}@@ -446,19 +484,29 @@ impl Webgraph {pub fn insert(&mut self, from: Node, to: Node, label: String) {let (from_id, to_id) = (self.id_or_assign(&from), self.id_or_assign(&to));- let label = label.chars().take(MAX_LABEL_LENGTH).collect::<String>();- self.live_segment.insert(from_id, to_id, label);++ let edge = Edge {+ from: from_id,+ to: to_id,+ label: label.chars().take(MAX_LABEL_LENGTH).collect(),+ };++ self.insert_batch.push(edge);++ if self.insert_batch.len() >= MAX_BATCH_SIZE {+ self.commit();+ }}pub fn merge(&mut self, mut other: Webgraph) {other.commit();- self.id2node.batch_put_owned(other.id2node.iter());+ self.id2node.batch_put(other.id2node.iter());for segment in other.segments {let id = segment.id();- let new_path = Path::new(&self.path).join("segments").join(segment.id());- std::fs::rename(segment.path(), &new_path).unwrap();+ let new_path = Path::new(&self.path).join("segments");+ std::fs::rename(segment.path(), &new_path.join(segment.id())).unwrap();self.meta.comitted_segments.push(segment.id());drop(segment);@@ -469,21 +517,11 @@ impl Webgraph {}pub fn commit(&mut self) {- if !self.live_segment.is_empty() {- let live_segment = std::mem::take(&mut self.live_segment);-- match (self.commit_mode, self.segments.first_mut()) {- (CommitMode::SingleSegment, Some(segment)) => {- segment.add(live_segment);- segment.flush();- }- _ => {- let segment = live_segment.commit(Path::new(&self.path).join("segments"));-- self.meta.comitted_segments.push(segment.id());- self.segments.push(segment);- }- }+ if !self.insert_batch.is_empty() {+ let seg = self.segments.last_mut().unwrap();+ seg.insert(&self.insert_batch);+ seg.flush();+ self.insert_batch.clear();}self.save_metadata();@@ -495,75 +533,55 @@ impl Webgraph {}pub fn ingoing_edges(&self, node: Node) -> Vec<FullEdge> {- self.inner_ingoing_edges(&node.id(), true)+ self.inner_edges(|segment| segment.ingoing_edges_with_label(&node.id()).collect_vec()).into_iter()- .map(|edge| FullEdge {- from: self.id2node(&edge.from).unwrap(),- to: self.id2node(&edge.to).unwrap(),- label: edge.label.loaded().unwrap(),+ .map(|e| FullEdge {+ from: self.id2node(&e.from).unwrap(),+ to: self.id2node(&e.to).unwrap(),+ label: e.label,}).collect()}- pub fn raw_ingoing_edges(&self, node: &NodeID) -> Vec<Edge> {- self.inner_ingoing_edges(node, false)+ pub fn raw_ingoing_edges(&self, node: &NodeID) -> Vec<Edge<()>> {+ self.inner_edges(|segment| segment.ingoing_edges(node).collect_vec())}- pub fn raw_ingoing_edges_with_labels(&self, node: &NodeID) -> Vec<Edge> {- self.inner_ingoing_edges(node, true)+ pub fn raw_ingoing_edges_with_labels(&self, node: &NodeID) -> Vec<Edge<String>> {+ self.inner_edges(|segment| segment.ingoing_edges_with_label(node).collect_vec())}- fn inner_ingoing_edges(&self, node: &NodeID, load_labels: bool) -> Vec<Edge> {- let mut edges: Vec<_> = self- .executor- .map(- |segment| segment.ingoing_edges(node, load_labels),- self.segments.iter(),- )- .unwrap()+ pub fn outgoing_edges(&self, node: Node) -> Vec<FullEdge> {+ self.inner_edges(|segment| segment.outgoing_edges_with_label(&node.id()).collect_vec()).into_iter()- .flatten()- .collect();-- edges.sort_unstable_by_key(|edge| edge.from);- edges.dedup_by_key(|edge| edge.from);-- edges+ .map(|e| FullEdge {+ from: self.id2node(&e.from).unwrap(),+ to: self.id2node(&e.to).unwrap(),+ label: e.label,+ })+ .collect()}- pub fn raw_outgoing_edges(&self, node: &NodeID) -> Vec<Edge> {- self.inner_outgoing_edges(node, false)+ pub fn raw_outgoing_edges(&self, node: &NodeID) -> Vec<Edge<()>> {+ self.inner_edges(|segment| segment.outgoing_edges(node).collect_vec())}- fn inner_outgoing_edges(&self, node: &NodeID, load_labels: bool) -> Vec<Edge> {- let mut edges: Vec<_> = self+ fn inner_edges<F, L>(&self, loader: F) -> Vec<Edge<L>>+ where+ L: Send + Sync,+ F: Sized + Sync + Fn(&StoredSegment) -> Vec<Edge<L>>,+ {+ let edges: Vec<_> = self.executor- .map(- |segment| segment.outgoing_edges(node, load_labels),- self.segments.iter(),- )+ .map(loader, self.segments.iter()).unwrap().into_iter().flatten().collect();- edges.sort_unstable_by_key(|edge| edge.to);- edges.dedup_by_key(|edge| edge.to);-edges}- pub fn outgoing_edges(&self, node: Node) -> Vec<FullEdge> {- self.inner_outgoing_edges(&node.id(), true)- .into_iter()- .map(|edge| FullEdge {- from: self.id2node(&edge.from).unwrap(),- to: self.id2node(&edge.to).unwrap(),- label: edge.label.loaded().unwrap(),- })- .collect()- }-pub fn id2node(&self, id: &NodeID) -> Option<Node> {self.id2node.get(id)}@@ -579,11 +597,11 @@ impl Webgraph {/// Iterate all edges in the graph at least once./// Some edges may be returned multiple times./// This happens if they are present in more than one segment.- pub fn edges(&self) -> impl Iterator<Item = Edge> + '_ {+ pub fn edges(&self) -> impl Iterator<Item = Edge<()>> + '_ {self.segments.iter().flat_map(|segment| segment.edges())}- pub fn par_edges(&self) -> impl ParallelIterator<Item = Edge> + '_ {+ pub fn par_edges(&self) -> impl ParallelIterator<Item = Edge<()>> + '_ {self.segments.par_iter().flat_map(|segment| segment.edges().par_bridge())@@ -649,44 +667,6 @@ impl Webgraph {}}-impl From<FrozenWebgraph> for Webgraph {- fn from(frozen: FrozenWebgraph) -> Self {- let path = match &frozen.root {- DirEntry::Folder { name, entries: _ } => name.clone(),- DirEntry::File {- name: _,- content: _,- } => {- panic!("Cannot open webgraph from a file - must be directory.")- }- };-- if Path::new(&path).exists() {- fs::remove_dir_all(&path).unwrap();- }-- directory::recreate_folder(&frozen.root).unwrap();-- WebgraphBuilder::new(path).open()- }-}--impl From<Webgraph> for FrozenWebgraph {- fn from(mut graph: Webgraph) -> Self {- graph.commit();- let path = graph.path.clone();- drop(graph);- let root = directory::scan_folder(path).unwrap();-- Self { root }- }-}--#[derive(Debug, Serialize, Deserialize)]-pub struct FrozenWebgraph {- pub root: DirEntry,-}-#[cfg(test)]mod test {use super::*;@@ -889,26 +869,6 @@ mod test {)}- #[test]- fn serialize_deserialize_bincode() {- let graph = test_graph();-- let path = graph.path.clone();- let frozen: FrozenWebgraph = graph.into();- let bytes = bincode::serialize(&frozen).unwrap();-- std::fs::remove_dir_all(path).unwrap();-- let deserialized_frozen: FrozenWebgraph = bincode::deserialize(&bytes).unwrap();- let graph: Webgraph = deserialized_frozen.into();-- let distances = graph.distances(Node::from("D"));-- assert_eq!(distances.get(&Node::from("C")), Some(&1));- assert_eq!(distances.get(&Node::from("A")), Some(&2));- assert_eq!(distances.get(&Node::from("B")), Some(&3));- }-#[test]fn node_lowercase_name() {let n = Node::from("TEST".to_string());@@ -936,13 +896,15 @@ mod test {let num_edges = edges.len();for (from, to, label) in test_edges() {- graph.insert(from, to, label);- graph.commit();+ let mut tmp_graph = WebgraphBuilder::new_memory().open();+ tmp_graph.insert(from, to, label);+ tmp_graph.commit();+ graph.merge(tmp_graph);}graph.commit();- assert_eq!(num_edges, graph.segments.len());+ assert!(graph.segments.len() >= num_edges);verify_graph(&graph);graph.merge_segments(2);@@ -951,23 +913,6 @@ mod test {verify_graph(&graph);}- #[test]- fn single_segment_commit_mode() {- let mut graph = WebgraphBuilder::new_memory()- .commit_mode(CommitMode::SingleSegment)- .open();-- for (from, to, label) in test_edges() {- graph.insert(from, to, label);- graph.commit();- }-- graph.commit();-- assert_eq!(graph.segments.len(), 1);- verify_graph(&graph);- }-#[test]fn cap_label_length() {let mut graph = WebgraphBuilder::new_memory().open();diff --git a/core/src/webgraph/segment.rs b/core/src/webgraph/segment.rsindex ac903622..c8b9548e 100644--- a/core/src/webgraph/segment.rs+++ b/core/src/webgraph/segment.rs@@ -14,28 +14,20 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.-use std::{- collections::{BTreeMap, HashSet},- path::Path,-};+use std::path::{Path, PathBuf};-use super::{store::Store, Edge, FullStoredEdge, Loaded, NodeID, SmallStoredEdge};+use super::{store::EdgeStore, Edge, NodeID};const FULL_ADJACENCY_STORE: &str = "full_adjacency";const FULL_REVERSED_ADJACENCY_STORE: &str = "full_reversed_adjacency";const SMALL_ADJACENCY_STORE: &str = "small_adjacency";const SMALL_REVERSED_ADJACENCY_STORE: &str = "small_reversed_adjacency";-struct MergePacket {- new_full_adjacency: Store<NodeID, HashSet<FullStoredEdge>>,- new_small_adjacency: Store<NodeID, HashSet<SmallStoredEdge>>,-}-pub struct StoredSegment {- full_adjacency: Store<NodeID, HashSet<FullStoredEdge>>,- full_reversed_adjacency: Store<NodeID, HashSet<FullStoredEdge>>,- small_adjacency: Store<NodeID, HashSet<SmallStoredEdge>>,- small_reversed_adjacency: Store<NodeID, HashSet<SmallStoredEdge>>,+ full_adjacency: EdgeStore<String>,+ full_reversed_adjacency: EdgeStore<String>,+ small_adjacency: EdgeStore<()>,+ small_reversed_adjacency: EdgeStore<()>,id: String,folder_path: String,}@@ -43,13 +35,27 @@ pub struct StoredSegment {impl StoredSegment {pub fn open<P: AsRef<Path>>(folder_path: P, id: String) -> Self {StoredSegment {- full_adjacency: Store::open(folder_path.as_ref().join(FULL_ADJACENCY_STORE)),- full_reversed_adjacency: Store::open(- folder_path.as_ref().join(FULL_REVERSED_ADJACENCY_STORE),+ full_adjacency: EdgeStore::open(+ folder_path.as_ref().join(&id).join(FULL_ADJACENCY_STORE),+ false,+ ),+ full_reversed_adjacency: EdgeStore::open(+ folder_path+ .as_ref()+ .join(&id)+ .join(FULL_REVERSED_ADJACENCY_STORE),+ true,),- small_adjacency: Store::open(folder_path.as_ref().join(SMALL_ADJACENCY_STORE)),- small_reversed_adjacency: Store::open(- folder_path.as_ref().join(SMALL_REVERSED_ADJACENCY_STORE),+ small_adjacency: EdgeStore::open(+ folder_path.as_ref().join(&id).join(SMALL_ADJACENCY_STORE),+ false,+ ),+ small_reversed_adjacency: EdgeStore::open(+ folder_path+ .as_ref()+ .join(&id)+ .join(SMALL_REVERSED_ADJACENCY_STORE),+ true,),folder_path: folder_path.as_ref()@@ -65,68 +71,26 @@ impl StoredSegment {self.small_adjacency.estimate_len()}- pub fn outgoing_edges(&self, node: &NodeID, load_label: bool) -> Vec<Edge> {- if load_label {- self.full_adjacency- .get(node)- .map(|edges| {- edges- .into_iter()- .map(move |edge| Edge {- from: *node,- to: edge.other,- label: Loaded::Some(edge.label),- })- .collect()- })- .unwrap_or_default()- } else {- self.small_adjacency- .get(node)- .map(|edges| {- edges- .into_iter()- .map(move |edge| Edge {- from: *node,- to: edge.other,- label: Loaded::NotYet,- })- .collect()- })- .unwrap_or_default()- }+ pub fn outgoing_edges_with_label(+ &self,+ node: &NodeID,+ ) -> impl Iterator<Item = Edge<String>> + '_ {+ self.full_adjacency.get(*node)}- pub fn ingoing_edges(&self, node: &NodeID, load_label: bool) -> Vec<Edge> {- if load_label {- self.full_reversed_adjacency- .get(node)- .map(|edges| {- edges- .into_iter()- .map(move |edge| Edge {- from: edge.other,- to: *node,- label: Loaded::Some(edge.label),- })- .collect()- })- .unwrap_or_default()- } else {- self.small_reversed_adjacency- .get(node)- .map(|edges| {- edges- .into_iter()- .map(move |edge| Edge {- from: edge.other,- to: *node,- label: Loaded::NotYet,- })- .collect()- })- .unwrap_or_default()- }+ pub fn outgoing_edges(&self, node: &NodeID) -> impl Iterator<Item = Edge<()>> + '_ {+ self.small_adjacency.get(*node)+ }++ pub fn ingoing_edges_with_label(+ &self,+ node: &NodeID,+ ) -> impl Iterator<Item = Edge<String>> + '_ {+ self.full_reversed_adjacency.get(*node)+ }++ pub fn ingoing_edges(&self, node: &NodeID) -> impl Iterator<Item = Edge<()>> + '_ {+ self.small_reversed_adjacency.get(*node)}pub fn flush(&mut self) {@@ -140,53 +104,24 @@ impl StoredSegment {self.id.clone()}- pub fn path(&self) -> &Path {- Path::new(&self.folder_path)+ pub fn path(&self) -> PathBuf {+ Path::new(&self.folder_path).join(&self.id)}- pub fn edges(&self) -> impl Iterator<Item = Edge> + '_ {- self.small_adjacency.iter().flat_map(move |(from, edges)| {- edges.into_iter().map(move |stored_edge| Edge {- from,- to: stored_edge.other,- label: Loaded::NotYet,- })- })+ pub fn edges(&self) -> impl Iterator<Item = Edge<()>> + '_ + Send + Sync {+ self.small_adjacency.iter()}- fn merge_adjacency<F1>(edges_fn: F1, packet: &MergePacket, segments: &[Self])- where- F1: Fn(&Self, &NodeID) -> Option<HashSet<FullStoredEdge>>,- {- if segments.is_empty() {- return;- }-- for segment in segments {- for node_id in segment- .small_adjacency- .keys()- .chain(segment.small_reversed_adjacency.keys())- {- let mut existing = packet.new_full_adjacency.get(&node_id).unwrap_or_default();-- if let Some(edges) = edges_fn(segment, &node_id) {- existing.extend(edges);- }-- packet.new_full_adjacency.put(&node_id, &existing);- packet.new_small_adjacency.put(- &node_id,- &existing- .into_iter()- .map(|edge| SmallStoredEdge { other: edge.other })- .collect::<HashSet<_>>(),- );- }- }+ fn merge_with(&self, other: &StoredSegment) {+ self.full_adjacency.merge_with(&other.full_adjacency);+ self.full_reversed_adjacency+ .merge_with(&other.full_reversed_adjacency);+ self.small_adjacency.merge_with(&other.small_adjacency);+ self.small_reversed_adjacency+ .merge_with(&other.small_reversed_adjacency);}- pub fn merge(segments: Vec<StoredSegment>) -> Self {+ pub fn merge(mut segments: Vec<StoredSegment>) -> Self {debug_assert!(!segments.is_empty());if segments.len() == 1 {@@ -194,171 +129,30 @@ impl StoredSegment {return segments.pop().unwrap();}- let new_segment_id = uuid::Uuid::new_v4().to_string();-- let new_path = segments[0].path().parent().unwrap().join(&new_segment_id);-- let new_full_adjacency = Store::open(new_path.join(FULL_ADJACENCY_STORE));- let new_small_adjacency = Store::open(new_path.join(SMALL_ADJACENCY_STORE));-- let packet = MergePacket {- new_full_adjacency,- new_small_adjacency,- };-- Self::merge_adjacency(- |segment: &StoredSegment, node_id: &NodeID| segment.full_adjacency.get(node_id),- &packet,- &segments,- );- let new_adjacency = packet.new_full_adjacency;- let new_small_adjacency = packet.new_small_adjacency;-- let new_full_reversed_adjacency = Store::open(new_path.join(FULL_REVERSED_ADJACENCY_STORE));- let new_small_reversed_adjacency =- Store::open(new_path.join(SMALL_REVERSED_ADJACENCY_STORE));-- let packet = MergePacket {- new_full_adjacency: new_full_reversed_adjacency,- new_small_adjacency: new_small_reversed_adjacency,- };-- Self::merge_adjacency(- |segment: &StoredSegment, node_id: &NodeID| {- segment.full_reversed_adjacency.get(node_id)- },- &packet,- &segments,- );-- let mut res = Self {- full_adjacency: new_adjacency,- small_adjacency: new_small_adjacency,- full_reversed_adjacency: packet.new_full_adjacency,- small_reversed_adjacency: packet.new_small_adjacency,- id: new_segment_id,- folder_path: new_path.to_str().unwrap().to_string(),- };-- res.flush();+ let segment = segments.remove(0);- res- }-- pub fn add(&mut self, live_segment: LiveSegment) {- for (node_id, adjacency) in live_segment.adjacency {- let mut existing = self.full_adjacency.get(&node_id).unwrap_or_default();- existing.extend(adjacency);- self.full_adjacency.put(&node_id, &existing);- self.small_adjacency.put(- &node_id,- &existing- .into_iter()- .map(|edge| SmallStoredEdge { other: edge.other })- .collect::<HashSet<_>>(),- );+ for other in segments {+ segment.merge_with(&other);}- for (node_id, adjacency) in live_segment.reversed_adjacency {- let mut existing = self- .full_reversed_adjacency- .get(&node_id)- .unwrap_or_default();- existing.extend(adjacency);- self.full_reversed_adjacency.put(&node_id, &existing);- self.small_reversed_adjacency.put(- &node_id,- &existing- .into_iter()- .map(|edge| SmallStoredEdge { other: edge.other })- .collect::<HashSet<_>>(),- );- }+ segment}-}--#[derive(Default)]-pub struct LiveSegment {- adjacency: BTreeMap<NodeID, HashSet<FullStoredEdge>>,- reversed_adjacency: BTreeMap<NodeID, HashSet<FullStoredEdge>>,-}--impl LiveSegment {- pub fn insert(&mut self, from: NodeID, to: NodeID, label: String) {- self.adjacency- .entry(from)- .or_default()- .insert(FullStoredEdge {- other: to,- label: label.clone(),- });-- self.reversed_adjacency- .entry(to)- .or_default()- .insert(FullStoredEdge { other: from, label });- }-- pub fn commit<P: AsRef<Path>>(self, folder_path: P) -> StoredSegment {- let segment_id = uuid::Uuid::new_v4().to_string();- let path = folder_path.as_ref().join(&segment_id);-- let small_adjacency: BTreeMap<NodeID, HashSet<SmallStoredEdge>> = self- .adjacency- .clone()- .into_iter()- .map(|(segment_node_id, edges)| {- let edges = edges- .into_iter()- .map(|edge| SmallStoredEdge { other: edge.other })- .collect();-- (segment_node_id, edges)- })- .collect();- let small_rev_adjacency: BTreeMap<NodeID, HashSet<SmallStoredEdge>> = self- .reversed_adjacency- .clone()- .into_iter()- .map(|(segment_node_id, edges)| {- let edges = edges- .into_iter()- .map(|edge| SmallStoredEdge { other: edge.other })- .collect();-- (segment_node_id, edges)+ pub fn insert(&mut self, edges: &[Edge<String>]) {+ let small_edges: Vec<_> = edges+ .iter()+ .map(|e| Edge {+ from: e.from,+ to: e.to,+ label: (),}).collect();- let full_adjacency = Store::open(path.join(FULL_ADJACENCY_STORE));- full_adjacency.batch_put(self.adjacency.iter());-- let full_reversed_adjacency = Store::open(path.join(FULL_REVERSED_ADJACENCY_STORE));- full_reversed_adjacency.batch_put(self.reversed_adjacency.iter());-- let small_adjacency_store = Store::open(path.join(SMALL_ADJACENCY_STORE));- small_adjacency_store.batch_put(small_adjacency.iter());-- let small_rev_adjacency_store = Store::open(path.join(SMALL_REVERSED_ADJACENCY_STORE));- small_rev_adjacency_store.batch_put(small_rev_adjacency.iter());-- let mut stored_segment = StoredSegment {- small_adjacency: small_adjacency_store,- small_reversed_adjacency: small_rev_adjacency_store,- full_adjacency,- full_reversed_adjacency,- folder_path: path.as_os_str().to_str().unwrap().to_string(),- id: segment_id,- };-- stored_segment.flush();-- stored_segment- }-- pub fn is_empty(&self) -> bool {- self.adjacency.is_empty()+ self.full_adjacency.put(edges.iter().cloned());+ self.full_reversed_adjacency.put(edges.iter().cloned());+ self.small_adjacency.put(small_edges.iter().cloned());+ self.small_reversed_adjacency+ .put(small_edges.iter().cloned());}}@@ -375,20 +169,39 @@ mod test {// ▼ │ │// 1─────►2◄┘- let mut store = LiveSegment::default();+ let mut store = StoredSegment::open(crate::gen_temp_path(), "test".to_string());++ let mut edges = Vec::new();let a = NodeID(0);let b = NodeID(1);let c = NodeID(2);- store.insert(a, b, String::new());- store.insert(b, c, String::new());- store.insert(c, a, String::new());- store.insert(a, c, String::new());-- let store: StoredSegment = store.commit(crate::gen_temp_path());-- let mut out: Vec<Edge> = store.outgoing_edges(&a, false);+ edges.push(Edge {+ from: a,+ to: b,+ label: String::new(),+ });+ edges.push(Edge {+ from: b,+ to: c,+ label: String::new(),+ });+ edges.push(Edge {+ from: c,+ to: a,+ label: String::new(),+ });+ edges.push(Edge {+ from: a,+ to: c,+ label: String::new(),+ });++ store.insert(&edges);+ store.flush();++ let mut out: Vec<_> = store.outgoing_edges(&a).collect();out.sort_by(|a, b| a.to.cmp(&b.to));@@ -398,59 +211,59 @@ mod test {Edge {from: a,to: b,- label: Loaded::NotYet+ label: ()},Edge {from: a,to: c,- label: Loaded::NotYet+ label: ()},]);- let mut out: Vec<Edge> = store.outgoing_edges(&b, false);+ let mut out: Vec<_> = store.outgoing_edges(&b).collect();out.sort_by(|a, b| a.to.cmp(&b.to));assert_eq!(out,vec![Edge {from: b,to: c,- label: Loaded::NotYet+ label: ()},]);- let mut out: Vec<Edge> = store.outgoing_edges(&c, false);+ let mut out: Vec<_> = store.outgoing_edges(&c).collect();out.sort_by(|a, b| a.to.cmp(&b.to));assert_eq!(out,vec![Edge {from: c,to: a,- label: Loaded::NotYet+ label: ()},]);- let out: Vec<Edge> = store.ingoing_edges(&a, false);+ let out: Vec<_> = store.ingoing_edges(&a).collect();assert_eq!(out,vec![Edge {from: c,to: a,- label: Loaded::NotYet+ label: ()},]);- let out: Vec<Edge> = store.ingoing_edges(&b, false);+ let out: Vec<_> = store.ingoing_edges(&b).collect();assert_eq!(out,vec![Edge {from: a,to: b,- label: Loaded::NotYet+ label: ()},]);- let mut out: Vec<Edge> = store.ingoing_edges(&c, false);+ let mut out: Vec<_> = store.ingoing_edges(&c).collect();out.sort_by(|a, b| a.from.cmp(&b.from));assert_eq!(out,@@ -458,12 +271,12 @@ mod test {Edge {from: a,to: c,- label: Loaded::NotYet+ label: ()},Edge {from: b,to: c,- label: Loaded::NotYet+ label: ()},]);diff --git a/core/src/webgraph/store.rs b/core/src/webgraph/store.rsindex 6008d84d..1fe89414 100644--- a/core/src/webgraph/store.rs+++ b/core/src/webgraph/store.rs@@ -16,30 +16,31 @@use std::path::Path;+use hashbrown::HashSet;use rocksdb::BlockBasedOptions;-pub struct Store<K, V> {+use super::{Edge, NodeID};++const MAX_BATCH_SIZE: usize = 1_000;++pub struct EdgeStore<L> {+ reversed: bool,db: rocksdb::DB,- _phantom: std::marker::PhantomData<(K, V)>,+ _phantom: std::marker::PhantomData<L>,}-impl<K, V> Store<K, V>+impl<L> EdgeStore<L>where- K: serde::de::DeserializeOwned + serde::Serialize + Send,- V: serde::de::DeserializeOwned + serde::Serialize,+ L: serde::Serialize + serde::de::DeserializeOwned + Send + Sync,{- pub fn open<P: AsRef<Path>>(path: P) -> Self {+ pub fn open<P: AsRef<Path>>(path: P, reversed: bool) -> Self {let mut options = rocksdb::Options::default();- options.set_max_open_files(1);-options.create_if_missing(true);let mut block_options = BlockBasedOptions::default();block_options.set_ribbon_filter(10.0);block_options.set_format_version(5);- block_options.disable_cache();-options.set_block_based_table_factory(&block_options);options.set_optimize_filters_for_hits(true);@@ -50,86 +51,122 @@ whereSelf {db,+ reversed,_phantom: std::marker::PhantomData,}}- pub fn get(&self, key: &K) -> Option<V> {- let bytes = bincode::serialize(key).unwrap();-- let mut readopts = rocksdb::ReadOptions::default();- readopts.set_readahead_size(4_194_304);-- self.db- .get_pinned_opt(bytes, &readopts)- .unwrap()- .map(|bytes| bincode::deserialize(&bytes).unwrap())- }-- pub fn put(&self, key: &K, value: &V) {- let key_bytes = bincode::serialize(key).unwrap();- let value_bytes = bincode::serialize(value).unwrap();-- let mut options = rocksdb::WriteOptions::default();- options.disable_wal(true);-- self.db.put_opt(key_bytes, value_bytes, &options).unwrap();+ pub fn get(&self, node: NodeID) -> impl Iterator<Item = Edge<L>> + '_ {+ let prefix_bytes = node.bit_128().to_be_bytes();+ let suffix = 0u128.to_be_bytes();++ let key_bytes = [prefix_bytes, suffix].concat();++ let iter = self.db.iterator(rocksdb::IteratorMode::From(+ &key_bytes,+ rocksdb::Direction::Forward,+ ));++ iter.take_while(move |r| {+ if let Ok((key, _)) = r.as_ref() {+ let cur_prefix = &key[..16];++ cur_prefix == prefix_bytes+ } else {+ false+ }+ })+ .filter_map(move |r| {+ let (key, value) = r.ok()?;++ let suffix = u128::from_be_bytes(key[16..32].try_into().unwrap());++ let label = bincode::deserialize(&value).ok()?;++ if self.reversed {+ Some(Edge {+ from: NodeID(suffix),+ to: node,+ label,+ })+ } else {+ Some(Edge {+ from: node,+ to: NodeID(suffix),+ label,+ })+ }+ })}- pub fn batch_put<'a>(&'a self, it: impl Iterator<Item = (&'a K, &'a V)>) {+ pub fn put(&self, edges: impl Iterator<Item = Edge<L>>) {let mut batch = rocksdb::WriteBatch::default();+ let mut batch_keys = HashSet::new();- for (key, value) in it {- let key_bytes = bincode::serialize(key).unwrap();- let value_bytes = bincode::serialize(value).unwrap();+ let mut opts = rocksdb::WriteOptions::default();+ opts.disable_wal(true);- batch.put(key_bytes, value_bytes);- }+ for edge in edges {+ let (prefix, suffix) = if self.reversed {+ (edge.to, edge.from)+ } else {+ (edge.from, edge.to)+ };- let mut options = rocksdb::WriteOptions::default();- options.disable_wal(true);+ let prefix_bytes = prefix.bit_128().to_be_bytes();+ let suffix_bytes = suffix.bit_128().to_be_bytes();- self.db.write_opt(batch, &options).unwrap();- }+ let key_bytes = [prefix_bytes, suffix_bytes].concat();- pub fn batch_put_owned(&self, it: impl Iterator<Item = (K, V)>) {- let mut batch = rocksdb::WriteBatch::default();+ if batch_keys.contains(&key_bytes) || self.db.get(&key_bytes).unwrap().is_some() {+ continue;+ }++ batch_keys.insert(key_bytes.clone());- for (key, value) in it {- let key_bytes = bincode::serialize(&key).unwrap();- let value_bytes = bincode::serialize(&value).unwrap();+ let value_bytes = bincode::serialize(&edge.label).unwrap();batch.put(key_bytes, value_bytes);- }- let mut options = rocksdb::WriteOptions::default();- options.disable_wal(true);+ if batch.len() >= MAX_BATCH_SIZE {+ self.db.write_opt(batch, &opts).unwrap();+ batch = rocksdb::WriteBatch::default();+ batch_keys.clear();+ }+ }- self.db.write_opt(batch, &options).unwrap();+ self.db.write_opt(batch, &opts).unwrap();}- pub fn keys(&self) -> impl Iterator<Item = K> + '_ {+ pub fn iter(&self) -> impl Iterator<Item = Edge<L>> + '_ + Send + Sync {let read_opts = rocksdb::ReadOptions::default();self.db.iterator_opt(rocksdb::IteratorMode::Start, read_opts).map(|res| {- let (key, _) = res.unwrap();- bincode::deserialize(&key).unwrap()- })- }+ let (key, val) = res.unwrap();- pub fn iter(&self) -> impl Iterator<Item = (K, V)> + '_ {- let read_opts = rocksdb::ReadOptions::default();+ let (from, to) = if self.reversed {+ let from = u128::from_be_bytes(key[16..32].try_into().unwrap());+ let from = NodeID(from);- self.db- .iterator_opt(rocksdb::IteratorMode::Start, read_opts)- .map(|res| {- let (key, val) = res.unwrap();- (- bincode::deserialize(&key).unwrap(),- bincode::deserialize(&val).unwrap(),- )+ let to = u128::from_be_bytes(key[0..16].try_into().unwrap());+ let to = NodeID(to);++ (from, to)+ } else {+ let from = u128::from_be_bytes(key[0..16].try_into().unwrap());+ let from = NodeID(from);++ let to = u128::from_be_bytes(key[16..32].try_into().unwrap());+ let to = NodeID(to);++ (from, to)+ };++ let label = bincode::deserialize(&val).unwrap();++ Edge { from, to, label }})}@@ -143,57 +180,63 @@ where.unwrap().unwrap_or(0) as usize}++ pub fn merge_with(&self, other: &EdgeStore<L>) {+ self.put(other.iter());+ }}#[cfg(test)]mod tests {use super::*;- #[derive(Debug, PartialEq, Eq, Clone, serde::Serialize, serde::Deserialize)]- struct TestStruct {- a: String,- b: i32,- }-#[test]fn test_insert() {- let kv = Store::<String, TestStruct>::open(crate::gen_temp_path().join("test-segment"));+ let kv: EdgeStore<String> =+ EdgeStore::open(crate::gen_temp_path().join("test-segment"), false);- assert!(kv.get(&"test".to_string()).is_none());-- let test_struct = TestStruct {- a: "test".to_string(),- b: 5,+ let e = Edge {+ from: NodeID(0),+ to: NodeID(1),+ label: "test".to_string(),};- kv.put(&"test".to_string(), &test_struct);+ kv.put(vec![e.clone()].into_iter());+kv.flush();- let archived = kv.get(&"test".to_string()).unwrap();- assert_eq!(test_struct, archived);+ let edges: Vec<_> = kv.get(NodeID(0)).collect();++ assert_eq!(edges.len(), 1);+ assert_eq!(&edges[0], &e);++ let edges: Vec<_> = kv.get(NodeID(1)).collect();++ assert_eq!(edges.len(), 0);}#[test]- fn test_re_open() {- let path = crate::gen_temp_path();- let segment_name = "test-segment";- let test_struct = TestStruct {- a: "test".to_string(),- b: 5,+ fn test_reversed() {+ let kv: EdgeStore<String> =+ EdgeStore::open(crate::gen_temp_path().join("test-segment"), true);++ let e = Edge {+ from: NodeID(0),+ to: NodeID(1),+ label: "test".to_string(),};- {- let kv = Store::<String, TestStruct>::open(path.join(segment_name));+ kv.put(vec![e.clone()].into_iter());- assert!(kv.get(&"test".to_string()).is_none());+ kv.flush();- kv.put(&"test".to_string(), &test_struct);- kv.flush();- }+ let edges: Vec<_> = kv.get(NodeID(0)).collect();++ assert_eq!(edges.len(), 0);- let kv = Store::<String, TestStruct>::open(path.join(segment_name));+ let edges: Vec<_> = kv.get(NodeID(1)).collect();- let archived = kv.get(&"test".to_string()).unwrap();- assert_eq!(test_struct, archived);+ assert_eq!(edges.len(), 1);+ assert_eq!(&edges[0], &e);}}diff --git a/justfile b/justfileindex d76b69d0..296a3af4 100644--- a/justfile+++ b/justfile@@ -31,7 +31,7 @@ export STRACT_CARGO_ARGS := env_var_or_default("STRACT_CARGO_ARGS", "")@configure *ARGS:just setup {{ARGS}}- just cargo run --release --all-features -- configure {{ARGS}}+ RUST_LOG="none,stract=info" just cargo run --release --all-features -- configure {{ARGS}}@setup_python_env:python3 -m venv .venv || true