web search done right
Re-write webgraph to be based on rocksdb. This allows us to create really big graphs (needed for page-level webgraphs).
Mikkel Denker3a87834547d4
diff --git a/Cargo.toml b/Cargo.tomlindex c53e50a8..89aae053 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -78,7 +78,7 @@ tokio-stream = "0.1.11"scylla = "0.7.0"chitchat = "0.5.0"memmap2 = "0.5.10"-rkyv = "0.7.41"+rkyv = "0.7.41"dashmap = "5.4.0"tch = "0.13.0"torch-sys = "0.13.0"diff --git a/core/examples/print_inbound_links.rs b/core/examples/print_inbound_links.rsindex f984edc0..06b7ab74 100644--- a/core/examples/print_inbound_links.rs+++ b/core/examples/print_inbound_links.rs@@ -28,7 +28,7 @@ pub fn main() {let node = Node::from(host).into_host();let node_id = graph.node2id(&node).unwrap();- let inbound_vec = inbound.get(node_id).unwrap();+ let inbound_vec = inbound.get(&node_id).unwrap();println!("{:?}", inbound_vec);for edge in graph.ingoing_edges(node) {diff --git a/core/src/entrypoint/indexer.rs b/core/src/entrypoint/indexer.rsindex 05814790..6dd0a80b 100644--- a/core/src/entrypoint/indexer.rs+++ b/core/src/entrypoint/indexer.rs@@ -132,7 +132,7 @@ pub fn process_job(job: &Job, worker: &IndexingWorker) -> Index {);let host_centrality = node_id- .and_then(|node_id| worker.centrality_store.harmonic.host.get(node_id))+ .and_then(|node_id| worker.centrality_store.harmonic.host.get(&node_id)).unwrap_or_default();if let Some(host_centrality_threshold) = job.host_centrality_threshold {@@ -176,7 +176,7 @@ pub fn process_job(job: &Job, worker: &IndexingWorker) -> Index {.get(&html.url().without_protocol().to_lowercase().into());let page_centrality = node_id- .and_then(|node_id| worker.centrality_store.harmonic.full.get(node_id))+ .and_then(|node_id| worker.centrality_store.harmonic.full.get(&node_id)).unwrap_or_default();let fetch_time_ms = record.metadata.fetch_time_ms as u64;@@ -189,8 +189,7 @@ pub fn process_job(job: &Job, worker: &IndexingWorker) -> Index {let node_id = worker.centrality_store.node2id- .get(&Node::from_url(html.url()).into_host())- .cloned();+ .get(&Node::from_url(html.url()).into_host());let mut dmoz_description = None;diff --git a/core/src/entrypoint/webgraph.rs b/core/src/entrypoint/webgraph.rsindex f6d2cd2a..68b0fb4c 100644--- a/core/src/entrypoint/webgraph.rs+++ b/core/src/entrypoint/webgraph.rs@@ -101,6 +101,10 @@ pub fn process_job(job: &Job) -> webgraph::Webgraph {.filter(|link| link.source.domain() != link.destination.domain()).filter(|link| link.matches_url_regex()){+ if link.source.domain() == link.destination.domain() {+ continue;+ }+trace!("inserting link {:?}", link);let mut source = Node::from(link.source);source.remove_protocol();diff --git a/core/src/query/optic.rs b/core/src/query/optic.rsindex 9caf1374..ee057897 100644--- a/core/src/query/optic.rs+++ b/core/src/query/optic.rs@@ -655,7 +655,7 @@ mod tests {pre_computed_score: 0.0,dmoz_description: None,- node_id: Some(*graph.node2id(&Node::from("www.a.com").into_host()).unwrap()),+ node_id: Some(graph.node2id(&Node::from("www.a.com").into_host()).unwrap()),}).expect("failed to insert webpage");index@@ -683,7 +683,7 @@ mod tests {pre_computed_score: 0.0,fetch_time_ms: 500,dmoz_description: None,- node_id: Some(*graph.node2id(&Node::from("www.b.com").into_host()).unwrap()),+ node_id: Some(graph.node2id(&Node::from("www.b.com").into_host()).unwrap()),}).expect("failed to insert webpage");index@@ -711,7 +711,7 @@ mod tests {pre_computed_score: 0.0,fetch_time_ms: 500,dmoz_description: None,- node_id: Some(*graph.node2id(&Node::from("www.c.com").into_host()).unwrap()),+ node_id: Some(graph.node2id(&Node::from("www.c.com").into_host()).unwrap()),}).expect("failed to insert webpage");diff --git a/core/src/ranking/centrality_store.rs b/core/src/ranking/centrality_store.rsindex 7cc995a9..af99a2e5 100644--- a/core/src/ranking/centrality_store.rs+++ b/core/src/ranking/centrality_store.rs@@ -14,13 +14,12 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.-use std::io::{BufReader, BufWriter, Read};-use std::{collections::BTreeMap, fs::File, path::Path};+use std::{fs::File, path::Path};+use rocksdb::BlockBasedOptions;use tracing::debug;use tracing::log::trace;-use crate::Result;use crate::{kv::{rocksdb_store::RocksDbStore, Kv},webgraph::{centrality::harmonic::HarmonicCentrality, Node, NodeID, Webgraph},@@ -28,6 +27,113 @@ use crate::{use super::inbound_similarity::InboundSimilarity;+pub struct Node2Id {+ db: rocksdb::DB,+}++impl Node2Id {+ pub fn open<P: AsRef<Path>>(path: P) -> Self {+ let mut options = rocksdb::Options::default();+ options.create_if_missing(true);+ options.increase_parallelism(8);+ options.set_write_buffer_size(256 * 1024 * 1024); // 256 MB memtable+ options.set_max_write_buffer_number(8);++ let mut block_options = BlockBasedOptions::default();+ block_options.set_bloom_filter(64.0, true);++ let cache = rocksdb::Cache::new_lru_cache(256 * 1024 * 1024).unwrap(); // 256 MB cache+ block_options.set_block_cache(&cache);++ options.set_block_based_table_factory(&block_options);++ let db = rocksdb::DB::open(&options, path.as_ref().to_str().unwrap()).unwrap();++ Self { db }+ }++ pub fn get(&self, key: &Node) -> Option<NodeID> {+ let bytes = bincode::serialize(key).unwrap();++ self.db+ .get(bytes)+ .unwrap()+ .map(|bytes| bincode::deserialize(&bytes).unwrap())+ }++ pub fn put(&self, key: &Node, value: &NodeID) {+ let key_bytes = bincode::serialize(key).unwrap();+ let value_bytes = bincode::serialize(value).unwrap();++ self.db.put(key_bytes, value_bytes).unwrap();+ }++ pub fn batch_put(&self, it: impl Iterator<Item = (Node, NodeID)>) {+ let mut batch = rocksdb::WriteBatch::default();++ for (key, value) in it {+ let key_bytes = bincode::serialize(&key).unwrap();+ let value_bytes = bincode::serialize(&value).unwrap();++ batch.put(key_bytes, value_bytes);+ }++ self.db.write(batch).unwrap();+ }++ pub fn contains(&self, key: &Node) -> bool {+ let bytes = bincode::serialize(key).unwrap();++ self.db.get(bytes).unwrap().is_some()+ }++ pub fn nodes(&self) -> impl Iterator<Item = Node> + '_ {+ let mut read_opts = rocksdb::ReadOptions::default();++ read_opts.set_readahead_size(4_194_304); // 4 MB++ self.db+ .iterator_opt(rocksdb::IteratorMode::Start, read_opts)+ .map(|res| {+ let (key, _) = res.unwrap();+ bincode::deserialize(&key).unwrap()+ })+ }++ pub fn ids(&self) -> impl Iterator<Item = NodeID> + '_ {+ let mut read_opts = rocksdb::ReadOptions::default();++ read_opts.set_readahead_size(4_194_304); // 4 MB++ self.db+ .iterator_opt(rocksdb::IteratorMode::Start, read_opts)+ .map(|res| {+ let (_, val) = res.unwrap();+ bincode::deserialize(&val).unwrap()+ })+ }++ pub fn iter(&self) -> impl Iterator<Item = (Node, NodeID)> + '_ {+ let mut read_opts = rocksdb::ReadOptions::default();++ read_opts.set_readahead_size(4_194_304); // 4 MB++ self.db+ .iterator_opt(rocksdb::IteratorMode::Start, read_opts)+ .map(|res| {+ let (key, val) = res.unwrap();+ (+ bincode::deserialize(&key).unwrap(),+ bincode::deserialize(&val).unwrap(),+ )+ })+ }++ pub fn flush(&self) {+ self.db.flush().unwrap();+ }+}+pub struct HarmonicCentralityStore {pub host: Box<dyn Kv<NodeID, f64>>,pub full: Box<dyn Kv<NodeID, f64>>,@@ -49,14 +155,14 @@ impl HarmonicCentralityStore {pub struct IndexerCentralityStore {pub harmonic: HarmonicCentralityStore,- pub node2id: BTreeMap<Node, NodeID>,+ pub node2id: Node2Id,}impl IndexerCentralityStore {pub fn open<P: AsRef<Path>>(path: P) -> Self {Self {harmonic: HarmonicCentralityStore::open(path.as_ref().join("harmonic")),- node2id: open_node2id(path.as_ref().join("node2id")).unwrap(),+ node2id: Node2Id::open(path.as_ref().join("node2id")),}}}@@ -72,7 +178,7 @@ impl From<CentralityStore> for IndexerCentralityStore {pub struct SearchCentralityStore {pub inbound_similarity: InboundSimilarity,- pub node2id: BTreeMap<Node, NodeID>,+ pub node2id: Node2Id,}impl SearchCentralityStore {@@ -80,7 +186,7 @@ impl SearchCentralityStore {Self {inbound_similarity: InboundSimilarity::open(path.as_ref().join("inbound_similarity")).unwrap(),- node2id: open_node2id(path.as_ref().join("node2id")).unwrap(),+ node2id: Node2Id::open(path.as_ref().join("node2id")),}}}@@ -97,20 +203,10 @@ impl From<CentralityStore> for SearchCentralityStore {pub struct CentralityStore {pub harmonic: HarmonicCentralityStore,pub inbound_similarity: InboundSimilarity,- pub node2id: BTreeMap<Node, NodeID>,+ pub node2id: Node2Id,pub base_path: String,}-fn open_node2id<P: AsRef<Path>>(path: P) -> Result<BTreeMap<Node, NodeID>> {- let file = File::open(path)?;- let mut reader = BufReader::new(file);-- let mut buf = Vec::new();- reader.read_to_end(&mut buf)?;-- Ok(bincode::deserialize(&buf)?)-}-impl CentralityStore {pub fn open<P: AsRef<Path>>(path: P) -> Self {Self {@@ -118,9 +214,7 @@ impl CentralityStore {inbound_similarity: InboundSimilarity::open(path.as_ref().join("inbound_similarity")).ok().unwrap_or_default(),- node2id: open_node2id(path.as_ref().join("node2id"))- .ok()- .unwrap_or_default(),+ node2id: Node2Id::open(path.as_ref().join("node2id")),base_path: path.as_ref().to_str().unwrap().to_string(),}}@@ -134,7 +228,7 @@ impl CentralityStore {.write(true).create(true).truncate(true)- .open(output_path.as_ref().join("harmonic_host.csv"))+ .open(output_path.as_ref().join("harmonic.csv")).unwrap();let mut host: Vec<_> = harmonic_centrality.host.into_iter().collect();@@ -142,7 +236,7 @@ impl CentralityStore {let mut wtr = csv::Writer::from_writer(csv_file);for (node, centrality) in host {let node_id = store.node2id.get(&node).unwrap();- store.harmonic.host.insert(*node_id, centrality);+ store.harmonic.host.insert(node_id, centrality);wtr.write_record(&[node.name, centrality.to_string()]).unwrap();}@@ -158,10 +252,7 @@ impl CentralityStore {pub fn build_harmonic<P: AsRef<Path>>(graph: &Webgraph, output_path: P) -> Self {let mut store = CentralityStore::open(output_path.as_ref());- store.node2id = graph- .node_ids()- .map(|(node, node_id)| (node.clone(), *node_id))- .collect();+ store.node2id.batch_put(graph.node_ids());let harmonic_centrality = HarmonicCentrality::calculate(graph);Self::store_host(&output_path, &mut store, harmonic_centrality);@@ -189,14 +280,6 @@ impl CentralityStore {.unwrap();trace!("saving node2id");- let mut file = BufWriter::new(- File::options()- .create(true)- .truncate(true)- .write(true)- .open(Path::new(&self.base_path).join("node2id"))- .unwrap(),- );- bincode::serialize_into(&mut file, &self.node2id).unwrap();+ self.node2id.flush();}}diff --git a/core/src/ranking/inbound_similarity.rs b/core/src/ranking/inbound_similarity.rsindex 9cdb3516..bd83cfab 100644--- a/core/src/ranking/inbound_similarity.rs+++ b/core/src/ranking/inbound_similarity.rs@@ -285,18 +285,18 @@ mod tests {for (node, centrality) in harmonic.host {harmonic_centrality_store.host- .insert(*graph.node2id(&node).unwrap(), centrality);+ .insert(graph.node2id(&node).unwrap(), centrality);}harmonic_centrality_store.host.flush();let inbound =InboundSimilarity::build_with_threshold(&graph, &harmonic_centrality_store, 1000);- let scorer = inbound.scorer(&[*graph.node2id(&Node::from("b.com")).unwrap()], &[]);+ let scorer = inbound.scorer(&[graph.node2id(&Node::from("b.com")).unwrap()], &[]);let e = graph.node2id(&Node::from("e.com")).unwrap();let d = graph.node2id(&Node::from("d.com")).unwrap();- assert!(scorer.score(e) > scorer.score(d));+ assert!(scorer.score(&e) > scorer.score(&d));}#[test]@@ -315,7 +315,7 @@ mod tests {for (node, centrality) in harmonic.host {harmonic_centrality_store.host- .insert(*graph.node2id(&node).unwrap(), centrality);+ .insert(graph.node2id(&node).unwrap(), centrality);}harmonic_centrality_store.host.flush();@@ -351,7 +351,7 @@ mod tests {page_centrality: 0.0,pre_computed_score: 0.0,- node_id: Some(*graph.node2id(&Node::from("e.com")).unwrap()),+ node_id: Some(graph.node2id(&Node::from("e.com")).unwrap()),dmoz_description: None,}).expect("failed to insert webpage");@@ -379,7 +379,7 @@ mod tests {page_centrality: 0.0,pre_computed_score: 0.0,- node_id: Some(*graph.node2id(&Node::from("d.com")).unwrap()),+ node_id: Some(graph.node2id(&Node::from("d.com")).unwrap()),dmoz_description: None,}).expect("failed to insert webpage");diff --git a/core/src/ranking/optics.rs b/core/src/ranking/optics.rsindex 1976ca3f..d5b05e11 100644--- a/core/src/ranking/optics.rs+++ b/core/src/ranking/optics.rs@@ -95,7 +95,6 @@ mod tests {node_id: Some(graph.node2id(&Node::from("https://www.first.com").into_host())- .copied().unwrap(),),})@@ -128,7 +127,6 @@ mod tests {node_id: Some(graph.node2id(&Node::from("https://www.second.com").into_host())- .copied().unwrap(),),})@@ -159,7 +157,7 @@ mod tests {dmoz_description: None,node_id: Some(- *graph+ graph.node2id(&Node::from("https://www.third.com").into_host()).unwrap(),),diff --git a/core/src/searcher/local.rs b/core/src/searcher/local.rsindex 40608036..3c89e904 100644--- a/core/src/searcher/local.rs+++ b/core/src/searcher/local.rs@@ -167,7 +167,7 @@ impl LocalSearcher {.liked.iter().map(|site| Node::from(site.clone()).into_host())- .filter_map(|node| store.node2id.get(&node).copied())+ .filter_map(|node| store.node2id.get(&node)).collect();let disliked_sites: Vec<_> = parsed_query@@ -175,7 +175,7 @@ impl LocalSearcher {.disliked.iter().map(|site| Node::from(site.clone()).into_host())- .filter_map(|node| store.node2id.get(&node).copied())+ .filter_map(|node| store.node2id.get(&node)).collect();aggregator.set_inbound_similarity(diff --git a/core/src/similar_sites.rs b/core/src/similar_sites.rsindex 363c8adc..dc30b08e 100644--- a/core/src/similar_sites.rs+++ b/core/src/similar_sites.rs@@ -74,7 +74,7 @@ impl SimilarSitesFinder {let nodes: Vec<_> = nodes.iter().map(|url| Node::from(url.to_string()).into_host())- .filter_map(|node| self.webgraph.node2id(&node).copied())+ .filter_map(|node| self.webgraph.node2id(&node)).collect();let scorer = self.inbound_similarity.scorer(&nodes, &[]);@@ -134,7 +134,7 @@ impl SimilarSitesFinder {scored_nodes.into_iter().map(|ScoredNodeID { node_id, score }| {- let node = self.webgraph.id2node(&node_id).unwrap().clone();+ let node = self.webgraph.id2node(&node_id).unwrap();ScoredNode { node, score }}).collect()@@ -142,7 +142,7 @@ impl SimilarSitesFinder {pub fn knows_about(&self, node: &Node) -> bool {match self.webgraph.node2id(node) {- Some(node_id) => self.inbound_similarity.knows_about(*node_id),+ Some(node_id) => self.inbound_similarity.knows_about(node_id),None => false,}}diff --git a/core/src/webgraph/centrality/betweenness.rs b/core/src/webgraph/centrality/betweenness.rsindex 742e115d..3bed8968 100644--- a/core/src/webgraph/centrality/betweenness.rs+++ b/core/src/webgraph/centrality/betweenness.rs@@ -32,7 +32,7 @@ fn calculate(graph: &Webgraph, with_progress: bool) -> (HashMap<Node, f64>, i32)let mut n = 0;let mut max_dist = 0;- let nodes: Vec<_> = graph.nodes().cloned().take(100_000).collect();+ let nodes: Vec<_> = graph.nodes().take(100_000).collect();let pb =if with_progress {@@ -128,7 +128,7 @@ fn calculate(graph: &Webgraph, with_progress: bool) -> (HashMap<Node, f64>, i32)(centrality.into_iter()- .map(|(id, centrality)| (graph.id2node(&id).unwrap().clone(), centrality / norm))+ .map(|(id, centrality)| (graph.id2node(&id).unwrap(), centrality / norm)).collect(),max_dist,)diff --git a/core/src/webgraph/centrality/harmonic.rs b/core/src/webgraph/centrality/harmonic.rsindex e8d3376f..37300a60 100644--- a/core/src/webgraph/centrality/harmonic.rs+++ b/core/src/webgraph/centrality/harmonic.rs@@ -207,7 +207,7 @@ fn calculate_centrality(graph: &Webgraph) -> HashMap<Node, f64> {.filter(|(_, centrality)| *centrality > 0.0).map(|(node_id, centrality)| {(- graph.id2node(&NodeID::from(node_id)).unwrap().clone(),+ graph.id2node(&NodeID::from(node_id)).unwrap(),centrality / norm_factor,)})diff --git a/core/src/webgraph/mod.rs b/core/src/webgraph/mod.rsindex 4674442c..16473596 100644--- a/core/src/webgraph/mod.rs+++ b/core/src/webgraph/mod.rs@@ -15,7 +15,6 @@// along with this program. If not, see <https://www.gnu.org/licenses/>.mod segment;-use rkyv::Archive;use serde::de::DeserializeOwned;use serde::{Deserialize, Serialize};use std::collections::{BTreeMap, BinaryHeap, HashSet};@@ -32,6 +31,7 @@ use crate::webpage::Url;pub mod centrality;mod store;use self::segment::{LiveSegment, SegmentNodeID, StoredSegment};+use self::store::Store;#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord, Hash)]pub struct NodeID(pub u64);@@ -42,8 +42,7 @@ impl From<u64> for NodeID {}}-#[derive(Debug, Clone, Archive, rkyv::Serialize, rkyv::Deserialize, PartialEq, Eq, Hash)]-#[archive_attr(derive(Eq, Hash, PartialEq, Debug))]+#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq, Eq, Hash)]pub(crate) struct StoredEdge {other: SegmentNodeID,label: String,@@ -182,8 +181,8 @@ wherelet mut queue = BinaryHeap::new();- queue.push(cmp::Reverse((0, *source_id)));- distances.insert(*source_id, 0);+ queue.push(cmp::Reverse((0, source_id)));+ distances.insert(source_id, 0);while let Some(state) = queue.pop() {let (cost, v) = state.0;@@ -212,7 +211,7 @@ impl ShortestPaths for Webgraph {fn distances(&self, source: Node) -> BTreeMap<Node, u8> {self.raw_distances(source).into_iter()- .map(|(id, dist)| (self.id2node(&id).expect("unknown node").clone(), dist))+ .map(|(id, dist)| (self.id2node(&id).expect("unknown node"), dist)).collect()}@@ -237,7 +236,7 @@ impl ShortestPaths for Webgraph {fn reversed_distances(&self, source: Node) -> BTreeMap<Node, u8> {self.raw_reversed_distances(source).into_iter()- .map(|(id, dist)| (self.id2node(&id).expect("unknown node").clone(), dist))+ .map(|(id, dist)| (self.id2node(&id).expect("unknown node"), dist)).collect()}}@@ -291,8 +290,8 @@ pub struct Webgraph {live_segment: LiveSegment,segments: Vec<StoredSegment>,executor: Arc<Executor>,- node2id: BTreeMap<Node, NodeID>,- id2node: BTreeMap<NodeID, Node>,+ node2id: Store<Node, NodeID>,+ id2node: Store<NodeID, Node>,meta: Meta,}@@ -346,8 +345,8 @@ impl Webgraph {live_segment: LiveSegment::default(),segments,executor: Arc::new(Executor::multi_thread("webgraph").unwrap()),- node2id: open_bin(path.as_ref().join("node2id.bin")),- id2node: open_bin(path.as_ref().join("id2node.bin")),+ node2id: Store::open(path.as_ref().join("node2id")),+ id2node: Store::open(path.as_ref().join("id2node")),meta,}}@@ -360,12 +359,12 @@ impl Webgraph {fn id_or_assign(&mut self, node: &Node) -> NodeID {match self.node2id(node) {- Some(id) => *id,+ Some(id) => id,None => {let id = self.id_and_increment();- self.node2id.insert(node.clone(), id);- self.id2node.insert(id, node.clone());+ self.node2id.put(node, &id);+ self.id2node.put(&id, node);id}@@ -382,11 +381,11 @@ impl Webgraph {let mut mapping = Vec::new();for (node, other_id) in other.node2id.iter() {- match self.node2id(node) {- Some(this_id) => mapping.push((*other_id, *this_id)),+ match self.node2id(&node) {+ Some(this_id) => mapping.push((other_id, this_id)),None => {- let new_id = self.id_or_assign(node);- mapping.push((*other_id, new_id));+ let new_id = self.id_or_assign(&node);+ mapping.push((other_id, new_id));}}}@@ -421,8 +420,8 @@ impl Webgraph {}self.save_metadata();- save_bin(&self.node2id, Path::new(&self.path).join("node2id.bin"));- save_bin(&self.id2node, Path::new(&self.path).join("id2node.bin"));+ self.node2id.flush();+ self.id2node.flush();if self.segments.len() > 2 * num_cpus::get() {self.merge_segments(num_cpus::get());@@ -431,11 +430,11 @@ impl Webgraph {pub fn ingoing_edges(&self, node: Node) -> Vec<FullEdge> {if let Some(node_id) = self.node2id(&node) {- self.inner_ingoing_edges(node_id, true)+ self.inner_ingoing_edges(&node_id, true).into_iter().map(|edge| FullEdge {- from: self.id2node(&edge.from).unwrap().clone(),- to: self.id2node(&edge.to).unwrap().clone(),+ from: self.id2node(&edge.from).unwrap(),+ to: self.id2node(&edge.to).unwrap(),label: edge.label.loaded().unwrap(),}).collect()@@ -478,11 +477,11 @@ impl Webgraph {pub fn outgoing_edges(&self, node: Node) -> Vec<FullEdge> {if let Some(node_id) = self.node2id(&node) {- self.inner_outgoing_edges(node_id, true)+ self.inner_outgoing_edges(&node_id, true).into_iter().map(|edge| FullEdge {- from: self.id2node(&edge.from).unwrap().clone(),- to: self.id2node(&edge.to).unwrap().clone(),+ from: self.id2node(&edge.from).unwrap(),+ to: self.id2node(&edge.to).unwrap(),label: edge.label.loaded().unwrap(),}).collect()@@ -491,19 +490,19 @@ impl Webgraph {}}- pub fn node2id(&self, node: &Node) -> Option<&NodeID> {+ pub fn node2id(&self, node: &Node) -> Option<NodeID> {self.node2id.get(node)}- pub fn id2node(&self, id: &NodeID) -> Option<&Node> {+ pub fn id2node(&self, id: &NodeID) -> Option<Node> {self.id2node.get(id)}- pub fn nodes(&self) -> impl Iterator<Item = &NodeID> + '_ {+ pub fn nodes(&self) -> impl Iterator<Item = NodeID> + '_ {self.node2id.values()}- pub fn node_ids(&self) -> impl Iterator<Item = (&Node, &NodeID)> + '_ {+ pub fn node_ids(&self) -> impl Iterator<Item = (Node, NodeID)> + '_ {self.node2id.iter()}diff --git a/core/src/webgraph/segment.rs b/core/src/webgraph/segment.rsindex 1c248e28..ac74f098 100644--- a/core/src/webgraph/segment.rs+++ b/core/src/webgraph/segment.rs@@ -49,16 +49,16 @@ impl From<u64> for SegmentNodeID {struct MergePacket {next_segment_node_id: SegmentNodeID,- new_id_mapping: BTreeMap<NodeID, SegmentNodeID>,- new_rev_id_mapping: BTreeMap<SegmentNodeID, NodeID>,+ new_id_mapping: Store<NodeID, SegmentNodeID>,+ new_rev_id_mapping: Store<SegmentNodeID, NodeID>,new_adjacency: Store<SegmentNodeID, HashSet<StoredEdge>>,}pub struct StoredSegment {adjacency: Store<SegmentNodeID, HashSet<StoredEdge>>,reversed_adjacency: Store<SegmentNodeID, HashSet<StoredEdge>>,- id_mapping: BTreeMap<NodeID, SegmentNodeID>,- rev_id_mapping: BTreeMap<SegmentNodeID, NodeID>,+ id_mapping: Store<NodeID, SegmentNodeID>,+ rev_id_mapping: Store<SegmentNodeID, NodeID>,meta: Meta,id: String,folder_path: String,@@ -67,11 +67,10 @@ pub struct StoredSegment {impl StoredSegment {pub fn open<P: AsRef<Path>>(folder_path: P, id: String) -> Self {StoredSegment {- adjacency: Store::open(folder_path.as_ref().join("adjacency"), &id).unwrap(),- reversed_adjacency: Store::open(folder_path.as_ref().join("reversed_adjacency"), &id)- .unwrap(),- id_mapping: open_bin(folder_path.as_ref().join("id_mapping.bin")),- rev_id_mapping: open_bin(folder_path.as_ref().join("rev_id_mapping.bin")),+ adjacency: Store::open(folder_path.as_ref().join("adjacency")),+ reversed_adjacency: Store::open(folder_path.as_ref().join("reversed_adjacency")),+ id_mapping: Store::open(folder_path.as_ref().join("id_mapping")),+ rev_id_mapping: Store::open(folder_path.as_ref().join("rev_id_mapping")),meta: open_bin(folder_path.as_ref().join("meta.bin")),folder_path: folder_path.as_ref()@@ -88,11 +87,11 @@ impl StoredSegment {}fn id_mapping(&self, node: &NodeID) -> Option<SegmentNodeID> {- self.id_mapping.get(node).copied()+ self.id_mapping.get(node)}fn rev_id_mapping(&self, node: &SegmentNodeID) -> Option<NodeID> {- self.rev_id_mapping.get(node).copied()+ self.rev_id_mapping.get(node)}pub fn outgoing_edges(&self, node: &NodeID, load_label: bool) -> Vec<Edge> {@@ -154,10 +153,11 @@ impl StoredSegment {}fn flush(&mut self) {- self.adjacency.flush().unwrap();- self.reversed_adjacency.flush().unwrap();- save_bin(&self.id_mapping, self.path().join("id_mapping.bin"));- save_bin(&self.rev_id_mapping, self.path().join("rev_id_mapping.bin"));+ self.adjacency.flush();+ self.reversed_adjacency.flush();+ self.id_mapping.flush();+ self.rev_id_mapping.flush();+save_bin(&self.meta, self.path().join("meta.bin"));}@@ -171,7 +171,7 @@ impl StoredSegment {pub fn edges(&self) -> impl Iterator<Item = Edge> + '_ {self.adjacency.iter().flat_map(move |(node_id, edges)| {- let from = self.rev_id_mapping(node_id).unwrap();+ let from = self.rev_id_mapping(&node_id).unwrap();edges.into_iter().map(move |stored_edge| Edge {from,@@ -200,7 +200,7 @@ impl StoredSegment {let mut node_seen_before = false;let mut j = 0;while j < i {- if segments[j].id_mapping.contains_key(node_id) {+ if segments[j].id_mapping.contains_key(&node_id) {node_seen_before = true;break;}@@ -208,15 +208,13 @@ impl StoredSegment {}if !node_seen_before {- if !packet.new_id_mapping.contains_key(node_id) {- let r = packet+ if !packet.new_id_mapping.contains_key(&node_id) {+ packet.new_id_mapping- .insert(*node_id, packet.next_segment_node_id);- debug_assert!(r.is_none());- let r = packet+ .put(&node_id, &packet.next_segment_node_id);+ packet.new_rev_id_mapping- .insert(packet.next_segment_node_id, *node_id);- debug_assert!(r.is_none());+ .put(&packet.next_segment_node_id, &node_id);packet.next_segment_node_id.0 += 1;}@@ -226,23 +224,21 @@ impl StoredSegment {while j < segments.len() {let segment = &segments[j];- if let Some(edges_j) = edges_fn(segment, node_id) {+ if let Some(edges_j) = edges_fn(segment, &node_id) {for edge in edges_j {let node = segment.rev_id_mapping(&edge.other).unwrap();- if let std::collections::btree_map::Entry::Vacant(e) =- packet.new_id_mapping.entry(node)- {- e.insert(packet.next_segment_node_id);- let r = packet+ if !packet.new_id_mapping.contains_key(&node) {+ packet+ .new_id_mapping+ .put(&node, &packet.next_segment_node_id);+ packet.new_rev_id_mapping- .insert(packet.next_segment_node_id, node);- debug_assert!(r.is_none());+ .put(&packet.next_segment_node_id, &node);packet.next_segment_node_id.0 += 1;}- let new_segment_node_id =- packet.new_id_mapping.get(&node).copied().unwrap();+ let new_segment_node_id = packet.new_id_mapping.get(&node).unwrap();let new_edge = StoredEdge {other: new_segment_node_id,@@ -257,11 +253,8 @@ impl StoredSegment {}if !edges.is_empty() {- let segment_node_id = packet.new_id_mapping.get(node_id).copied().unwrap();- packet- .new_adjacency- .insert(segment_node_id, &edges)- .unwrap();+ let segment_node_id = packet.new_id_mapping.get(&node_id).unwrap();+ packet.new_adjacency.put(&segment_node_id, &edges);}}}@@ -276,12 +269,13 @@ impl StoredSegment {let new_path = segments[0].path().parent().unwrap().join(&new_segment_id);- let new_adjacency = Store::open(new_path.join("adjacency"), &new_segment_id).unwrap();+ let new_adjacency = Store::open(new_path.join("adjacency"));let next_segment_node_id = SegmentNodeID(0);- let new_id_mapping = BTreeMap::new();- let new_rev_id_mapping = BTreeMap::new();+ let new_id_mapping = Store::open(new_path.join("id_mapping"));+ let new_rev_id_mapping = Store::open(new_path.join("rev_id_mapping"));+let mut packet = MergePacket {next_segment_node_id,new_id_mapping,@@ -292,7 +286,7 @@ impl StoredSegment {Self::merge_adjacency(|segment: &StoredSegment, node_id: &NodeID| {segment.id_mapping.get(node_id).map(|segment_node_id| {- segment.adjacency.get(segment_node_id).unwrap_or_default()+ segment.adjacency.get(&segment_node_id).unwrap_or_default()})},&mut packet,@@ -300,8 +294,7 @@ impl StoredSegment {);let new_adjacency = packet.new_adjacency;- let new_reversed_adjacency =- Store::open(new_path.join("reversed_adjacency"), &new_segment_id).unwrap();+ let new_reversed_adjacency = Store::open(new_path.join("reversed_adjacency"));let mut packet = MergePacket {next_segment_node_id: packet.next_segment_node_id,@@ -315,7 +308,7 @@ impl StoredSegment {segment.id_mapping.get(node_id).map(|segment_node_id| {segment.reversed_adjacency- .get(segment_node_id)+ .get(&segment_node_id).unwrap_or_default()})},@@ -352,10 +345,16 @@ impl StoredSegment {}}- for (segment_id, new_id) in new_mappings {- self.id_mapping.insert(new_id, segment_id);- self.rev_id_mapping.insert(segment_id, new_id);- }+ self.id_mapping.batch_put(+ new_mappings+ .iter()+ .map(|(segment_id, new_id)| (new_id, segment_id)),+ );+ self.rev_id_mapping.batch_put(+ new_mappings+ .iter()+ .map(|(segment_id, new_id)| (segment_id, new_id)),+ );self.flush();}@@ -409,24 +408,24 @@ impl LiveSegment {let segment_id = uuid::Uuid::new_v4().to_string();let path = folder_path.as_ref().join(&segment_id);- let mut adjacency = Store::open(path.join("adjacency"), &segment_id).unwrap();+ let adjacency = Store::open(path.join("adjacency"));+ adjacency.batch_put(self.adjacency.iter());- for (from, edges) in self.adjacency {- adjacency.insert(from, &edges).unwrap();- }+ let reversed_adjacency = Store::open(path.join("reversed_adjacency"));- let mut reversed_adjacency =- Store::open(path.join("reversed_adjacency"), &segment_id).unwrap();+ reversed_adjacency.batch_put(self.reversed_adjacency.iter());- for (to, edges) in self.reversed_adjacency {- reversed_adjacency.insert(to, &edges).unwrap();- }+ let id_mapping = Store::open(path.join("id_mapping"));+ id_mapping.batch_put(self.id_mapping.iter());++ let rev_id_mapping = Store::open(path.join("rev_id_mapping"));+ rev_id_mapping.batch_put(self.rev_id_mapping.iter());let mut stored_segment = StoredSegment {adjacency,reversed_adjacency,- id_mapping: self.id_mapping,- rev_id_mapping: self.rev_id_mapping,+ id_mapping,+ rev_id_mapping,meta: Meta {num_nodes: self.next_id,},diff --git a/core/src/webgraph/store.rs b/core/src/webgraph/store.rsindex e7394953..e2dbe043 100644--- a/core/src/webgraph/store.rs+++ b/core/src/webgraph/store.rs@@ -14,228 +14,135 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.-use std::{- collections::BTreeMap,- fs::OpenOptions,- io::BufReader,- ops::Range,- path::{Path, PathBuf},-};+use std::path::Path;-use rkyv::{ser::serializers::AllocSerializer, AlignedVec};--type Result<T> = std::result::Result<T, Error>;--const SCRATCH_SIZE: usize = 4096;-const LOCATIONS_NAME: &str = "locations.bin";-const MMAP_NAME: &str = "data.bin";--#[derive(Debug, thiserror::Error)]-pub enum Error {- #[error("key already exists")]- KeyExists,-- #[error("io error: {0}")]- Io(std::io::Error),-- #[error("bincode error: {0}")]- Bincode(bincode::Error),-}+use rocksdb::BlockBasedOptions;pub struct Store<K, V> {- locations: BTreeMap<K, Range<usize>>,- folder_path: PathBuf,- cur_end_range: usize,- map: memmap2::MmapMut,- has_new_inserts: bool,- segment_name: String,- _values: std::marker::PhantomData<V>,+ db: rocksdb::DB,+ _phantom: std::marker::PhantomData<(K, V)>,}impl<K, V> Store<K, V>where- K: serde::de::DeserializeOwned + serde::Serialize + Ord,- V: rkyv::Archive + 'static,- V::Archived: rkyv::Deserialize<V, rkyv::de::deserializers::SharedDeserializeMap> + 'static,+ K: serde::de::DeserializeOwned + serde::Serialize,+ V: serde::de::DeserializeOwned + serde::Serialize,{- pub fn open<P: AsRef<Path>>(folder_path: P, segment_name: &str) -> Result<Self> {- if folder_path.as_ref().is_file() {- return Err(Error::Io(std::io::Error::new(- std::io::ErrorKind::NotFound,- format!("{} is not a directory", folder_path.as_ref().display()),- )));- }-- std::fs::create_dir_all(folder_path.as_ref()).map_err(Error::Io)?;-- let locations_path = folder_path- .as_ref()- .join(format!("{segment_name}-{LOCATIONS_NAME}"));- let f = OpenOptions::new()- .read(true)- .write(true)- .create(true)- .open(locations_path)- .map_err(Error::Io)?;-- let mut reader = BufReader::new(f);- let locations: BTreeMap<K, Range<usize>> =- bincode::deserialize_from(&mut reader).unwrap_or_default();-- let cur_end_range = locations.values().map(|range| range.end).max().unwrap_or(0);-- let mmap_path = folder_path- .as_ref()- .join(format!("{segment_name}-{MMAP_NAME}"));- let map = unsafe {- let file = OpenOptions::new()- .read(true)- .write(true)- .create(true)- .open(mmap_path)- .map_err(Error::Io)?;-- memmap2::MmapOptions::new()- .map_mut(&file)- .map_err(Error::Io)?- };+ pub fn open<P: AsRef<Path>>(path: P) -> Self {+ let mut options = rocksdb::Options::default();+ options.create_if_missing(true);+ options.increase_parallelism(8);+ options.set_write_buffer_size(256 * 1024 * 1024); // 256 MB memtable+ options.set_max_write_buffer_number(8);- Ok(Self {- locations,- cur_end_range,- map,- folder_path: folder_path.as_ref().to_path_buf(),- has_new_inserts: false,- _values: std::marker::PhantomData,- segment_name: segment_name.to_string(),- })- }+ let mut block_options = BlockBasedOptions::default();+ block_options.set_bloom_filter(64.0, true);- pub fn get(&self, key: &K) -> Option<V> {- let range = self.locations.get(key)?;- let bytes = &self.map[range.clone()];+ let cache = rocksdb::Cache::new_lru_cache(256 * 1024 * 1024).unwrap(); // 256 MB cache+ block_options.set_block_cache(&cache);- let mut aligned = AlignedVec::with_capacity(bytes.len());- aligned.extend_from_slice(bytes);+ options.set_block_based_table_factory(&block_options);- // SAFETY:- // We ensure the target `V` is `'static` and contains only owned data so it's safe to- // temporarily extend the lifetime so we can allocate the type entirely.- let v = unsafe { rkyv::util::from_bytes_unchecked::<V>(&aligned).unwrap() };+ let db = rocksdb::DB::open(&options, path.as_ref().to_str().unwrap()).unwrap();- Some(v)+ Self {+ db,+ _phantom: std::marker::PhantomData,+ }}- pub fn iter(&self) -> impl Iterator<Item = (&K, V)> + '_ {- self.locations.iter().map(move |(key, range)| {- let bytes = &self.map[range.clone()];+ pub fn get(&self, key: &K) -> Option<V> {+ let bytes = bincode::serialize(key).unwrap();- let mut aligned = AlignedVec::with_capacity(bytes.len());- aligned.extend_from_slice(bytes);+ self.db+ .get(bytes)+ .unwrap()+ .map(|bytes| bincode::deserialize(&bytes).unwrap())+ }- // SAFETY:- // We ensure the target `V` is `'static` and contains only owned data so it's safe to- // temporarily extend the lifetime so we can allocate the type entirely.- let v = unsafe { rkyv::util::from_bytes_unchecked::<V>(&aligned).unwrap() };+ pub fn put(&self, key: &K, value: &V) {+ let key_bytes = bincode::serialize(key).unwrap();+ let value_bytes = bincode::serialize(value).unwrap();- (key, v)- })+ self.db.put(key_bytes, value_bytes).unwrap();}- pub fn insert(&mut self, key: K, value: &V) -> Result<()>- where- V: rkyv::Serialize<AllocSerializer<SCRATCH_SIZE>>,- {- if self.locations.contains_key(&key) {- return Err(Error::KeyExists);+ pub fn batch_put<'a>(&'a self, it: impl Iterator<Item = (&'a K, &'a V)>) {+ let mut batch = rocksdb::WriteBatch::default();++ for (key, value) in it {+ let key_bytes = bincode::serialize(&key).unwrap();+ let value_bytes = bincode::serialize(&value).unwrap();++ batch.put(key_bytes, value_bytes);}- let bytes = rkyv::to_bytes::<_, SCRATCH_SIZE>(value).unwrap();- self.insert_raw(key, &bytes[..])+ self.db.write(batch).unwrap();}- fn insert_raw(&mut self, key: K, bytes: &[u8]) -> Result<()> {- let old_end = self.cur_end_range;+ pub fn contains_key(&self, key: &K) -> bool {+ let bytes = bincode::serialize(key).unwrap();- let new_end = bytes.len() + self.cur_end_range;+ self.db.get(bytes).unwrap().is_some()+ }- if new_end > self.map.len() {- self.resize(new_end)?;- }+ pub fn keys(&self) -> impl Iterator<Item = K> + '_ {+ let mut read_opts = rocksdb::ReadOptions::default();- self.map[old_end..new_end].copy_from_slice(bytes);- self.locations.insert(key, old_end..new_end);- self.cur_end_range = new_end;- self.has_new_inserts = true;+ read_opts.set_readahead_size(4_194_304); // 4 MB- Ok(())+ self.db+ .iterator_opt(rocksdb::IteratorMode::Start, read_opts)+ .map(|res| {+ let (key, _) = res.unwrap();+ bincode::deserialize(&key).unwrap()+ })}- fn resize(&mut self, size: usize) -> Result<()> {- if size > self.map.len() {- self.flush_map()?;-- let file = OpenOptions::new()- .read(true)- .write(true)- .create(true)- .open(- self.folder_path- .join(format!("{}-{MMAP_NAME}", self.segment_name)),- )- .map_err(Error::Io)?;-- file.set_len(size as u64).map_err(Error::Io)?;+ pub fn values(&self) -> impl Iterator<Item = V> + '_ {+ let mut read_opts = rocksdb::ReadOptions::default();- self.map = unsafe {- memmap2::MmapOptions::new()- .map_mut(&file)- .map_err(Error::Io)?- };- }+ read_opts.set_readahead_size(4_194_304); // 4 MB- Ok(())+ self.db+ .iterator_opt(rocksdb::IteratorMode::Start, read_opts)+ .map(|res| {+ let (_, val) = res.unwrap();+ bincode::deserialize(&val).unwrap()+ })}- fn flush_map(&mut self) -> Result<()> {- if self.has_new_inserts {- self.map.flush().map_err(Error::Io)?;- }+ pub fn iter(&self) -> impl Iterator<Item = (K, V)> + '_ {+ let mut read_opts = rocksdb::ReadOptions::default();- Ok(())- }+ read_opts.set_readahead_size(4_194_304); // 4 MB- pub fn flush(&mut self) -> Result<()> {- if self.has_new_inserts {- self.flush_map()?;-- let f = OpenOptions::new()- .read(true)- .write(true)- .create(true)- .open(- self.folder_path- .join(format!("{}-{}", self.segment_name, LOCATIONS_NAME)),+ self.db+ .iterator_opt(rocksdb::IteratorMode::Start, read_opts)+ .map(|res| {+ let (key, val) = res.unwrap();+ (+ bincode::deserialize(&key).unwrap(),+ bincode::deserialize(&val).unwrap(),)- .map_err(Error::Io)?;+ })+ }- bincode::serialize_into(f, &self.locations).map_err(Error::Bincode)?;- }- self.has_new_inserts = false;+ pub fn flush(&self) {+ self.db.flush().unwrap();+ }- Ok(())+ pub fn remove(&self, key: &K) {+ let bytes = bincode::serialize(key).unwrap();+ self.db.delete(bytes).unwrap();}}#[cfg(test)]mod tests {- use rkyv::Archive;-use super::*;- #[derive(Debug, PartialEq, Eq, Clone, Archive, rkyv::Serialize, rkyv::Deserialize)]- #[archive(compare(PartialEq))]- #[archive_attr(derive(Debug))]+ #[derive(Debug, PartialEq, Eq, Clone, serde::Serialize, serde::Deserialize)]struct TestStruct {a: String,b: i32,@@ -243,8 +150,7 @@ mod tests {#[test]fn test_insert() {- let mut kv =- Store::<String, TestStruct>::open(crate::gen_temp_path(), "test-segment").unwrap();+ let kv = Store::<String, TestStruct>::open(crate::gen_temp_path().join("test-segment"));assert!(kv.get(&"test".to_string()).is_none());@@ -253,8 +159,8 @@ mod tests {b: 5,};- kv.insert("test".to_string(), &test_struct).unwrap();- kv.flush().unwrap();+ kv.put(&"test".to_string(), &test_struct);+ kv.flush();let archived = kv.get(&"test".to_string()).unwrap();assert_eq!(test_struct, archived);@@ -270,43 +176,17 @@ mod tests {};{- let mut kv = Store::<String, TestStruct>::open(path.clone(), segment_name).unwrap();+ let kv = Store::<String, TestStruct>::open(path.join(segment_name));assert!(kv.get(&"test".to_string()).is_none());- kv.insert("test".to_string(), &test_struct).unwrap();- kv.flush().unwrap();+ kv.put(&"test".to_string(), &test_struct);+ kv.flush();}- let kv = Store::<String, TestStruct>::open(path, segment_name).unwrap();+ let kv = Store::<String, TestStruct>::open(path.join(segment_name));let archived = kv.get(&"test".to_string()).unwrap();assert_eq!(test_struct, archived);}-- #[test]- fn test_insert_existing_key() {- let mut kv =- Store::<String, TestStruct>::open(crate::gen_temp_path(), "test-segment").unwrap();-- assert!(kv.get(&"test".to_string()).is_none());-- let test_struct = TestStruct {- a: "test".to_string(),- b: 5,- };-- kv.insert("test".to_string(), &test_struct).unwrap();- kv.flush().unwrap();-- let archived = kv.get(&"test".to_string()).unwrap();- assert_eq!(test_struct, archived);-- let test_struct = TestStruct {- a: "test".to_string(),- b: 6,- };-- assert!(kv.insert("test".to_string(), &test_struct).is_err());- }}