web search done right
Faster webgraph reads. We now have a webgraph writer and a stationary webgraph optimized for reads. Webgraph reads turned out to be too slow for very big graphs which slowed down the indexer. By splitting the webgraph into a writer (backed by rocksdb) and a read (memory mapped file) we can get very fast writes and reads. The downside is that we now have to convert from the writer to the reader when we are done constructing the webgraph, but it seems to be pretty fast.
Mikkel Denker7cf1325d045f
diff --git a/core/src/entrypoint/configure.rs b/core/src/entrypoint/configure.rsindex 66933a52..102dfda4 100644--- a/core/src/entrypoint/configure.rs+++ b/core/src/entrypoint/configure.rs@@ -84,12 +84,15 @@ fn create_webgraph() -> Result<()> {let mut worker = webgraph::WebgraphWorker {redirect: None,- host_graph: webgraph::open_host_graph(&out_path_host),- page_graph: webgraph::open_page_graph(&out_path_page),+ host_graph: webgraph::open_host_graph_writer(&out_path_host),+ page_graph: webgraph::open_page_graph_writer(&out_path_page),};worker.process_job(&job);+ worker.host_graph.finalize();+ worker.page_graph.finalize();+Ok(())}diff --git a/core/src/entrypoint/indexer.rs b/core/src/entrypoint/indexer.rsindex 846f6b77..cd29d257 100644--- a/core/src/entrypoint/indexer.rs+++ b/core/src/entrypoint/indexer.rs@@ -146,7 +146,7 @@ pub fn process_job(job: &Job, worker: &IndexingWorker) -> Index {let node = Node::from(html.url());let host_node_id = node.clone().into_host().id();- let host_centrality = worker+ let mut host_centrality = worker.host_centrality_store.harmonic.get(&host_node_id)@@ -184,6 +184,7 @@ pub fn process_job(job: &Job, worker: &IndexingWorker) -> Index {.raw_ingoing_edges_with_labels(&Node::from(html.url()).id()).into_iter().map(|edge| edge.label)+ .filter(|label| !label.is_empty()).collect()}).unwrap_or_default();@@ -208,6 +209,14 @@ pub fn process_job(job: &Job, worker: &IndexingWorker) -> Index {has_backlinks = true;}+ if !page_centrality.is_normal() {+ page_centrality = 0.0;+ }++ if !host_centrality.is_normal() {+ host_centrality = 0.0;+ }+let fetch_time_ms = record.metadata.fetch_time_ms as u64;trace!("inserting webpage: {:?}", html.url());diff --git a/core/src/entrypoint/webgraph.rs b/core/src/entrypoint/webgraph.rsindex 6c248d5f..596aee2b 100644--- a/core/src/entrypoint/webgraph.rs+++ b/core/src/entrypoint/webgraph.rs@@ -19,7 +19,7 @@ use crate::{crawler::crawl_db::RedirectDb,entrypoint::download_all_warc_files,mapreduce::Worker,- webgraph::{self, Node, WebgraphBuilder},+ webgraph::{self, Node, WebgraphWriter},webpage::{url_ext::UrlExt, Html},Result,};@@ -67,21 +67,17 @@ pub struct Job {pub warc_paths: Vec<String>,}-pub fn open_host_graph<P: AsRef<Path>>(path: P) -> webgraph::Webgraph {- WebgraphBuilder::new(path)- .deduplication(webgraph::Deduplication::OnlyQuery)- .open()+pub fn open_host_graph_writer<P: AsRef<Path>>(path: P) -> webgraph::WebgraphWriter {+ WebgraphWriter::new(path, crate::executor::Executor::single_thread())}-pub fn open_page_graph<P: AsRef<Path>>(path: P) -> webgraph::Webgraph {- WebgraphBuilder::new(path)- .deduplication(webgraph::Deduplication::OnlyQuery)- .open()+pub fn open_page_graph_writer<P: AsRef<Path>>(path: P) -> webgraph::WebgraphWriter {+ WebgraphWriter::new(path, crate::executor::Executor::single_thread())}pub struct WebgraphWorker {- pub host_graph: webgraph::Webgraph,- pub page_graph: webgraph::Webgraph,+ pub host_graph: webgraph::WebgraphWriter,+ pub page_graph: webgraph::WebgraphWriter,pub redirect: Option<Arc<RedirectDb>>,}@@ -149,8 +145,6 @@ impl WebgraphWorker {self.host_graph.commit();self.page_graph.commit();}- self.host_graph.merge_segments(1);- self.page_graph.merge_segments(1);info!("{} done", name);}@@ -199,8 +193,8 @@ impl Webgraph {let mut worker = WebgraphWorker {redirect: redirect.clone(),- host_graph: open_host_graph(host_path),- page_graph: open_page_graph(page_path),+ host_graph: open_host_graph_writer(host_path),+ page_graph: open_page_graph_writer(page_path),};let jobs = jobs.clone();@@ -209,7 +203,7 @@ impl Webgraph {worker.process_job(job);}- (worker.host_graph, worker.page_graph)+ (worker.host_graph.finalize(), worker.page_graph.finalize())}));}diff --git a/core/src/main.rs b/core/src/main.rsindex c57410e2..2d979fc3 100644--- a/core/src/main.rs+++ b/core/src/main.rs@@ -158,13 +158,10 @@ enum WebgraphOptions {/// Create a new webgraph.Create { config_path: String },- /// Merge multiple webgraphs into a single graph with the specified number of segments.- /// Can also be used to merge segments of a single webgraph.+ /// Merge multiple webgraphs into a single graph.Merge {#[clap(required = true)]paths: Vec<String>,- #[clap(long)]- num_segments: Option<usize>,},/// Deploy the webgraph server. The webgraph server is responsible for serving the webgraph to the search servers.@@ -246,10 +243,7 @@ fn main() -> Result<()> {let config = load_toml_config(config_path);entrypoint::Webgraph::run(&config)?;}- WebgraphOptions::Merge {- mut paths,- num_segments,- } => {+ WebgraphOptions::Merge { mut paths } => {let mut webgraph = WebgraphBuilder::new(paths.remove(0)).open();for other_path in paths {@@ -257,10 +251,6 @@ fn main() -> Result<()> {webgraph.merge(other);std::fs::remove_dir_all(other_path).unwrap();}-- if let Some(num_segments) = num_segments {- webgraph.merge_segments(num_segments)- }}WebgraphOptions::Server { config_path } => {let config: config::WebgraphServerConfig = load_toml_config(config_path);diff --git a/core/src/query/optic.rs b/core/src/query/optic.rsindex e5439f26..bbbbcc99 100644--- a/core/src/query/optic.rs+++ b/core/src/query/optic.rs@@ -240,7 +240,7 @@ mod tests {index::Index,ranking::centrality_store::CentralityStore,searcher::{LocalSearcher, SearchQuery},- webgraph::{Node, WebgraphBuilder},+ webgraph::{Node, WebgraphWriter},webpage::{Html, Webpage},};@@ -622,37 +622,38 @@ mod tests {fn liked_sites() {let mut index = Index::temporary().expect("Unable to open index");- let mut graph = WebgraphBuilder::new_memory().open();+ let mut writer =+ WebgraphWriter::new(gen_temp_path(), crate::executor::Executor::single_thread());- graph.insert(+ writer.insert(Node::from("https://www.e.com").into_host(),Node::from("https://www.a.com").into_host(),String::new(),);- graph.insert(+ writer.insert(Node::from("https://www.a.com").into_host(),Node::from("https://www.e.com").into_host(),String::new(),);- graph.insert(+ writer.insert(Node::from("https://www.c.com").into_host(),Node::from("https://www.c.com").into_host(),String::new(),);- graph.insert(+ writer.insert(Node::from("https://www.b.com").into_host(),Node::from("https://www.e.com").into_host(),String::new(),);- graph.insert(+ writer.insert(Node::from("https://www.e.com").into_host(),Node::from("https://www.b.com").into_host(),String::new(),);- graph.commit();+ let graph = writer.finalize();let centrality_store = CentralityStore::build(&graph, gen_temp_path());diff --git a/core/src/ranking/inbound_similarity.rs b/core/src/ranking/inbound_similarity.rsindex 263583a2..15c35331 100644--- a/core/src/ranking/inbound_similarity.rs+++ b/core/src/ranking/inbound_similarity.rs@@ -214,7 +214,7 @@ mod tests {rand_words,ranking::centrality_store::CentralityStore,searcher::{LocalSearcher, SearchQuery},- webgraph::{Node, WebgraphBuilder},+ webgraph::{Node, WebgraphWriter},webpage::{Html, Webpage},};@@ -222,19 +222,20 @@ mod tests {#[test]fn it_favors_liked_sites() {- let mut graph = WebgraphBuilder::new_memory().open();+ let mut wrt =+ WebgraphWriter::new(gen_temp_path(), crate::executor::Executor::single_thread());- graph.insert(Node::from("a.com"), Node::from("b.com"), String::new());- graph.insert(Node::from("c.com"), Node::from("d.com"), String::new());- graph.insert(Node::from("a.com"), Node::from("e.com"), String::new());+ wrt.insert(Node::from("a.com"), Node::from("b.com"), String::new());+ wrt.insert(Node::from("c.com"), Node::from("d.com"), String::new());+ wrt.insert(Node::from("a.com"), Node::from("e.com"), String::new());- graph.insert(Node::from("z.com"), Node::from("a.com"), String::new());- graph.insert(Node::from("z.com"), Node::from("b.com"), String::new());- graph.insert(Node::from("z.com"), Node::from("c.com"), String::new());- graph.insert(Node::from("z.com"), Node::from("d.com"), String::new());- graph.insert(Node::from("z.com"), Node::from("e.com"), String::new());+ wrt.insert(Node::from("z.com"), Node::from("a.com"), String::new());+ wrt.insert(Node::from("z.com"), Node::from("b.com"), String::new());+ wrt.insert(Node::from("z.com"), Node::from("c.com"), String::new());+ wrt.insert(Node::from("z.com"), Node::from("d.com"), String::new());+ wrt.insert(Node::from("z.com"), Node::from("e.com"), String::new());- graph.commit();+ let graph = wrt.finalize();let inbound = InboundSimilarity::build(&graph);@@ -247,14 +248,17 @@ mod tests {#[test]fn it_ranks_search_results() {- let mut graph = WebgraphBuilder::new_memory().open();+ let mut wrt = WebgraphWriter::new(+ crate::gen_temp_path(),+ crate::executor::Executor::single_thread(),+ );- graph.insert(Node::from("b.com"), Node::from("a.com"), String::new());- graph.insert(Node::from("c.com"), Node::from("d.com"), String::new());- graph.insert(Node::from("b.com"), Node::from("e.com"), String::new());- graph.insert(Node::from("c.com"), Node::from("b.com"), String::new());+ wrt.insert(Node::from("b.com"), Node::from("a.com"), String::new());+ wrt.insert(Node::from("c.com"), Node::from("d.com"), String::new());+ wrt.insert(Node::from("b.com"), Node::from("e.com"), String::new());+ wrt.insert(Node::from("c.com"), Node::from("b.com"), String::new());- graph.commit();+ let graph = wrt.finalize();let inbound = InboundSimilarity::build(&graph);diff --git a/core/src/ranking/optics.rs b/core/src/ranking/optics.rsindex e6dafbc3..1d42c1fc 100644--- a/core/src/ranking/optics.rs+++ b/core/src/ranking/optics.rs@@ -23,7 +23,7 @@ mod tests {index::Index,ranking::centrality_store::CentralityStore,searcher::{LocalSearcher, SearchQuery},- webgraph::{Node, WebgraphBuilder},+ webgraph::{Node, WebgraphWriter},webpage::{Html, Webpage},};const CONTENT: &str = "this is the best example website ever this is the best example website ever this is the best example website ever this is the best example website ever this is the best example website ever this is the best example website ever";@@ -32,50 +32,51 @@ mod tests {fn site_rankings() {let mut index = Index::temporary().expect("Unable to open index");- let mut graph = WebgraphBuilder::new_memory().open();+ let mut wrt =+ WebgraphWriter::new(gen_temp_path(), crate::executor::Executor::single_thread());- graph.insert(+ wrt.insert(Node::from("https://www.first.com").into_host(),Node::from("https://www.nan.com").into_host(),String::new(),);- graph.insert(+ wrt.insert(Node::from("https://www.nan.com").into_host(),Node::from("https://www.first.com").into_host(),String::new(),);- graph.insert(+ wrt.insert(Node::from("https://www.third.com").into_host(),Node::from("https://www.third.com").into_host(),String::new(),);- graph.insert(+ wrt.insert(Node::from("https://www.nan.com").into_host(),Node::from("https://www.second.com").into_host(),String::new(),);- graph.insert(+ wrt.insert(Node::from("https://www.second.com").into_host(),Node::from("https://www.nan.com").into_host(),String::new(),);- graph.insert(+ wrt.insert(Node::from("https://www.second.com").into_host(),Node::from("https://www.third.com").into_host(),String::new(),);- graph.insert(+ wrt.insert(Node::from("https://www.extra.com").into_host(),Node::from("https://www.first.com").into_host(),String::new(),);- graph.insert(+ wrt.insert(Node::from("https://www.second.com").into_host(),Node::from("https://www.extra.com").into_host(),String::new(),);- graph.commit();+ let graph = wrt.finalize();index.insert(Webpage {diff --git a/core/src/webgraph/centrality/betweenness.rs b/core/src/webgraph/centrality/betweenness.rsindex 4de8fc4f..4688b4ca 100644--- a/core/src/webgraph/centrality/betweenness.rs+++ b/core/src/webgraph/centrality/betweenness.rs@@ -163,35 +163,25 @@ impl Betweenness {mod tests {use maplit::hashmap;- use crate::webgraph::WebgraphBuilder;+ use crate::webgraph::WebgraphWriter;use super::*;fn create_path_graph(n: usize) -> Webgraph {- // ┌────┐- // │ │- // ┌───A◄─┐ │- // │ │ │- // ▼ │ │- // B─────►C◄┘- // ▲- // │- // │- // D-- let mut graph = WebgraphBuilder::new_memory().open();+ let mut writer = WebgraphWriter::new(+ crate::gen_temp_path(),+ crate::executor::Executor::single_thread(),+ );for i in 0..n - 1 {- graph.insert(+ writer.insert(Node::from(i.to_string()),Node::from((i + 1).to_string()),String::new(),);}- graph.commit();-- graph+ writer.finalize()}#[test]diff --git a/core/src/webgraph/centrality/harmonic.rs b/core/src/webgraph/centrality/harmonic.rsindex 3c961830..0af635b3 100644--- a/core/src/webgraph/centrality/harmonic.rs+++ b/core/src/webgraph/centrality/harmonic.rs@@ -237,9 +237,9 @@ impl HarmonicCentrality {#[cfg(test)]mod tests {use super::*;- use crate::webgraph::{Node, WebgraphBuilder};+ use crate::webgraph::{Node, WebgraphWriter};- fn test_graph() -> Webgraph {+ fn test_edges() -> Vec<(Node, Node, String)> {// ┌────┐// │ │// ┌───A◄─┐ │@@ -250,96 +250,107 @@ mod tests {// │// │// D+ vec![+ (Node::from("A"), Node::from("B"), String::new()),+ (Node::from("B"), Node::from("C"), String::new()),+ (Node::from("A"), Node::from("C"), String::new()),+ (Node::from("C"), Node::from("A"), String::new()),+ (Node::from("D"), Node::from("C"), String::new()),+ ]+ }- let mut graph = WebgraphBuilder::new_memory().open();-- graph.insert(Node::from("A"), Node::from("B"), String::new());- graph.insert(Node::from("B"), Node::from("C"), String::new());- graph.insert(Node::from("A"), Node::from("C"), String::new());- graph.insert(Node::from("C"), Node::from("A"), String::new());- graph.insert(Node::from("D"), Node::from("C"), String::new());+ fn test_graph() -> Webgraph {+ let mut writer = WebgraphWriter::new(+ crate::gen_temp_path(),+ crate::executor::Executor::single_thread(),+ );- graph.commit();+ for (from, to, label) in test_edges() {+ writer.insert(from, to, label);+ }- graph+ writer.finalize()}#[test]fn host_harmonic_centrality() {- let mut graph = WebgraphBuilder::new_memory().open();+ let mut writer = WebgraphWriter::new(+ crate::gen_temp_path(),+ crate::executor::Executor::single_thread(),+ );- graph.insert(+ writer.insert(Node::from("A.com/1").into_host(),Node::from("A.com/2").into_host(),String::new(),);- graph.insert(+ writer.insert(Node::from("A.com/1").into_host(),Node::from("A.com/3").into_host(),String::new(),);- graph.insert(+ writer.insert(Node::from("A.com/1").into_host(),Node::from("A.com/4").into_host(),String::new(),);- graph.insert(+ writer.insert(Node::from("A.com/2").into_host(),Node::from("A.com/1").into_host(),String::new(),);- graph.insert(+ writer.insert(Node::from("A.com/2").into_host(),Node::from("A.com/3").into_host(),String::new(),);- graph.insert(+ writer.insert(Node::from("A.com/2").into_host(),Node::from("A.com/4").into_host(),String::new(),);- graph.insert(+ writer.insert(Node::from("A.com/3").into_host(),Node::from("A.com/1").into_host(),String::new(),);- graph.insert(+ writer.insert(Node::from("A.com/3").into_host(),Node::from("A.com/2").into_host(),String::new(),);- graph.insert(+ writer.insert(Node::from("A.com/3").into_host(),Node::from("A.com/4").into_host(),String::new(),);- graph.insert(+ writer.insert(Node::from("A.com/4").into_host(),Node::from("A.com/1").into_host(),String::new(),);- graph.insert(+ writer.insert(Node::from("A.com/4").into_host(),Node::from("A.com/2").into_host(),String::new(),);- graph.insert(+ writer.insert(Node::from("A.com/4").into_host(),Node::from("A.com/3").into_host(),String::new(),);- graph.insert(+ writer.insert(Node::from("C.com").into_host(),Node::from("B.com").into_host(),String::new(),);- graph.insert(+ writer.insert(Node::from("D.com").into_host(),Node::from("B.com").into_host(),String::new(),);- graph.commit();+ let graph = writer.finalize();let centrality = HarmonicCentrality::calculate(&graph);@@ -367,51 +378,37 @@ mod tests {#[test]fn additional_edges_ignored() {- let mut graph = test_graph();-+ let graph = test_graph();let centrality = HarmonicCentrality::calculate(&graph);- graph.insert(Node::from("A"), Node::from("B"), "1".to_string());- graph.commit();- graph.insert(Node::from("A"), Node::from("B"), "2".to_string());- graph.commit();- graph.insert(Node::from("A"), Node::from("B"), "3".to_string());- graph.commit();- graph.insert(Node::from("A"), Node::from("B"), "4".to_string());- graph.commit();- graph.insert(Node::from("A"), Node::from("B"), "5".to_string());- graph.commit();- graph.insert(Node::from("A"), Node::from("B"), "6".to_string());- graph.commit();- graph.insert(Node::from("A"), Node::from("B"), "7".to_string());- graph.commit();-- let centrality_extra = HarmonicCentrality::calculate(&graph);+ let mut other = WebgraphWriter::new(+ crate::gen_temp_path(),+ crate::executor::Executor::single_thread(),+ );- assert_eq!(centrality.0, centrality_extra.0);- }+ for (from, to, label) in test_edges() {+ other.insert(from, to, label);+ }- #[test]- fn same_centrality_after_segment_merge() {- let mut graph = WebgraphBuilder::new_memory().open();-- graph.insert(Node::from("A"), Node::from("B"), String::new());- graph.commit();- graph.insert(Node::from("B"), Node::from("C"), String::new());- graph.commit();- graph.insert(Node::from("A"), Node::from("C"), String::new());- graph.commit();- graph.insert(Node::from("C"), Node::from("A"), String::new());- graph.commit();- graph.insert(Node::from("D"), Node::from("C"), String::new());- graph.commit();-- graph.merge_segments(1);- let centrality = HarmonicCentrality::calculate(&graph);+ other.insert(Node::from("A"), Node::from("B"), "1".to_string());+ other.commit();+ other.insert(Node::from("A"), Node::from("B"), "2".to_string());+ other.commit();+ other.insert(Node::from("A"), Node::from("B"), "3".to_string());+ other.commit();+ other.insert(Node::from("A"), Node::from("B"), "4".to_string());+ other.commit();+ other.insert(Node::from("A"), Node::from("B"), "5".to_string());+ other.commit();+ other.insert(Node::from("A"), Node::from("B"), "6".to_string());+ other.commit();+ other.insert(Node::from("A"), Node::from("B"), "7".to_string());+ other.commit();++ let graph = other.finalize();- let orig_graph = test_graph();- let orig_centrality = HarmonicCentrality::calculate(&orig_graph);+ let centrality_extra = HarmonicCentrality::calculate(&graph);- assert_eq!(centrality.0, orig_centrality.0);+ assert_eq!(centrality.0, centrality_extra.0);}}diff --git a/core/src/webgraph/mod.rs b/core/src/webgraph/mod.rsindex dc1f96c4..0acb8d9b 100644--- a/core/src/webgraph/mod.rs+++ b/core/src/webgraph/mod.rs@@ -15,14 +15,13 @@// along with this program. If not, see <https://www.gnu.org/licenses/>.mod segment;-use std::collections::{BTreeMap, BinaryHeap, HashSet};+use std::collections::{BTreeMap, BinaryHeap};use std::fs::File;use std::io::{BufReader, BufWriter, Read, Write};use std::path::Path;use std::sync::Arc;use std::{cmp, fs};-use itertools::Itertools;use rayon::prelude::*;use serde::{Deserialize, Serialize};use url::Url;@@ -33,7 +32,7 @@ use crate::intmap;pub mod centrality;mod store;-use self::segment::StoredSegment;+use self::segment::{Segment, SegmentWriter};pub const MAX_LABEL_LENGTH: usize = 1024;@@ -230,52 +229,26 @@ pub fn normalize_url(url: &Url) -> String {normalized}-#[derive(Default, Clone, Copy)]-pub enum Deduplication {- /// Do not deduplicate the inserted edges in the segment. Only deduplicate- /// the edges when querying all the segments.- OnlyQuery,- /// Deduplicate the inserted edges in the segmnt by first checking if the- /// edge is already present in the segment.- #[default]- QueryAndInserts,-}-pub struct WebgraphBuilder {path: Box<Path>,executor: Executor,- deduplication: Deduplication,}impl WebgraphBuilder {- #[cfg(test)]- pub fn new_memory() -> Self {- use crate::gen_temp_path;-- let path = gen_temp_path();- Self::new(path)- }-pub fn new<P: AsRef<Path>>(path: P) -> Self {Self {path: path.as_ref().into(),executor: Executor::multi_thread("webgraph").unwrap(),- deduplication: Deduplication::default(),}}- pub fn deduplication(mut self, deduplication: Deduplication) -> Self {- self.deduplication = deduplication;- self- }-pub fn single_threaded(mut self) -> Self {self.executor = Executor::single_thread();self}pub fn open(self) -> Webgraph {- Webgraph::open(self.path, self.executor, self.deduplication)+ Webgraph::open(self.path, self.executor)}}@@ -362,9 +335,35 @@ struct Meta {comitted_segments: Vec<SegmentID>,}-struct SegmentMergeCandidate {- segment: StoredSegment,- merges: Vec<StoredSegment>,+impl Meta {+ fn open<P: AsRef<Path>>(path: P) -> Self {+ let mut reader = BufReader::new(+ File::options()+ .create(true)+ .write(true)+ .read(true)+ .open(path)+ .unwrap(),+ );+ let mut buf = String::new();+ reader.read_to_string(&mut buf).unwrap();+ serde_json::from_str(&buf).unwrap_or_default()+ }++ fn save<P: AsRef<Path>>(&self, path: P) {+ let mut writer = BufWriter::new(+ File::options()+ .create(true)+ .write(true)+ .read(true)+ .truncate(true)+ .open(path)+ .unwrap(),+ );++ let json = serde_json::to_string_pretty(&self).unwrap();+ writer.write_all(json.as_bytes()).unwrap();+ }}struct Id2NodeDb {@@ -447,84 +446,52 @@ impl Id2NodeDb {}}-pub struct Webgraph {+pub struct WebgraphWriter {pub path: String,- segments: Vec<StoredSegment>,- executor: Arc<Executor>,+ segment: SegmentWriter,insert_batch: Vec<Edge<String>>,id2node: Id2NodeDb,+ executor: Executor,meta: Meta,- deduplication: Deduplication,}-impl Webgraph {+impl WebgraphWriter {fn meta<P: AsRef<Path>>(path: P) -> Meta {let meta_path = path.as_ref().join("metadata.json");- let mut reader = BufReader::new(- File::options()- .create(true)- .write(true)- .read(true)- .open(meta_path)- .unwrap(),- );- let mut buf = String::new();- reader.read_to_string(&mut buf).unwrap();- serde_json::from_str(&buf).unwrap_or_default()+ Meta::open(meta_path)}fn save_metadata(&mut self) {let path = Path::new(&self.path).join("metadata.json");- let mut writer = BufWriter::new(- File::options()- .create(true)- .write(true)- .read(true)- .truncate(true)- .open(path)- .unwrap(),- );-- let json = serde_json::to_string_pretty(&self.meta).unwrap();- writer.write_all(json.as_bytes()).unwrap();+ self.meta.save(path);}- fn open<P: AsRef<Path>>(path: P, executor: Executor, deduplication: Deduplication) -> Self {+ pub fn new<P: AsRef<Path>>(path: P, executor: Executor) -> Self {fs::create_dir_all(&path).unwrap();let mut meta = Self::meta(&path);+ meta.comitted_segments.clear();fs::create_dir_all(path.as_ref().join("segments")).unwrap();- let mut segments = Vec::new();- for segment in &meta.comitted_segments {- segments.push(StoredSegment::open(- path.as_ref().join("segments"),- segment.clone(),- deduplication,- ));- }-- if segments.is_empty() {- segments.push(StoredSegment::open(- path.as_ref().join("segments"),- uuid::Uuid::new_v4().to_string(),- deduplication,- ));+ let id = uuid::Uuid::new_v4().to_string();+ let segment = SegmentWriter::open(path.as_ref().join("segments"), id.clone());- meta.comitted_segments.push(segments[0].id());- }+ meta.comitted_segments.push(id);Self {path: path.as_ref().as_os_str().to_str().unwrap().to_string(),- deduplication,- segments,- executor: Arc::new(executor),+ segment,id2node: Id2NodeDb::open(path.as_ref().join("id2node")),insert_batch: Vec::with_capacity(store::MAX_BATCH_SIZE),+ executor,meta,}}+ pub fn id2node(&self, id: &NodeID) -> Option<Node> {+ self.id2node.get(id)+ }+fn id_or_assign(&mut self, node: &Node) -> NodeID {let id = node.id();@@ -551,9 +518,68 @@ impl Webgraph {}}- pub fn merge(&mut self, mut other: Webgraph) {- other.commit();+ pub fn commit(&mut self) {+ if !self.insert_batch.is_empty() {+ self.segment.insert(&self.insert_batch);+ self.segment.flush();+ self.insert_batch.clear();+ }++ self.save_metadata();+ self.id2node.flush();+ }++ pub fn finalize(mut self) -> Webgraph {+ self.commit();++ Webgraph {+ path: self.path,+ segments: vec![self.segment.finalize()],+ executor: self.executor.into(),+ id2node: self.id2node,+ meta: self.meta,+ }+ }+}++pub struct Webgraph {+ pub path: String,+ segments: Vec<Segment>,+ executor: Arc<Executor>,+ id2node: Id2NodeDb,+ meta: Meta,+}++impl Webgraph {+ fn meta<P: AsRef<Path>>(path: P) -> Meta {+ let meta_path = path.as_ref().join("metadata.json");+ Meta::open(meta_path)+ }++ fn open<P: AsRef<Path>>(path: P, executor: Executor) -> Self {+ fs::create_dir_all(&path).unwrap();+ let meta = Self::meta(&path);++ fs::create_dir_all(path.as_ref().join("segments")).unwrap();++ let mut segments = Vec::new();+ for segment in &meta.comitted_segments {+ segments.push(Segment::open(+ path.as_ref().join("segments"),+ segment.clone(),+ ));+ }++ Self {+ path: path.as_ref().as_os_str().to_str().unwrap().to_string(),+ segments,+ executor: Arc::new(executor),+ id2node: Id2NodeDb::open(path.as_ref().join("id2node")),+ meta,+ }+ }+ pub fn merge(&mut self, other: Webgraph) {self.id2node.batch_put(other.id2node.iter());for segment in other.segments {@@ -563,41 +589,20 @@ impl Webgraph {self.meta.comitted_segments.push(segment.id());drop(segment);- self.segments- .push(StoredSegment::open(new_path, id, self.deduplication));- }-- self.commit();- }-- pub fn commit(&mut self) {- if !self.insert_batch.is_empty() {- let seg = self.segments.last_mut().unwrap();- seg.insert(&self.insert_batch);- seg.flush();- self.insert_batch.clear();+ self.segments.push(Segment::open(new_path, id));}- self.save_metadata();self.id2node.flush();-- if self.segments.len() > 2 * num_cpus::get() {- self.merge_segments(num_cpus::get());- }}pub fn ingoing_edges(&self, node: Node) -> Vec<FullEdge> {- let dedup = match self.deduplication {- Deduplication::OnlyQuery | Deduplication::QueryAndInserts => {- |edges: &mut Vec<Edge<String>>| {- edges.sort_by_key(|e| e.from);- edges.dedup_by_key(|e| e.from);- }- }+ let dedup = |edges: &mut Vec<Edge<String>>| {+ edges.sort_by_key(|e| e.from);+ edges.dedup_by_key(|e| e.from);};self.inner_edges(- |segment| segment.ingoing_edges_with_label(&node.id()).collect_vec(),+ |segment| segment.ingoing_edges_with_label(&node.id()),dedup,).into_iter()@@ -610,46 +615,31 @@ impl Webgraph {}pub fn raw_ingoing_edges(&self, node: &NodeID) -> Vec<Edge<()>> {- let dedup = match self.deduplication {- Deduplication::OnlyQuery | Deduplication::QueryAndInserts => {- |edges: &mut Vec<Edge<()>>| {- edges.sort_by_key(|e| e.from);- edges.dedup_by_key(|e| e.from);- }- }+ let dedup = |edges: &mut Vec<Edge<()>>| {+ edges.sort_by_key(|e| e.from);+ edges.dedup_by_key(|e| e.from);};- self.inner_edges(|segment| segment.ingoing_edges(node).collect_vec(), dedup)+ self.inner_edges(|segment| segment.ingoing_edges(node), dedup)}pub fn raw_ingoing_edges_with_labels(&self, node: &NodeID) -> Vec<Edge<String>> {- let dedup = match self.deduplication {- Deduplication::OnlyQuery | Deduplication::QueryAndInserts => {- |edges: &mut Vec<Edge<String>>| {- edges.sort_by_key(|e| e.from);- edges.dedup_by_key(|e| e.from);- }- }+ let dedup = |edges: &mut Vec<Edge<String>>| {+ edges.sort_by_key(|e| e.from);+ edges.dedup_by_key(|e| e.from);};- self.inner_edges(- |segment| segment.ingoing_edges_with_label(node).collect_vec(),- dedup,- )+ self.inner_edges(|segment| segment.ingoing_edges_with_label(node), dedup)}pub fn outgoing_edges(&self, node: Node) -> Vec<FullEdge> {- let dedup = match self.deduplication {- Deduplication::OnlyQuery | Deduplication::QueryAndInserts => {- |edges: &mut Vec<Edge<String>>| {- edges.sort_by_key(|e| e.to);- edges.dedup_by_key(|e| e.to);- }- }+ let dedup = |edges: &mut Vec<Edge<String>>| {+ edges.sort_by_key(|e| e.to);+ edges.dedup_by_key(|e| e.to);};self.inner_edges(- |segment| segment.outgoing_edges_with_label(&node.id()).collect_vec(),+ |segment| segment.outgoing_edges_with_label(&node.id()),dedup,).into_iter()@@ -662,22 +652,18 @@ impl Webgraph {}pub fn raw_outgoing_edges(&self, node: &NodeID) -> Vec<Edge<()>> {- let dedup = match self.deduplication {- Deduplication::OnlyQuery | Deduplication::QueryAndInserts => {- |edges: &mut Vec<Edge<()>>| {- edges.sort_by_key(|e| e.to);- edges.dedup_by_key(|e| e.to);- }- }+ let dedup = |edges: &mut Vec<Edge<()>>| {+ edges.sort_by_key(|e| e.to);+ edges.dedup_by_key(|e| e.to);};- self.inner_edges(|segment| segment.outgoing_edges(node).collect_vec(), dedup)+ self.inner_edges(|segment| segment.outgoing_edges(node), dedup)}fn inner_edges<F1, F2, L>(&self, loader: F1, dedup: F2) -> Vec<Edge<L>>whereL: EdgeLabel,- F1: Sized + Sync + Fn(&StoredSegment) -> Vec<Edge<L>>,+ F1: Sized + Sync + Fn(&Segment) -> Vec<Edge<L>>,F2: Fn(&mut Vec<Edge<L>>),{let mut edges: Vec<_> = self@@ -717,65 +703,6 @@ impl Webgraph {.par_iter().flat_map(|segment| segment.edges().par_bridge())}-- pub fn merge_segments(&mut self, num_segments: usize) {- if num_segments >= self.segments.len() {- return;- }-- self.segments- .sort_by_key(|segment| std::cmp::Reverse(segment.estimate_num_nodes()));-- let mut candidates = Vec::with_capacity(num_segments);-- for segment in self.segments.drain(0..num_segments) {- candidates.push(SegmentMergeCandidate {- segment,- merges: Vec::new(),- });- }-- let num_candidates = candidates.len();-- for (next_candidate, segment) in self.segments.drain(0..).enumerate() {- candidates[next_candidate % num_candidates]- .merges- .push(segment);- }-- self.segments = self- .executor- .map(- |mut candidate| {- let mut segments = vec![candidate.segment];- segments.append(&mut candidate.merges);-- StoredSegment::merge(segments)- },- candidates.into_iter(),- )- .unwrap();-- self.meta.comitted_segments = self.segments.iter().map(|segment| segment.id()).collect();- self.save_metadata();-- self.garbage_collect();- }-- fn garbage_collect(&self) {- let path = Path::new(&self.path).join("segments");- let segments: HashSet<_> = self.meta.comitted_segments.iter().cloned().collect();-- for path in fs::read_dir(path).unwrap() {- let path = path.unwrap();- let file_name = path.file_name();- let name = file_name.as_os_str().to_str().unwrap();-- if !segments.contains(name) {- fs::remove_dir_all(path.path()).unwrap();- }- }- }}#[cfg(test)]@@ -804,7 +731,7 @@ mod test {// │// D- let mut graph = WebgraphBuilder::new_memory().open();+ let mut graph = WebgraphWriter::new(crate::gen_temp_path(), Executor::single_thread());for (from, to, label) in test_edges() {graph.insert(from, to, label);@@ -812,69 +739,7 @@ mod test {graph.commit();- graph- }-- fn verify_graph(graph: &Webgraph) {- let mut res = graph.outgoing_edges(Node::from("A"));- res.sort_by(|a, b| a.to.name.cmp(&b.to.name));-- assert_eq!(- res,- vec![- FullEdge {- from: Node::from("A"),- to: Node::from("B"),- label: String::new()- },- FullEdge {- from: Node::from("A"),- to: Node::from("C"),- label: String::new()- }- ]- );-- let mut res = graph.ingoing_edges(Node::from("C"));- res.sort_by(|a, b| a.from.name.cmp(&b.from.name));-- assert_eq!(- res,- vec![- FullEdge {- from: Node::from("A"),- to: Node::from("C"),- label: String::new()- },- FullEdge {- from: Node::from("B"),- to: Node::from("C"),- label: String::new()- },- FullEdge {- from: Node::from("D"),- to: Node::from("C"),- label: String::new()- },- ]- );-- let res = graph.outgoing_edges(Node::from("D"));-- assert_eq!(- res,- vec![FullEdge {- from: Node::from("D"),- to: Node::from("C"),- label: String::new()- },]- );-- let distances = graph.distances(Node::from("D"));-- assert_eq!(distances.get(&Node::from("C")), Some(&1));- assert_eq!(distances.get(&Node::from("A")), Some(&2));- assert_eq!(distances.get(&Node::from("B")), Some(&3));+ graph.finalize()}#[test]@@ -924,10 +789,9 @@ mod test {(Node::from("F"), Node::from("G"), String::new()),(Node::from("G"), Node::from("H"), String::new()),] {- let mut graph = WebgraphBuilder::new_memory().open();- graph.insert(from.clone(), to.clone(), label.clone());- graph.commit();- graphs.push(graph);+ let mut wrt = WebgraphWriter::new(crate::gen_temp_path(), Executor::single_thread());+ wrt.insert(from.clone(), to.clone(), label.clone());+ graphs.push(wrt.finalize());}let mut graph = graphs.pop().unwrap();@@ -940,12 +804,6 @@ mod test {graph.distances(Node::from("A")).get(&Node::from("H")),Some(&7));-- graph.merge_segments(1);- assert_eq!(- graph.distances(Node::from("A")).get(&Node::from("H")),- Some(&7)- )}#[test]fn merge_cycle() {@@ -956,10 +814,9 @@ mod test {(Node::from("B"), Node::from("C"), String::new()),(Node::from("C"), Node::from("A"), String::new()),] {- let mut graph = WebgraphBuilder::new_memory().open();- graph.insert(from.clone(), to.clone(), label.clone());- graph.commit();- graphs.push(graph);+ let mut wrt = WebgraphWriter::new(crate::gen_temp_path(), Executor::single_thread());+ wrt.insert(from.clone(), to.clone(), label.clone());+ graphs.push(wrt.finalize());}let mut graph = graphs.pop().unwrap();@@ -972,12 +829,6 @@ mod test {graph.distances(Node::from("A")).get(&Node::from("C")),Some(&2));-- graph.merge_segments(1);- assert_eq!(- graph.distances(Node::from("A")).get(&Node::from("C")),- Some(&2)- )}#[test]@@ -999,42 +850,17 @@ mod test {assert_eq!(&n.name, "example.com/?test=");}- #[test]- fn merge_segments() {- let mut graph = WebgraphBuilder::new_memory().open();-- let edges = test_edges();- let num_edges = edges.len();-- for (from, to, label) in test_edges() {- let mut tmp_graph = WebgraphBuilder::new_memory().open();- tmp_graph.insert(from, to, label);- tmp_graph.commit();- graph.merge(tmp_graph);- }-- graph.commit();-- assert!(graph.segments.len() >= num_edges);- verify_graph(&graph);-- graph.merge_segments(2);- assert_eq!(graph.segments.len(), 2);-- verify_graph(&graph);- }-#[test]fn cap_label_length() {- let mut graph = WebgraphBuilder::new_memory().open();+ let mut writer = WebgraphWriter::new(crate::gen_temp_path(), Executor::single_thread());- graph.insert(+ writer.insert(Node::from("A"),Node::from("B"),"a".repeat(MAX_LABEL_LENGTH + 1),);- graph.commit();+ let graph = writer.finalize();assert_eq!(graph.segments.len(), 1);assert_eq!(diff --git a/core/src/webgraph/segment.rs b/core/src/webgraph/segment.rsindex 93f32a0d..1914c34a 100644--- a/core/src/webgraph/segment.rs+++ b/core/src/webgraph/segment.rs@@ -16,38 +16,34 @@use std::path::{Path, PathBuf};-use super::{store::EdgeStore, Deduplication, Edge, NodeID};+use super::{+ store::{EdgeStore, EdgeStoreWriter},+ Edge, NodeID,+};const ADJACENCY_STORE: &str = "adjacency";const REVERSED_ADJACENCY_STORE: &str = "reversed_adjacency";-pub struct StoredSegment {- full_adjacency: EdgeStore,- full_reversed_adjacency: EdgeStore,+pub struct SegmentWriter {+ full_adjacency: EdgeStoreWriter,+ full_reversed_adjacency: EdgeStoreWriter,id: String,folder_path: String,}-impl StoredSegment {- pub fn open<P: AsRef<Path>>(folder_path: P, id: String, dedup: Deduplication) -> Self {- let dedup_insert = match dedup {- Deduplication::OnlyQuery => false,- Deduplication::QueryAndInserts => true,- };-- StoredSegment {- full_adjacency: EdgeStore::open(+impl SegmentWriter {+ pub fn open<P: AsRef<Path>>(folder_path: P, id: String) -> Self {+ SegmentWriter {+ full_adjacency: EdgeStoreWriter::open(folder_path.as_ref().join(&id).join(ADJACENCY_STORE),false,- dedup_insert,),- full_reversed_adjacency: EdgeStore::open(+ full_reversed_adjacency: EdgeStoreWriter::open(folder_path.as_ref().join(&id).join(REVERSED_ADJACENCY_STORE),true,- dedup_insert,),folder_path: folder_path.as_ref()@@ -59,30 +55,15 @@ impl StoredSegment {}}- pub fn estimate_num_nodes(&self) -> usize {- self.full_adjacency.estimate_len()- }-- pub fn outgoing_edges_with_label(- &self,- node: &NodeID,- ) -> impl Iterator<Item = Edge<String>> + '_ {- self.full_adjacency.get(*node)- }-- pub fn outgoing_edges(&self, node: &NodeID) -> impl Iterator<Item = Edge<()>> + '_ {- self.full_adjacency.get(*node)- }-- pub fn ingoing_edges_with_label(- &self,- node: &NodeID,- ) -> impl Iterator<Item = Edge<String>> + '_ {- self.full_reversed_adjacency.get(*node)- }+ pub fn finalize(mut self) -> Segment {+ self.flush();- pub fn ingoing_edges(&self, node: &NodeID) -> impl Iterator<Item = Edge<()>> + '_ {- self.full_reversed_adjacency.get(*node)+ Segment {+ full_adjacency: self.full_adjacency.finalize(),+ full_reversed_adjacency: self.full_reversed_adjacency.finalize(),+ folder_path: self.folder_path,+ id: self.id,+ }}pub fn flush(&mut self) {@@ -90,44 +71,69 @@ impl StoredSegment {self.full_reversed_adjacency.flush();}- pub fn id(&self) -> String {- self.id.clone()+ pub fn insert(&mut self, edges: &[Edge<String>]) {+ self.full_adjacency.put(edges.iter());+ self.full_reversed_adjacency.put(edges.iter());}+}- pub fn path(&self) -> PathBuf {- Path::new(&self.folder_path).join(&self.id)- }+pub struct Segment {+ full_adjacency: EdgeStore,+ full_reversed_adjacency: EdgeStore,+ id: String,+ folder_path: String,+}- pub fn edges(&self) -> impl Iterator<Item = Edge<()>> + '_ + Send + Sync {- self.full_adjacency.iter()+impl Segment {+ pub fn open<P: AsRef<Path>>(folder_path: P, id: String) -> Self {+ Segment {+ full_adjacency: EdgeStore::open(+ folder_path.as_ref().join(&id).join(ADJACENCY_STORE),+ false,+ ),+ full_reversed_adjacency: EdgeStore::open(+ folder_path+ .as_ref()+ .join(&id)+ .join(REVERSED_ADJACENCY_STORE),+ true,+ ),+ folder_path: folder_path+ .as_ref()+ .as_os_str()+ .to_str()+ .unwrap()+ .to_string(),+ id,+ }}- fn merge_with(&self, other: &StoredSegment) {- self.full_adjacency.merge_with(&other.full_adjacency);- self.full_reversed_adjacency- .merge_with(&other.full_reversed_adjacency);+ pub fn outgoing_edges_with_label(&self, node: &NodeID) -> Vec<Edge<String>> {+ self.full_adjacency.get_with_label(node)}- pub fn merge(mut segments: Vec<StoredSegment>) -> Self {- debug_assert!(!segments.is_empty());+ pub fn outgoing_edges(&self, node: &NodeID) -> Vec<Edge<()>> {+ self.full_adjacency.get_without_label(node)+ }- if segments.len() == 1 {- let mut segments = segments;- return segments.pop().unwrap();- }+ pub fn ingoing_edges_with_label(&self, node: &NodeID) -> Vec<Edge<String>> {+ self.full_reversed_adjacency.get_with_label(node)+ }- let segment = segments.remove(0);+ pub fn ingoing_edges(&self, node: &NodeID) -> Vec<Edge<()>> {+ self.full_reversed_adjacency.get_without_label(node)+ }- for other in segments {- segment.merge_with(&other);- }+ pub fn id(&self) -> String {+ self.id.clone()+ }- segment+ pub fn path(&self) -> PathBuf {+ Path::new(&self.folder_path).join(&self.id)}- pub fn insert(&mut self, edges: &[Edge<String>]) {- self.full_adjacency.put(edges.iter());- self.full_reversed_adjacency.put(edges.iter());+ pub fn edges(&self) -> impl Iterator<Item = Edge<()>> + '_ + Send + Sync {+ self.full_adjacency.iter_without_label()}}@@ -144,11 +150,7 @@ mod test {// ▼ │ │// 1─────►2◄┘- let mut store = StoredSegment::open(- crate::gen_temp_path(),- "test".to_string(),- Deduplication::OnlyQuery,- );+ let mut writer = SegmentWriter::open(crate::gen_temp_path(), "test".to_string());let mut edges = Vec::new();@@ -177,10 +179,10 @@ mod test {label: String::new(),});- store.insert(&edges);- store.flush();+ writer.insert(&edges);+ let segment = writer.finalize();- let mut out: Vec<_> = store.outgoing_edges(&a).collect();+ let mut out: Vec<_> = segment.outgoing_edges(&a);out.sort_by(|a, b| a.to.cmp(&b.to));@@ -200,7 +202,7 @@ mod test {]);- let mut out: Vec<_> = store.outgoing_edges(&b).collect();+ let mut out: Vec<_> = segment.outgoing_edges(&b);out.sort_by(|a, b| a.to.cmp(&b.to));assert_eq!(out,@@ -211,7 +213,7 @@ mod test {},]);- let mut out: Vec<_> = store.outgoing_edges(&c).collect();+ let mut out: Vec<_> = segment.outgoing_edges(&c);out.sort_by(|a, b| a.to.cmp(&b.to));assert_eq!(out,@@ -222,7 +224,7 @@ mod test {},]);- let out: Vec<_> = store.ingoing_edges(&a).collect();+ let out: Vec<_> = segment.ingoing_edges(&a);assert_eq!(out,vec![Edge {@@ -232,7 +234,7 @@ mod test {},]);- let out: Vec<_> = store.ingoing_edges(&b).collect();+ let out: Vec<_> = segment.ingoing_edges(&b);assert_eq!(out,vec![Edge {@@ -242,7 +244,7 @@ mod test {},]);- let mut out: Vec<_> = store.ingoing_edges(&c).collect();+ let mut out: Vec<_> = segment.ingoing_edges(&c);out.sort_by(|a, b| a.from.cmp(&b.from));assert_eq!(out,diff --git a/core/src/webgraph/store.rs b/core/src/webgraph/store.rsindex 00ba2a84..4f11cf82 100644--- a/core/src/webgraph/store.rs+++ b/core/src/webgraph/store.rs@@ -14,23 +14,23 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.-use std::path::Path;+use std::{fs::File, io::Write, ops::Range, path::Path};-use hashbrown::HashSet;+use itertools::Itertools;+use memmap2::Mmap;use rocksdb::BlockBasedOptions;use super::{Edge, EdgeLabel, NodeID};pub const MAX_BATCH_SIZE: usize = 100_000;-pub struct EdgeStore {+pub struct EdgeStoreWriter {reversed: bool,- dedup: bool,db: rocksdb::DB,}-impl EdgeStore {- pub fn open<P: AsRef<Path>>(path: P, reversed: bool, dedup: bool) -> Self {+impl EdgeStoreWriter {+ pub fn open<P: AsRef<Path>>(path: P, reversed: bool) -> Self {let mut options = rocksdb::Options::default();options.create_if_missing(true);@@ -61,61 +61,13 @@ impl EdgeStore {options.set_block_based_table_factory(&block_options);options.set_compression_type(rocksdb::DBCompressionType::Lz4);- let db = rocksdb::DB::open(&options, path.as_ref().to_str().unwrap()).unwrap();-- Self {- db,- reversed,- dedup,- }- }-- pub fn get<L: EdgeLabel>(&self, node: NodeID) -> impl Iterator<Item = Edge<L>> + '_ {- let prefix_bytes = node.bit_128().to_be_bytes();- let suffix = 0u128.to_be_bytes();-- let key_bytes = [prefix_bytes, suffix].concat();-- let iter = self.db.iterator(rocksdb::IteratorMode::From(- &key_bytes,- rocksdb::Direction::Forward,- ));-- iter.take_while(move |r| {- if let Ok((key, _)) = r.as_ref() {- let cur_prefix = &key[..16];-- cur_prefix == prefix_bytes- } else {- false- }- })- .filter_map(move |r| {- let (key, value) = r.ok()?;-- let suffix = u128::from_be_bytes(key[16..32].try_into().unwrap());-- let label = L::from_bytes(&value).ok()?;+ let db = rocksdb::DB::open(&options, path.as_ref().join("writer")).unwrap();- if self.reversed {- Some(Edge {- from: NodeID(suffix),- to: node,- label,- })- } else {- Some(Edge {- from: node,- to: NodeID(suffix),- label,- })- }- })+ Self { db, reversed }}pub fn put<'a, L: EdgeLabel + 'a>(&'a self, edges: impl Iterator<Item = &'a Edge<L>>) {let mut batch = rocksdb::WriteBatch::default();- let mut batch_keys = HashSet::new();let mut opts = rocksdb::WriteOptions::default();opts.disable_wal(true);@@ -131,15 +83,6 @@ impl EdgeStore {let suffix_bytes = suffix.bit_128().to_be_bytes();let key_bytes = [prefix_bytes, suffix_bytes].concat();-- if self.dedup {- if batch_keys.contains(&key_bytes) || self.db.get(&key_bytes).unwrap().is_some() {- continue;- }-- batch_keys.insert(key_bytes.clone());- }-let value_bytes = edge.label.to_bytes().unwrap();batch.put(key_bytes, value_bytes);@@ -147,7 +90,6 @@ impl EdgeStore {if batch.len() >= MAX_BATCH_SIZE {self.db.write_opt(batch, &opts).unwrap();batch = rocksdb::WriteBatch::default();- batch_keys.clear();}}@@ -190,26 +132,328 @@ impl EdgeStore {self.db.flush().unwrap();}- pub fn estimate_len(&self) -> usize {- self.db- .property_int_value("rocksdb.estimate-num-keys")- .unwrap()- .unwrap_or(0) as usize+ pub fn finalize(self) -> EdgeStore {+ let s = EdgeStore::build(self.db.path().parent().unwrap(), self.reversed, self.iter());++ // delete the writer db+ let p = self.db.path().to_owned();+ drop(self.db);+ std::fs::remove_dir_all(p).unwrap();++ s}+}++pub struct EdgeStore {+ reversed: bool,+ ranges: rocksdb::DB, // column[nodes] = nodeid -> (start, end); column[labels] = nodeid -> (start, end)+ edge_labels_file: File,+ edge_labels_len: usize,+ edge_labels: Mmap,+ edge_nodes_file: File,+ edge_nodes_len: usize,+ edge_nodes: Mmap,+}++impl EdgeStore {+ pub fn open<P: AsRef<Path>>(path: P, reversed: bool) -> Self {+ let mut options = rocksdb::Options::default();+ options.create_if_missing(true);++ options.set_max_background_jobs(8);+ options.increase_parallelism(8);+ options.set_max_subcompactions(8);++ options.set_allow_mmap_reads(true);+ options.set_allow_mmap_writes(true);- pub fn merge_with(&self, other: &EdgeStore) {- let mut batch = Vec::with_capacity(MAX_BATCH_SIZE);+ options.set_level_zero_slowdown_writes_trigger(-1);+ options.set_level_zero_stop_writes_trigger(-1);++ let cache = rocksdb::Cache::new_lru_cache(256 * 1024 * 1024); // 256 mb++ // some recommended settings (https://github.com/facebook/rocksdb/wiki/Setup-Options-and-Basic-Tuning)+ options.set_level_compaction_dynamic_level_bytes(true);+ options.set_bytes_per_sync(1048576);+ let mut block_options = BlockBasedOptions::default();+ block_options.set_block_size(16 * 1024);+ block_options.set_format_version(5);+ block_options.set_cache_index_and_filter_blocks(true);+ block_options.set_pin_l0_filter_and_index_blocks_in_cache(true);- for edge in other.iter::<String>() {+ block_options.set_ribbon_filter(10.0);++ options.set_block_based_table_factory(&block_options);+ options.set_row_cache(&cache);+ options.set_compression_type(rocksdb::DBCompressionType::Lz4);+ options.optimize_for_point_lookup(512);++ let ranges = match rocksdb::DB::open_cf_with_opts(+ &options,+ path.as_ref().join("ranges"),+ [("nodes", options.clone()), ("labels", options.clone())],+ ) {+ Ok(db) => db,+ Err(_) => {+ let mut ranges = rocksdb::DB::open(&options, path.as_ref().join("ranges")).unwrap();++ ranges.create_cf("nodes", &options).unwrap();+ ranges.create_cf("labels", &options).unwrap();++ ranges+ }+ };++ let edge_labels_file = File::options()+ .read(true)+ .create(true)+ .write(true)+ .open(path.as_ref().join("labels"))+ .unwrap();+ let edge_labels = unsafe { Mmap::map(&edge_labels_file).unwrap() };+ let edge_labels_len = edge_labels.len();++ let edge_nodes_file = File::options()+ .read(true)+ .create(true)+ .write(true)+ .open(path.as_ref().join("nodes"))+ .unwrap();+ let edge_nodes = unsafe { Mmap::map(&edge_nodes_file).unwrap() };+ let edge_nodes_len = edge_nodes.len();++ Self {+ reversed,+ ranges,+ edge_labels,+ edge_labels_len,+ edge_labels_file,+ edge_nodes,+ edge_nodes_file,+ edge_nodes_len,+ }+ }++ /// Insert a batch of edges into the store.+ /// The edges *must* have been de-duplicated by their from/to node.+ /// I.e. if the store is not reversed, there should only ever be a single+ /// put for each from node, and vice versa.+ fn put(&mut self, edges: &[Edge<String>]) {+ if edges.is_empty() {+ return;+ }++ let node = if self.reversed {+ edges[0].to+ } else {+ edges[0].from+ };+ let node_bytes = node.bit_128().to_be_bytes();++ let node_cf = self.ranges.cf_handle("nodes").unwrap();+ let label_cf = self.ranges.cf_handle("labels").unwrap();++ debug_assert!(self.ranges.get_cf(node_cf, node_bytes).unwrap().is_none());+ debug_assert!(self.ranges.get_cf(label_cf, node_bytes).unwrap().is_none());++ let mut edge_labels = Vec::new();+ let mut edge_nodes = Vec::new();++ for edge in edges {+ edge_labels.push(edge.label.clone());+ edge_nodes.push(if self.reversed { edge.from } else { edge.to });+ }++ let edge_labels_bytes = bincode::serialize(&edge_labels).unwrap();+ let edge_nodes_bytes = bincode::serialize(&edge_nodes).unwrap();++ let label_range = self.edge_labels_len..(self.edge_labels_len + edge_labels_bytes.len());+ let node_range = self.edge_nodes_len..(self.edge_nodes_len + edge_nodes_bytes.len());++ self.edge_labels_len += edge_labels_bytes.len();+ self.edge_nodes_len += edge_nodes_bytes.len();++ self.edge_labels_file.write_all(&edge_labels_bytes).unwrap();+ self.edge_nodes_file.write_all(&edge_nodes_bytes).unwrap();++ let mut opt = rocksdb::WriteOptions::default();+ opt.disable_wal(true);++ self.ranges+ .put_cf_opt(+ node_cf,+ node_bytes,+ bincode::serialize(&node_range).unwrap(),+ &opt,+ )+ .unwrap();++ self.ranges+ .put_cf_opt(+ label_cf,+ node_bytes,+ bincode::serialize(&label_range).unwrap(),+ &opt,+ )+ .unwrap();+ }++ /// Build a new edge store from a set of edges. The edges must be sorted by+ /// either the from or to node, depending on the value of `reversed`.+ fn build<P: AsRef<Path>>(+ path: P,+ reversed: bool,+ edges: impl Iterator<Item = Edge<String>>,+ ) -> Self {+ let mut s = Self::open(path, reversed);++ // create batches of consecutive edges with the same from/to node+ let mut batch = Vec::new();+ let mut last_node = None;+ for edge in edges {+ if let Some(last_node) = last_node {+ if (reversed && edge.to != last_node) || (!reversed && edge.from != last_node) {+ s.put(&batch);+ batch.clear();+ }+ }++ last_node = Some(if reversed { edge.to } else { edge.from });batch.push(edge);+ }- if batch.len() >= MAX_BATCH_SIZE {- self.put(batch.iter());- batch.clear();+ if !batch.is_empty() {+ s.put(&batch);+ }++ s.flush();++ s+ }++ fn flush(&mut self) {+ self.ranges.flush().unwrap();++ self.edge_nodes_file.flush().unwrap();+ self.edge_labels_file.flush().unwrap();++ self.edge_nodes = unsafe { Mmap::map(&self.edge_nodes_file).unwrap() };+ self.edge_labels = unsafe { Mmap::map(&self.edge_labels_file).unwrap() };++ self.edge_nodes_len = self.edge_nodes.len();+ self.edge_labels_len = self.edge_labels.len();+ }++ pub fn get_with_label(&self, node: &NodeID) -> Vec<Edge<String>> {+ let node_bytes = node.bit_128().to_be_bytes();++ let node_cf = self.ranges.cf_handle("nodes").unwrap();+ let edge_cf = self.ranges.cf_handle("labels").unwrap();++ match (+ self.ranges.get_cf(node_cf, node_bytes).unwrap(),+ self.ranges.get_cf(edge_cf, node_bytes).unwrap(),+ ) {+ (Some(node_range_bytes), Some(edge_range_bytes)) => {+ let node_range = bincode::deserialize::<Range<usize>>(&node_range_bytes).unwrap();+ let edge_range = bincode::deserialize::<Range<usize>>(&edge_range_bytes).unwrap();++ let edge_labels = &self.edge_labels[edge_range];+ let edge_labels: Vec<String> = bincode::deserialize(edge_labels).unwrap();++ let edge_nodes = &self.edge_nodes[node_range];+ let edge_nodes: Vec<NodeID> = bincode::deserialize(edge_nodes).unwrap();++ edge_labels+ .into_iter()+ .zip_eq(edge_nodes)+ .map(|(label, other)| {+ if self.reversed {+ Edge {+ from: other,+ to: *node,+ label,+ }+ } else {+ Edge {+ from: *node,+ to: other,+ label,+ }+ }+ })+ .collect()}+ _ => Vec::new(),}+ }- self.put(batch.iter());+ pub fn get_without_label(&self, node: &NodeID) -> Vec<Edge<()>> {+ let node_bytes = node.bit_128().to_be_bytes();++ let node_cf = self.ranges.cf_handle("nodes").unwrap();++ match self.ranges.get_cf(node_cf, node_bytes).unwrap() {+ Some(node_range_bytes) => {+ let node_range = bincode::deserialize::<Range<usize>>(&node_range_bytes).unwrap();++ let edge_nodes = &self.edge_nodes[node_range];+ let edge_nodes: Vec<NodeID> = bincode::deserialize(edge_nodes).unwrap();++ edge_nodes+ .into_iter()+ .map(|other| {+ if self.reversed {+ Edge {+ from: other,+ to: *node,+ label: (),+ }+ } else {+ Edge {+ from: *node,+ to: other,+ label: (),+ }+ }+ })+ .collect()+ }+ _ => Vec::new(),+ }+ }++ pub fn iter_without_label(&self) -> impl Iterator<Item = Edge<()>> + '_ + Send + Sync {+ let node_cf = self.ranges.cf_handle("nodes").unwrap();++ self.ranges+ .iterator_cf(node_cf, rocksdb::IteratorMode::Start)+ .flat_map(move |res| {+ let (key, val) = res.unwrap();++ let node = u128::from_be_bytes((*key).try_into().unwrap());+ let node = NodeID(node);++ let node_range = bincode::deserialize::<Range<usize>>(&val).unwrap();+ let edge_nodes = &self.edge_nodes[node_range];+ let edge_nodes: Vec<NodeID> = bincode::deserialize(edge_nodes).unwrap();++ edge_nodes.into_iter().map(move |other| {+ if self.reversed {+ Edge {+ from: other,+ to: node,+ label: (),+ }+ } else {+ Edge {+ from: node,+ to: other,+ label: (),+ }+ }+ })+ })}}@@ -219,8 +463,8 @@ mod tests {#[test]fn test_insert() {- let kv: EdgeStore =- EdgeStore::open(crate::gen_temp_path().join("test-segment"), false, false);+ let kv: EdgeStoreWriter =+ EdgeStoreWriter::open(crate::gen_temp_path().join("test-segment"), false);let e = Edge {from: NodeID(0),@@ -230,22 +474,22 @@ mod tests {kv.put([e.clone()].iter());- kv.flush();+ let store = kv.finalize();- let edges: Vec<_> = kv.get(NodeID(0)).collect();+ let edges: Vec<_> = store.get_with_label(&NodeID(0));assert_eq!(edges.len(), 1);assert_eq!(&edges[0], &e);- let edges: Vec<Edge<String>> = kv.get(NodeID(1)).collect();+ let edges: Vec<_> = store.get_with_label(&NodeID(1));assert_eq!(edges.len(), 0);}#[test]fn test_reversed() {- let kv: EdgeStore =- EdgeStore::open(crate::gen_temp_path().join("test-segment"), true, false);+ let kv: EdgeStoreWriter =+ EdgeStoreWriter::open(crate::gen_temp_path().join("test-segment"), true);let e = Edge {from: NodeID(0),@@ -255,14 +499,12 @@ mod tests {kv.put([e.clone()].iter());- kv.flush();-- let edges: Vec<Edge<String>> = kv.get(NodeID(0)).collect();+ let store = kv.finalize();+ let edges: Vec<_> = store.get_with_label(&NodeID(0));assert_eq!(edges.len(), 0);- let edges: Vec<Edge<String>> = kv.get(NodeID(1)).collect();-+ let edges: Vec<_> = store.get_with_label(&NodeID(1));assert_eq!(edges.len(), 1);assert_eq!(&edges[0], &e);}diff --git a/core/src/webpage/mod.rs b/core/src/webpage/mod.rsindex 6cab2121..b10c3306 100644--- a/core/src/webpage/mod.rs+++ b/core/src/webpage/mod.rs@@ -113,6 +113,7 @@ impl<const N: usize> Preprocessor<N> {}}+#[derive(Debug)]pub struct Webpage {pub html: Html,pub backlink_labels: Vec<String>,