web search done right
webgraph compression
Mikkel Denker2a607ee88e55
diff --git a/Cargo.lock b/Cargo.lockindex 4dd9d8ec..d758183b 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -2329,6 +2329,9 @@ name = "lz4_flex"version = "0.11.1"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "3ea9b256699eda7b0387ffbc776dd625e28bde3918446381781245b7a50349d8"+dependencies = [+ "twox-hash",+][[package]]name = "mac"@@ -4290,6 +4293,7 @@ dependencies = ["libc","logos","lz-str",+ "lz4_flex 0.11.1","maplit","md5","memmap",diff --git a/Cargo.toml b/Cargo.tomlindex 53464fac..4c14270f 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -11,98 +11,99 @@ debug = true[workspace.dependencies]aes-gcm = "0.10.2"+anyhow = {version = "1.0.72", features = ["backtrace"]}+async-channel = "1.8.0"+async-stream = "0.3.3"+async-trait = "0.1.57"+axum = "0.6.9"+axum-extra = {version = "0.6.0", features = ["spa"]}+axum-macros = "0.3.4"+base64 = "0.13.0"bincode = "1.3.3"-clap = { version = "3.1.18", features = ["derive"] }+bitvec = "1.0.1"+bytemuck = {version = "1.13.1", features = ["derive"]}+byteorder = "1.4.3"+bzip2 = "0.4.4"+chitchat = "0.5.0"+chrono = {version = "0.4.23", features = ["serde"]}+clap = {version = "3.1.18", features = ["derive"]}+criterion = "0.3.6"+crossbeam-channel = "0.5.6"+csv = "1.1.6"+dashmap = {version = "5.4.0", features = ["rayon"]}encoding_rs = "0.8.31"+eventsource-stream = "0.2.3"+fend-core = "1.2.2"flate2 = "1.0.23"-indicatif = { version = "0.16.2", features = ["rayon"] }+fst = {version = "0.4.7", features = ["levenshtein"]}+futures = "0.3.21"+half = {version = "2.2.1", features = ["serde"]}+hashbrown = {version = "0.14.0", features = ["serde", "rkyv"]}+http = "0.2.8"+image = "0.24.3"+indicatif = {version = "0.16.2", features = ["rayon"]}+insta = "1.31"itertools = "0.10.3"-rayon = "1.5.3"-half = { version = "2.2.1", features = ["serde"] }-eventsource-stream = "0.2.3"-reqwest-eventsource = "0.4.0"-reqwest = { version = "0.11.16", features = ["blocking", "stream", "json"] }-serde = { version = "1.0.137", features = ["rc", "derive"] }-serde_json = "1.0.81"-tantivy = "0.21.0"-thiserror = "1.0.31"-toml = "0.5.9"-tracing = "0.1.34"-tracing-subscriber = { version = "0.3.11", features = ["env-filter"] }-whatlang = "0.16.0"-tokio = { version = "1.23.1", features = ["full"] }-axum = "0.6.9"-axum-extra = { version = "0.6.0", features = ["spa"] }-axum-macros = "0.3.4"+lalrpop = {version = "0.19.8", features = ["lexer"]}+lalrpop-util = {version = "0.19.8", features = ["lexer"]}+libc = "0.2.142"logos = "0.12.1"-csv = "1.1.6"-fst = { version = "0.4.7", features = ["levenshtein"] }-rocksdb = { version = "0.21.0", features = ["default", "jemalloc", "io-uring"] }-image = "0.24.3"-chrono = { version = "0.4.23", features = ["serde"] }-uuid = "1.1.2"-futures = "0.3.21"-quick-xml = "0.23.0"-rio_turtle = "0.8.4"-rio_api = "0.8.4"-bzip2 = "0.4.4"-parse_wiki_text = "0.1.5"+lz-str = "0.2.1"+lz4_flex = "0.11.1"+maplit = "1.0.2"md5 = "0.7.0"memmap = "0.7.0"+memmap2 = "0.5.10"+mime = "0.3.17"+min-max-heap = "1.3.0"+num_cpus = "1.15.0"once_cell = "1.13.1"+parse_wiki_text = "0.1.5"+proptest = "1.2.0"+proptest-derive = "0.3.0"+publicsuffix = "2.2.3"+quick-xml = "0.23.0"+rand = "0.8.5"+rayon = "1.5.3"regex = "1.6.0"-http = "0.2.8"-tower-http = { version = "0.4.0", features = ["compression-gzip", "cors"] }-lalrpop-util = { version = "0.19.8", features = ["lexer"] }-lalrpop = { version = "0.19.8", features = ["lexer"] }-serde_urlencoded = "0.7.1"+reqwest = {version = "0.11.16", features = ["blocking", "stream", "json"]}+reqwest-eventsource = "0.4.0"+rio_api = "0.8.4"+rio_turtle = "0.8.4"+rkyv = {version = "0.7.42", default_features = false, features = [+ "std",+ "validation",+ "size_64",+]}+robotstxt-with-cache = "0.4.0"+rocksdb = {version = "0.21.0", features = ["default", "jemalloc", "io-uring"]}+rusqlite = {version = "0.29.0", features = [+ "bundled",+ "modern-full",+ "buildtime_bindgen",+]}+rust-s3 = {version = "0.33.0", features = ["blocking", "tokio"]}rust-stemmers = "1.2.0"-min-max-heap = "1.3.0"-base64 = "0.13.0"-async-trait = "0.1.57"-urlencoding = "2.1.2"-bitvec = "1.0.1"-criterion = "0.3.6"-maplit = "1.0.2"-rand = "0.8.5"-crossbeam-channel = "0.5.6"-num_cpus = "1.15.0"-tokenizers = "0.13.2"-byteorder = "1.4.3"-async-stream = "0.3.3"-tokio-stream = "0.1.11"+safetensors = "0.3.1"scylla = "0.7.0"-chitchat = "0.5.0"-memmap2 = "0.5.10"-dashmap = { version = "5.4.0", features = ["rayon"] }+serde = {version = "1.0.137", features = ["rc", "derive"]}+serde_json = "1.0.81"+serde_urlencoded = "0.7.1"+tantivy = "0.21.0"tch = "0.13.0"-torch-sys = "0.13.0"-libc = "0.2.142"-lz-str = "0.2.1"-safetensors = "0.3.1"-robotstxt-with-cache = "0.4.0"-rust-s3 = { version = "0.33.0", features = ["blocking", "tokio"] }-rusqlite = { version = "0.29.0", features = [- "bundled",- "modern-full",- "buildtime_bindgen",-] }-async-channel = "1.8.0"-url = { version = "2.4.0", features = ["serde"] }-anyhow = { version = "1.0.72", features = ["backtrace"] }-bytemuck = { version = "1.13.1", features = ["derive"] }-proptest = "1.2.0"-proptest-derive = "0.3.0"-insta = "1.31"-utoipa = { version = "3.4.4", features = ["axum_extras"] }-utoipa-swagger-ui = { version = "3.1.4", features = ["axum"] }+thiserror = "1.0.31"tikv-jemallocator = "0.5"-publicsuffix = "2.2.3"-hashbrown = { version = "0.14.0", features = ["serde", "rkyv"] }-rkyv = { version = "0.7.42", default_features = false, features = [- "std",- "validation",- "size_64",-] }-mime = "0.3.17"-fend-core = "1.2.2"+tokenizers = "0.13.2"+tokio = {version = "1.23.1", features = ["full"]}+tokio-stream = "0.1.11"+toml = "0.5.9"+torch-sys = "0.13.0"+tower-http = {version = "0.4.0", features = ["compression-gzip", "cors"]}+tracing = "0.1.34"+tracing-subscriber = {version = "0.3.11", features = ["env-filter"]}+url = {version = "2.4.0", features = ["serde"]}+urlencoding = "2.1.2"+utoipa = {version = "3.4.4", features = ["axum_extras"]}+utoipa-swagger-ui = {version = "3.1.4", features = ["axum"]}+uuid = "1.1.2"+whatlang = "0.16.0"diff --git a/core/Cargo.toml b/core/Cargo.tomlindex cf610dc0..147537ef 100644--- a/core/Cargo.toml+++ b/core/Cargo.toml@@ -1,18 +1,18 @@[package]-name = "stract"-version = "0.1.0"-edition = "2021"-license = "AGPL-3.0-or-later"authors = ['Stract ApS. <[email protected]>']default-run = "stract"+edition = "2021"+license = "AGPL-3.0-or-later"+name = "stract"+version = "0.1.0"# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html[features]+cors = []default = ["cors", "with_alice"]dev = ["cors"]prod = ["cors"]-cors = []with_alice = [][[bin]]@@ -20,138 +20,139 @@ name = "stract"path = "src/main.rs"[dependencies]-bincode = { workspace = true }-clap = { workspace = true }-encoding_rs = { workspace = true }-flate2 = { workspace = true }-indicatif = { workspace = true }-itertools = { workspace = true }-rayon = { workspace = true }-reqwest = { workspace = true }-serde = { workspace = true }-serde_json = { workspace = true }-tantivy = { workspace = true }-thiserror = { workspace = true }-toml = { workspace = true }-tracing = { workspace = true }-tracing-subscriber = { workspace = true }-whatlang = { workspace = true }-tokio = { workspace = true }-axum = { workspace = true }-axum-extra = { workspace = true }-logos = { workspace = true }-csv = { workspace = true }-fst = { workspace = true }-rocksdb = { workspace = true }-image = { workspace = true }-chrono = { workspace = true }-uuid = { workspace = true }-futures = { workspace = true }-quick-xml = { workspace = true }-bzip2 = { workspace = true }-parse_wiki_text = { workspace = true }-md5 = { workspace = true }-memmap = { workspace = true }-regex = { workspace = true }-http = { workspace = true }-kuchiki = { path = "../kuchiki" }-serde_urlencoded = { workspace = true }-rust-stemmers = { workspace = true }-min-max-heap = { workspace = true }-base64 = { workspace = true }-async-trait = { workspace = true }-axum-macros = { workspace = true }-urlencoding = { workspace = true }-bitvec = { workspace = true }-tower-http = { workspace = true }-once_cell = { workspace = true }-optics = { path = "../optics" }-crossbeam-channel = { workspace = true }-num_cpus = { workspace = true }-tokenizers = { workspace = true }-lalrpop-util = { workspace = true }-byteorder = { workspace = true }-async-stream = { workspace = true }-tokio-stream = { workspace = true }-scylla = { workspace = true }-chitchat = { workspace = true }-memmap2 = { workspace = true }-dashmap = { workspace = true }-tch = { workspace = true }-torch-sys = { workspace = true }-libc = { workspace = true }-lz-str = { workspace = true }-safetensors = { workspace = true }-eventsource-stream = { workspace = true }-rand = { workspace = true }-half = { workspace = true }-aes-gcm = { workspace = true }-reqwest-eventsource = { workspace = true }-robotstxt-with-cache = { workspace = true }-rust-s3 = { workspace = true }-async-channel = { workspace = true }-url = { workspace = true }-anyhow = { workspace = true }-bytemuck = { workspace = true }-utoipa = { workspace = true }-utoipa-swagger-ui = { workspace = true }-publicsuffix = { workspace = true }-rkyv = { workspace = true }-hashbrown = { workspace = true }-mime = { workspace = true }-fend-core = { workspace = true }-rio_turtle = { workspace = true }-rio_api = { workspace = true }+aes-gcm = {workspace = true}+anyhow = {workspace = true}+async-channel = {workspace = true}+async-stream = {workspace = true}+async-trait = {workspace = true}+axum = {workspace = true}+axum-extra = {workspace = true}+axum-macros = {workspace = true}+base64 = {workspace = true}+bincode = {workspace = true}+bitvec = {workspace = true}+bytemuck = {workspace = true}+byteorder = {workspace = true}+bzip2 = {workspace = true}+chitchat = {workspace = true}+chrono = {workspace = true}+clap = {workspace = true}+crossbeam-channel = {workspace = true}+csv = {workspace = true}+dashmap = {workspace = true}+encoding_rs = {workspace = true}+eventsource-stream = {workspace = true}+fend-core = {workspace = true}+flate2 = {workspace = true}+fst = {workspace = true}+futures = {workspace = true}+half = {workspace = true}+hashbrown = {workspace = true}+http = {workspace = true}+image = {workspace = true}+indicatif = {workspace = true}+itertools = {workspace = true}+kuchiki = {path = "../kuchiki"}+lalrpop-util = {workspace = true}+libc = {workspace = true}+logos = {workspace = true}+lz-str = {workspace = true}+lz4_flex = {workspace = true}+md5 = {workspace = true}+memmap = {workspace = true}+memmap2 = {workspace = true}+mime = {workspace = true}+min-max-heap = {workspace = true}+num_cpus = {workspace = true}+once_cell = {workspace = true}+optics = {path = "../optics"}+parse_wiki_text = {workspace = true}+publicsuffix = {workspace = true}+quick-xml = {workspace = true}+rand = {workspace = true}+rayon = {workspace = true}+regex = {workspace = true}+reqwest = {workspace = true}+reqwest-eventsource = {workspace = true}+rio_api = {workspace = true}+rio_turtle = {workspace = true}+rkyv = {workspace = true}+robotstxt-with-cache = {workspace = true}+rocksdb = {workspace = true}+rust-s3 = {workspace = true}+rust-stemmers = {workspace = true}+safetensors = {workspace = true}+scylla = {workspace = true}+serde = {workspace = true}+serde_json = {workspace = true}+serde_urlencoded = {workspace = true}+tantivy = {workspace = true}+tch = {workspace = true}+thiserror = {workspace = true}+tokenizers = {workspace = true}+tokio = {workspace = true}+tokio-stream = {workspace = true}+toml = {workspace = true}+torch-sys = {workspace = true}+tower-http = {workspace = true}+tracing = {workspace = true}+tracing-subscriber = {workspace = true}+url = {workspace = true}+urlencoding = {workspace = true}+utoipa = {workspace = true}+utoipa-swagger-ui = {workspace = true}+uuid = {workspace = true}+whatlang = {workspace = true}[target.'cfg(not(target_env = "msvc"))'.dependencies]-tikv-jemallocator = { workspace = true }+tikv-jemallocator = {workspace = true}[build-dependencies]-lalrpop = { workspace = true }+lalrpop = {workspace = true}[dev-dependencies]-criterion = { workspace = true }-maplit = { workspace = true }-proptest = { workspace = true }-proptest-derive = { workspace = true }-insta = { workspace = true }+criterion = {workspace = true}+insta = {workspace = true}+maplit = {workspace = true}+proptest = {workspace = true}+proptest-derive = {workspace = true}[[bench]]-name = "search-preindexed"harness = false+name = "search-preindexed"[[bench]]-name = "search-preindexed-optic"harness = false+name = "search-preindexed-optic"[[bench]]-name = "harmonic-centrality"harness = false+name = "harmonic-centrality"[[bench]]-name = "abstractive-summarizer"harness = false+name = "abstractive-summarizer"[[bench]]-name = "build-similarity"harness = false+name = "build-similarity"[[bench]]-name = "spell-correction"harness = false+name = "spell-correction"[[bench]]-name = "crawl-coordinator"harness = false+name = "crawl-coordinator"[[bench]]-name = "similar-sites"harness = false+name = "similar-sites"[[bench]]-name = "hyperloglog"harness = false+name = "hyperloglog"[[bench]]-name = "naive-bayes"harness = false+name = "naive-bayes"diff --git a/core/src/crawler/crawl_db.rs b/core/src/crawler/crawl_db.rsindex 9c974f0d..9d27c9a3 100644--- a/core/src/crawler/crawl_db.rs+++ b/core/src/crawler/crawl_db.rs@@ -495,6 +495,9 @@ impl DomainStateDb {options.set_allow_mmap_writes(true);options.set_max_subcompactions(8);+ options.set_level_zero_slowdown_writes_trigger(-1);+ options.set_level_zero_stop_writes_trigger(-1);+let db = rocksdb::DB::open(&options, path.as_ref())?;Ok(Self { db })@@ -643,12 +646,13 @@ impl CrawlDb {for (domain, urls) in domains.into_iter() {let mut domain_state = self.domain_state.get(&domain)?.unwrap_or_default();+ let all_urls: HashMap<_, _> = self.urls.get_all_urls(&domain)?.into_iter().collect();let mut url_states = Vec::new();for url in urls {- let mut url_state = match self.urls.get(&domain, &UrlString::from(&url.url))? {- Some(state) => state,+ let mut url_state = match all_urls.get(&UrlString::from(&url.url)) {+ Some(state) => state.clone(),None => UrlState::default(),};diff --git a/core/src/entrypoint/centrality.rs b/core/src/entrypoint/centrality.rsindex 0c56634d..530fc566 100644--- a/core/src/entrypoint/centrality.rs+++ b/core/src/entrypoint/centrality.rs@@ -16,20 +16,27 @@use std::path::Path;-use crate::{ranking::centrality_store::CentralityStore, webgraph::WebgraphBuilder};+use crate::{+ ranking::centrality_store::CentralityStore,+ webgraph::{Compression, WebgraphBuilder},+};pub struct Centrality {}impl Centrality {pub fn build_harmonic<P: AsRef<Path>>(webgraph_path: P, output_path: P) {tracing::info!("Building harmonic centrality");- let graph = WebgraphBuilder::new(webgraph_path).open();+ let graph = WebgraphBuilder::new(webgraph_path)+ .compression(Compression::Lz4)+ .open();CentralityStore::build_harmonic(&graph, output_path);}pub fn build_similarity<P: AsRef<Path>>(webgraph_path: P, output_path: P) {tracing::info!("Building inbound similarity");- let graph = WebgraphBuilder::new(webgraph_path).open();+ let graph = WebgraphBuilder::new(webgraph_path)+ .compression(Compression::Lz4)+ .open();CentralityStore::build_similarity(&graph, output_path);}}diff --git a/core/src/entrypoint/webgraph.rs b/core/src/entrypoint/webgraph.rsindex 596aee2b..aff872dd 100644--- a/core/src/entrypoint/webgraph.rs+++ b/core/src/entrypoint/webgraph.rs@@ -68,11 +68,19 @@ pub struct Job {}pub fn open_host_graph_writer<P: AsRef<Path>>(path: P) -> webgraph::WebgraphWriter {- WebgraphWriter::new(path, crate::executor::Executor::single_thread())+ WebgraphWriter::new(+ path,+ crate::executor::Executor::single_thread(),+ webgraph::Compression::Lz4,+ )}pub fn open_page_graph_writer<P: AsRef<Path>>(path: P) -> webgraph::WebgraphWriter {- WebgraphWriter::new(path, crate::executor::Executor::single_thread())+ WebgraphWriter::new(+ path,+ crate::executor::Executor::single_thread(),+ webgraph::Compression::Lz4,+ )}pub struct WebgraphWorker {diff --git a/core/src/entrypoint/webgraph_server.rs b/core/src/entrypoint/webgraph_server.rsindex 3c075e0d..d7707216 100644--- a/core/src/entrypoint/webgraph_server.rs+++ b/core/src/entrypoint/webgraph_server.rs@@ -34,6 +34,7 @@ use crate::ranking::inbound_similarity::InboundSimilarity;use crate::searcher::DistributedSearcher;use crate::similar_sites::SimilarSitesFinder;use crate::sonic_service;+use crate::webgraph::Compression;use crate::webgraph::FullEdge;use crate::webgraph::Node;use crate::webgraph::Webgraph;@@ -189,8 +190,16 @@ pub async fn run(config: config::WebgraphServerConfig) -> Result<()> {);let searcher = DistributedSearcher::new(cluster);- let host_graph = Arc::new(WebgraphBuilder::new(config.host_graph_path).open());- let page_graph = Arc::new(WebgraphBuilder::new(config.page_graph_path).open());+ let host_graph = Arc::new(+ WebgraphBuilder::new(config.host_graph_path)+ .compression(Compression::Lz4)+ .open(),+ );+ let page_graph = Arc::new(+ WebgraphBuilder::new(config.page_graph_path)+ .compression(Compression::Lz4)+ .open(),+ );let inbound_similarity = InboundSimilarity::open(config.inbound_similarity_path)?;let similar_sites_finder = SimilarSitesFinder::new(diff --git a/core/src/query/optic.rs b/core/src/query/optic.rsindex bbbbcc99..948b8906 100644--- a/core/src/query/optic.rs+++ b/core/src/query/optic.rs@@ -622,8 +622,11 @@ mod tests {fn liked_sites() {let mut index = Index::temporary().expect("Unable to open index");- let mut writer =- WebgraphWriter::new(gen_temp_path(), crate::executor::Executor::single_thread());+ let mut writer = WebgraphWriter::new(+ gen_temp_path(),+ crate::executor::Executor::single_thread(),+ crate::webgraph::Compression::default(),+ );writer.insert(Node::from("https://www.e.com").into_host(),diff --git a/core/src/ranking/inbound_similarity.rs b/core/src/ranking/inbound_similarity.rsindex 15c35331..7f09233d 100644--- a/core/src/ranking/inbound_similarity.rs+++ b/core/src/ranking/inbound_similarity.rs@@ -222,8 +222,11 @@ mod tests {#[test]fn it_favors_liked_sites() {- let mut wrt =- WebgraphWriter::new(gen_temp_path(), crate::executor::Executor::single_thread());+ let mut wrt = WebgraphWriter::new(+ gen_temp_path(),+ crate::executor::Executor::single_thread(),+ crate::webgraph::Compression::default(),+ );wrt.insert(Node::from("a.com"), Node::from("b.com"), String::new());wrt.insert(Node::from("c.com"), Node::from("d.com"), String::new());@@ -251,6 +254,7 @@ mod tests {let mut wrt = WebgraphWriter::new(crate::gen_temp_path(),crate::executor::Executor::single_thread(),+ crate::webgraph::Compression::default(),);wrt.insert(Node::from("b.com"), Node::from("a.com"), String::new());diff --git a/core/src/ranking/optics.rs b/core/src/ranking/optics.rsindex 1d42c1fc..3ed1b3b3 100644--- a/core/src/ranking/optics.rs+++ b/core/src/ranking/optics.rs@@ -32,8 +32,11 @@ mod tests {fn site_rankings() {let mut index = Index::temporary().expect("Unable to open index");- let mut wrt =- WebgraphWriter::new(gen_temp_path(), crate::executor::Executor::single_thread());+ let mut wrt = WebgraphWriter::new(+ gen_temp_path(),+ crate::executor::Executor::single_thread(),+ crate::webgraph::Compression::default(),+ );wrt.insert(Node::from("https://www.first.com").into_host(),diff --git a/core/src/webgraph/centrality/betweenness.rs b/core/src/webgraph/centrality/betweenness.rsindex 4688b4ca..2fe45cd5 100644--- a/core/src/webgraph/centrality/betweenness.rs+++ b/core/src/webgraph/centrality/betweenness.rs@@ -171,6 +171,7 @@ mod tests {let mut writer = WebgraphWriter::new(crate::gen_temp_path(),crate::executor::Executor::single_thread(),+ crate::webgraph::Compression::default(),);for i in 0..n - 1 {diff --git a/core/src/webgraph/centrality/harmonic.rs b/core/src/webgraph/centrality/harmonic.rsindex 0af635b3..c41a5e30 100644--- a/core/src/webgraph/centrality/harmonic.rs+++ b/core/src/webgraph/centrality/harmonic.rs@@ -263,6 +263,7 @@ mod tests {let mut writer = WebgraphWriter::new(crate::gen_temp_path(),crate::executor::Executor::single_thread(),+ crate::webgraph::Compression::default(),);for (from, to, label) in test_edges() {@@ -277,6 +278,7 @@ mod tests {let mut writer = WebgraphWriter::new(crate::gen_temp_path(),crate::executor::Executor::single_thread(),+ crate::webgraph::Compression::default(),);writer.insert(@@ -384,6 +386,7 @@ mod tests {let mut other = WebgraphWriter::new(crate::gen_temp_path(),crate::executor::Executor::single_thread(),+ crate::webgraph::Compression::default(),);for (from, to, label) in test_edges() {diff --git a/core/src/webgraph/mod.rs b/core/src/webgraph/mod.rsindex 816a5f9a..7a43d3da 100644--- a/core/src/webgraph/mod.rs+++ b/core/src/webgraph/mod.rs@@ -229,9 +229,33 @@ pub fn normalize_url(url: &Url) -> String {normalized}+#[derive(Default, Debug, Clone, Copy)]+pub enum Compression {+ None,+ #[default]+ Lz4,+}++impl Compression {+ pub fn compress(&self, bytes: &[u8]) -> Vec<u8> {+ match self {+ Compression::None => bytes.to_vec(),+ Compression::Lz4 => lz4_flex::compress_prepend_size(bytes),+ }+ }++ pub fn decompress(&self, bytes: &[u8]) -> Vec<u8> {+ match self {+ Compression::None => bytes.to_vec(),+ Compression::Lz4 => lz4_flex::decompress_size_prepended(bytes).unwrap(),+ }+ }+}+pub struct WebgraphBuilder {path: Box<Path>,executor: Executor,+ compression: Compression,}impl WebgraphBuilder {@@ -239,6 +263,7 @@ impl WebgraphBuilder {Self {path: path.as_ref().into(),executor: Executor::multi_thread("webgraph").unwrap(),+ compression: Compression::default(),}}@@ -247,8 +272,13 @@ impl WebgraphBuilder {self}+ pub fn compression(mut self, compression: Compression) -> Self {+ self.compression = compression;+ self+ }+pub fn open(self) -> Webgraph {- Webgraph::open(self.path, self.executor)+ Webgraph::open(self.path, self.executor, self.compression)}}@@ -453,6 +483,7 @@ pub struct WebgraphWriter {id2node: Id2NodeDb,executor: Executor,meta: Meta,+ compression: Compression,}impl WebgraphWriter {@@ -466,7 +497,7 @@ impl WebgraphWriter {self.meta.save(path);}- pub fn new<P: AsRef<Path>>(path: P, executor: Executor) -> Self {+ pub fn new<P: AsRef<Path>>(path: P, executor: Executor, compression: Compression) -> Self {fs::create_dir_all(&path).unwrap();let mut meta = Self::meta(&path);meta.comitted_segments.clear();@@ -474,7 +505,7 @@ impl WebgraphWriter {fs::create_dir_all(path.as_ref().join("segments")).unwrap();let id = uuid::Uuid::new_v4().to_string();- let segment = SegmentWriter::open(path.as_ref().join("segments"), id.clone());+ let segment = SegmentWriter::open(path.as_ref().join("segments"), id.clone(), compression);meta.comitted_segments.push(id);@@ -485,6 +516,7 @@ impl WebgraphWriter {insert_batch: Vec::with_capacity(store::MAX_BATCH_SIZE),executor,meta,+ compression,}}@@ -538,6 +570,7 @@ impl WebgraphWriter {executor: self.executor.into(),id2node: self.id2node,meta: self.meta,+ compression: self.compression,}}}@@ -548,6 +581,7 @@ pub struct Webgraph {executor: Arc<Executor>,id2node: Id2NodeDb,meta: Meta,+ compression: Compression,}impl Webgraph {@@ -561,7 +595,7 @@ impl Webgraph {self.meta.save(path);}- fn open<P: AsRef<Path>>(path: P, executor: Executor) -> Self {+ fn open<P: AsRef<Path>>(path: P, executor: Executor, compression: Compression) -> Self {fs::create_dir_all(&path).unwrap();let meta = Self::meta(&path);@@ -572,6 +606,7 @@ impl Webgraph {segments.push(Segment::open(path.as_ref().join("segments"),segment.clone(),+ compression,));}@@ -581,6 +616,7 @@ impl Webgraph {executor: Arc::new(executor),id2node: Id2NodeDb::open(path.as_ref().join("id2node")),meta,+ compression,}}@@ -594,7 +630,8 @@ impl Webgraph {self.meta.comitted_segments.push(segment.id());drop(segment);- self.segments.push(Segment::open(new_path, id));+ self.segments+ .push(Segment::open(new_path, id, self.compression));}self.save_metadata();@@ -737,7 +774,11 @@ mod test {// │// D- let mut graph = WebgraphWriter::new(crate::gen_temp_path(), Executor::single_thread());+ let mut graph = WebgraphWriter::new(+ crate::gen_temp_path(),+ Executor::single_thread(),+ Compression::default(),+ );for (from, to, label) in test_edges() {graph.insert(from, to, label);@@ -795,7 +836,11 @@ mod test {(Node::from("F"), Node::from("G"), String::new()),(Node::from("G"), Node::from("H"), String::new()),] {- let mut wrt = WebgraphWriter::new(crate::gen_temp_path(), Executor::single_thread());+ let mut wrt = WebgraphWriter::new(+ crate::gen_temp_path(),+ Executor::single_thread(),+ Compression::default(),+ );wrt.insert(from.clone(), to.clone(), label.clone());graphs.push(wrt.finalize());}@@ -806,8 +851,6 @@ mod test {graph.merge(other);}- dbg!(&graph.path);-assert_eq!(graph.distances(Node::from("A")).get(&Node::from("H")),Some(&7)@@ -822,7 +865,11 @@ mod test {(Node::from("B"), Node::from("C"), String::new()),(Node::from("C"), Node::from("A"), String::new()),] {- let mut wrt = WebgraphWriter::new(crate::gen_temp_path(), Executor::single_thread());+ let mut wrt = WebgraphWriter::new(+ crate::gen_temp_path(),+ Executor::single_thread(),+ Compression::default(),+ );wrt.insert(from.clone(), to.clone(), label.clone());graphs.push(wrt.finalize());}@@ -860,7 +907,11 @@ mod test {#[test]fn cap_label_length() {- let mut writer = WebgraphWriter::new(crate::gen_temp_path(), Executor::single_thread());+ let mut writer = WebgraphWriter::new(+ crate::gen_temp_path(),+ Executor::single_thread(),+ Compression::default(),+ );writer.insert(Node::from("A"),diff --git a/core/src/webgraph/segment.rs b/core/src/webgraph/segment.rsindex 1914c34a..26a8ebee 100644--- a/core/src/webgraph/segment.rs+++ b/core/src/webgraph/segment.rs@@ -18,7 +18,7 @@ use std::path::{Path, PathBuf};use super::{store::{EdgeStore, EdgeStoreWriter},- Edge, NodeID,+ Compression, Edge, NodeID,};const ADJACENCY_STORE: &str = "adjacency";@@ -32,10 +32,11 @@ pub struct SegmentWriter {}impl SegmentWriter {- pub fn open<P: AsRef<Path>>(folder_path: P, id: String) -> Self {+ pub fn open<P: AsRef<Path>>(folder_path: P, id: String, compression: Compression) -> Self {SegmentWriter {full_adjacency: EdgeStoreWriter::open(folder_path.as_ref().join(&id).join(ADJACENCY_STORE),+ compression,false,),full_reversed_adjacency: EdgeStoreWriter::open(@@ -43,6 +44,7 @@ impl SegmentWriter {.as_ref().join(&id).join(REVERSED_ADJACENCY_STORE),+ compression,true,),folder_path: folder_path@@ -85,11 +87,12 @@ pub struct Segment {}impl Segment {- pub fn open<P: AsRef<Path>>(folder_path: P, id: String) -> Self {+ pub fn open<P: AsRef<Path>>(folder_path: P, id: String, compression: Compression) -> Self {Segment {full_adjacency: EdgeStore::open(folder_path.as_ref().join(&id).join(ADJACENCY_STORE),false,+ compression,),full_reversed_adjacency: EdgeStore::open(folder_path@@ -97,6 +100,7 @@ impl Segment {.join(&id).join(REVERSED_ADJACENCY_STORE),true,+ compression,),folder_path: folder_path.as_ref()@@ -150,7 +154,11 @@ mod test {// ▼ │ │// 1─────►2◄┘- let mut writer = SegmentWriter::open(crate::gen_temp_path(), "test".to_string());+ let mut writer = SegmentWriter::open(+ crate::gen_temp_path(),+ "test".to_string(),+ Compression::default(),+ );let mut edges = Vec::new();diff --git a/core/src/webgraph/store.rs b/core/src/webgraph/store.rsindex 4a91c885..dc8fb350 100644--- a/core/src/webgraph/store.rs+++ b/core/src/webgraph/store.rs@@ -20,17 +20,18 @@ use itertools::Itertools;use memmap2::Mmap;use rocksdb::BlockBasedOptions;-use super::{Edge, EdgeLabel, NodeID};+use super::{Compression, Edge, EdgeLabel, NodeID};pub const MAX_BATCH_SIZE: usize = 100_000;pub struct EdgeStoreWriter {reversed: bool,db: rocksdb::DB,+ compression: Compression,}impl EdgeStoreWriter {- pub fn open<P: AsRef<Path>>(path: P, reversed: bool) -> Self {+ pub fn open<P: AsRef<Path>>(path: P, compression: Compression, reversed: bool) -> Self {let mut options = rocksdb::Options::default();options.create_if_missing(true);@@ -63,7 +64,11 @@ impl EdgeStoreWriter {let db = rocksdb::DB::open(&options, path.as_ref().join("writer")).unwrap();- Self { db, reversed }+ Self {+ db,+ reversed,+ compression,+ }}pub fn put<'a, L: EdgeLabel + 'a>(&'a self, edges: impl Iterator<Item = &'a Edge<L>>) {@@ -133,7 +138,12 @@ impl EdgeStoreWriter {}pub fn finalize(self) -> EdgeStore {- let s = EdgeStore::build(self.db.path().parent().unwrap(), self.reversed, self.iter());+ let s = EdgeStore::build(+ self.db.path().parent().unwrap(),+ self.compression,+ self.reversed,+ self.iter(),+ );// delete the writer dblet p = self.db.path().to_owned();@@ -148,16 +158,20 @@ pub struct EdgeStore {reversed: bool,ranges: rocksdb::DB, // column[nodes] = nodeid -> (start, end); column[labels] = nodeid -> (start, end)_cache: rocksdb::Cache,+edge_labels_file: File,edge_labels_len: usize,edge_labels: Mmap,+edge_nodes_file: File,edge_nodes_len: usize,edge_nodes: Mmap,++ compression: Compression,}impl EdgeStore {- pub fn open<P: AsRef<Path>>(path: P, reversed: bool) -> Self {+ pub fn open<P: AsRef<Path>>(path: P, reversed: bool, compression: Compression) -> Self {let mut options = rocksdb::Options::default();options.create_if_missing(true);@@ -233,6 +247,7 @@ impl EdgeStore {edge_nodes,edge_nodes_file,edge_nodes_len,+ compression,}}@@ -269,6 +284,9 @@ impl EdgeStore {let edge_labels_bytes = bincode::serialize(&edge_labels).unwrap();let edge_nodes_bytes = bincode::serialize(&edge_nodes).unwrap();+ let edge_labels_bytes = self.compression.compress(&edge_labels_bytes);+ let edge_nodes_bytes = self.compression.compress(&edge_nodes_bytes);+let label_range = self.edge_labels_len..(self.edge_labels_len + edge_labels_bytes.len());let node_range = self.edge_nodes_len..(self.edge_nodes_len + edge_nodes_bytes.len());@@ -304,10 +322,11 @@ impl EdgeStore {/// either the from or to node, depending on the value of `reversed`.fn build<P: AsRef<Path>>(path: P,+ compression: Compression,reversed: bool,edges: impl Iterator<Item = Edge<String>>,) -> Self {- let mut s = Self::open(path, reversed);+ let mut s = Self::open(path, reversed, compression);// create batches of consecutive edges with the same from/to nodelet mut batch = Vec::new();@@ -367,10 +386,12 @@ impl EdgeStore {let edge_range = bincode::deserialize::<Range<usize>>(&edge_range_bytes).unwrap();let edge_labels = &self.edge_labels[edge_range];- let edge_labels: Vec<String> = bincode::deserialize(edge_labels).unwrap();+ let edge_labels = self.compression.decompress(edge_labels);+ let edge_labels: Vec<String> = bincode::deserialize(&edge_labels).unwrap();let edge_nodes = &self.edge_nodes[node_range];- let edge_nodes: Vec<NodeID> = bincode::deserialize(edge_nodes).unwrap();+ let edge_nodes = self.compression.decompress(edge_nodes);+ let edge_nodes: Vec<NodeID> = bincode::deserialize(&edge_nodes).unwrap();edge_labels.into_iter()@@ -406,7 +427,8 @@ impl EdgeStore {let node_range = bincode::deserialize::<Range<usize>>(&node_range_bytes).unwrap();let edge_nodes = &self.edge_nodes[node_range];- let edge_nodes: Vec<NodeID> = bincode::deserialize(edge_nodes).unwrap();+ let edge_nodes = self.compression.decompress(edge_nodes);+ let edge_nodes: Vec<NodeID> = bincode::deserialize(&edge_nodes).unwrap();edge_nodes.into_iter()@@ -444,7 +466,8 @@ impl EdgeStore {let node_range = bincode::deserialize::<Range<usize>>(&val).unwrap();let edge_nodes = &self.edge_nodes[node_range];- let edge_nodes: Vec<NodeID> = bincode::deserialize(edge_nodes).unwrap();+ let edge_nodes = self.compression.decompress(edge_nodes);+ let edge_nodes: Vec<NodeID> = bincode::deserialize(&edge_nodes).unwrap();edge_nodes.into_iter().map(move |other| {if self.reversed {@@ -471,8 +494,11 @@ mod tests {#[test]fn test_insert() {- let kv: EdgeStoreWriter =- EdgeStoreWriter::open(crate::gen_temp_path().join("test-segment"), false);+ let kv: EdgeStoreWriter = EdgeStoreWriter::open(+ crate::gen_temp_path().join("test-segment"),+ Compression::default(),+ false,+ );let e = Edge {from: NodeID(0),@@ -496,8 +522,11 @@ mod tests {#[test]fn test_reversed() {- let kv: EdgeStoreWriter =- EdgeStoreWriter::open(crate::gen_temp_path().join("test-segment"), true);+ let kv: EdgeStoreWriter = EdgeStoreWriter::open(+ crate::gen_temp_path().join("test-segment"),+ Compression::default(),+ true,+ );let e = Edge {from: NodeID(0),