Skip to content

web search done right

webgraph compression

Mikkel Denker2a607ee88e55
diff --git a/Cargo.lock b/Cargo.lock
index 4dd9d8ec..d758183b 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -2329,6 +2329,9 @@ name = "lz4_flex"
version = "0.11.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3ea9b256699eda7b0387ffbc776dd625e28bde3918446381781245b7a50349d8"
+dependencies = [
+ "twox-hash",
+]
[[package]]
name = "mac"
@@ -4290,6 +4293,7 @@ dependencies = [
"libc",
"logos",
"lz-str",
+ "lz4_flex 0.11.1",
"maplit",
"md5",
"memmap",
diff --git a/Cargo.toml b/Cargo.toml
index 53464fac..4c14270f 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -11,98 +11,99 @@ debug = true
[workspace.dependencies]
aes-gcm = "0.10.2"
+anyhow = {version = "1.0.72", features = ["backtrace"]}
+async-channel = "1.8.0"
+async-stream = "0.3.3"
+async-trait = "0.1.57"
+axum = "0.6.9"
+axum-extra = {version = "0.6.0", features = ["spa"]}
+axum-macros = "0.3.4"
+base64 = "0.13.0"
bincode = "1.3.3"
-clap = { version = "3.1.18", features = ["derive"] }
+bitvec = "1.0.1"
+bytemuck = {version = "1.13.1", features = ["derive"]}
+byteorder = "1.4.3"
+bzip2 = "0.4.4"
+chitchat = "0.5.0"
+chrono = {version = "0.4.23", features = ["serde"]}
+clap = {version = "3.1.18", features = ["derive"]}
+criterion = "0.3.6"
+crossbeam-channel = "0.5.6"
+csv = "1.1.6"
+dashmap = {version = "5.4.0", features = ["rayon"]}
encoding_rs = "0.8.31"
+eventsource-stream = "0.2.3"
+fend-core = "1.2.2"
flate2 = "1.0.23"
-indicatif = { version = "0.16.2", features = ["rayon"] }
+fst = {version = "0.4.7", features = ["levenshtein"]}
+futures = "0.3.21"
+half = {version = "2.2.1", features = ["serde"]}
+hashbrown = {version = "0.14.0", features = ["serde", "rkyv"]}
+http = "0.2.8"
+image = "0.24.3"
+indicatif = {version = "0.16.2", features = ["rayon"]}
+insta = "1.31"
itertools = "0.10.3"
-rayon = "1.5.3"
-half = { version = "2.2.1", features = ["serde"] }
-eventsource-stream = "0.2.3"
-reqwest-eventsource = "0.4.0"
-reqwest = { version = "0.11.16", features = ["blocking", "stream", "json"] }
-serde = { version = "1.0.137", features = ["rc", "derive"] }
-serde_json = "1.0.81"
-tantivy = "0.21.0"
-thiserror = "1.0.31"
-toml = "0.5.9"
-tracing = "0.1.34"
-tracing-subscriber = { version = "0.3.11", features = ["env-filter"] }
-whatlang = "0.16.0"
-tokio = { version = "1.23.1", features = ["full"] }
-axum = "0.6.9"
-axum-extra = { version = "0.6.0", features = ["spa"] }
-axum-macros = "0.3.4"
+lalrpop = {version = "0.19.8", features = ["lexer"]}
+lalrpop-util = {version = "0.19.8", features = ["lexer"]}
+libc = "0.2.142"
logos = "0.12.1"
-csv = "1.1.6"
-fst = { version = "0.4.7", features = ["levenshtein"] }
-rocksdb = { version = "0.21.0", features = ["default", "jemalloc", "io-uring"] }
-image = "0.24.3"
-chrono = { version = "0.4.23", features = ["serde"] }
-uuid = "1.1.2"
-futures = "0.3.21"
-quick-xml = "0.23.0"
-rio_turtle = "0.8.4"
-rio_api = "0.8.4"
-bzip2 = "0.4.4"
-parse_wiki_text = "0.1.5"
+lz-str = "0.2.1"
+lz4_flex = "0.11.1"
+maplit = "1.0.2"
md5 = "0.7.0"
memmap = "0.7.0"
+memmap2 = "0.5.10"
+mime = "0.3.17"
+min-max-heap = "1.3.0"
+num_cpus = "1.15.0"
once_cell = "1.13.1"
+parse_wiki_text = "0.1.5"
+proptest = "1.2.0"
+proptest-derive = "0.3.0"
+publicsuffix = "2.2.3"
+quick-xml = "0.23.0"
+rand = "0.8.5"
+rayon = "1.5.3"
regex = "1.6.0"
-http = "0.2.8"
-tower-http = { version = "0.4.0", features = ["compression-gzip", "cors"] }
-lalrpop-util = { version = "0.19.8", features = ["lexer"] }
-lalrpop = { version = "0.19.8", features = ["lexer"] }
-serde_urlencoded = "0.7.1"
+reqwest = {version = "0.11.16", features = ["blocking", "stream", "json"]}
+reqwest-eventsource = "0.4.0"
+rio_api = "0.8.4"
+rio_turtle = "0.8.4"
+rkyv = {version = "0.7.42", default_features = false, features = [
+ "std",
+ "validation",
+ "size_64",
+]}
+robotstxt-with-cache = "0.4.0"
+rocksdb = {version = "0.21.0", features = ["default", "jemalloc", "io-uring"]}
+rusqlite = {version = "0.29.0", features = [
+ "bundled",
+ "modern-full",
+ "buildtime_bindgen",
+]}
+rust-s3 = {version = "0.33.0", features = ["blocking", "tokio"]}
rust-stemmers = "1.2.0"
-min-max-heap = "1.3.0"
-base64 = "0.13.0"
-async-trait = "0.1.57"
-urlencoding = "2.1.2"
-bitvec = "1.0.1"
-criterion = "0.3.6"
-maplit = "1.0.2"
-rand = "0.8.5"
-crossbeam-channel = "0.5.6"
-num_cpus = "1.15.0"
-tokenizers = "0.13.2"
-byteorder = "1.4.3"
-async-stream = "0.3.3"
-tokio-stream = "0.1.11"
+safetensors = "0.3.1"
scylla = "0.7.0"
-chitchat = "0.5.0"
-memmap2 = "0.5.10"
-dashmap = { version = "5.4.0", features = ["rayon"] }
+serde = {version = "1.0.137", features = ["rc", "derive"]}
+serde_json = "1.0.81"
+serde_urlencoded = "0.7.1"
+tantivy = "0.21.0"
tch = "0.13.0"
-torch-sys = "0.13.0"
-libc = "0.2.142"
-lz-str = "0.2.1"
-safetensors = "0.3.1"
-robotstxt-with-cache = "0.4.0"
-rust-s3 = { version = "0.33.0", features = ["blocking", "tokio"] }
-rusqlite = { version = "0.29.0", features = [
- "bundled",
- "modern-full",
- "buildtime_bindgen",
-] }
-async-channel = "1.8.0"
-url = { version = "2.4.0", features = ["serde"] }
-anyhow = { version = "1.0.72", features = ["backtrace"] }
-bytemuck = { version = "1.13.1", features = ["derive"] }
-proptest = "1.2.0"
-proptest-derive = "0.3.0"
-insta = "1.31"
-utoipa = { version = "3.4.4", features = ["axum_extras"] }
-utoipa-swagger-ui = { version = "3.1.4", features = ["axum"] }
+thiserror = "1.0.31"
tikv-jemallocator = "0.5"
-publicsuffix = "2.2.3"
-hashbrown = { version = "0.14.0", features = ["serde", "rkyv"] }
-rkyv = { version = "0.7.42", default_features = false, features = [
- "std",
- "validation",
- "size_64",
-] }
-mime = "0.3.17"
-fend-core = "1.2.2"
+tokenizers = "0.13.2"
+tokio = {version = "1.23.1", features = ["full"]}
+tokio-stream = "0.1.11"
+toml = "0.5.9"
+torch-sys = "0.13.0"
+tower-http = {version = "0.4.0", features = ["compression-gzip", "cors"]}
+tracing = "0.1.34"
+tracing-subscriber = {version = "0.3.11", features = ["env-filter"]}
+url = {version = "2.4.0", features = ["serde"]}
+urlencoding = "2.1.2"
+utoipa = {version = "3.4.4", features = ["axum_extras"]}
+utoipa-swagger-ui = {version = "3.1.4", features = ["axum"]}
+uuid = "1.1.2"
+whatlang = "0.16.0"
diff --git a/core/Cargo.toml b/core/Cargo.toml
index cf610dc0..147537ef 100644
--- a/core/Cargo.toml
+++ b/core/Cargo.toml
@@ -1,18 +1,18 @@
[package]
-name = "stract"
-version = "0.1.0"
-edition = "2021"
-license = "AGPL-3.0-or-later"
authors = ['Stract ApS. <[email protected]>']
default-run = "stract"
+edition = "2021"
+license = "AGPL-3.0-or-later"
+name = "stract"
+version = "0.1.0"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[features]
+cors = []
default = ["cors", "with_alice"]
dev = ["cors"]
prod = ["cors"]
-cors = []
with_alice = []
[[bin]]
@@ -20,138 +20,139 @@ name = "stract"
path = "src/main.rs"
[dependencies]
-bincode = { workspace = true }
-clap = { workspace = true }
-encoding_rs = { workspace = true }
-flate2 = { workspace = true }
-indicatif = { workspace = true }
-itertools = { workspace = true }
-rayon = { workspace = true }
-reqwest = { workspace = true }
-serde = { workspace = true }
-serde_json = { workspace = true }
-tantivy = { workspace = true }
-thiserror = { workspace = true }
-toml = { workspace = true }
-tracing = { workspace = true }
-tracing-subscriber = { workspace = true }
-whatlang = { workspace = true }
-tokio = { workspace = true }
-axum = { workspace = true }
-axum-extra = { workspace = true }
-logos = { workspace = true }
-csv = { workspace = true }
-fst = { workspace = true }
-rocksdb = { workspace = true }
-image = { workspace = true }
-chrono = { workspace = true }
-uuid = { workspace = true }
-futures = { workspace = true }
-quick-xml = { workspace = true }
-bzip2 = { workspace = true }
-parse_wiki_text = { workspace = true }
-md5 = { workspace = true }
-memmap = { workspace = true }
-regex = { workspace = true }
-http = { workspace = true }
-kuchiki = { path = "../kuchiki" }
-serde_urlencoded = { workspace = true }
-rust-stemmers = { workspace = true }
-min-max-heap = { workspace = true }
-base64 = { workspace = true }
-async-trait = { workspace = true }
-axum-macros = { workspace = true }
-urlencoding = { workspace = true }
-bitvec = { workspace = true }
-tower-http = { workspace = true }
-once_cell = { workspace = true }
-optics = { path = "../optics" }
-crossbeam-channel = { workspace = true }
-num_cpus = { workspace = true }
-tokenizers = { workspace = true }
-lalrpop-util = { workspace = true }
-byteorder = { workspace = true }
-async-stream = { workspace = true }
-tokio-stream = { workspace = true }
-scylla = { workspace = true }
-chitchat = { workspace = true }
-memmap2 = { workspace = true }
-dashmap = { workspace = true }
-tch = { workspace = true }
-torch-sys = { workspace = true }
-libc = { workspace = true }
-lz-str = { workspace = true }
-safetensors = { workspace = true }
-eventsource-stream = { workspace = true }
-rand = { workspace = true }
-half = { workspace = true }
-aes-gcm = { workspace = true }
-reqwest-eventsource = { workspace = true }
-robotstxt-with-cache = { workspace = true }
-rust-s3 = { workspace = true }
-async-channel = { workspace = true }
-url = { workspace = true }
-anyhow = { workspace = true }
-bytemuck = { workspace = true }
-utoipa = { workspace = true }
-utoipa-swagger-ui = { workspace = true }
-publicsuffix = { workspace = true }
-rkyv = { workspace = true }
-hashbrown = { workspace = true }
-mime = { workspace = true }
-fend-core = { workspace = true }
-rio_turtle = { workspace = true }
-rio_api = { workspace = true }
+aes-gcm = {workspace = true}
+anyhow = {workspace = true}
+async-channel = {workspace = true}
+async-stream = {workspace = true}
+async-trait = {workspace = true}
+axum = {workspace = true}
+axum-extra = {workspace = true}
+axum-macros = {workspace = true}
+base64 = {workspace = true}
+bincode = {workspace = true}
+bitvec = {workspace = true}
+bytemuck = {workspace = true}
+byteorder = {workspace = true}
+bzip2 = {workspace = true}
+chitchat = {workspace = true}
+chrono = {workspace = true}
+clap = {workspace = true}
+crossbeam-channel = {workspace = true}
+csv = {workspace = true}
+dashmap = {workspace = true}
+encoding_rs = {workspace = true}
+eventsource-stream = {workspace = true}
+fend-core = {workspace = true}
+flate2 = {workspace = true}
+fst = {workspace = true}
+futures = {workspace = true}
+half = {workspace = true}
+hashbrown = {workspace = true}
+http = {workspace = true}
+image = {workspace = true}
+indicatif = {workspace = true}
+itertools = {workspace = true}
+kuchiki = {path = "../kuchiki"}
+lalrpop-util = {workspace = true}
+libc = {workspace = true}
+logos = {workspace = true}
+lz-str = {workspace = true}
+lz4_flex = {workspace = true}
+md5 = {workspace = true}
+memmap = {workspace = true}
+memmap2 = {workspace = true}
+mime = {workspace = true}
+min-max-heap = {workspace = true}
+num_cpus = {workspace = true}
+once_cell = {workspace = true}
+optics = {path = "../optics"}
+parse_wiki_text = {workspace = true}
+publicsuffix = {workspace = true}
+quick-xml = {workspace = true}
+rand = {workspace = true}
+rayon = {workspace = true}
+regex = {workspace = true}
+reqwest = {workspace = true}
+reqwest-eventsource = {workspace = true}
+rio_api = {workspace = true}
+rio_turtle = {workspace = true}
+rkyv = {workspace = true}
+robotstxt-with-cache = {workspace = true}
+rocksdb = {workspace = true}
+rust-s3 = {workspace = true}
+rust-stemmers = {workspace = true}
+safetensors = {workspace = true}
+scylla = {workspace = true}
+serde = {workspace = true}
+serde_json = {workspace = true}
+serde_urlencoded = {workspace = true}
+tantivy = {workspace = true}
+tch = {workspace = true}
+thiserror = {workspace = true}
+tokenizers = {workspace = true}
+tokio = {workspace = true}
+tokio-stream = {workspace = true}
+toml = {workspace = true}
+torch-sys = {workspace = true}
+tower-http = {workspace = true}
+tracing = {workspace = true}
+tracing-subscriber = {workspace = true}
+url = {workspace = true}
+urlencoding = {workspace = true}
+utoipa = {workspace = true}
+utoipa-swagger-ui = {workspace = true}
+uuid = {workspace = true}
+whatlang = {workspace = true}
[target.'cfg(not(target_env = "msvc"))'.dependencies]
-tikv-jemallocator = { workspace = true }
+tikv-jemallocator = {workspace = true}
[build-dependencies]
-lalrpop = { workspace = true }
+lalrpop = {workspace = true}
[dev-dependencies]
-criterion = { workspace = true }
-maplit = { workspace = true }
-proptest = { workspace = true }
-proptest-derive = { workspace = true }
-insta = { workspace = true }
+criterion = {workspace = true}
+insta = {workspace = true}
+maplit = {workspace = true}
+proptest = {workspace = true}
+proptest-derive = {workspace = true}
[[bench]]
-name = "search-preindexed"
harness = false
+name = "search-preindexed"
[[bench]]
-name = "search-preindexed-optic"
harness = false
+name = "search-preindexed-optic"
[[bench]]
-name = "harmonic-centrality"
harness = false
+name = "harmonic-centrality"
[[bench]]
-name = "abstractive-summarizer"
harness = false
+name = "abstractive-summarizer"
[[bench]]
-name = "build-similarity"
harness = false
+name = "build-similarity"
[[bench]]
-name = "spell-correction"
harness = false
+name = "spell-correction"
[[bench]]
-name = "crawl-coordinator"
harness = false
+name = "crawl-coordinator"
[[bench]]
-name = "similar-sites"
harness = false
+name = "similar-sites"
[[bench]]
-name = "hyperloglog"
harness = false
+name = "hyperloglog"
[[bench]]
-name = "naive-bayes"
harness = false
+name = "naive-bayes"
diff --git a/core/src/crawler/crawl_db.rs b/core/src/crawler/crawl_db.rs
index 9c974f0d..9d27c9a3 100644
--- a/core/src/crawler/crawl_db.rs
+++ b/core/src/crawler/crawl_db.rs
@@ -495,6 +495,9 @@ impl DomainStateDb {
options.set_allow_mmap_writes(true);
options.set_max_subcompactions(8);
+ options.set_level_zero_slowdown_writes_trigger(-1);
+ options.set_level_zero_stop_writes_trigger(-1);
+
let db = rocksdb::DB::open(&options, path.as_ref())?;
Ok(Self { db })
@@ -643,12 +646,13 @@ impl CrawlDb {
for (domain, urls) in domains.into_iter() {
let mut domain_state = self.domain_state.get(&domain)?.unwrap_or_default();
+ let all_urls: HashMap<_, _> = self.urls.get_all_urls(&domain)?.into_iter().collect();
let mut url_states = Vec::new();
for url in urls {
- let mut url_state = match self.urls.get(&domain, &UrlString::from(&url.url))? {
- Some(state) => state,
+ let mut url_state = match all_urls.get(&UrlString::from(&url.url)) {
+ Some(state) => state.clone(),
None => UrlState::default(),
};
diff --git a/core/src/entrypoint/centrality.rs b/core/src/entrypoint/centrality.rs
index 0c56634d..530fc566 100644
--- a/core/src/entrypoint/centrality.rs
+++ b/core/src/entrypoint/centrality.rs
@@ -16,20 +16,27 @@
use std::path::Path;
-use crate::{ranking::centrality_store::CentralityStore, webgraph::WebgraphBuilder};
+use crate::{
+ ranking::centrality_store::CentralityStore,
+ webgraph::{Compression, WebgraphBuilder},
+};
pub struct Centrality {}
impl Centrality {
pub fn build_harmonic<P: AsRef<Path>>(webgraph_path: P, output_path: P) {
tracing::info!("Building harmonic centrality");
- let graph = WebgraphBuilder::new(webgraph_path).open();
+ let graph = WebgraphBuilder::new(webgraph_path)
+ .compression(Compression::Lz4)
+ .open();
CentralityStore::build_harmonic(&graph, output_path);
}
pub fn build_similarity<P: AsRef<Path>>(webgraph_path: P, output_path: P) {
tracing::info!("Building inbound similarity");
- let graph = WebgraphBuilder::new(webgraph_path).open();
+ let graph = WebgraphBuilder::new(webgraph_path)
+ .compression(Compression::Lz4)
+ .open();
CentralityStore::build_similarity(&graph, output_path);
}
}
diff --git a/core/src/entrypoint/webgraph.rs b/core/src/entrypoint/webgraph.rs
index 596aee2b..aff872dd 100644
--- a/core/src/entrypoint/webgraph.rs
+++ b/core/src/entrypoint/webgraph.rs
@@ -68,11 +68,19 @@ pub struct Job {
}
pub fn open_host_graph_writer<P: AsRef<Path>>(path: P) -> webgraph::WebgraphWriter {
- WebgraphWriter::new(path, crate::executor::Executor::single_thread())
+ WebgraphWriter::new(
+ path,
+ crate::executor::Executor::single_thread(),
+ webgraph::Compression::Lz4,
+ )
}
pub fn open_page_graph_writer<P: AsRef<Path>>(path: P) -> webgraph::WebgraphWriter {
- WebgraphWriter::new(path, crate::executor::Executor::single_thread())
+ WebgraphWriter::new(
+ path,
+ crate::executor::Executor::single_thread(),
+ webgraph::Compression::Lz4,
+ )
}
pub struct WebgraphWorker {
diff --git a/core/src/entrypoint/webgraph_server.rs b/core/src/entrypoint/webgraph_server.rs
index 3c075e0d..d7707216 100644
--- a/core/src/entrypoint/webgraph_server.rs
+++ b/core/src/entrypoint/webgraph_server.rs
@@ -34,6 +34,7 @@ use crate::ranking::inbound_similarity::InboundSimilarity;
use crate::searcher::DistributedSearcher;
use crate::similar_sites::SimilarSitesFinder;
use crate::sonic_service;
+use crate::webgraph::Compression;
use crate::webgraph::FullEdge;
use crate::webgraph::Node;
use crate::webgraph::Webgraph;
@@ -189,8 +190,16 @@ pub async fn run(config: config::WebgraphServerConfig) -> Result<()> {
);
let searcher = DistributedSearcher::new(cluster);
- let host_graph = Arc::new(WebgraphBuilder::new(config.host_graph_path).open());
- let page_graph = Arc::new(WebgraphBuilder::new(config.page_graph_path).open());
+ let host_graph = Arc::new(
+ WebgraphBuilder::new(config.host_graph_path)
+ .compression(Compression::Lz4)
+ .open(),
+ );
+ let page_graph = Arc::new(
+ WebgraphBuilder::new(config.page_graph_path)
+ .compression(Compression::Lz4)
+ .open(),
+ );
let inbound_similarity = InboundSimilarity::open(config.inbound_similarity_path)?;
let similar_sites_finder = SimilarSitesFinder::new(
diff --git a/core/src/query/optic.rs b/core/src/query/optic.rs
index bbbbcc99..948b8906 100644
--- a/core/src/query/optic.rs
+++ b/core/src/query/optic.rs
@@ -622,8 +622,11 @@ mod tests {
fn liked_sites() {
let mut index = Index::temporary().expect("Unable to open index");
- let mut writer =
- WebgraphWriter::new(gen_temp_path(), crate::executor::Executor::single_thread());
+ let mut writer = WebgraphWriter::new(
+ gen_temp_path(),
+ crate::executor::Executor::single_thread(),
+ crate::webgraph::Compression::default(),
+ );
writer.insert(
Node::from("https://www.e.com").into_host(),
diff --git a/core/src/ranking/inbound_similarity.rs b/core/src/ranking/inbound_similarity.rs
index 15c35331..7f09233d 100644
--- a/core/src/ranking/inbound_similarity.rs
+++ b/core/src/ranking/inbound_similarity.rs
@@ -222,8 +222,11 @@ mod tests {
#[test]
fn it_favors_liked_sites() {
- let mut wrt =
- WebgraphWriter::new(gen_temp_path(), crate::executor::Executor::single_thread());
+ let mut wrt = WebgraphWriter::new(
+ gen_temp_path(),
+ crate::executor::Executor::single_thread(),
+ crate::webgraph::Compression::default(),
+ );
wrt.insert(Node::from("a.com"), Node::from("b.com"), String::new());
wrt.insert(Node::from("c.com"), Node::from("d.com"), String::new());
@@ -251,6 +254,7 @@ mod tests {
let mut wrt = WebgraphWriter::new(
crate::gen_temp_path(),
crate::executor::Executor::single_thread(),
+ crate::webgraph::Compression::default(),
);
wrt.insert(Node::from("b.com"), Node::from("a.com"), String::new());
diff --git a/core/src/ranking/optics.rs b/core/src/ranking/optics.rs
index 1d42c1fc..3ed1b3b3 100644
--- a/core/src/ranking/optics.rs
+++ b/core/src/ranking/optics.rs
@@ -32,8 +32,11 @@ mod tests {
fn site_rankings() {
let mut index = Index::temporary().expect("Unable to open index");
- let mut wrt =
- WebgraphWriter::new(gen_temp_path(), crate::executor::Executor::single_thread());
+ let mut wrt = WebgraphWriter::new(
+ gen_temp_path(),
+ crate::executor::Executor::single_thread(),
+ crate::webgraph::Compression::default(),
+ );
wrt.insert(
Node::from("https://www.first.com").into_host(),
diff --git a/core/src/webgraph/centrality/betweenness.rs b/core/src/webgraph/centrality/betweenness.rs
index 4688b4ca..2fe45cd5 100644
--- a/core/src/webgraph/centrality/betweenness.rs
+++ b/core/src/webgraph/centrality/betweenness.rs
@@ -171,6 +171,7 @@ mod tests {
let mut writer = WebgraphWriter::new(
crate::gen_temp_path(),
crate::executor::Executor::single_thread(),
+ crate::webgraph::Compression::default(),
);
for i in 0..n - 1 {
diff --git a/core/src/webgraph/centrality/harmonic.rs b/core/src/webgraph/centrality/harmonic.rs
index 0af635b3..c41a5e30 100644
--- a/core/src/webgraph/centrality/harmonic.rs
+++ b/core/src/webgraph/centrality/harmonic.rs
@@ -263,6 +263,7 @@ mod tests {
let mut writer = WebgraphWriter::new(
crate::gen_temp_path(),
crate::executor::Executor::single_thread(),
+ crate::webgraph::Compression::default(),
);
for (from, to, label) in test_edges() {
@@ -277,6 +278,7 @@ mod tests {
let mut writer = WebgraphWriter::new(
crate::gen_temp_path(),
crate::executor::Executor::single_thread(),
+ crate::webgraph::Compression::default(),
);
writer.insert(
@@ -384,6 +386,7 @@ mod tests {
let mut other = WebgraphWriter::new(
crate::gen_temp_path(),
crate::executor::Executor::single_thread(),
+ crate::webgraph::Compression::default(),
);
for (from, to, label) in test_edges() {
diff --git a/core/src/webgraph/mod.rs b/core/src/webgraph/mod.rs
index 816a5f9a..7a43d3da 100644
--- a/core/src/webgraph/mod.rs
+++ b/core/src/webgraph/mod.rs
@@ -229,9 +229,33 @@ pub fn normalize_url(url: &Url) -> String {
normalized
}
+#[derive(Default, Debug, Clone, Copy)]
+pub enum Compression {
+ None,
+ #[default]
+ Lz4,
+}
+
+impl Compression {
+ pub fn compress(&self, bytes: &[u8]) -> Vec<u8> {
+ match self {
+ Compression::None => bytes.to_vec(),
+ Compression::Lz4 => lz4_flex::compress_prepend_size(bytes),
+ }
+ }
+
+ pub fn decompress(&self, bytes: &[u8]) -> Vec<u8> {
+ match self {
+ Compression::None => bytes.to_vec(),
+ Compression::Lz4 => lz4_flex::decompress_size_prepended(bytes).unwrap(),
+ }
+ }
+}
+
pub struct WebgraphBuilder {
path: Box<Path>,
executor: Executor,
+ compression: Compression,
}
impl WebgraphBuilder {
@@ -239,6 +263,7 @@ impl WebgraphBuilder {
Self {
path: path.as_ref().into(),
executor: Executor::multi_thread("webgraph").unwrap(),
+ compression: Compression::default(),
}
}
@@ -247,8 +272,13 @@ impl WebgraphBuilder {
self
}
+ pub fn compression(mut self, compression: Compression) -> Self {
+ self.compression = compression;
+ self
+ }
+
pub fn open(self) -> Webgraph {
- Webgraph::open(self.path, self.executor)
+ Webgraph::open(self.path, self.executor, self.compression)
}
}
@@ -453,6 +483,7 @@ pub struct WebgraphWriter {
id2node: Id2NodeDb,
executor: Executor,
meta: Meta,
+ compression: Compression,
}
impl WebgraphWriter {
@@ -466,7 +497,7 @@ impl WebgraphWriter {
self.meta.save(path);
}
- pub fn new<P: AsRef<Path>>(path: P, executor: Executor) -> Self {
+ pub fn new<P: AsRef<Path>>(path: P, executor: Executor, compression: Compression) -> Self {
fs::create_dir_all(&path).unwrap();
let mut meta = Self::meta(&path);
meta.comitted_segments.clear();
@@ -474,7 +505,7 @@ impl WebgraphWriter {
fs::create_dir_all(path.as_ref().join("segments")).unwrap();
let id = uuid::Uuid::new_v4().to_string();
- let segment = SegmentWriter::open(path.as_ref().join("segments"), id.clone());
+ let segment = SegmentWriter::open(path.as_ref().join("segments"), id.clone(), compression);
meta.comitted_segments.push(id);
@@ -485,6 +516,7 @@ impl WebgraphWriter {
insert_batch: Vec::with_capacity(store::MAX_BATCH_SIZE),
executor,
meta,
+ compression,
}
}
@@ -538,6 +570,7 @@ impl WebgraphWriter {
executor: self.executor.into(),
id2node: self.id2node,
meta: self.meta,
+ compression: self.compression,
}
}
}
@@ -548,6 +581,7 @@ pub struct Webgraph {
executor: Arc<Executor>,
id2node: Id2NodeDb,
meta: Meta,
+ compression: Compression,
}
impl Webgraph {
@@ -561,7 +595,7 @@ impl Webgraph {
self.meta.save(path);
}
- fn open<P: AsRef<Path>>(path: P, executor: Executor) -> Self {
+ fn open<P: AsRef<Path>>(path: P, executor: Executor, compression: Compression) -> Self {
fs::create_dir_all(&path).unwrap();
let meta = Self::meta(&path);
@@ -572,6 +606,7 @@ impl Webgraph {
segments.push(Segment::open(
path.as_ref().join("segments"),
segment.clone(),
+ compression,
));
}
@@ -581,6 +616,7 @@ impl Webgraph {
executor: Arc::new(executor),
id2node: Id2NodeDb::open(path.as_ref().join("id2node")),
meta,
+ compression,
}
}
@@ -594,7 +630,8 @@ impl Webgraph {
self.meta.comitted_segments.push(segment.id());
drop(segment);
- self.segments.push(Segment::open(new_path, id));
+ self.segments
+ .push(Segment::open(new_path, id, self.compression));
}
self.save_metadata();
@@ -737,7 +774,11 @@ mod test {
// │
// D
- let mut graph = WebgraphWriter::new(crate::gen_temp_path(), Executor::single_thread());
+ let mut graph = WebgraphWriter::new(
+ crate::gen_temp_path(),
+ Executor::single_thread(),
+ Compression::default(),
+ );
for (from, to, label) in test_edges() {
graph.insert(from, to, label);
@@ -795,7 +836,11 @@ mod test {
(Node::from("F"), Node::from("G"), String::new()),
(Node::from("G"), Node::from("H"), String::new()),
] {
- let mut wrt = WebgraphWriter::new(crate::gen_temp_path(), Executor::single_thread());
+ let mut wrt = WebgraphWriter::new(
+ crate::gen_temp_path(),
+ Executor::single_thread(),
+ Compression::default(),
+ );
wrt.insert(from.clone(), to.clone(), label.clone());
graphs.push(wrt.finalize());
}
@@ -806,8 +851,6 @@ mod test {
graph.merge(other);
}
- dbg!(&graph.path);
-
assert_eq!(
graph.distances(Node::from("A")).get(&Node::from("H")),
Some(&7)
@@ -822,7 +865,11 @@ mod test {
(Node::from("B"), Node::from("C"), String::new()),
(Node::from("C"), Node::from("A"), String::new()),
] {
- let mut wrt = WebgraphWriter::new(crate::gen_temp_path(), Executor::single_thread());
+ let mut wrt = WebgraphWriter::new(
+ crate::gen_temp_path(),
+ Executor::single_thread(),
+ Compression::default(),
+ );
wrt.insert(from.clone(), to.clone(), label.clone());
graphs.push(wrt.finalize());
}
@@ -860,7 +907,11 @@ mod test {
#[test]
fn cap_label_length() {
- let mut writer = WebgraphWriter::new(crate::gen_temp_path(), Executor::single_thread());
+ let mut writer = WebgraphWriter::new(
+ crate::gen_temp_path(),
+ Executor::single_thread(),
+ Compression::default(),
+ );
writer.insert(
Node::from("A"),
diff --git a/core/src/webgraph/segment.rs b/core/src/webgraph/segment.rs
index 1914c34a..26a8ebee 100644
--- a/core/src/webgraph/segment.rs
+++ b/core/src/webgraph/segment.rs
@@ -18,7 +18,7 @@ use std::path::{Path, PathBuf};
use super::{
store::{EdgeStore, EdgeStoreWriter},
- Edge, NodeID,
+ Compression, Edge, NodeID,
};
const ADJACENCY_STORE: &str = "adjacency";
@@ -32,10 +32,11 @@ pub struct SegmentWriter {
}
impl SegmentWriter {
- pub fn open<P: AsRef<Path>>(folder_path: P, id: String) -> Self {
+ pub fn open<P: AsRef<Path>>(folder_path: P, id: String, compression: Compression) -> Self {
SegmentWriter {
full_adjacency: EdgeStoreWriter::open(
folder_path.as_ref().join(&id).join(ADJACENCY_STORE),
+ compression,
false,
),
full_reversed_adjacency: EdgeStoreWriter::open(
@@ -43,6 +44,7 @@ impl SegmentWriter {
.as_ref()
.join(&id)
.join(REVERSED_ADJACENCY_STORE),
+ compression,
true,
),
folder_path: folder_path
@@ -85,11 +87,12 @@ pub struct Segment {
}
impl Segment {
- pub fn open<P: AsRef<Path>>(folder_path: P, id: String) -> Self {
+ pub fn open<P: AsRef<Path>>(folder_path: P, id: String, compression: Compression) -> Self {
Segment {
full_adjacency: EdgeStore::open(
folder_path.as_ref().join(&id).join(ADJACENCY_STORE),
false,
+ compression,
),
full_reversed_adjacency: EdgeStore::open(
folder_path
@@ -97,6 +100,7 @@ impl Segment {
.join(&id)
.join(REVERSED_ADJACENCY_STORE),
true,
+ compression,
),
folder_path: folder_path
.as_ref()
@@ -150,7 +154,11 @@ mod test {
// ▼ │ │
// 1─────►2◄┘
- let mut writer = SegmentWriter::open(crate::gen_temp_path(), "test".to_string());
+ let mut writer = SegmentWriter::open(
+ crate::gen_temp_path(),
+ "test".to_string(),
+ Compression::default(),
+ );
let mut edges = Vec::new();
diff --git a/core/src/webgraph/store.rs b/core/src/webgraph/store.rs
index 4a91c885..dc8fb350 100644
--- a/core/src/webgraph/store.rs
+++ b/core/src/webgraph/store.rs
@@ -20,17 +20,18 @@ use itertools::Itertools;
use memmap2::Mmap;
use rocksdb::BlockBasedOptions;
-use super::{Edge, EdgeLabel, NodeID};
+use super::{Compression, Edge, EdgeLabel, NodeID};
pub const MAX_BATCH_SIZE: usize = 100_000;
pub struct EdgeStoreWriter {
reversed: bool,
db: rocksdb::DB,
+ compression: Compression,
}
impl EdgeStoreWriter {
- pub fn open<P: AsRef<Path>>(path: P, reversed: bool) -> Self {
+ pub fn open<P: AsRef<Path>>(path: P, compression: Compression, reversed: bool) -> Self {
let mut options = rocksdb::Options::default();
options.create_if_missing(true);
@@ -63,7 +64,11 @@ impl EdgeStoreWriter {
let db = rocksdb::DB::open(&options, path.as_ref().join("writer")).unwrap();
- Self { db, reversed }
+ Self {
+ db,
+ reversed,
+ compression,
+ }
}
pub fn put<'a, L: EdgeLabel + 'a>(&'a self, edges: impl Iterator<Item = &'a Edge<L>>) {
@@ -133,7 +138,12 @@ impl EdgeStoreWriter {
}
pub fn finalize(self) -> EdgeStore {
- let s = EdgeStore::build(self.db.path().parent().unwrap(), self.reversed, self.iter());
+ let s = EdgeStore::build(
+ self.db.path().parent().unwrap(),
+ self.compression,
+ self.reversed,
+ self.iter(),
+ );
// delete the writer db
let p = self.db.path().to_owned();
@@ -148,16 +158,20 @@ pub struct EdgeStore {
reversed: bool,
ranges: rocksdb::DB, // column[nodes] = nodeid -> (start, end); column[labels] = nodeid -> (start, end)
_cache: rocksdb::Cache,
+
edge_labels_file: File,
edge_labels_len: usize,
edge_labels: Mmap,
+
edge_nodes_file: File,
edge_nodes_len: usize,
edge_nodes: Mmap,
+
+ compression: Compression,
}
impl EdgeStore {
- pub fn open<P: AsRef<Path>>(path: P, reversed: bool) -> Self {
+ pub fn open<P: AsRef<Path>>(path: P, reversed: bool, compression: Compression) -> Self {
let mut options = rocksdb::Options::default();
options.create_if_missing(true);
@@ -233,6 +247,7 @@ impl EdgeStore {
edge_nodes,
edge_nodes_file,
edge_nodes_len,
+ compression,
}
}
@@ -269,6 +284,9 @@ impl EdgeStore {
let edge_labels_bytes = bincode::serialize(&edge_labels).unwrap();
let edge_nodes_bytes = bincode::serialize(&edge_nodes).unwrap();
+ let edge_labels_bytes = self.compression.compress(&edge_labels_bytes);
+ let edge_nodes_bytes = self.compression.compress(&edge_nodes_bytes);
+
let label_range = self.edge_labels_len..(self.edge_labels_len + edge_labels_bytes.len());
let node_range = self.edge_nodes_len..(self.edge_nodes_len + edge_nodes_bytes.len());
@@ -304,10 +322,11 @@ impl EdgeStore {
/// either the from or to node, depending on the value of `reversed`.
fn build<P: AsRef<Path>>(
path: P,
+ compression: Compression,
reversed: bool,
edges: impl Iterator<Item = Edge<String>>,
) -> Self {
- let mut s = Self::open(path, reversed);
+ let mut s = Self::open(path, reversed, compression);
// create batches of consecutive edges with the same from/to node
let mut batch = Vec::new();
@@ -367,10 +386,12 @@ impl EdgeStore {
let edge_range = bincode::deserialize::<Range<usize>>(&edge_range_bytes).unwrap();
let edge_labels = &self.edge_labels[edge_range];
- let edge_labels: Vec<String> = bincode::deserialize(edge_labels).unwrap();
+ let edge_labels = self.compression.decompress(edge_labels);
+ let edge_labels: Vec<String> = bincode::deserialize(&edge_labels).unwrap();
let edge_nodes = &self.edge_nodes[node_range];
- let edge_nodes: Vec<NodeID> = bincode::deserialize(edge_nodes).unwrap();
+ let edge_nodes = self.compression.decompress(edge_nodes);
+ let edge_nodes: Vec<NodeID> = bincode::deserialize(&edge_nodes).unwrap();
edge_labels
.into_iter()
@@ -406,7 +427,8 @@ impl EdgeStore {
let node_range = bincode::deserialize::<Range<usize>>(&node_range_bytes).unwrap();
let edge_nodes = &self.edge_nodes[node_range];
- let edge_nodes: Vec<NodeID> = bincode::deserialize(edge_nodes).unwrap();
+ let edge_nodes = self.compression.decompress(edge_nodes);
+ let edge_nodes: Vec<NodeID> = bincode::deserialize(&edge_nodes).unwrap();
edge_nodes
.into_iter()
@@ -444,7 +466,8 @@ impl EdgeStore {
let node_range = bincode::deserialize::<Range<usize>>(&val).unwrap();
let edge_nodes = &self.edge_nodes[node_range];
- let edge_nodes: Vec<NodeID> = bincode::deserialize(edge_nodes).unwrap();
+ let edge_nodes = self.compression.decompress(edge_nodes);
+ let edge_nodes: Vec<NodeID> = bincode::deserialize(&edge_nodes).unwrap();
edge_nodes.into_iter().map(move |other| {
if self.reversed {
@@ -471,8 +494,11 @@ mod tests {
#[test]
fn test_insert() {
- let kv: EdgeStoreWriter =
- EdgeStoreWriter::open(crate::gen_temp_path().join("test-segment"), false);
+ let kv: EdgeStoreWriter = EdgeStoreWriter::open(
+ crate::gen_temp_path().join("test-segment"),
+ Compression::default(),
+ false,
+ );
let e = Edge {
from: NodeID(0),
@@ -496,8 +522,11 @@ mod tests {
#[test]
fn test_reversed() {
- let kv: EdgeStoreWriter =
- EdgeStoreWriter::open(crate::gen_temp_path().join("test-segment"), true);
+ let kv: EdgeStoreWriter = EdgeStoreWriter::open(
+ crate::gen_temp_path().join("test-segment"),
+ Compression::default(),
+ true,
+ );
let e = Edge {
from: NodeID(0),