Skip to content

web search done right

turn off insert deduplication in host graph. The keys are de-duplicated in rocksdb during compaction

Mikkel Denker6879043767c2
diff --git a/core/src/entrypoint/webgraph.rs b/core/src/entrypoint/webgraph.rs
index b6cf6d24..6c248d5f 100644
--- a/core/src/entrypoint/webgraph.rs
+++ b/core/src/entrypoint/webgraph.rs
@@ -68,7 +68,9 @@ pub struct Job {
}
pub fn open_host_graph<P: AsRef<Path>>(path: P) -> webgraph::Webgraph {
- WebgraphBuilder::new(path).open()
+ WebgraphBuilder::new(path)
+ .deduplication(webgraph::Deduplication::OnlyQuery)
+ .open()
}
pub fn open_page_graph<P: AsRef<Path>>(path: P) -> webgraph::Webgraph {
diff --git a/core/src/webgraph/store.rs b/core/src/webgraph/store.rs
index 11654651..c995354c 100644
--- a/core/src/webgraph/store.rs
+++ b/core/src/webgraph/store.rs
@@ -40,6 +40,7 @@ impl EdgeStore {
options.set_allow_mmap_reads(true);
options.set_allow_mmap_writes(true);
+ options.set_write_buffer_size(128 * 1024 * 1024); // 128 MB
// some recommended settings (https://github.com/facebook/rocksdb/wiki/Setup-Options-and-Basic-Tuning)
options.set_level_compaction_dynamic_level_bytes(true);