web search done right
turn off insert deduplication in host graph. The keys are de-duplicated in rocksdb during compaction
Mikkel Denker6879043767c2
diff --git a/core/src/entrypoint/webgraph.rs b/core/src/entrypoint/webgraph.rsindex b6cf6d24..6c248d5f 100644--- a/core/src/entrypoint/webgraph.rs+++ b/core/src/entrypoint/webgraph.rs@@ -68,7 +68,9 @@ pub struct Job {}pub fn open_host_graph<P: AsRef<Path>>(path: P) -> webgraph::Webgraph {- WebgraphBuilder::new(path).open()+ WebgraphBuilder::new(path)+ .deduplication(webgraph::Deduplication::OnlyQuery)+ .open()}pub fn open_page_graph<P: AsRef<Path>>(path: P) -> webgraph::Webgraph {diff --git a/core/src/webgraph/store.rs b/core/src/webgraph/store.rsindex 11654651..c995354c 100644--- a/core/src/webgraph/store.rs+++ b/core/src/webgraph/store.rs@@ -40,6 +40,7 @@ impl EdgeStore {options.set_allow_mmap_reads(true);options.set_allow_mmap_writes(true);+ options.set_write_buffer_size(128 * 1024 * 1024); // 128 MB// some recommended settings (https://github.com/facebook/rocksdb/wiki/Setup-Options-and-Basic-Tuning)options.set_level_compaction_dynamic_level_bytes(true);