web search done right
larger leves in webgraph store
Mikkel Denker097b8e4abbe7
diff --git a/core/src/crawler/crawl_db.rs b/core/src/crawler/crawl_db.rsindex 8b12ebb1..d5a962c8 100644--- a/core/src/crawler/crawl_db.rs+++ b/core/src/crawler/crawl_db.rs@@ -246,6 +246,8 @@ impl UrlStateDbShard {options.set_write_buffer_size(512 * 1024 * 1024);options.set_allow_mmap_reads(true);options.set_allow_mmap_writes(true);+ options.set_target_file_size_base(512 * 1024 * 1024); // 512 MB+ options.set_target_file_size_multiplier(10);// some recommended settings (https://github.com/facebook/rocksdb/wiki/Setup-Options-and-Basic-Tuning)options.set_level_compaction_dynamic_level_bytes(true);diff --git a/core/src/webgraph/store.rs b/core/src/webgraph/store.rsindex c995354c..11a0d22f 100644--- a/core/src/webgraph/store.rs+++ b/core/src/webgraph/store.rs@@ -41,6 +41,8 @@ impl EdgeStore {options.set_allow_mmap_reads(true);options.set_allow_mmap_writes(true);options.set_write_buffer_size(128 * 1024 * 1024); // 128 MB+ options.set_target_file_size_base(512 * 1024 * 1024); // 512 MB+ options.set_target_file_size_multiplier(10);// some recommended settings (https://github.com/facebook/rocksdb/wiki/Setup-Options-and-Basic-Tuning)options.set_level_compaction_dynamic_level_bytes(true);