Skip to content

web search done right

larger leves in webgraph store

Mikkel Denker097b8e4abbe7
diff --git a/core/src/crawler/crawl_db.rs b/core/src/crawler/crawl_db.rs
index 8b12ebb1..d5a962c8 100644
--- a/core/src/crawler/crawl_db.rs
+++ b/core/src/crawler/crawl_db.rs
@@ -246,6 +246,8 @@ impl UrlStateDbShard {
options.set_write_buffer_size(512 * 1024 * 1024);
options.set_allow_mmap_reads(true);
options.set_allow_mmap_writes(true);
+ options.set_target_file_size_base(512 * 1024 * 1024); // 512 MB
+ options.set_target_file_size_multiplier(10);
// some recommended settings (https://github.com/facebook/rocksdb/wiki/Setup-Options-and-Basic-Tuning)
options.set_level_compaction_dynamic_level_bytes(true);
diff --git a/core/src/webgraph/store.rs b/core/src/webgraph/store.rs
index c995354c..11a0d22f 100644
--- a/core/src/webgraph/store.rs
+++ b/core/src/webgraph/store.rs
@@ -41,6 +41,8 @@ impl EdgeStore {
options.set_allow_mmap_reads(true);
options.set_allow_mmap_writes(true);
options.set_write_buffer_size(128 * 1024 * 1024); // 128 MB
+ options.set_target_file_size_base(512 * 1024 * 1024); // 512 MB
+ options.set_target_file_size_multiplier(10);
// some recommended settings (https://github.com/facebook/rocksdb/wiki/Setup-Options-and-Basic-Tuning)
options.set_level_compaction_dynamic_level_bytes(true);