web search done right
increase rocksdb parallelism and version number for crawldb
Mikkel Denker73c5dc21c787
diff --git a/core/src/crawler/crawl_db.rs b/core/src/crawler/crawl_db.rsindex 44a03de6..3045ac9d 100644--- a/core/src/crawler/crawl_db.rs+++ b/core/src/crawler/crawl_db.rs@@ -137,7 +137,8 @@ impl RedirectDb {options.create_if_missing(true);- let block_options = rocksdb::BlockBasedOptions::default();+ let mut block_options = rocksdb::BlockBasedOptions::default();+ block_options.set_format_version(5);options.set_block_based_table_factory(&block_options);@@ -183,9 +184,11 @@ impl RangesDb {let mut block_options = rocksdb::BlockBasedOptions::default();block_options.set_ribbon_filter(10.0);+ block_options.set_format_version(5);options.set_block_based_table_factory(&block_options);options.set_max_background_jobs(8);+ options.increase_parallelism(8);options.set_write_buffer_size(512 * 1024 * 1024);options.set_allow_mmap_reads(true);options.set_allow_mmap_writes(true);@@ -255,12 +258,14 @@ impl UrlStateDbShard {let mut block_options = rocksdb::BlockBasedOptions::default();block_options.set_ribbon_filter(10.0);+ block_options.set_format_version(5);let cache = rocksdb::Cache::new_lru_cache(1024 * 1024 * 1024)?; // 1GBblock_options.set_block_cache(&cache);options.set_block_based_table_factory(&block_options);options.set_max_background_jobs(8);+ options.increase_parallelism(8);options.set_write_buffer_size(512 * 1024 * 1024);options.set_allow_mmap_reads(true);options.set_allow_mmap_writes(true);@@ -468,10 +473,12 @@ impl DomainStateDb {let mut block_options = rocksdb::BlockBasedOptions::default();block_options.set_ribbon_filter(10.0);+ block_options.set_format_version(5);options.set_block_based_table_factory(&block_options);options.set_optimize_filters_for_hits(true);options.set_max_background_jobs(8);+ options.increase_parallelism(8);options.set_write_buffer_size(512 * 1024 * 1024);options.set_allow_mmap_reads(true);options.set_allow_mmap_writes(true);