web search done right
[webgrah] zstd was even too slow for the id2node database. this sets the compression to lz4 instead. a lot of time also seems to be spend for xxh3 hashing in rocksdb. don't know exactly where this comes from, but lets try to disable ribbon filters for id2node database as well as this probably uses xxh3 hashing.
Mikkel Denkerbe288d11ba11
diff --git a/core/src/webgraph/mod.rs b/core/src/webgraph/mod.rsindex f8b02237..974be3fb 100644--- a/core/src/webgraph/mod.rs+++ b/core/src/webgraph/mod.rs@@ -487,7 +487,6 @@ impl Id2NodeDb {opts.set_target_file_size_multiplier(10);let mut block_opts = rocksdb::BlockBasedOptions::default();- block_opts.set_ribbon_filter(5.0);// some recommended settings (https://github.com/facebook/rocksdb/wiki/Setup-Options-and-Basic-Tuning)opts.set_level_compaction_dynamic_level_bytes(true);@@ -498,7 +497,7 @@ impl Id2NodeDb {block_opts.set_pin_l0_filter_and_index_blocks_in_cache(true);opts.set_block_based_table_factory(&block_opts);- opts.set_compression_type(rocksdb::DBCompressionType::Zstd);+ opts.set_compression_type(rocksdb::DBCompressionType::Lz4);let db = rocksdb::DB::open(&opts, path).unwrap();