Skip to content

web search done right

rocksdb bloom filter

Mikkel Denker3979d1f6349f
diff --git a/core/src/crawler/crawl_db.rs b/core/src/crawler/crawl_db.rs
index da8011e6..7c3d4160 100644
--- a/core/src/crawler/crawl_db.rs
+++ b/core/src/crawler/crawl_db.rs
@@ -24,6 +24,7 @@ use std::{
use lru::LruCache;
use rand::Rng;
+use rocksdb::BlockBasedOptions;
use crate::webpage::Url;
@@ -46,6 +47,7 @@ pub enum DomainStatus {
struct IdTable<T> {
t2id: rocksdb::DB,
id2t: rocksdb::DB,
+
next_id: u64,
t2id_cache: LruCache<T, u64>,
@@ -63,6 +65,12 @@ where
options.create_if_missing(true);
options.increase_parallelism(3);
+ let mut block_options = BlockBasedOptions::default();
+
+ block_options.set_bloom_filter(64.0, true);
+
+ options.set_block_based_table_factory(&block_options);
+
// create dir if not exists
std::fs::create_dir_all(path.as_ref())?;