web search done right
rocksdb bloom filter
Mikkel Denker3979d1f6349f
diff --git a/core/src/crawler/crawl_db.rs b/core/src/crawler/crawl_db.rsindex da8011e6..7c3d4160 100644--- a/core/src/crawler/crawl_db.rs+++ b/core/src/crawler/crawl_db.rs@@ -24,6 +24,7 @@ use std::{use lru::LruCache;use rand::Rng;+use rocksdb::BlockBasedOptions;use crate::webpage::Url;@@ -46,6 +47,7 @@ pub enum DomainStatus {struct IdTable<T> {t2id: rocksdb::DB,id2t: rocksdb::DB,+next_id: u64,t2id_cache: LruCache<T, u64>,@@ -63,6 +65,12 @@ whereoptions.create_if_missing(true);options.increase_parallelism(3);+ let mut block_options = BlockBasedOptions::default();++ block_options.set_bloom_filter(64.0, true);++ options.set_block_based_table_factory(&block_options);+// create dir if not existsstd::fs::create_dir_all(path.as_ref())?;