Skip to content

web search done right

rocksdb bloom filters caused OOM errors. Updated filter sizes to more sane defaults

Mikkel Denker2e885e9d1137
diff --git a/configs/crawler/crawler.toml b/configs/crawler/crawler.toml
index e0e5ff62..c9874527 100644
--- a/configs/crawler/crawler.toml
+++ b/configs/crawler/crawler.toml
@@ -1,11 +1,11 @@
-num_workers = 10
+num_worker_threads = 10
politeness_factor = 1.0
timeout_seconds = 30
coordinator_host = "0.0.0.0:8080"
[user_agent]
token = "StractBot"
-full = "Mozilla/5.0 (compatible; StractBot/0.1; +https://trystract.com/webmasters)"
+full = "Mozilla/5.0 (compatible; StractBot/0.1; open source search engine; +https://trystract.com/webmasters)"
[s3]
bucket = "crawl"
diff --git a/core/src/config/mod.rs b/core/src/config/mod.rs
index 9b14ef0a..8a3a9f84 100644
--- a/core/src/config/mod.rs
+++ b/core/src/config/mod.rs
@@ -269,7 +269,7 @@ pub struct UserAgent {
#[derive(Debug, Serialize, Deserialize, Clone)]
pub struct CrawlerConfig {
- pub num_workers: usize,
+ pub num_worker_threads: usize,
pub user_agent: UserAgent,
#[serde(default = "defaults::Crawler::robots_txt_cache_sec")]
diff --git a/core/src/crawler/crawl_db.rs b/core/src/crawler/crawl_db.rs
index ac410a5e..4e4fa9ac 100644
--- a/core/src/crawler/crawl_db.rs
+++ b/core/src/crawler/crawl_db.rs
@@ -68,7 +68,7 @@ where
options.set_max_write_buffer_number(8);
let mut block_options = BlockBasedOptions::default();
- block_options.set_bloom_filter(64.0, true);
+ block_options.set_ribbon_filter(10.0);
options.set_block_based_table_factory(&block_options);
@@ -86,8 +86,8 @@ where
id2t,
next_id: 0,
- t2id_cache: LruCache::new(NonZeroUsize::new(1_000_000).unwrap()),
- id2t_cache: LruCache::new(NonZeroUsize::new(1_000_000).unwrap()),
+ t2id_cache: LruCache::new(NonZeroUsize::new(500_000).unwrap()),
+ id2t_cache: LruCache::new(NonZeroUsize::new(500_000).unwrap()),
_marker: std::marker::PhantomData,
})
@@ -303,7 +303,7 @@ impl RedirectDb {
options.create_if_missing(true);
let mut block_options = BlockBasedOptions::default();
- block_options.set_bloom_filter(64.0, true);
+ block_options.set_ribbon_filter(10.0);
options.set_block_based_table_factory(&block_options);
let inner = rocksdb::DB::open(&options, path.as_ref())?;
diff --git a/core/src/crawler/mod.rs b/core/src/crawler/mod.rs
index 06db7eba..8beef4a4 100644
--- a/core/src/crawler/mod.rs
+++ b/core/src/crawler/mod.rs
@@ -25,7 +25,7 @@ use url::Url;
use crate::config::CrawlerConfig;
-use self::{warc_writer::WarcWriter, worker::Worker};
+use self::{warc_writer::WarcWriter, worker::WorkerThread};
pub mod coordinator;
pub mod crawl_db;
@@ -148,8 +148,8 @@ impl Crawler {
let mut handles = Vec::new();
let coordinator_host = config.coordinator_host.parse()?;
- for _ in 0..config.num_workers {
- let worker = Worker::new(
+ for _ in 0..config.num_worker_threads {
+ let worker = WorkerThread::new(
Arc::clone(&pending_commands),
Arc::clone(&writer),
Arc::clone(&results),
diff --git a/core/src/crawler/worker.rs b/core/src/crawler/worker.rs
index 58a4a1be..9a2d0f2a 100644
--- a/core/src/crawler/worker.rs
+++ b/core/src/crawler/worker.rs
@@ -38,7 +38,7 @@ use super::{
WarcWriter, WorkerJob,
};
-pub struct Worker {
+pub struct WorkerThread {
current_job: Option<WorkerJob>,
pending_commands: Arc<Mutex<VecDeque<Command>>>,
writer: Arc<WarcWriter>,
@@ -51,7 +51,7 @@ pub struct Worker {
robotstxt: RobotsTxtManager,
}
-impl Worker {
+impl WorkerThread {
pub fn new(
pending_commands: Arc<Mutex<VecDeque<Command>>>,
writer: Arc<WarcWriter>,
@@ -82,7 +82,7 @@ impl Worker {
results,
current_job: None,
pending_commands,
- num_jobs_per_fetch: config.num_workers,
+ num_jobs_per_fetch: config.num_worker_threads,
politeness_factor: config.politeness_factor,
config,
coordinator_host,
diff --git a/core/src/ranking/centrality_store.rs b/core/src/ranking/centrality_store.rs
index 4d2df7b9..cad01f5b 100644
--- a/core/src/ranking/centrality_store.rs
+++ b/core/src/ranking/centrality_store.rs
@@ -43,7 +43,7 @@ impl Node2Id {
options.set_max_write_buffer_number(8);
let mut block_options = BlockBasedOptions::default();
- block_options.set_bloom_filter(64.0, true);
+ block_options.set_ribbon_filter(10.0);
let cache = rocksdb::Cache::new_lru_cache(256 * 1024 * 1024).unwrap(); // 256 MB cache
block_options.set_block_cache(&cache);
diff --git a/core/src/webgraph/mod.rs b/core/src/webgraph/mod.rs
index 3796cd31..9593928a 100644
--- a/core/src/webgraph/mod.rs
+++ b/core/src/webgraph/mod.rs
@@ -393,7 +393,7 @@ impl Webgraph {
segments,
executor: Arc::new(Executor::multi_thread("webgraph").unwrap()),
id2node: Store::open(path.as_ref().join("id2node")),
- id2node_cache: Mutex::new(LruCache::new(500_000.try_into().unwrap())),
+ id2node_cache: Mutex::new(LruCache::new(10_000.try_into().unwrap())),
meta,
}
}
diff --git a/core/src/webgraph/store.rs b/core/src/webgraph/store.rs
index 97c85e44..6be615da 100644
--- a/core/src/webgraph/store.rs
+++ b/core/src/webgraph/store.rs
@@ -39,9 +39,9 @@ where
options.set_max_write_buffer_number(8);
let mut block_options = BlockBasedOptions::default();
- block_options.set_bloom_filter(128.0, true);
+ block_options.set_ribbon_filter(5.0);
- let cache = rocksdb::Cache::new_lru_cache(512 * 1024 * 1024).unwrap(); // 512 MB cache
+ let cache = rocksdb::Cache::new_lru_cache(256 * 1024 * 1024).unwrap(); // 256 MB cache
block_options.set_block_cache(&cache);
block_options.set_block_size(128 * 1024); // 128 KB block size
diff --git a/frontend/src/pages/crawler.mdx b/frontend/src/pages/crawler.mdx
index 0c036f00..1aade6ab 100644
--- a/frontend/src/pages/crawler.mdx
+++ b/frontend/src/pages/crawler.mdx
@@ -6,7 +6,7 @@ title: Stract Crawler
# Stract Crawler
StractBot is the name of our crawler that collects pages from the web in order to index them into the search engine.
It is written in Rust and the source code can be found [here](https://github.com/StractOrg/stract/tree/main/core/src/crawler).
-The crawler uses the user agent `Mozilla/5.0 (compatible; StractBot/0.1; +https://trystract.com/webmasters)`.
+The crawler uses the user agent `Mozilla/5.0 (compatible; StractBot/0.1; open source search engine; +https://trystract.com/webmasters)`.
## Politeness
StractBot is a polite crawler. It respects the [robots.txt](https://en.wikipedia.org/wiki/Robots.txt) file of the website it is crawling and tries to not overload the server.