web search done right
rocksdb bloom filters caused OOM errors. Updated filter sizes to more sane defaults
Mikkel Denker2e885e9d1137
diff --git a/configs/crawler/crawler.toml b/configs/crawler/crawler.tomlindex e0e5ff62..c9874527 100644--- a/configs/crawler/crawler.toml+++ b/configs/crawler/crawler.toml@@ -1,11 +1,11 @@-num_workers = 10+num_worker_threads = 10politeness_factor = 1.0timeout_seconds = 30coordinator_host = "0.0.0.0:8080"[user_agent]token = "StractBot"-full = "Mozilla/5.0 (compatible; StractBot/0.1; +https://trystract.com/webmasters)"+full = "Mozilla/5.0 (compatible; StractBot/0.1; open source search engine; +https://trystract.com/webmasters)"[s3]bucket = "crawl"diff --git a/core/src/config/mod.rs b/core/src/config/mod.rsindex 9b14ef0a..8a3a9f84 100644--- a/core/src/config/mod.rs+++ b/core/src/config/mod.rs@@ -269,7 +269,7 @@ pub struct UserAgent {#[derive(Debug, Serialize, Deserialize, Clone)]pub struct CrawlerConfig {- pub num_workers: usize,+ pub num_worker_threads: usize,pub user_agent: UserAgent,#[serde(default = "defaults::Crawler::robots_txt_cache_sec")]diff --git a/core/src/crawler/crawl_db.rs b/core/src/crawler/crawl_db.rsindex ac410a5e..4e4fa9ac 100644--- a/core/src/crawler/crawl_db.rs+++ b/core/src/crawler/crawl_db.rs@@ -68,7 +68,7 @@ whereoptions.set_max_write_buffer_number(8);let mut block_options = BlockBasedOptions::default();- block_options.set_bloom_filter(64.0, true);+ block_options.set_ribbon_filter(10.0);options.set_block_based_table_factory(&block_options);@@ -86,8 +86,8 @@ whereid2t,next_id: 0,- t2id_cache: LruCache::new(NonZeroUsize::new(1_000_000).unwrap()),- id2t_cache: LruCache::new(NonZeroUsize::new(1_000_000).unwrap()),+ t2id_cache: LruCache::new(NonZeroUsize::new(500_000).unwrap()),+ id2t_cache: LruCache::new(NonZeroUsize::new(500_000).unwrap()),_marker: std::marker::PhantomData,})@@ -303,7 +303,7 @@ impl RedirectDb {options.create_if_missing(true);let mut block_options = BlockBasedOptions::default();- block_options.set_bloom_filter(64.0, true);+ block_options.set_ribbon_filter(10.0);options.set_block_based_table_factory(&block_options);let inner = rocksdb::DB::open(&options, path.as_ref())?;diff --git a/core/src/crawler/mod.rs b/core/src/crawler/mod.rsindex 06db7eba..8beef4a4 100644--- a/core/src/crawler/mod.rs+++ b/core/src/crawler/mod.rs@@ -25,7 +25,7 @@ use url::Url;use crate::config::CrawlerConfig;-use self::{warc_writer::WarcWriter, worker::Worker};+use self::{warc_writer::WarcWriter, worker::WorkerThread};pub mod coordinator;pub mod crawl_db;@@ -148,8 +148,8 @@ impl Crawler {let mut handles = Vec::new();let coordinator_host = config.coordinator_host.parse()?;- for _ in 0..config.num_workers {- let worker = Worker::new(+ for _ in 0..config.num_worker_threads {+ let worker = WorkerThread::new(Arc::clone(&pending_commands),Arc::clone(&writer),Arc::clone(&results),diff --git a/core/src/crawler/worker.rs b/core/src/crawler/worker.rsindex 58a4a1be..9a2d0f2a 100644--- a/core/src/crawler/worker.rs+++ b/core/src/crawler/worker.rs@@ -38,7 +38,7 @@ use super::{WarcWriter, WorkerJob,};-pub struct Worker {+pub struct WorkerThread {current_job: Option<WorkerJob>,pending_commands: Arc<Mutex<VecDeque<Command>>>,writer: Arc<WarcWriter>,@@ -51,7 +51,7 @@ pub struct Worker {robotstxt: RobotsTxtManager,}-impl Worker {+impl WorkerThread {pub fn new(pending_commands: Arc<Mutex<VecDeque<Command>>>,writer: Arc<WarcWriter>,@@ -82,7 +82,7 @@ impl Worker {results,current_job: None,pending_commands,- num_jobs_per_fetch: config.num_workers,+ num_jobs_per_fetch: config.num_worker_threads,politeness_factor: config.politeness_factor,config,coordinator_host,diff --git a/core/src/ranking/centrality_store.rs b/core/src/ranking/centrality_store.rsindex 4d2df7b9..cad01f5b 100644--- a/core/src/ranking/centrality_store.rs+++ b/core/src/ranking/centrality_store.rs@@ -43,7 +43,7 @@ impl Node2Id {options.set_max_write_buffer_number(8);let mut block_options = BlockBasedOptions::default();- block_options.set_bloom_filter(64.0, true);+ block_options.set_ribbon_filter(10.0);let cache = rocksdb::Cache::new_lru_cache(256 * 1024 * 1024).unwrap(); // 256 MB cacheblock_options.set_block_cache(&cache);diff --git a/core/src/webgraph/mod.rs b/core/src/webgraph/mod.rsindex 3796cd31..9593928a 100644--- a/core/src/webgraph/mod.rs+++ b/core/src/webgraph/mod.rs@@ -393,7 +393,7 @@ impl Webgraph {segments,executor: Arc::new(Executor::multi_thread("webgraph").unwrap()),id2node: Store::open(path.as_ref().join("id2node")),- id2node_cache: Mutex::new(LruCache::new(500_000.try_into().unwrap())),+ id2node_cache: Mutex::new(LruCache::new(10_000.try_into().unwrap())),meta,}}diff --git a/core/src/webgraph/store.rs b/core/src/webgraph/store.rsindex 97c85e44..6be615da 100644--- a/core/src/webgraph/store.rs+++ b/core/src/webgraph/store.rs@@ -39,9 +39,9 @@ whereoptions.set_max_write_buffer_number(8);let mut block_options = BlockBasedOptions::default();- block_options.set_bloom_filter(128.0, true);+ block_options.set_ribbon_filter(5.0);- let cache = rocksdb::Cache::new_lru_cache(512 * 1024 * 1024).unwrap(); // 512 MB cache+ let cache = rocksdb::Cache::new_lru_cache(256 * 1024 * 1024).unwrap(); // 256 MB cacheblock_options.set_block_cache(&cache);block_options.set_block_size(128 * 1024); // 128 KB block sizediff --git a/frontend/src/pages/crawler.mdx b/frontend/src/pages/crawler.mdxindex 0c036f00..1aade6ab 100644--- a/frontend/src/pages/crawler.mdx+++ b/frontend/src/pages/crawler.mdx@@ -6,7 +6,7 @@ title: Stract Crawler# Stract CrawlerStractBot is the name of our crawler that collects pages from the web in order to index them into the search engine.It is written in Rust and the source code can be found [here](https://github.com/StractOrg/stract/tree/main/core/src/crawler).-The crawler uses the user agent `Mozilla/5.0 (compatible; StractBot/0.1; +https://trystract.com/webmasters)`.+The crawler uses the user agent `Mozilla/5.0 (compatible; StractBot/0.1; open source search engine; +https://trystract.com/webmasters)`.## PolitenessStractBot is a polite crawler. It respects the [robots.txt](https://en.wikipedia.org/wiki/Robots.txt) file of the website it is crawling and tries to not overload the server.