web search done right
Disable WAL in rocksdb for crawl coordinator. We don't care about data loss in this part. If a crawl coordinator fails, we will have to restart the crawl anyway. Full speeeeeeed
Mikkel Denkera6e7d85cab61
diff --git a/core/src/crawler/coordinator.rs b/core/src/crawler/coordinator.rsindex 9bb813c6..c7168ba1 100644--- a/core/src/crawler/coordinator.rs+++ b/core/src/crawler/coordinator.rs@@ -86,6 +86,7 @@ impl CrawlCoordinator {.unwrap();tracing::debug!("inserted responses in {:?}", start.elapsed());+ dbg!(start.elapsed());Ok(())}diff --git a/core/src/crawler/crawl_db.rs b/core/src/crawler/crawl_db.rsindex e9051ef5..bfce167c 100644--- a/core/src/crawler/crawl_db.rs+++ b/core/src/crawler/crawl_db.rs@@ -63,10 +63,11 @@ wherepub fn open<P: AsRef<Path>>(path: P) -> Result<Self> {let mut options = rocksdb::Options::default();options.create_if_missing(true);- options.increase_parallelism(3);+ options.increase_parallelism(8);+ options.set_write_buffer_size(256 * 1024 * 1024); // 256 MB memtable+ options.set_max_write_buffer_number(8);let mut block_options = BlockBasedOptions::default();-block_options.set_bloom_filter(64.0, true);options.set_block_based_table_factory(&block_options);@@ -131,8 +132,12 @@ whereids.push(id);}- self.id2t.write(batch_id2t)?;- self.t2id.write(batch_t2id)?;+ let mut write_options = rocksdb::WriteOptions::default();+ write_options.set_sync(false);+ write_options.disable_wal(true);++ self.id2t.write_opt(batch_id2t, &write_options)?;+ self.t2id.write_opt(batch_t2id, &write_options)?;Ok(ids)}@@ -160,8 +165,13 @@ wherelet id = self.next_id;self.next_id += 1;let id_bytes = bincode::serialize(&id)?;- self.t2id.put(&item_bytes, &id_bytes)?;- self.id2t.put(&id_bytes, &item_bytes)?;++ let mut write_options = rocksdb::WriteOptions::default();+ write_options.set_sync(false);+ write_options.disable_wal(true);++ self.t2id.put_opt(&item_bytes, &id_bytes, &write_options)?;+ self.id2t.put_opt(&id_bytes, &item_bytes, &write_options)?;// update cacheself.t2id_cache.put(item.clone(), id);@@ -286,7 +296,12 @@ impl RedirectDb {pub fn put(&self, from: &Url, to: &Url) -> Result<()> {let url_bytes = bincode::serialize(from)?;let redirect_bytes = bincode::serialize(to)?;- self.inner.put(url_bytes, redirect_bytes)?;++ let mut write_options = rocksdb::WriteOptions::default();+ write_options.set_sync(false);+ write_options.disable_wal(true);+ self.inner+ .put_opt(url_bytes, redirect_bytes, &write_options)?;Ok(())}diff --git a/core/src/crawler/worker.rs b/core/src/crawler/worker.rsindex 050b3b65..a3893cbe 100644--- a/core/src/crawler/worker.rs+++ b/core/src/crawler/worker.rs@@ -294,7 +294,11 @@ impl Worker {self.politeness_factor = MAX_POLITENESS_FACTOR;}- tracing::warn!("politeness factor increased to {}", self.politeness_factor);+ tracing::warn!(+ "politeness factor increased to {} for {}",+ self.politeness_factor,+ &url+ );}tracing::debug!("failed to fetch url ({}): {}", &url, datum.status_code);