web search done right
Improve crawldb insert performance. Introduce a limit to maximum number of newly discovered domains per crawled domain. Also enables 'no_slowdown' for the inserts, even though the LOGs don't indicate that any write stalls is happening.
Mikkel Denkerbb21f41cd86c
diff --git a/core/examples/crawldb_insert.rs b/core/examples/crawldb_insert.rsnew file mode 100644index 00000000..2639cf3c--- /dev/null+++ b/core/examples/crawldb_insert.rs@@ -0,0 +1,52 @@+use anyhow::Result;+use hashbrown::HashMap;+use stract::crawler::{crawl_db::CrawlDb, Domain, UrlString, UrlToInsert};++fn random_domains() -> Vec<String> {+ let mut domains = Vec::new();++ for i in 0..1024 {+ domains.push(format!("domain-{}", i));+ }++ domains+}++fn random_urls(domain: &str) -> Vec<String> {+ let mut urls = Vec::new();++ for i in 0..256 {+ urls.push(format!("https://{domain}/{i}"));+ }++ urls+}++fn main() -> Result<()> {+ let mut db = CrawlDb::open("data/crawldb")?;++ loop {+ let domains = random_domains();++ let mut urls = HashMap::new();++ for domain in &domains {+ let rand_urls = random_urls(domain);++ urls.insert(+ Domain::from(domain.clone()),+ rand_urls+ .into_iter()+ .map(|url| UrlToInsert {+ url: UrlString(url),+ weight: 1.0,+ })+ .collect(),+ );+ }++ let start = std::time::Instant::now();+ db.insert_urls(urls)?;+ println!("inserted in {:?}", start.elapsed());+ }+}diff --git a/core/src/crawler/crawl_db.rs b/core/src/crawler/crawl_db.rsindex 2f657859..68a29dcc 100644--- a/core/src/crawler/crawl_db.rs+++ b/core/src/crawler/crawl_db.rs@@ -16,12 +16,11 @@use hashbrown::HashMap;use rand::Rng;-use std::hash::Hash;use std::path::PathBuf;use std::{collections::BinaryHeap, path::Path};use url::Url;-use super::{Domain, DomainCrawled, Job, Result, UrlToInsert, MAX_URL_LEN_BYTES};+use super::{Domain, DomainCrawled, Job, Result, UrlString, UrlToInsert, MAX_URL_LEN_BYTES};const MAX_URL_DB_SIZE_BYTES: u64 = 20 * 1024 * 1024 * 1024; // 20GB@@ -154,8 +153,6 @@ impl UrlStateDbShard {options.set_target_file_size_base(512 * 1024 * 1024); // 512 MBoptions.set_target_file_size_multiplier(10);- options.set_max_write_buffer_number(4);- options.set_min_write_buffer_number_to_merge(2);options.set_level_zero_slowdown_writes_trigger(-1);options.set_level_zero_stop_writes_trigger(-1);@@ -172,7 +169,6 @@ impl UrlStateDbShard {block_options.set_pin_l0_filter_and_index_blocks_in_cache(true);options.set_block_based_table_factory(&block_options);- options.set_optimize_filters_for_hits(true);options.set_compression_type(rocksdb::DBCompressionType::None);@@ -211,6 +207,7 @@ impl UrlStateDbShard {let mut write_options = rocksdb::WriteOptions::default();write_options.disable_wal(true);+ write_options.set_no_slowdown(true);self.db.write_opt(rocksdb_batch, &write_options)?;@@ -461,6 +458,7 @@ impl DomainStateDb {let mut write_options = rocksdb::WriteOptions::default();write_options.disable_wal(true);+ write_options.set_no_slowdown(true);self.db.write_opt(rocksdb_batch, &write_options)?;@@ -480,30 +478,6 @@ impl DomainStateDb {}}-#[derive(- Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, serde::Serialize, serde::Deserialize,-)]-struct UrlString(String);--impl From<&Url> for UrlString {- fn from(url: &Url) -> Self {- Self(url.as_str().to_string())- }-}--impl From<Url> for UrlString {- fn from(url: Url) -> Self {- Self(url.as_str().to_string())- }-}--impl TryFrom<&UrlString> for Url {- type Error = anyhow::Error;- fn try_from(url: &UrlString) -> Result<Self, Self::Error> {- Ok(Url::parse(&url.0)?)- }-}-pub struct CrawlDb {domain_state: DomainStateDb,urls: UrlStateDb,@@ -526,10 +500,7 @@ impl CrawlDb {let mut url_states = self.urls.multi_get(domain_urls.iter()- .flat_map(|(domain, urls)| {- urls.iter()- .map(|url| (domain.clone(), UrlString::from(&url.url)))- })+ .flat_map(|(domain, urls)| urls.iter().map(|url| (domain.clone(), url.url.clone()))).collect::<Vec<_>>().as_slice(),)?;@@ -544,12 +515,12 @@ impl CrawlDb {continue;}- match url_states.get_mut(&UrlString::from(&url.url)) {+ match url_states.get_mut(&url.url) {Some(state) => {state.weight += url.weight;domain_state.weight += url.weight;if url.weight > 0.0 {- updated_url_states.push((UrlString::from(&url.url), state.clone()));+ updated_url_states.push((url.url.clone(), state.clone()));}}None => {@@ -557,7 +528,7 @@ impl CrawlDb {state.weight += url.weight;domain_state.weight += url.weight;- updated_url_states.push((UrlString::from(&url.url), state));+ updated_url_states.push((url.url.clone(), state));}};}@@ -704,7 +675,7 @@ mod tests {for url in urls {let domain = Domain::from(url);let url_to_insert = UrlToInsert {- url: url.clone(),+ url: UrlString::from(url),weight: 0.0,};diff --git a/core/src/crawler/mod.rs b/core/src/crawler/mod.rsindex c4bf392e..93cb640f 100644--- a/core/src/crawler/mod.rs+++ b/core/src/crawler/mod.rs@@ -38,7 +38,10 @@ mod worker;pub use coordinator::CrawlCoordinator;pub const MAX_URL_LEN_BYTES: usize = 8192;-pub const MAX_URLS_FOR_DOMAIN_PER_INSERT: usize = 500;+pub const MAX_URLS_FOR_DOMAIN_PER_INSERT: usize = 256;+/// Number of new domains that can be discovered for each domain crawled.+pub const MAX_DOMAIN_DISCOVERY_FACTOR: usize = 4;+pub const MAX_OUTGOING_URLS_PER_PAGE: usize = 200;#[derive(thiserror::Error, Debug)]@@ -135,9 +138,38 @@ pub struct JobResponse {pub weight_budget: f64,}+#[derive(+ Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, serde::Serialize, serde::Deserialize,+)]+pub struct UrlString(pub String);+impl UrlString {+ fn as_str(&self) -> &str {+ &self.0+ }+}++impl From<&Url> for UrlString {+ fn from(url: &Url) -> Self {+ Self(url.as_str().to_string())+ }+}++impl From<Url> for UrlString {+ fn from(url: Url) -> Self {+ Self(url.as_str().to_string())+ }+}++impl TryFrom<&UrlString> for Url {+ type Error = anyhow::Error;+ fn try_from(url: &UrlString) -> Result<Self, Self::Error> {+ Ok(Url::parse(&url.0)?)+ }+}+#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]pub struct UrlToInsert {- pub url: Url,+ pub url: UrlString,pub weight: f64,}diff --git a/core/src/crawler/router.rs b/core/src/crawler/router.rsindex 2f9cb68f..02665c51 100644--- a/core/src/crawler/router.rs+++ b/core/src/crawler/router.rs@@ -10,7 +10,10 @@ use crate::{entrypoint::crawler::coordinator::{CoordinatorService, GetJobs, InsertUrls, MarkJobsComplete},};-use super::{Domain, Job, JobResponse, MAX_URLS_FOR_DOMAIN_PER_INSERT, MAX_URL_LEN_BYTES};+use super::{+ Domain, Job, JobResponse, UrlString, MAX_DOMAIN_DISCOVERY_FACTOR,+ MAX_URLS_FOR_DOMAIN_PER_INSERT, MAX_URL_LEN_BYTES,+};struct RemoteCoordinator {addr: SocketAddr,@@ -84,7 +87,10 @@ impl Router {.or_default().entry(domain).or_default()- .push(UrlToInsert { url, weight: 0.0 });+ .push(UrlToInsert {+ url: UrlString::from(url),+ weight: 0.0,+ });}let mut futures = Vec::new();@@ -112,11 +118,11 @@ impl Router {}pub async fn add_responses(&self, responses: &[JobResponse]) -> Result<()> {- let mut domain_urls: HashMap<Domain, Vec<UrlToInsert>> = HashMap::new();+ let mut domain_urls: HashMap<Domain, HashMap<UrlString, f64>> = HashMap::new();let mut domain_budgets: Vec<DomainCrawled> = Vec::new();for res in responses {- let mut urls: Vec<(Domain, Url)> = res+ let urls: Vec<(Domain, Url)> = res.discovered_urls.iter().map(|url| {@@ -130,9 +136,6 @@ impl Router {.filter(|(domain, _)| res.domain != *domain).count() as f64;- urls.sort_unstable_by(|(_, a), (_, b)| a.as_str().cmp(b.as_str()));- urls.dedup_by(|(_, a), (_, b)| a.as_str() == b.as_str());-let mut used_budget = 0.0;for (domain, url) in urls {@@ -156,7 +159,9 @@ impl Router {continue;}- urls.push(UrlToInsert { url, weight });+ urls.entry(UrlString::from(url.clone()))+ .and_modify(|w| *w = (*w + weight).min(1.0))+ .or_insert(weight);}domain_budgets.push(DomainCrawled {@@ -169,12 +174,17 @@ impl Router {HashMap::new();for (domain, urls) in domain_urls {+ let urls = urls+ .into_iter()+ .map(|(url, weight)| UrlToInsert { url, weight })+ .collect();let coordinator_index = self.coordinator_index(&domain);- coordinator_urls- .entry(coordinator_index)- .or_default()- .insert(domain, urls);+ let domains = coordinator_urls.entry(coordinator_index).or_default();++ if domains.len() < MAX_DOMAIN_DISCOVERY_FACTOR * responses.len() {+ domains.insert(domain, urls);+ }}let mut futures = Vec::new();diff --git a/core/src/entrypoint/crawler.rs b/core/src/entrypoint/crawler.rsindex 63e57b4e..e314f2ec 100644--- a/core/src/entrypoint/crawler.rs+++ b/core/src/entrypoint/crawler.rs@@ -121,7 +121,10 @@ pub mod coordinator {type Response = ();async fn handle(self, server: &CoordinatorService) -> sonic::Result<Self::Response> {- server.coordinator.insert_urls(self.urls)?;+ if let Err(e) = server.coordinator.insert_urls(self.urls) {+ tracing::error!("failed to insert urls: {}", e);+ }+Ok(())}}@@ -141,7 +144,10 @@ pub mod coordinator {type Response = ();async fn handle(self, server: &CoordinatorService) -> sonic::Result<Self::Response> {- server.coordinator.mark_jobs_complete(&self.domains)?;+ if let Err(e) = server.coordinator.mark_jobs_complete(&self.domains) {+ tracing::error!("failed to mark jobs complete: {}", e);+ }+Ok(())}}