Skip to content

web search done right

Improve crawldb insert performance. Introduce a limit to maximum number of newly discovered domains per crawled domain. Also enables 'no_slowdown' for the inserts, even though the LOGs don't indicate that any write stalls is happening.

Mikkel Denkerbb21f41cd86c
diff --git a/core/examples/crawldb_insert.rs b/core/examples/crawldb_insert.rs
new file mode 100644
index 00000000..2639cf3c
--- /dev/null
+++ b/core/examples/crawldb_insert.rs
@@ -0,0 +1,52 @@
+use anyhow::Result;
+use hashbrown::HashMap;
+use stract::crawler::{crawl_db::CrawlDb, Domain, UrlString, UrlToInsert};
+
+fn random_domains() -> Vec<String> {
+ let mut domains = Vec::new();
+
+ for i in 0..1024 {
+ domains.push(format!("domain-{}", i));
+ }
+
+ domains
+}
+
+fn random_urls(domain: &str) -> Vec<String> {
+ let mut urls = Vec::new();
+
+ for i in 0..256 {
+ urls.push(format!("https://{domain}/{i}"));
+ }
+
+ urls
+}
+
+fn main() -> Result<()> {
+ let mut db = CrawlDb::open("data/crawldb")?;
+
+ loop {
+ let domains = random_domains();
+
+ let mut urls = HashMap::new();
+
+ for domain in &domains {
+ let rand_urls = random_urls(domain);
+
+ urls.insert(
+ Domain::from(domain.clone()),
+ rand_urls
+ .into_iter()
+ .map(|url| UrlToInsert {
+ url: UrlString(url),
+ weight: 1.0,
+ })
+ .collect(),
+ );
+ }
+
+ let start = std::time::Instant::now();
+ db.insert_urls(urls)?;
+ println!("inserted in {:?}", start.elapsed());
+ }
+}
diff --git a/core/src/crawler/crawl_db.rs b/core/src/crawler/crawl_db.rs
index 2f657859..68a29dcc 100644
--- a/core/src/crawler/crawl_db.rs
+++ b/core/src/crawler/crawl_db.rs
@@ -16,12 +16,11 @@
use hashbrown::HashMap;
use rand::Rng;
-use std::hash::Hash;
use std::path::PathBuf;
use std::{collections::BinaryHeap, path::Path};
use url::Url;
-use super::{Domain, DomainCrawled, Job, Result, UrlToInsert, MAX_URL_LEN_BYTES};
+use super::{Domain, DomainCrawled, Job, Result, UrlString, UrlToInsert, MAX_URL_LEN_BYTES};
const MAX_URL_DB_SIZE_BYTES: u64 = 20 * 1024 * 1024 * 1024; // 20GB
@@ -154,8 +153,6 @@ impl UrlStateDbShard {
options.set_target_file_size_base(512 * 1024 * 1024); // 512 MB
options.set_target_file_size_multiplier(10);
- options.set_max_write_buffer_number(4);
- options.set_min_write_buffer_number_to_merge(2);
options.set_level_zero_slowdown_writes_trigger(-1);
options.set_level_zero_stop_writes_trigger(-1);
@@ -172,7 +169,6 @@ impl UrlStateDbShard {
block_options.set_pin_l0_filter_and_index_blocks_in_cache(true);
options.set_block_based_table_factory(&block_options);
- options.set_optimize_filters_for_hits(true);
options.set_compression_type(rocksdb::DBCompressionType::None);
@@ -211,6 +207,7 @@ impl UrlStateDbShard {
let mut write_options = rocksdb::WriteOptions::default();
write_options.disable_wal(true);
+ write_options.set_no_slowdown(true);
self.db.write_opt(rocksdb_batch, &write_options)?;
@@ -461,6 +458,7 @@ impl DomainStateDb {
let mut write_options = rocksdb::WriteOptions::default();
write_options.disable_wal(true);
+ write_options.set_no_slowdown(true);
self.db.write_opt(rocksdb_batch, &write_options)?;
@@ -480,30 +478,6 @@ impl DomainStateDb {
}
}
-#[derive(
- Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, serde::Serialize, serde::Deserialize,
-)]
-struct UrlString(String);
-
-impl From<&Url> for UrlString {
- fn from(url: &Url) -> Self {
- Self(url.as_str().to_string())
- }
-}
-
-impl From<Url> for UrlString {
- fn from(url: Url) -> Self {
- Self(url.as_str().to_string())
- }
-}
-
-impl TryFrom<&UrlString> for Url {
- type Error = anyhow::Error;
- fn try_from(url: &UrlString) -> Result<Self, Self::Error> {
- Ok(Url::parse(&url.0)?)
- }
-}
-
pub struct CrawlDb {
domain_state: DomainStateDb,
urls: UrlStateDb,
@@ -526,10 +500,7 @@ impl CrawlDb {
let mut url_states = self.urls.multi_get(
domain_urls
.iter()
- .flat_map(|(domain, urls)| {
- urls.iter()
- .map(|url| (domain.clone(), UrlString::from(&url.url)))
- })
+ .flat_map(|(domain, urls)| urls.iter().map(|url| (domain.clone(), url.url.clone())))
.collect::<Vec<_>>()
.as_slice(),
)?;
@@ -544,12 +515,12 @@ impl CrawlDb {
continue;
}
- match url_states.get_mut(&UrlString::from(&url.url)) {
+ match url_states.get_mut(&url.url) {
Some(state) => {
state.weight += url.weight;
domain_state.weight += url.weight;
if url.weight > 0.0 {
- updated_url_states.push((UrlString::from(&url.url), state.clone()));
+ updated_url_states.push((url.url.clone(), state.clone()));
}
}
None => {
@@ -557,7 +528,7 @@ impl CrawlDb {
state.weight += url.weight;
domain_state.weight += url.weight;
- updated_url_states.push((UrlString::from(&url.url), state));
+ updated_url_states.push((url.url.clone(), state));
}
};
}
@@ -704,7 +675,7 @@ mod tests {
for url in urls {
let domain = Domain::from(url);
let url_to_insert = UrlToInsert {
- url: url.clone(),
+ url: UrlString::from(url),
weight: 0.0,
};
diff --git a/core/src/crawler/mod.rs b/core/src/crawler/mod.rs
index c4bf392e..93cb640f 100644
--- a/core/src/crawler/mod.rs
+++ b/core/src/crawler/mod.rs
@@ -38,7 +38,10 @@ mod worker;
pub use coordinator::CrawlCoordinator;
pub const MAX_URL_LEN_BYTES: usize = 8192;
-pub const MAX_URLS_FOR_DOMAIN_PER_INSERT: usize = 500;
+pub const MAX_URLS_FOR_DOMAIN_PER_INSERT: usize = 256;
+/// Number of new domains that can be discovered for each domain crawled.
+pub const MAX_DOMAIN_DISCOVERY_FACTOR: usize = 4;
+
pub const MAX_OUTGOING_URLS_PER_PAGE: usize = 200;
#[derive(thiserror::Error, Debug)]
@@ -135,9 +138,38 @@ pub struct JobResponse {
pub weight_budget: f64,
}
+#[derive(
+ Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, serde::Serialize, serde::Deserialize,
+)]
+pub struct UrlString(pub String);
+impl UrlString {
+ fn as_str(&self) -> &str {
+ &self.0
+ }
+}
+
+impl From<&Url> for UrlString {
+ fn from(url: &Url) -> Self {
+ Self(url.as_str().to_string())
+ }
+}
+
+impl From<Url> for UrlString {
+ fn from(url: Url) -> Self {
+ Self(url.as_str().to_string())
+ }
+}
+
+impl TryFrom<&UrlString> for Url {
+ type Error = anyhow::Error;
+ fn try_from(url: &UrlString) -> Result<Self, Self::Error> {
+ Ok(Url::parse(&url.0)?)
+ }
+}
+
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
pub struct UrlToInsert {
- pub url: Url,
+ pub url: UrlString,
pub weight: f64,
}
diff --git a/core/src/crawler/router.rs b/core/src/crawler/router.rs
index 2f9cb68f..02665c51 100644
--- a/core/src/crawler/router.rs
+++ b/core/src/crawler/router.rs
@@ -10,7 +10,10 @@ use crate::{
entrypoint::crawler::coordinator::{CoordinatorService, GetJobs, InsertUrls, MarkJobsComplete},
};
-use super::{Domain, Job, JobResponse, MAX_URLS_FOR_DOMAIN_PER_INSERT, MAX_URL_LEN_BYTES};
+use super::{
+ Domain, Job, JobResponse, UrlString, MAX_DOMAIN_DISCOVERY_FACTOR,
+ MAX_URLS_FOR_DOMAIN_PER_INSERT, MAX_URL_LEN_BYTES,
+};
struct RemoteCoordinator {
addr: SocketAddr,
@@ -84,7 +87,10 @@ impl Router {
.or_default()
.entry(domain)
.or_default()
- .push(UrlToInsert { url, weight: 0.0 });
+ .push(UrlToInsert {
+ url: UrlString::from(url),
+ weight: 0.0,
+ });
}
let mut futures = Vec::new();
@@ -112,11 +118,11 @@ impl Router {
}
pub async fn add_responses(&self, responses: &[JobResponse]) -> Result<()> {
- let mut domain_urls: HashMap<Domain, Vec<UrlToInsert>> = HashMap::new();
+ let mut domain_urls: HashMap<Domain, HashMap<UrlString, f64>> = HashMap::new();
let mut domain_budgets: Vec<DomainCrawled> = Vec::new();
for res in responses {
- let mut urls: Vec<(Domain, Url)> = res
+ let urls: Vec<(Domain, Url)> = res
.discovered_urls
.iter()
.map(|url| {
@@ -130,9 +136,6 @@ impl Router {
.filter(|(domain, _)| res.domain != *domain)
.count() as f64;
- urls.sort_unstable_by(|(_, a), (_, b)| a.as_str().cmp(b.as_str()));
- urls.dedup_by(|(_, a), (_, b)| a.as_str() == b.as_str());
-
let mut used_budget = 0.0;
for (domain, url) in urls {
@@ -156,7 +159,9 @@ impl Router {
continue;
}
- urls.push(UrlToInsert { url, weight });
+ urls.entry(UrlString::from(url.clone()))
+ .and_modify(|w| *w = (*w + weight).min(1.0))
+ .or_insert(weight);
}
domain_budgets.push(DomainCrawled {
@@ -169,12 +174,17 @@ impl Router {
HashMap::new();
for (domain, urls) in domain_urls {
+ let urls = urls
+ .into_iter()
+ .map(|(url, weight)| UrlToInsert { url, weight })
+ .collect();
let coordinator_index = self.coordinator_index(&domain);
- coordinator_urls
- .entry(coordinator_index)
- .or_default()
- .insert(domain, urls);
+ let domains = coordinator_urls.entry(coordinator_index).or_default();
+
+ if domains.len() < MAX_DOMAIN_DISCOVERY_FACTOR * responses.len() {
+ domains.insert(domain, urls);
+ }
}
let mut futures = Vec::new();
diff --git a/core/src/entrypoint/crawler.rs b/core/src/entrypoint/crawler.rs
index 63e57b4e..e314f2ec 100644
--- a/core/src/entrypoint/crawler.rs
+++ b/core/src/entrypoint/crawler.rs
@@ -121,7 +121,10 @@ pub mod coordinator {
type Response = ();
async fn handle(self, server: &CoordinatorService) -> sonic::Result<Self::Response> {
- server.coordinator.insert_urls(self.urls)?;
+ if let Err(e) = server.coordinator.insert_urls(self.urls) {
+ tracing::error!("failed to insert urls: {}", e);
+ }
+
Ok(())
}
}
@@ -141,7 +144,10 @@ pub mod coordinator {
type Response = ();
async fn handle(self, server: &CoordinatorService) -> sonic::Result<Self::Response> {
- server.coordinator.mark_jobs_complete(&self.domains)?;
+ if let Err(e) = server.coordinator.mark_jobs_complete(&self.domains) {
+ tracing::error!("failed to mark jobs complete: {}", e);
+ }
+
Ok(())
}
}