Skip to content

web search done right

Shard url state database. Some domains had a very (!) large number of urls. A lot of time was spend reading and writing the urlstates to/from disk for these domains. We now shard the urls and choose a random shard when sampling. If there are no valid urls in the shard, then the job will simply contain 0 urls, the worker will quickly finish the job and request a new domain to crawl.

Mikkel Denker87b6699e7ae0
diff --git a/Cargo.toml b/Cargo.toml
index dd136093..9852fc51 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -97,5 +97,5 @@ utoipa = { version = "3.4.4", features = ["axum_extras"] }
utoipa-swagger-ui = { version = "3.1.4", features = ["axum"] }
tikv-jemallocator = "0.5"
publicsuffix = "2.2.3"
-rkyv = { version = "0.7.42", features = ["validation"] }
+rkyv = { version = "0.7.42", default_features = false, features = ["std", "validation", "size_64"] }
diff --git a/core/examples/crawl_coordinator_memory_stress.rs b/core/examples/crawl_coordinator_memory_stress.rs
index 6f7143c3..4c2803ce 100644
--- a/core/examples/crawl_coordinator_memory_stress.rs
+++ b/core/examples/crawl_coordinator_memory_stress.rs
@@ -16,7 +16,7 @@ fn rand_domain() -> Domain {
// and appending .com
let mut rng = rand::thread_rng();
let mut domain = String::new();
- for _ in 0..rng.gen_range(1..10) {
+ for _ in 0..rng.gen_range(1..3) {
domain.push(rng.gen_range(b'a'..=b'z') as char);
}
domain.push_str(".com");
diff --git a/core/src/crawler/coordinator.rs b/core/src/crawler/coordinator.rs
index 29a20d54..8fc3049b 100644
--- a/core/src/crawler/coordinator.rs
+++ b/core/src/crawler/coordinator.rs
@@ -85,7 +85,6 @@ impl CrawlCoordinator {
let mut db = self.db.lock().unwrap();
db.insert_urls(responses).unwrap();
- db.update_url_status(responses).unwrap();
for response in responses.iter() {
db.set_domain_status(&response.domain, DomainStatus::Pending)
diff --git a/core/src/crawler/crawl_db.rs b/core/src/crawler/crawl_db.rs
index 6e76fdc7..370f5a14 100644
--- a/core/src/crawler/crawl_db.rs
+++ b/core/src/crawler/crawl_db.rs
@@ -26,6 +26,8 @@ use url::Url;
use super::{Domain, Job, JobResponse, Result, UrlResponse};
+const URLS_PER_SHARD: usize = 5_000;
+
#[derive(Clone, PartialEq, Eq, Serialize, Deserialize, Archive)]
#[archive(check_bytes)]
pub enum UrlStatus {
@@ -103,6 +105,7 @@ struct UrlState {
struct DomainState {
weight: f64,
status: DomainStatus,
+ max_shard_id: u64,
}
pub struct RedirectDb {
@@ -149,6 +152,13 @@ impl RedirectDb {
}
}
+#[derive(Clone, Serialize, Deserialize, Archive)]
+#[archive(check_bytes)]
+struct DomainShard {
+ domain: Domain,
+ shard_id: u64,
+}
+
struct UrlToInsert {
url: Url,
different_domain: bool,
@@ -183,7 +193,7 @@ impl UrlStateDb {
Ok(Self { db })
}
- pub fn get(&mut self, key: &Domain) -> Result<Option<BTreeMap<UrlString, UrlState>>> {
+ pub fn get(&mut self, key: &DomainShard) -> Result<Option<BTreeMap<UrlString, UrlState>>> {
let key_bytes = rkyv::to_bytes::<_, 1024>(key)?;
let value_bytes = self.db.get(key_bytes)?;
@@ -199,7 +209,7 @@ impl UrlStateDb {
}
}
- pub fn put(&mut self, key: &Domain, value: &BTreeMap<UrlString, UrlState>) -> Result<()> {
+ pub fn put(&mut self, key: &DomainShard, value: &BTreeMap<UrlString, UrlState>) -> Result<()> {
let key_bytes = rkyv::to_bytes::<_, 1024>(key)?;
let value_bytes = rkyv::to_bytes::<_, 1024>(value)?;
@@ -301,25 +311,21 @@ impl From<&UrlString> for Url {
}
pub struct CrawlDb {
- redirects: RedirectDb,
-
domain_state: DomainStateDb,
-
urls: UrlStateDb,
+ redirects: RedirectDb,
}
impl CrawlDb {
pub fn open<P: AsRef<Path>>(path: P) -> Result<Self> {
- // if Path::new(path.as_ref()).exists() {
- // return Err(anyhow::anyhow!(
- // "crawl db already exists and might be in incorrect state"
- // ));
- // }
-
- let redirects = RedirectDb::open(path.as_ref().join("redirects"))?;
+ if Path::new(path.as_ref()).exists() {
+ return Err(anyhow::anyhow!(
+ "crawl db already exists and might be in incorrect state"
+ ));
+ }
Ok(Self {
- redirects,
+ redirects: RedirectDb::open(path.as_ref().join("redirects"))?,
domain_state: DomainStateDb::open(path.as_ref().join("domains"))?,
urls: UrlStateDb::open(path.as_ref().join("urls"))?,
})
@@ -334,10 +340,16 @@ impl CrawlDb {
&DomainState {
weight: 0.0,
status: DomainStatus::Pending,
+ max_shard_id: 0,
},
)?;
- let mut urls = self.urls.get(&domain)?.unwrap_or_default();
+ let sharded_domain = DomainShard {
+ domain,
+ shard_id: 0,
+ };
+
+ let mut urls = self.urls.get(&sharded_domain)?.unwrap_or_default();
urls.insert(
url.into(),
@@ -347,7 +359,7 @@ impl CrawlDb {
},
);
- self.urls.put(&domain, &urls)?;
+ self.urls.put(&sharded_domain, &urls)?;
}
Ok(())
@@ -366,6 +378,12 @@ impl CrawlDb {
different_domain,
});
}
+
+ for url_res in &res.url_responses {
+ if let UrlResponse::Redirected { url, new_url } = url_res {
+ self.redirects.put(url, new_url).unwrap();
+ }
+ }
});
for (domain, urls) in domains.into_iter() {
@@ -375,6 +393,7 @@ impl CrawlDb {
let state = DomainState {
weight: 0.0,
status: DomainStatus::Pending,
+ max_shard_id: 0,
};
self.domain_state.put(&domain, &state)?;
@@ -382,7 +401,19 @@ impl CrawlDb {
}
};
- let mut url_states = self.urls.get(&domain)?.unwrap_or_default();
+ let mut sharded_domain = DomainShard {
+ domain,
+ shard_id: domain_state.max_shard_id,
+ };
+
+ let mut url_states = self.urls.get(&sharded_domain)?.unwrap_or_default();
+
+ if url_states.len() >= URLS_PER_SHARD {
+ domain_state.max_shard_id += 1;
+ sharded_domain.shard_id = domain_state.max_shard_id;
+
+ url_states = BTreeMap::new();
+ }
for url in urls {
let mut url_state = url_states
@@ -404,94 +435,10 @@ impl CrawlDb {
url_states.insert(url.url.into(), url_state);
}
- self.domain_state.put(&domain, &domain_state)?;
-
- self.urls.put(&domain, &url_states)?;
- }
-
- Ok(())
- }
-
- pub fn update_url_status(&mut self, job_responses: &[JobResponse]) -> Result<()> {
- let mut url_responses: HashMap<Domain, Vec<UrlResponse>> = HashMap::new();
-
- for res in job_responses {
- for url_response in &res.url_responses {
- match url_response {
- UrlResponse::Success { url } => {
- let domain = Domain::from(url);
- url_responses
- .entry(domain)
- .or_default()
- .push(url_response.clone());
- }
- UrlResponse::Failed {
- url,
- status_code: _,
- } => {
- let domain = Domain::from(url);
- url_responses
- .entry(domain)
- .or_default()
- .push(url_response.clone());
- }
- UrlResponse::Redirected { url, new_url: _ } => {
- let domain = Domain::from(url);
- url_responses
- .entry(domain)
- .or_default()
- .push(url_response.clone());
- }
- }
- }
- }
-
- for (domain, responses) in url_responses {
- if self.domain_state.get(&domain)?.is_none() {
- self.domain_state.put(
- &domain,
- &DomainState {
- weight: 0.0,
- status: DomainStatus::Pending,
- },
- )?;
- }
-
- let mut url_states = self.urls.get(&domain)?.unwrap_or_default();
- for response in responses {
- match response {
- UrlResponse::Success { url } => {
- let mut url_state = url_states
- .get(&UrlString::from(&url))
- .cloned()
- .unwrap_or(UrlState {
- weight: 0.0,
- status: UrlStatus::Pending,
- });
-
- url_state.status = UrlStatus::Done;
-
- url_states.insert(UrlString::from(&url), url_state);
- }
- UrlResponse::Failed { url, status_code } => {
- let mut url_state = url_states
- .get(&UrlString::from(&url))
- .cloned()
- .unwrap_or(UrlState {
- weight: 0.0,
- status: UrlStatus::Pending,
- });
-
- url_state.status = UrlStatus::Failed { status_code };
- url_states.insert(url.into(), url_state);
- }
- UrlResponse::Redirected { url, new_url } => {
- self.redirects.put(&url, &new_url).ok();
- }
- }
- }
+ self.domain_state
+ .put(&sharded_domain.domain, &domain_state)?;
- self.urls.put(&domain, &url_states)?;
+ self.urls.put(&sharded_domain, &url_states)?;
}
Ok(())
@@ -501,6 +448,7 @@ impl CrawlDb {
let mut domain_state = self.domain_state.get(domain)?.unwrap_or(DomainState {
weight: 0.0,
status,
+ max_shard_id: 0,
});
domain_state.status = status;
@@ -534,7 +482,16 @@ impl CrawlDb {
pub fn prepare_jobs(&mut self, domains: &[Domain], urls_per_job: usize) -> Result<Vec<Job>> {
let mut jobs = Vec::with_capacity(domains.len());
for domain in domains {
- let mut urls = self.urls.get(domain)?.unwrap_or_default();
+ let state = self.domain_state.get(domain)?.unwrap();
+
+ let shard_id = rand::thread_rng().gen_range(0..=state.max_shard_id);
+
+ let shard = DomainShard {
+ domain: domain.clone(),
+ shard_id,
+ };
+
+ let mut urls = self.urls.get(&shard)?.unwrap_or_default();
let available_urls: Vec<_> = urls
.iter()
.filter_map(|(url, state)| {
@@ -586,7 +543,7 @@ impl CrawlDb {
jobs.push(job);
- self.urls.put(domain, &urls)?;
+ self.urls.put(&shard, &urls)?;
}
Ok(jobs)