web search done right
if a crawl job had no discovered urls, the domain might not have any valid range of of urls which would cause the coordinator to crash
Mikkel Denkerb0f2ca73683e
diff --git a/core/src/crawler/coordinator.rs b/core/src/crawler/coordinator.rsindex 16446b24..f19b214e 100644--- a/core/src/crawler/coordinator.rs+++ b/core/src/crawler/coordinator.rs@@ -84,16 +84,15 @@ impl CrawlCoordinator {let mut db = self.db.lock().unwrap();let start = Instant::now();- let nonempty_domains = db.insert_urls(responses).unwrap();+ let nonempty_domains = db.insert_urls(responses)?;for domain in &nonempty_domains {- db.set_domain_status(domain, DomainStatus::Pending).unwrap();+ db.set_domain_status(domain, DomainStatus::Pending)?;}for response in responses.iter() {if !nonempty_domains.contains(&response.domain) {- db.set_domain_status(&response.domain, DomainStatus::NoUncrawledUrls)- .unwrap();+ db.set_domain_status(&response.domain, DomainStatus::NoUncrawledUrls)?;}}diff --git a/core/src/crawler/crawl_db.rs b/core/src/crawler/crawl_db.rsindex e86b9396..df063907 100644--- a/core/src/crawler/crawl_db.rs+++ b/core/src/crawler/crawl_db.rs@@ -18,7 +18,6 @@ use dashmap::DashMap;use hashbrown::{HashMap, HashSet};use rand::Rng;use rayon::prelude::*;-use rkyv::Deserialize;use std::hash::Hash;use std::ops::Range;use std::path::PathBuf;@@ -31,7 +30,7 @@ use url::Url;use super::{Domain, Job, JobResponse, Result, UrlResponse};-const MAX_URL_DB_SIZE_BYTES: u64 = 5 * 1024 * 1024 * 1024; // 5GB+const MAX_URL_DB_SIZE_BYTES: u64 = 10 * 1024 * 1024 * 1024; // 10GB#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]pub enum UrlStatus {@@ -41,8 +40,7 @@ pub enum UrlStatus {Done,}-#[derive(Debug, Clone, Copy, PartialEq, Eq, rkyv::Serialize, rkyv::Deserialize, rkyv::Archive)]-#[archive(check_bytes)]+#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)]pub enum DomainStatus {Pending,NoUncrawledUrls,@@ -111,8 +109,7 @@ impl Default for UrlState {}}-#[derive(Clone, rkyv::Serialize, rkyv::Deserialize, rkyv::Archive)]-#[archive(check_bytes)]+#[derive(Clone, serde::Serialize, serde::Deserialize)]struct DomainState {weight: f64,status: DomainStatus,@@ -352,7 +349,9 @@ impl UrlStateDbShard {self.db.write_opt(batch, &write_options)?;- self.ranges.put(domain, range.as_ref().unwrap())?;+ if let Some(range) = range {+ self.ranges.put(domain, &range)?;+ }Ok(())}@@ -371,8 +370,11 @@ impl UrlStateDbShard {Ok(iter.take_while(|r| {- let (key, _) = r.as_ref().unwrap();- &**key <= end.as_slice()+ if let Ok((key, _)) = r.as_ref() {+ &**key <= end.as_slice()+ } else {+ false+ }}).filter_map(|r| {let (key, value) = r.ok()?;@@ -395,7 +397,7 @@ impl UrlStateDbShard {self.approx_size_bytes = self.db.property_int_value(rocksdb::properties::TOTAL_SST_FILES_SIZE)?- .unwrap()+ .unwrap_or_default().into();}@@ -523,21 +525,19 @@ impl DomainStateDb {}fn get(&self, domain: &Domain) -> Result<Option<DomainState>> {- let domain_bytes = rkyv::to_bytes::<_, 256>(domain)?;+ let domain_bytes = bincode::serialize(&domain)?;let value_bytes = self.db.get(domain_bytes)?;if let Some(value_bytes) = &value_bytes {- let archived = rkyv::check_archived_root::<DomainState>(&value_bytes[..]).unwrap();- let value = archived.deserialize(&mut rkyv::Infallible).unwrap();- return Ok(Some(value));+ return Ok(Some(bincode::deserialize(&value_bytes[..])?));}Ok(None)}fn put(&self, domain: &Domain, state: &DomainState) -> Result<()> {- let domain_bytes = rkyv::to_bytes::<_, 256>(domain)?;- let state_bytes = rkyv::to_bytes::<_, 256>(state)?;+ let domain_bytes = bincode::serialize(domain)?;+ let state_bytes = bincode::serialize(state)?;let mut write_options = rocksdb::WriteOptions::default();write_options.disable_wal(true);@@ -551,11 +551,8 @@ impl DomainStateDb {iter.filter_map(|r| {let (key, value) = r.ok()?;- let domain_archive = rkyv::check_archived_root::<Domain>(&key[..]).ok()?;- let state_archive = rkyv::check_archived_root::<DomainState>(&value[..]).ok()?;-- let domain: Domain = domain_archive.deserialize(&mut rkyv::Infallible).ok()?;- let state: DomainState = state_archive.deserialize(&mut rkyv::Infallible).ok()?;+ let domain = bincode::deserialize(&key[..]).ok()?;+ let state = bincode::deserialize(&value[..]).ok()?;Some((domain, state))})@@ -642,7 +639,7 @@ impl CrawlDb {for url_res in &res.url_responses {if let UrlResponse::Redirected { url, new_url } = url_res {- self.redirects.put(url, new_url).unwrap();+ self.redirects.put(url, new_url).ok();}}});@@ -721,7 +718,7 @@ impl CrawlDb {);for domain in sampled.iter() {- let mut state = self.domain_state.get(domain)?.unwrap();+ let mut state = self.domain_state.get(domain)?.unwrap_or_default();state.status = DomainStatus::CrawlInProgress;self.domain_state.put(domain, &state)?;}@@ -753,7 +750,7 @@ impl CrawlDb {let mut new_url_states = Vec::new();for url in &sampled {- let mut state = self.urls.get(domain, url)?.unwrap();+ let mut state = self.urls.get(domain, url)?.unwrap_or_default();state.status = UrlStatus::Crawling;new_url_states.push(((*url).clone(), state));@@ -761,7 +758,7 @@ impl CrawlDb {self.urls.put_batch(domain, &new_url_states)?;- let mut domain_state = self.domain_state.get(domain)?.unwrap();+ let mut domain_state = self.domain_state.get(domain)?.unwrap_or_default();domain_state.weight = urls.iter()