Skip to content

web search done right

insert crawler job responses in batches

Mikkel Denkere1c16e3f2012
diff --git a/core/src/crawler/coordinator.rs b/core/src/crawler/coordinator.rs
index 9bb813c6..824f000e 100644
--- a/core/src/crawler/coordinator.rs
+++ b/core/src/crawler/coordinator.rs
@@ -36,6 +36,7 @@ pub struct CrawlCoordinator {
num_crawled_urls: AtomicU64,
num_urls_to_crawl: u64,
call_counter: Mutex<CallCounter>,
+ response_queue: Mutex<Vec<JobResponse>>,
}
impl CrawlCoordinator {
@@ -55,6 +56,7 @@ impl CrawlCoordinator {
num_urls_to_crawl,
num_crawled_urls: AtomicU64::new(0),
call_counter: Mutex::new(CallCounter::new(Duration::from_secs(10))),
+ response_queue: Mutex::new(Vec::new()),
})
}
@@ -70,20 +72,30 @@ impl CrawlCoordinator {
}
pub fn add_response(&self, response: &JobResponse) -> Result<()> {
+ self.log_crawls_per_second(response.url_responses.len());
+ self.num_crawled_urls
+ .fetch_add(response.url_responses.len() as u64, Ordering::SeqCst);
+
+ self.response_queue.lock().unwrap().push(response.clone());
+
+ Ok(())
+ }
+
+ fn flush_responses(&self) -> Result<()> {
let start = Instant::now();
+ let mut response_queue = self.response_queue.lock().unwrap();
let mut db = self.db.lock().unwrap();
- self.log_crawls_per_second(response.url_responses.len());
-
- self.num_crawled_urls.fetch_add(1, Ordering::SeqCst);
+ db.insert_urls(&response_queue).unwrap();
- db.insert_urls(&response.domain, &response.discovered_urls)
- .unwrap();
+ db.update_url_status(&response_queue).unwrap();
- db.update_url_status(&response.url_responses).unwrap();
+ for response in response_queue.iter() {
+ db.set_domain_status(&response.domain, DomainStatus::Pending)
+ .unwrap();
+ }
- db.set_domain_status(&response.domain, DomainStatus::Pending)
- .unwrap();
+ response_queue.clear();
tracing::debug!("inserted responses in {:?}", start.elapsed());
@@ -96,6 +108,7 @@ impl CrawlCoordinator {
pub fn sample_jobs(&self, num_jobs: usize) -> Result<Vec<Job>> {
let start = Instant::now();
+ self.flush_responses()?;
let mut db = self.db.lock().unwrap();
let domains = db.sample_domains(num_jobs)?;
diff --git a/core/src/crawler/crawl_db.rs b/core/src/crawler/crawl_db.rs
index 13dbc9ff..9656e2ee 100644
--- a/core/src/crawler/crawl_db.rs
+++ b/core/src/crawler/crawl_db.rs
@@ -21,13 +21,14 @@ use std::{
path::Path,
};
+use itertools::Itertools;
use lru::LruCache;
use rand::Rng;
use rocksdb::BlockBasedOptions;
use crate::webpage::Url;
-use super::{Domain, Job, Result, UrlResponse};
+use super::{Domain, Job, JobResponse, Result, UrlResponse};
#[derive(Clone, PartialEq, Eq)]
pub enum UrlStatus {
@@ -305,6 +306,11 @@ impl RedirectDb {
}
}
+struct UrlToInsert {
+ url: Url,
+ different_domain: bool,
+}
+
pub struct CrawlDb {
url_ids: IdTable<Url>,
domain_ids: IdTable<Domain>,
@@ -355,13 +361,19 @@ impl CrawlDb {
Ok(())
}
- pub fn insert_urls(&mut self, crawled_domain: &Domain, urls: &[Url]) -> Result<()> {
- let mut domains: HashMap<Domain, Vec<Url>> = HashMap::new();
+ pub fn insert_urls(&mut self, responses: &[JobResponse]) -> Result<()> {
+ let mut domains: HashMap<Domain, Vec<UrlToInsert>> = HashMap::new();
- for url in urls {
- let domain: Domain = url.domain().to_string().into();
+ for res in responses {
+ for url in &res.discovered_urls {
+ let domain: Domain = url.domain().to_string().into();
+ let different_domain = res.domain != domain;
- domains.entry(domain).or_default().push(url.clone());
+ domains.entry(domain).or_default().push(UrlToInsert {
+ url: url.clone(),
+ different_domain,
+ });
+ }
}
let domain_ids: Vec<DomainId> = self
@@ -371,33 +383,31 @@ impl CrawlDb {
.map(DomainId::from)
.collect();
- self.url_ids.bulk_ids(domains.values().flatten().cloned())?;
+ self.url_ids
+ .bulk_ids(domains.values().flatten().map(|u| u.url.clone()))?;
- for (domain_id, (domain, urls)) in domain_ids.into_iter().zip(domains.into_iter()) {
- for url in urls {
- let url_id: UrlId = self.url_ids.id(url.clone())?.into();
+ for (domain_id, urls) in domain_ids.into_iter().zip_eq(domains.values()) {
+ let domain_state = self
+ .domain_state
+ .entry(domain_id)
+ .or_insert_with(|| DomainState {
+ weight: 0.0,
+ status: DomainStatus::Pending,
+ });
- let domain_state =
- self.domain_state
- .entry(domain_id)
- .or_insert_with(|| DomainState {
- weight: 0.0,
- status: DomainStatus::Pending,
- });
+ let url_states = self.urls.entry(domain_id).or_default();
+
+ for url in urls {
+ let url_id: UrlId = self.url_ids.id(url.url.clone())?.into();
domain_state.status = DomainStatus::Pending;
- let url_state = self
- .urls
- .entry(domain_id)
- .or_default()
- .entry(url_id)
- .or_insert_with(|| UrlState {
- weight: 0.0,
- status: UrlStatus::Pending,
- });
-
- if &domain != crawled_domain {
+ let url_state = url_states.entry(url_id).or_insert_with(|| UrlState {
+ weight: 0.0,
+ status: UrlStatus::Pending,
+ });
+
+ if url.different_domain {
url_state.weight += 1.0;
}
@@ -410,11 +420,45 @@ impl CrawlDb {
Ok(())
}
- pub fn update_url_status(&mut self, url_responses: &[UrlResponse]) -> Result<()> {
+ pub fn update_url_status(&mut self, job_responses: &[JobResponse]) -> Result<()> {
+ let mut url_responses: HashMap<Domain, Vec<UrlResponse>> = HashMap::new();
+
+ for res in job_responses {
+ for url_response in &res.url_responses {
+ match url_response {
+ UrlResponse::Success { url } => {
+ let domain: Domain = url.domain().to_string().into();
+ url_responses
+ .entry(domain)
+ .or_default()
+ .push(url_response.clone());
+ }
+ UrlResponse::Failed {
+ url,
+ status_code: _,
+ } => {
+ let domain: Domain = url.domain().to_string().into();
+ url_responses
+ .entry(domain)
+ .or_default()
+ .push(url_response.clone());
+ }
+ UrlResponse::Redirected { url, new_url: _ } => {
+ let domain: Domain = url.domain().to_string().into();
+ url_responses
+ .entry(domain)
+ .or_default()
+ .push(url_response.clone());
+ }
+ }
+ }
+ }
+
// bulk register urls
self.url_ids.bulk_ids(
url_responses
- .iter()
+ .values()
+ .flatten()
.flat_map(|res| match res {
UrlResponse::Success { url } => vec![url].into_iter(),
UrlResponse::Failed {
@@ -426,64 +470,47 @@ impl CrawlDb {
.cloned(),
)?;
- for response in url_responses {
- match response {
- UrlResponse::Success { url } => {
- let domain: Domain = url.domain().to_string().into();
- let domain_id: DomainId = self.domain_ids.id(domain.clone())?.into();
- let url_id: UrlId = self.url_ids.id(url.clone())?.into();
-
- let domain_state =
- self.domain_state
- .entry(domain_id)
- .or_insert_with(|| DomainState {
- weight: 0.0,
- status: DomainStatus::Pending,
- });
- domain_state.status = DomainStatus::Pending;
-
- let url_state = self
- .urls
- .entry(domain_id)
- .or_default()
- .entry(url_id)
- .or_insert_with(|| UrlState {
+ // bulk register domains
+ self.domain_ids.bulk_ids(url_responses.keys().cloned())?;
+
+ for (domain, responses) in url_responses {
+ let domain_id: DomainId = self.domain_ids.id(domain.clone())?.into();
+ let domain_state = self
+ .domain_state
+ .entry(domain_id)
+ .or_insert_with(|| DomainState {
+ weight: 0.0,
+ status: DomainStatus::Pending,
+ });
+ domain_state.status = DomainStatus::Pending;
+
+ let url_states = self.urls.entry(domain_id).or_default();
+
+ for response in responses {
+ match response {
+ UrlResponse::Success { url } => {
+ let url_id: UrlId = self.url_ids.id(url.clone())?.into();
+
+ let url_state = url_states.entry(url_id).or_insert_with(|| UrlState {
weight: 0.0,
status: UrlStatus::Pending,
});
- url_state.status = UrlStatus::Done;
- }
- UrlResponse::Failed { url, status_code } => {
- let domain: Domain = url.domain().to_string().into();
- let domain_id: DomainId = self.domain_ids.id(domain.clone())?.into();
- let url_id: UrlId = self.url_ids.id(url.clone())?.into();
-
- let domain_state =
- self.domain_state
- .entry(domain_id)
- .or_insert_with(|| DomainState {
- weight: 0.0,
- status: DomainStatus::Pending,
- });
- domain_state.status = DomainStatus::Pending;
-
- let url_state = self
- .urls
- .entry(domain_id)
- .or_default()
- .entry(url_id)
- .or_insert_with(|| UrlState {
+ url_state.status = UrlStatus::Done;
+ }
+ UrlResponse::Failed { url, status_code } => {
+ let url_id: UrlId = self.url_ids.id(url.clone())?.into();
+
+ let url_state = url_states.entry(url_id).or_insert_with(|| UrlState {
weight: 0.0,
status: UrlStatus::Pending,
});
- url_state.status = UrlStatus::Failed {
- status_code: *status_code,
- };
- }
- UrlResponse::Redirected { url, new_url } => {
- self.redirects.put(url, new_url)?;
+ url_state.status = UrlStatus::Failed { status_code };
+ }
+ UrlResponse::Redirected { url, new_url } => {
+ self.redirects.put(&url, &new_url)?;
+ }
}
}
}
diff --git a/core/src/warc.rs b/core/src/warc.rs
index 4341d786..b3085e31 100644
--- a/core/src/warc.rs
+++ b/core/src/warc.rs
@@ -186,7 +186,8 @@ impl WarcFile {
expiration: None,
},
)?
- .with_path_style();
+ .with_path_style()
+ .with_request_timeout(Duration::from_secs(30 * 60));
let res = bucket.get_object_blocking(warc_path)?;