web search done right
insert crawler job responses in batches
Mikkel Denkere1c16e3f2012
diff --git a/core/src/crawler/coordinator.rs b/core/src/crawler/coordinator.rsindex 9bb813c6..824f000e 100644--- a/core/src/crawler/coordinator.rs+++ b/core/src/crawler/coordinator.rs@@ -36,6 +36,7 @@ pub struct CrawlCoordinator {num_crawled_urls: AtomicU64,num_urls_to_crawl: u64,call_counter: Mutex<CallCounter>,+ response_queue: Mutex<Vec<JobResponse>>,}impl CrawlCoordinator {@@ -55,6 +56,7 @@ impl CrawlCoordinator {num_urls_to_crawl,num_crawled_urls: AtomicU64::new(0),call_counter: Mutex::new(CallCounter::new(Duration::from_secs(10))),+ response_queue: Mutex::new(Vec::new()),})}@@ -70,20 +72,30 @@ impl CrawlCoordinator {}pub fn add_response(&self, response: &JobResponse) -> Result<()> {+ self.log_crawls_per_second(response.url_responses.len());+ self.num_crawled_urls+ .fetch_add(response.url_responses.len() as u64, Ordering::SeqCst);++ self.response_queue.lock().unwrap().push(response.clone());++ Ok(())+ }++ fn flush_responses(&self) -> Result<()> {let start = Instant::now();+ let mut response_queue = self.response_queue.lock().unwrap();let mut db = self.db.lock().unwrap();- self.log_crawls_per_second(response.url_responses.len());-- self.num_crawled_urls.fetch_add(1, Ordering::SeqCst);+ db.insert_urls(&response_queue).unwrap();- db.insert_urls(&response.domain, &response.discovered_urls)- .unwrap();+ db.update_url_status(&response_queue).unwrap();- db.update_url_status(&response.url_responses).unwrap();+ for response in response_queue.iter() {+ db.set_domain_status(&response.domain, DomainStatus::Pending)+ .unwrap();+ }- db.set_domain_status(&response.domain, DomainStatus::Pending)- .unwrap();+ response_queue.clear();tracing::debug!("inserted responses in {:?}", start.elapsed());@@ -96,6 +108,7 @@ impl CrawlCoordinator {pub fn sample_jobs(&self, num_jobs: usize) -> Result<Vec<Job>> {let start = Instant::now();+ self.flush_responses()?;let mut db = self.db.lock().unwrap();let domains = db.sample_domains(num_jobs)?;diff --git a/core/src/crawler/crawl_db.rs b/core/src/crawler/crawl_db.rsindex 13dbc9ff..9656e2ee 100644--- a/core/src/crawler/crawl_db.rs+++ b/core/src/crawler/crawl_db.rs@@ -21,13 +21,14 @@ use std::{path::Path,};+use itertools::Itertools;use lru::LruCache;use rand::Rng;use rocksdb::BlockBasedOptions;use crate::webpage::Url;-use super::{Domain, Job, Result, UrlResponse};+use super::{Domain, Job, JobResponse, Result, UrlResponse};#[derive(Clone, PartialEq, Eq)]pub enum UrlStatus {@@ -305,6 +306,11 @@ impl RedirectDb {}}+struct UrlToInsert {+ url: Url,+ different_domain: bool,+}+pub struct CrawlDb {url_ids: IdTable<Url>,domain_ids: IdTable<Domain>,@@ -355,13 +361,19 @@ impl CrawlDb {Ok(())}- pub fn insert_urls(&mut self, crawled_domain: &Domain, urls: &[Url]) -> Result<()> {- let mut domains: HashMap<Domain, Vec<Url>> = HashMap::new();+ pub fn insert_urls(&mut self, responses: &[JobResponse]) -> Result<()> {+ let mut domains: HashMap<Domain, Vec<UrlToInsert>> = HashMap::new();- for url in urls {- let domain: Domain = url.domain().to_string().into();+ for res in responses {+ for url in &res.discovered_urls {+ let domain: Domain = url.domain().to_string().into();+ let different_domain = res.domain != domain;- domains.entry(domain).or_default().push(url.clone());+ domains.entry(domain).or_default().push(UrlToInsert {+ url: url.clone(),+ different_domain,+ });+ }}let domain_ids: Vec<DomainId> = self@@ -371,33 +383,31 @@ impl CrawlDb {.map(DomainId::from).collect();- self.url_ids.bulk_ids(domains.values().flatten().cloned())?;+ self.url_ids+ .bulk_ids(domains.values().flatten().map(|u| u.url.clone()))?;- for (domain_id, (domain, urls)) in domain_ids.into_iter().zip(domains.into_iter()) {- for url in urls {- let url_id: UrlId = self.url_ids.id(url.clone())?.into();+ for (domain_id, urls) in domain_ids.into_iter().zip_eq(domains.values()) {+ let domain_state = self+ .domain_state+ .entry(domain_id)+ .or_insert_with(|| DomainState {+ weight: 0.0,+ status: DomainStatus::Pending,+ });- let domain_state =- self.domain_state- .entry(domain_id)- .or_insert_with(|| DomainState {- weight: 0.0,- status: DomainStatus::Pending,- });+ let url_states = self.urls.entry(domain_id).or_default();++ for url in urls {+ let url_id: UrlId = self.url_ids.id(url.url.clone())?.into();domain_state.status = DomainStatus::Pending;- let url_state = self- .urls- .entry(domain_id)- .or_default()- .entry(url_id)- .or_insert_with(|| UrlState {- weight: 0.0,- status: UrlStatus::Pending,- });-- if &domain != crawled_domain {+ let url_state = url_states.entry(url_id).or_insert_with(|| UrlState {+ weight: 0.0,+ status: UrlStatus::Pending,+ });++ if url.different_domain {url_state.weight += 1.0;}@@ -410,11 +420,45 @@ impl CrawlDb {Ok(())}- pub fn update_url_status(&mut self, url_responses: &[UrlResponse]) -> Result<()> {+ pub fn update_url_status(&mut self, job_responses: &[JobResponse]) -> Result<()> {+ let mut url_responses: HashMap<Domain, Vec<UrlResponse>> = HashMap::new();++ for res in job_responses {+ for url_response in &res.url_responses {+ match url_response {+ UrlResponse::Success { url } => {+ let domain: Domain = url.domain().to_string().into();+ url_responses+ .entry(domain)+ .or_default()+ .push(url_response.clone());+ }+ UrlResponse::Failed {+ url,+ status_code: _,+ } => {+ let domain: Domain = url.domain().to_string().into();+ url_responses+ .entry(domain)+ .or_default()+ .push(url_response.clone());+ }+ UrlResponse::Redirected { url, new_url: _ } => {+ let domain: Domain = url.domain().to_string().into();+ url_responses+ .entry(domain)+ .or_default()+ .push(url_response.clone());+ }+ }+ }+ }+// bulk register urlsself.url_ids.bulk_ids(url_responses- .iter()+ .values()+ .flatten().flat_map(|res| match res {UrlResponse::Success { url } => vec![url].into_iter(),UrlResponse::Failed {@@ -426,64 +470,47 @@ impl CrawlDb {.cloned(),)?;- for response in url_responses {- match response {- UrlResponse::Success { url } => {- let domain: Domain = url.domain().to_string().into();- let domain_id: DomainId = self.domain_ids.id(domain.clone())?.into();- let url_id: UrlId = self.url_ids.id(url.clone())?.into();-- let domain_state =- self.domain_state- .entry(domain_id)- .or_insert_with(|| DomainState {- weight: 0.0,- status: DomainStatus::Pending,- });- domain_state.status = DomainStatus::Pending;-- let url_state = self- .urls- .entry(domain_id)- .or_default()- .entry(url_id)- .or_insert_with(|| UrlState {+ // bulk register domains+ self.domain_ids.bulk_ids(url_responses.keys().cloned())?;++ for (domain, responses) in url_responses {+ let domain_id: DomainId = self.domain_ids.id(domain.clone())?.into();+ let domain_state = self+ .domain_state+ .entry(domain_id)+ .or_insert_with(|| DomainState {+ weight: 0.0,+ status: DomainStatus::Pending,+ });+ domain_state.status = DomainStatus::Pending;++ let url_states = self.urls.entry(domain_id).or_default();++ for response in responses {+ match response {+ UrlResponse::Success { url } => {+ let url_id: UrlId = self.url_ids.id(url.clone())?.into();++ let url_state = url_states.entry(url_id).or_insert_with(|| UrlState {weight: 0.0,status: UrlStatus::Pending,});- url_state.status = UrlStatus::Done;- }- UrlResponse::Failed { url, status_code } => {- let domain: Domain = url.domain().to_string().into();- let domain_id: DomainId = self.domain_ids.id(domain.clone())?.into();- let url_id: UrlId = self.url_ids.id(url.clone())?.into();-- let domain_state =- self.domain_state- .entry(domain_id)- .or_insert_with(|| DomainState {- weight: 0.0,- status: DomainStatus::Pending,- });- domain_state.status = DomainStatus::Pending;-- let url_state = self- .urls- .entry(domain_id)- .or_default()- .entry(url_id)- .or_insert_with(|| UrlState {+ url_state.status = UrlStatus::Done;+ }+ UrlResponse::Failed { url, status_code } => {+ let url_id: UrlId = self.url_ids.id(url.clone())?.into();++ let url_state = url_states.entry(url_id).or_insert_with(|| UrlState {weight: 0.0,status: UrlStatus::Pending,});- url_state.status = UrlStatus::Failed {- status_code: *status_code,- };- }- UrlResponse::Redirected { url, new_url } => {- self.redirects.put(url, new_url)?;+ url_state.status = UrlStatus::Failed { status_code };+ }+ UrlResponse::Redirected { url, new_url } => {+ self.redirects.put(&url, &new_url)?;+ }}}}diff --git a/core/src/warc.rs b/core/src/warc.rsindex 4341d786..b3085e31 100644--- a/core/src/warc.rs+++ b/core/src/warc.rs@@ -186,7 +186,8 @@ impl WarcFile {expiration: None,},)?- .with_path_style();+ .with_path_style()+ .with_request_timeout(Duration::from_secs(30 * 60));let res = bucket.get_object_blocking(warc_path)?;