web search done right
reduce number of connections from crawl workers to coordinator
Mikkel Denker93dcb506913a
diff --git a/core/benches/crawl-coordinator.rs b/core/benches/crawl-coordinator.rsindex 5df87e7b..721902a6 100644--- a/core/benches/crawl-coordinator.rs+++ b/core/benches/crawl-coordinator.rs@@ -1776,7 +1776,7 @@ Url::from("https://www.apple.com/legal/internet-services/itunes/us/terms.html".tc.bench_function("Add response to coordinator", |b| {b.iter(|| {let response = responses.choose(&mut rng).unwrap();- coordinator.add_response(response).unwrap();+ coordinator.add_responses(&[response.clone()]).unwrap();})});diff --git a/core/src/crawler/coordinator.rs b/core/src/crawler/coordinator.rsindex 824f000e..ba06e265 100644--- a/core/src/crawler/coordinator.rs+++ b/core/src/crawler/coordinator.rs@@ -36,7 +36,6 @@ pub struct CrawlCoordinator {num_crawled_urls: AtomicU64,num_urls_to_crawl: u64,call_counter: Mutex<CallCounter>,- response_queue: Mutex<Vec<JobResponse>>,}impl CrawlCoordinator {@@ -56,7 +55,6 @@ impl CrawlCoordinator {num_urls_to_crawl,num_crawled_urls: AtomicU64::new(0),call_counter: Mutex::new(CallCounter::new(Duration::from_secs(10))),- response_queue: Mutex::new(Vec::new()),})}@@ -71,32 +69,19 @@ impl CrawlCoordinator {tracing::info!("avg crawls per second: {}", call_counter.avg_per_second());}- pub fn add_response(&self, response: &JobResponse) -> Result<()> {- self.log_crawls_per_second(response.url_responses.len());- self.num_crawled_urls- .fetch_add(response.url_responses.len() as u64, Ordering::SeqCst);-- self.response_queue.lock().unwrap().push(response.clone());-- Ok(())- }-- fn flush_responses(&self) -> Result<()> {+ pub fn add_responses(&self, responses: &[JobResponse]) -> Result<()> {let start = Instant::now();- let mut response_queue = self.response_queue.lock().unwrap();+ self.log_crawls_per_second(responses.iter().map(|res| res.url_responses.len()).sum());let mut db = self.db.lock().unwrap();- db.insert_urls(&response_queue).unwrap();-- db.update_url_status(&response_queue).unwrap();+ db.insert_urls(&responses).unwrap();+ db.update_url_status(&responses).unwrap();- for response in response_queue.iter() {+ for response in responses.iter() {db.set_domain_status(&response.domain, DomainStatus::Pending).unwrap();}- response_queue.clear();-tracing::debug!("inserted responses in {:?}", start.elapsed());Ok(())@@ -108,7 +93,7 @@ impl CrawlCoordinator {pub fn sample_jobs(&self, num_jobs: usize) -> Result<Vec<Job>> {let start = Instant::now();- self.flush_responses()?;+let mut db = self.db.lock().unwrap();let domains = db.sample_domains(num_jobs)?;diff --git a/core/src/crawler/mod.rs b/core/src/crawler/mod.rsindex db6576d4..b220238a 100644--- a/core/src/crawler/mod.rs+++ b/core/src/crawler/mod.rs@@ -165,6 +165,7 @@ pub struct Crawler {impl Crawler {pub async fn new(config: CrawlerConfig) -> Result<Self> {let pending_commands = Arc::new(Mutex::new(VecDeque::new()));+ let results = Arc::new(Mutex::new(Vec::new()));let writer = Arc::new(WarcWriter::new(config.s3.clone()));let timeout = Duration::from_secs(config.timeout_seconds);let mut handles = Vec::new();@@ -174,6 +175,7 @@ impl Crawler {let worker = Worker::new(Arc::clone(&pending_commands),Arc::clone(&writer),+ Arc::clone(&results),config.clone(),timeout,coordinator_host,@@ -198,13 +200,14 @@ impl Crawler {#[derive(serde::Serialize, serde::Deserialize)]pub enum Request {- NewJobs { num_jobs: usize },- CrawlResult { job_response: JobResponse },+ NewJobs {+ responses: Vec<JobResponse>,+ num_jobs: usize,+ },}#[derive(serde::Serialize, serde::Deserialize)]pub enum Response {NewJobs { jobs: Vec<Job> },Done,- Ok,}diff --git a/core/src/crawler/worker.rs b/core/src/crawler/worker.rsindex c8b19609..8cd6b3e5 100644--- a/core/src/crawler/worker.rs+++ b/core/src/crawler/worker.rs@@ -41,6 +41,7 @@ pub struct Worker {current_job: Option<WorkerJob>,pending_commands: Arc<Mutex<VecDeque<Command>>>,writer: Arc<WarcWriter>,+ results: Arc<Mutex<Vec<JobResponse>>>,client: reqwest::Client,config: CrawlerConfig,politeness_factor: f32,@@ -53,6 +54,7 @@ impl Worker {pub fn new(pending_commands: Arc<Mutex<VecDeque<Command>>>,writer: Arc<WarcWriter>,+ results: Arc<Mutex<Vec<JobResponse>>>,config: CrawlerConfig,timeout: Duration,coordinator_host: SocketAddr,@@ -73,6 +75,7 @@ impl Worker {Ok(Self {writer,client,+ results,current_job: None,pending_commands,num_jobs_per_fetch: config.num_workers,@@ -113,9 +116,12 @@ impl Worker {}} else {let mut conn = self.coordinator_conn();+ let results = self.results.lock().await.drain(..).collect::<Vec<_>>();+let res = conn.send_with_timeout::<_, Response>(&Request::NewJobs {+ responses: results,num_jobs: 2 * self.num_jobs_per_fetch,},Duration::from_secs(60 * 60),@@ -217,21 +223,13 @@ impl Worker {url_responses.push(res.response);}- // send response to coordinatorlet job_response = JobResponse {domain: job.domain,url_responses,discovered_urls: discovered_urls.into_iter().collect(),};- let mut conn = self.coordinator_conn();- let _ = conn- .send_with_timeout::<_, Response>(- &Request::CrawlResult { job_response },- Duration::from_secs(60),- )- .await- .expect("Failed to send crawl result to coordinator");+ self.results.lock().await.push(job_response);}async fn process_url(&mut self, url: Url) -> ProcessedUrl {diff --git a/core/src/entrypoint/crawler.rs b/core/src/entrypoint/crawler.rsindex 83336833..f718262c 100644--- a/core/src/entrypoint/crawler.rs+++ b/core/src/entrypoint/crawler.rs@@ -46,7 +46,12 @@ pub async fn coordinator(config: config::CrawlCoordinatorConfig) -> Result<()> {loop {if let Ok(req) = server.accept::<crate::crawler::Request>().await {match &req.body {- crate::crawler::Request::NewJobs { num_jobs } => {+ crate::crawler::Request::NewJobs {+ responses,+ num_jobs,+ } => {+ coordinator.add_responses(responses)?;+if coordinator.is_done() {tracing::info!("Crawl is done. Waiting for workers to finish.");req.respond(sonic::Response::Content(crate::crawler::Response::Done))@@ -62,13 +67,6 @@ pub async fn coordinator(config: config::CrawlCoordinatorConfig) -> Result<()> {.ok();}}- crate::crawler::Request::CrawlResult { job_response } => {- coordinator.add_response(job_response)?;-- req.respond(sonic::Response::Content(crate::crawler::Response::Ok))- .await- .ok();- }}}}