Skip to content

web search done right

reduce number of connections from crawl workers to coordinator

Mikkel Denker93dcb506913a
diff --git a/core/benches/crawl-coordinator.rs b/core/benches/crawl-coordinator.rs
index 5df87e7b..721902a6 100644
--- a/core/benches/crawl-coordinator.rs
+++ b/core/benches/crawl-coordinator.rs
@@ -1776,7 +1776,7 @@ Url::from("https://www.apple.com/legal/internet-services/itunes/us/terms.html".t
c.bench_function("Add response to coordinator", |b| {
b.iter(|| {
let response = responses.choose(&mut rng).unwrap();
- coordinator.add_response(response).unwrap();
+ coordinator.add_responses(&[response.clone()]).unwrap();
})
});
diff --git a/core/src/crawler/coordinator.rs b/core/src/crawler/coordinator.rs
index 824f000e..ba06e265 100644
--- a/core/src/crawler/coordinator.rs
+++ b/core/src/crawler/coordinator.rs
@@ -36,7 +36,6 @@ pub struct CrawlCoordinator {
num_crawled_urls: AtomicU64,
num_urls_to_crawl: u64,
call_counter: Mutex<CallCounter>,
- response_queue: Mutex<Vec<JobResponse>>,
}
impl CrawlCoordinator {
@@ -56,7 +55,6 @@ impl CrawlCoordinator {
num_urls_to_crawl,
num_crawled_urls: AtomicU64::new(0),
call_counter: Mutex::new(CallCounter::new(Duration::from_secs(10))),
- response_queue: Mutex::new(Vec::new()),
})
}
@@ -71,32 +69,19 @@ impl CrawlCoordinator {
tracing::info!("avg crawls per second: {}", call_counter.avg_per_second());
}
- pub fn add_response(&self, response: &JobResponse) -> Result<()> {
- self.log_crawls_per_second(response.url_responses.len());
- self.num_crawled_urls
- .fetch_add(response.url_responses.len() as u64, Ordering::SeqCst);
-
- self.response_queue.lock().unwrap().push(response.clone());
-
- Ok(())
- }
-
- fn flush_responses(&self) -> Result<()> {
+ pub fn add_responses(&self, responses: &[JobResponse]) -> Result<()> {
let start = Instant::now();
- let mut response_queue = self.response_queue.lock().unwrap();
+ self.log_crawls_per_second(responses.iter().map(|res| res.url_responses.len()).sum());
let mut db = self.db.lock().unwrap();
- db.insert_urls(&response_queue).unwrap();
-
- db.update_url_status(&response_queue).unwrap();
+ db.insert_urls(&responses).unwrap();
+ db.update_url_status(&responses).unwrap();
- for response in response_queue.iter() {
+ for response in responses.iter() {
db.set_domain_status(&response.domain, DomainStatus::Pending)
.unwrap();
}
- response_queue.clear();
-
tracing::debug!("inserted responses in {:?}", start.elapsed());
Ok(())
@@ -108,7 +93,7 @@ impl CrawlCoordinator {
pub fn sample_jobs(&self, num_jobs: usize) -> Result<Vec<Job>> {
let start = Instant::now();
- self.flush_responses()?;
+
let mut db = self.db.lock().unwrap();
let domains = db.sample_domains(num_jobs)?;
diff --git a/core/src/crawler/mod.rs b/core/src/crawler/mod.rs
index db6576d4..b220238a 100644
--- a/core/src/crawler/mod.rs
+++ b/core/src/crawler/mod.rs
@@ -165,6 +165,7 @@ pub struct Crawler {
impl Crawler {
pub async fn new(config: CrawlerConfig) -> Result<Self> {
let pending_commands = Arc::new(Mutex::new(VecDeque::new()));
+ let results = Arc::new(Mutex::new(Vec::new()));
let writer = Arc::new(WarcWriter::new(config.s3.clone()));
let timeout = Duration::from_secs(config.timeout_seconds);
let mut handles = Vec::new();
@@ -174,6 +175,7 @@ impl Crawler {
let worker = Worker::new(
Arc::clone(&pending_commands),
Arc::clone(&writer),
+ Arc::clone(&results),
config.clone(),
timeout,
coordinator_host,
@@ -198,13 +200,14 @@ impl Crawler {
#[derive(serde::Serialize, serde::Deserialize)]
pub enum Request {
- NewJobs { num_jobs: usize },
- CrawlResult { job_response: JobResponse },
+ NewJobs {
+ responses: Vec<JobResponse>,
+ num_jobs: usize,
+ },
}
#[derive(serde::Serialize, serde::Deserialize)]
pub enum Response {
NewJobs { jobs: Vec<Job> },
Done,
- Ok,
}
diff --git a/core/src/crawler/worker.rs b/core/src/crawler/worker.rs
index c8b19609..8cd6b3e5 100644
--- a/core/src/crawler/worker.rs
+++ b/core/src/crawler/worker.rs
@@ -41,6 +41,7 @@ pub struct Worker {
current_job: Option<WorkerJob>,
pending_commands: Arc<Mutex<VecDeque<Command>>>,
writer: Arc<WarcWriter>,
+ results: Arc<Mutex<Vec<JobResponse>>>,
client: reqwest::Client,
config: CrawlerConfig,
politeness_factor: f32,
@@ -53,6 +54,7 @@ impl Worker {
pub fn new(
pending_commands: Arc<Mutex<VecDeque<Command>>>,
writer: Arc<WarcWriter>,
+ results: Arc<Mutex<Vec<JobResponse>>>,
config: CrawlerConfig,
timeout: Duration,
coordinator_host: SocketAddr,
@@ -73,6 +75,7 @@ impl Worker {
Ok(Self {
writer,
client,
+ results,
current_job: None,
pending_commands,
num_jobs_per_fetch: config.num_workers,
@@ -113,9 +116,12 @@ impl Worker {
}
} else {
let mut conn = self.coordinator_conn();
+ let results = self.results.lock().await.drain(..).collect::<Vec<_>>();
+
let res = conn
.send_with_timeout::<_, Response>(
&Request::NewJobs {
+ responses: results,
num_jobs: 2 * self.num_jobs_per_fetch,
},
Duration::from_secs(60 * 60),
@@ -217,21 +223,13 @@ impl Worker {
url_responses.push(res.response);
}
- // send response to coordinator
let job_response = JobResponse {
domain: job.domain,
url_responses,
discovered_urls: discovered_urls.into_iter().collect(),
};
- let mut conn = self.coordinator_conn();
- let _ = conn
- .send_with_timeout::<_, Response>(
- &Request::CrawlResult { job_response },
- Duration::from_secs(60),
- )
- .await
- .expect("Failed to send crawl result to coordinator");
+ self.results.lock().await.push(job_response);
}
async fn process_url(&mut self, url: Url) -> ProcessedUrl {
diff --git a/core/src/entrypoint/crawler.rs b/core/src/entrypoint/crawler.rs
index 83336833..f718262c 100644
--- a/core/src/entrypoint/crawler.rs
+++ b/core/src/entrypoint/crawler.rs
@@ -46,7 +46,12 @@ pub async fn coordinator(config: config::CrawlCoordinatorConfig) -> Result<()> {
loop {
if let Ok(req) = server.accept::<crate::crawler::Request>().await {
match &req.body {
- crate::crawler::Request::NewJobs { num_jobs } => {
+ crate::crawler::Request::NewJobs {
+ responses,
+ num_jobs,
+ } => {
+ coordinator.add_responses(responses)?;
+
if coordinator.is_done() {
tracing::info!("Crawl is done. Waiting for workers to finish.");
req.respond(sonic::Response::Content(crate::crawler::Response::Done))
@@ -62,13 +67,6 @@ pub async fn coordinator(config: config::CrawlCoordinatorConfig) -> Result<()> {
.ok();
}
}
- crate::crawler::Request::CrawlResult { job_response } => {
- coordinator.add_response(job_response)?;
-
- req.respond(sonic::Response::Content(crate::crawler::Response::Ok))
- .await
- .ok();
- }
}
}
}