web search done right
Each worker now fetches 2*num_workers jobs. This will decrease the pressure on the coordinator
Mikkel Denker565e7e1e70d5
diff --git a/core/src/crawler/worker.rs b/core/src/crawler/worker.rsindex a3893cbe..39c4ab39 100644--- a/core/src/crawler/worker.rs+++ b/core/src/crawler/worker.rs@@ -125,7 +125,7 @@ impl Worker {let res = conn.send_with_timeout::<_, Response>(&Request::NewJobs {- num_jobs: self.num_jobs_per_fetch,+ num_jobs: 2 * self.num_jobs_per_fetch,},Duration::from_secs(60 * 60),)