Skip to content

web search done right

Each worker now fetches 2*num_workers jobs. This will decrease the pressure on the coordinator

Mikkel Denker565e7e1e70d5
diff --git a/core/src/crawler/worker.rs b/core/src/crawler/worker.rs
index a3893cbe..39c4ab39 100644
--- a/core/src/crawler/worker.rs
+++ b/core/src/crawler/worker.rs
@@ -125,7 +125,7 @@ impl Worker {
let res = conn
.send_with_timeout::<_, Response>(
&Request::NewJobs {
- num_jobs: self.num_jobs_per_fetch,
+ num_jobs: 2 * self.num_jobs_per_fetch,
},
Duration::from_secs(60 * 60),
)