web search done right
100 urls per job seems more reasonable
Mikkel Denker8aab7bb96a29
diff --git a/core/src/crawler/coordinator.rs b/core/src/crawler/coordinator.rsindex ecf47d17..1f8f229a 100644--- a/core/src/crawler/coordinator.rs+++ b/core/src/crawler/coordinator.rs@@ -30,7 +30,7 @@ use std::{time::{Duration, Instant},};-const DEFAULT_JOB_URLS: usize = 1000;+const DEFAULT_JOB_URLS: usize = 100;pub struct CrawlCoordinator {db: CrawlDb,