web search done right
[crawler] normalize url before fetch
Mikkel Denker5968185136a7
diff --git a/crates/core/src/crawler/worker.rs b/crates/core/src/crawler/worker.rsindex 5ae578a2..82a42252 100644--- a/crates/core/src/crawler/worker.rs+++ b/crates/core/src/crawler/worker.rs@@ -201,8 +201,13 @@ impl<S: DatumStream> JobExecutor<S> {let urls: VecDeque<_> = urls.into_iter()- .take(self.job.wandering_urls.saturating_sub(self.wandered_urls) as usize).map(|(url, _)| url)+ .map(|mut url| {+ url.normalize();+ url+ })+ .filter(|url| !self.crawled_urls.contains(url))+ .take(self.job.wandering_urls.saturating_sub(self.wandered_urls) as usize).map(|url| WeightedUrl { url, weight: 0.0 }).map(RetrieableUrl::from).collect();@@ -535,6 +540,9 @@ impl<S: DatumStream> JobExecutor<S> {}async fn crawl_url(&mut self, url: Url) -> Result<CrawlDatum> {+ let mut url = url;+ url.normalize();+if self.crawled_urls.contains(&url) {bail!("url already crawled: {}", url);}