web search done right
normalize urls before they are crawled for better duplicates detection
Mikkel Denker0114e8d9eb5b
diff --git a/crates/core/src/crawler/worker.rs b/crates/core/src/crawler/worker.rsindex f245377d..3ed5945b 100644--- a/crates/core/src/crawler/worker.rs+++ b/crates/core/src/crawler/worker.rs@@ -305,7 +305,9 @@ impl<S: DatumStream> JobExecutor<S> {pub async fn process_urls(&mut self, mut urls: VecDeque<RetrieableUrl>) {tracing::debug!("processing {} urls", urls.len());- while let Some(retryable_url) = urls.pop_front() {+ while let Some(mut retryable_url) = urls.pop_front() {+ retryable_url.weighted_url.url.normalize_in_place();+if let UrlVisit::Skip = self.verify_url(&retryable_url).await {tracing::debug!("skipping url: {}", retryable_url.url());continue;diff --git a/crates/core/src/live_index/crawler/crawlable_site.rs b/crates/core/src/live_index/crawler/crawlable_site.rsindex 2930e55c..e58f3329 100644--- a/crates/core/src/live_index/crawler/crawlable_site.rs+++ b/crates/core/src/live_index/crawler/crawlable_site.rs@@ -212,6 +212,10 @@ impl CrawlableSiteGuard {.into_iter().filter(|u| u.url.icann_domain() == icann_domain).unique_by(|u| u.url.clone())+ .map(|mut u| {+ u.url.normalize_in_place();+ u+ }).collect();urls.retain(|url| !self.crawled_db.has_crawled(&url.url).unwrap_or(false));