Skip to content

web search done right

normalize urls before they are crawled for better duplicates detection

Mikkel Denker0114e8d9eb5b
diff --git a/crates/core/src/crawler/worker.rs b/crates/core/src/crawler/worker.rs
index f245377d..3ed5945b 100644
--- a/crates/core/src/crawler/worker.rs
+++ b/crates/core/src/crawler/worker.rs
@@ -305,7 +305,9 @@ impl<S: DatumStream> JobExecutor<S> {
pub async fn process_urls(&mut self, mut urls: VecDeque<RetrieableUrl>) {
tracing::debug!("processing {} urls", urls.len());
- while let Some(retryable_url) = urls.pop_front() {
+ while let Some(mut retryable_url) = urls.pop_front() {
+ retryable_url.weighted_url.url.normalize_in_place();
+
if let UrlVisit::Skip = self.verify_url(&retryable_url).await {
tracing::debug!("skipping url: {}", retryable_url.url());
continue;
diff --git a/crates/core/src/live_index/crawler/crawlable_site.rs b/crates/core/src/live_index/crawler/crawlable_site.rs
index 2930e55c..e58f3329 100644
--- a/crates/core/src/live_index/crawler/crawlable_site.rs
+++ b/crates/core/src/live_index/crawler/crawlable_site.rs
@@ -212,6 +212,10 @@ impl CrawlableSiteGuard {
.into_iter()
.filter(|u| u.url.icann_domain() == icann_domain)
.unique_by(|u| u.url.clone())
+ .map(|mut u| {
+ u.url.normalize_in_place();
+ u
+ })
.collect();
urls.retain(|url| !self.crawled_db.has_crawled(&url.url).unwrap_or(false));