web search done right
normalize urls
Mikkel Denkerd61986c25513
diff --git a/crates/core/src/live_index/crawler/crawlable_site.rs b/crates/core/src/live_index/crawler/crawlable_site.rsindex 3c80a516..2930e55c 100644--- a/crates/core/src/live_index/crawler/crawlable_site.rs+++ b/crates/core/src/live_index/crawler/crawlable_site.rs@@ -235,7 +235,8 @@ impl CrawlableSiteGuard {);for crawlable_url in &urls {- self.crawled_db.insert(&crawlable_url.url)?;+ self.crawled_db+ .insert(&crawlable_url.url.clone().normalize())?;}let crawl_data = Arc::new(tokio::sync::Mutex::new(Vec::new()));