Skip to content

web search done right

normalize urls

Mikkel Denkerd61986c25513
diff --git a/crates/core/src/live_index/crawler/crawlable_site.rs b/crates/core/src/live_index/crawler/crawlable_site.rs
index 3c80a516..2930e55c 100644
--- a/crates/core/src/live_index/crawler/crawlable_site.rs
+++ b/crates/core/src/live_index/crawler/crawlable_site.rs
@@ -235,7 +235,8 @@ impl CrawlableSiteGuard {
);
for crawlable_url in &urls {
- self.crawled_db.insert(&crawlable_url.url)?;
+ self.crawled_db
+ .insert(&crawlable_url.url.clone().normalize())?;
}
let crawl_data = Arc::new(tokio::sync::Mutex::new(Vec::new()));