Skip to content

web search done right

[crawler] don't re-crawl url when it redirects to another url

Mikkel Denkerca9b249992f1
diff --git a/crates/core/src/crawler/worker.rs b/crates/core/src/crawler/worker.rs
index 82a42252..43b4a734 100644
--- a/crates/core/src/crawler/worker.rs
+++ b/crates/core/src/crawler/worker.rs
@@ -555,6 +555,7 @@ impl<S: DatumStream> JobExecutor<S> {
// we want to delay before returning the error
let res = res?;
+ self.crawled_urls.insert(url.clone());
let payload_type = self.check_headers(&res)?;
if let Some(datum) = self.redirect_datum(&res, &url, payload_type, fetch_time)? {