web search done right
[crawler] don't re-crawl url when it redirects to another url
Mikkel Denkerca9b249992f1
diff --git a/crates/core/src/crawler/worker.rs b/crates/core/src/crawler/worker.rsindex 82a42252..43b4a734 100644--- a/crates/core/src/crawler/worker.rs+++ b/crates/core/src/crawler/worker.rs@@ -555,6 +555,7 @@ impl<S: DatumStream> JobExecutor<S> {// we want to delay before returning the errorlet res = res?;+ self.crawled_urls.insert(url.clone());let payload_type = self.check_headers(&res)?;if let Some(datum) = self.redirect_datum(&res, &url, payload_type, fetch_time)? {