Skip to content

web search done right

Retry urls that return 429 during crawl

Mikkel Denker43bd806fef60
diff --git a/core/src/crawler/mod.rs b/core/src/crawler/mod.rs
index cafddf8e..77e4f786 100644
--- a/core/src/crawler/mod.rs
+++ b/core/src/crawler/mod.rs
@@ -121,13 +121,30 @@ pub struct JobResponse {
pub discovered_urls: Vec<Url>,
}
+struct RetrieableUrl {
+ url: Url,
+ retries: u8,
+}
+
+impl From<Url> for RetrieableUrl {
+ fn from(url: Url) -> Self {
+ Self { url, retries: 0 }
+ }
+}
+
struct WorkerJob {
- job: Job,
+ pub domain: Domain,
+ pub fetch_sitemap: bool,
+ pub urls: VecDeque<RetrieableUrl>,
}
impl From<Job> for WorkerJob {
fn from(value: Job) -> Self {
- Self { job: value }
+ Self {
+ domain: value.domain,
+ fetch_sitemap: value.fetch_sitemap,
+ urls: value.urls.into_iter().map(RetrieableUrl::from).collect(),
+ }
}
}
diff --git a/core/src/crawler/worker.rs b/core/src/crawler/worker.rs
index 0bce76dd..7177f810 100644
--- a/core/src/crawler/worker.rs
+++ b/core/src/crawler/worker.rs
@@ -41,6 +41,7 @@ const MIN_CRAWL_DELAY_MS: u64 = 200;
const MAX_CRAWL_DELAY_MS: u64 = 30_000;
const MAX_POLITENESS_FACTOR: f32 = 1024.0;
const MIN_POLITENESS_FACTOR: f32 = 1.0;
+const MAX_URL_SLOWDOWN_RETRY: u8 = 3;
pub struct Worker {
current_job: Option<WorkerJob>,
@@ -160,41 +161,42 @@ impl Worker {
async fn process_job(&mut self) {
self.politeness_factor = self.default_politeness_factor;
- let job = self.current_job.take().unwrap();
- tracing::info!("Processing job: {:?}", job.job.domain);
+ let mut job = self.current_job.take().unwrap();
+ tracing::info!("Processing job: {:?}", job.domain);
let mut url_responses: Vec<UrlResponse> = Vec::new();
let mut discovered_urls: HashSet<Url> = HashSet::new();
let mut crawled_sitemaps: HashSet<Site> = HashSet::new();
- for url in job.job.urls {
- if url.is_empty() || url.site().is_empty() {
+ while let Some(retryable_url) = job.urls.pop_front() {
+ if retryable_url.retries > MAX_URL_SLOWDOWN_RETRY {
+ continue;
+ }
+
+ if retryable_url.url.is_empty() || retryable_url.url.site().is_empty() {
continue;
}
if !self
.robotstxt
- .is_allowed(&url, &self.user_agent.token)
+ .is_allowed(&retryable_url.url, &self.user_agent.token)
.await
{
continue;
}
- let site = Site(url.site().to_string());
- if job.job.fetch_sitemap && !crawled_sitemaps.contains(&site) {
+ let site = Site(retryable_url.url.site().to_string());
+ if job.fetch_sitemap && !crawled_sitemaps.contains(&site) {
crawled_sitemaps.insert(site.clone());
- if let Ok(Some(sitemap)) = self.robotstxt.sitemap(&url).await {
+ if let Ok(Some(sitemap)) = self.robotstxt.sitemap(&retryable_url.url).await {
let sitemap_urls = self.urls_from_sitemap(sitemap, 0, 5).await;
discovered_urls.extend(sitemap_urls.into_iter());
}
}
- let res = self.process_url(url).await;
-
- discovered_urls.extend(res.new_urls.into_iter());
- url_responses.push(res.response);
+ let res = self.process_url(retryable_url.url.clone()).await;
let mut delay = res.fetch_time.mul_f32(self.politeness_factor);
@@ -207,11 +209,27 @@ impl Worker {
}
tokio::time::sleep(delay).await;
+
+ if let UrlResponse::Failed {
+ url: _,
+ status_code,
+ } = res.response
+ {
+ if matches!(status_code, Some(429)) {
+ let mut retryable_url = retryable_url;
+ retryable_url.retries += 1;
+ job.urls.push_back(retryable_url);
+ continue;
+ }
+ }
+
+ discovered_urls.extend(res.new_urls.into_iter());
+ url_responses.push(res.response);
}
// send response to coordinator
let job_response = JobResponse {
- domain: job.job.domain,
+ domain: job.domain,
url_responses,
discovered_urls: discovered_urls.into_iter().collect(),
};