web search done right
start politeness factor at 2, decrease iff. we don't receive any 429 responses. also increase max delay to 180 seconds
Mikkel Denker8d0ad573a724
diff --git a/crates/core/src/config/defaults.rs b/crates/core/src/config/defaults.rsindex d506be8d..8c1211e7 100644--- a/crates/core/src/config/defaults.rs+++ b/crates/core/src/config/defaults.rs@@ -101,8 +101,16 @@ impl Crawler {60 * 60}- pub fn politeness_factor() -> f32 {- 1.0+ pub fn min_politeness_factor() -> u32 {+ 0+ }++ pub fn start_politeness_factor() -> u32 {+ 2+ }++ pub fn max_politeness_factor() -> u32 {+ 11}pub fn min_crawl_delay_ms() -> u64 {@@ -110,11 +118,7 @@ impl Crawler {}pub fn max_crawl_delay_ms() -> u64 {- 60_000- }-- pub fn max_politeness_factor() -> f32 {- 2048.0+ 180_000}pub fn max_url_slowdown_retry() -> u8 {diff --git a/crates/core/src/config/mod.rs b/crates/core/src/config/mod.rsindex ffebf2ad..7295ba60 100644--- a/crates/core/src/config/mod.rs+++ b/crates/core/src/config/mod.rs@@ -345,8 +345,11 @@ pub struct CrawlerConfig {#[serde(default = "defaults::Crawler::robots_txt_cache_sec")]pub robots_txt_cache_sec: u64,- #[serde(default = "defaults::Crawler::politeness_factor")]- pub politeness_factor: f32,+ #[serde(default = "defaults::Crawler::min_politeness_factor")]+ pub min_politeness_factor: u32,++ #[serde(default = "defaults::Crawler::start_politeness_factor")]+ pub start_politeness_factor: u32,#[serde(default = "defaults::Crawler::min_crawl_delay_ms")]pub min_crawl_delay_ms: u64,@@ -355,7 +358,7 @@ pub struct CrawlerConfig {pub max_crawl_delay_ms: u64,#[serde(default = "defaults::Crawler::max_politeness_factor")]- pub max_politeness_factor: f32,+ pub max_politeness_factor: u32,#[serde(default = "defaults::Crawler::max_url_slowdown_retry")]pub max_url_slowdown_retry: u8,@@ -471,14 +474,16 @@ pub struct LiveIndexConfig {pub user_agent: UserAgent,#[serde(default = "defaults::Crawler::robots_txt_cache_sec")]pub robots_txt_cache_sec: u64,- #[serde(default = "defaults::Crawler::politeness_factor")]- pub politeness_factor: f32,+ #[serde(default = "defaults::Crawler::min_politeness_factor")]+ pub min_politeness_factor: u32,+ #[serde(default = "defaults::Crawler::start_politeness_factor")]+ pub start_politeness_factor: u32,#[serde(default = "defaults::Crawler::min_crawl_delay_ms")]pub min_crawl_delay_ms: u64,#[serde(default = "defaults::Crawler::max_crawl_delay_ms")]pub max_crawl_delay_ms: u64,#[serde(default = "defaults::Crawler::max_politeness_factor")]- pub max_politeness_factor: f32,+ pub max_politeness_factor: u32,#[serde(default = "defaults::Crawler::max_url_slowdown_retry")]pub max_url_slowdown_retry: u8,#[serde(default = "defaults::Crawler::max_redirects")]diff --git a/crates/core/src/crawler/worker.rs b/crates/core/src/crawler/worker.rsindex a07eaea4..82b28255 100644--- a/crates/core/src/crawler/worker.rs+++ b/crates/core/src/crawler/worker.rs@@ -129,7 +129,8 @@ impl WorkerThread {pub struct JobExecutor<S: DatumStream> {writer: Arc<S>,client: reqwest::Client,- politeness_factor: f32,+ has_gotten_429_response: bool,+ politeness_factor: u32,robotstxt: RobotsTxtManager,crawled_urls: HashSet<Url>,crawled_sitemaps: HashSet<Site>,@@ -137,7 +138,8 @@ pub struct JobExecutor<S: DatumStream> {min_crawl_delay: Duration,max_crawl_delay: Duration,max_url_slowdown_retry: u8,- max_politeness_factor: f32,+ min_politeness_factor: u32,+ max_politeness_factor: u32,wander_prioritiser: WanderPrioritiser,wandered_urls: u64,job: WorkerJob,@@ -152,7 +154,8 @@ impl<S: DatumStream> JobExecutor<S> {) -> Self {Self {writer,- politeness_factor: config.politeness_factor,+ politeness_factor: config.start_politeness_factor,+ min_politeness_factor: config.min_politeness_factor,robotstxt: RobotsTxtManager::new(&config),client,crawled_urls: HashSet::new(),@@ -164,6 +167,7 @@ impl<S: DatumStream> JobExecutor<S> {max_url_slowdown_retry: config.max_url_slowdown_retry,max_politeness_factor: config.max_politeness_factor,wander_prioritiser: WanderPrioritiser::new(),+ has_gotten_429_response: false,job,}}@@ -307,6 +311,10 @@ impl<S: DatumStream> JobExecutor<S> {match res {Ok(res) => {+ if !self.has_gotten_429_response {+ self.decrease_politeness();+ }+let weight = retryable_url.weighted_url.weight;for new_url in res.new_urls {@@ -325,6 +333,8 @@ impl<S: DatumStream> JobExecutor<S> {status_code,headers,}) if status_code == 429 => {+ self.has_gotten_429_response = true;+if headers.contains_key("retry-after") {let retry_after = headers["retry-after"].to_str()@@ -352,13 +362,26 @@ impl<S: DatumStream> JobExecutor<S> {}fn increase_politeness(&mut self) {- self.politeness_factor *= 2.0;+ self.politeness_factor += 1;if self.politeness_factor > self.max_politeness_factor {self.politeness_factor = self.max_politeness_factor;}- tracing::warn!("politeness factor increased to {}", self.politeness_factor);+ tracing::warn!(+ "politeness factor increased to {} for {}",+ self.politeness_factor,+ self.job.domain.as_str()+ );+ }++ fn decrease_politeness(&mut self) {+ if self.min_politeness_factor >= self.politeness_factor {+ self.politeness_factor = self.min_politeness_factor;+ return;+ }++ self.politeness_factor = self.politeness_factor.saturating_sub(1);}fn new_urls(&self, html: &Html) -> Vec<Url> {@@ -431,7 +454,7 @@ impl<S: DatumStream> JobExecutor<S> {fn delay_duration(&self) -> Duration {let mut delay = self.min_crawl_delay;- delay = delay.mul_f32(self.politeness_factor);+ delay = delay.mul_f64(2.0_f64.powi(self.politeness_factor as i32));if delay > self.max_crawl_delay {delay = self.max_crawl_delay;@@ -447,7 +470,7 @@ impl<S: DatumStream> JobExecutor<S> {delay = self.min_crawl_delay;}- delay = delay.mul_f32(self.politeness_factor);+ delay = delay.mul_f64(2.0_f64.powi(self.politeness_factor as i32));if delay > self.max_crawl_delay {delay = self.max_crawl_delay;diff --git a/crates/core/src/live_index/mod.rs b/crates/core/src/live_index/mod.rsindex b8df3d39..98ec42e8 100644--- a/crates/core/src/live_index/mod.rs+++ b/crates/core/src/live_index/mod.rs@@ -63,10 +63,11 @@ impl From<&LiveIndexConfig> for CrawlerConfig {num_worker_threads: 1, // no impactuser_agent: live.user_agent.clone(),robots_txt_cache_sec: live.robots_txt_cache_sec,- politeness_factor: live.politeness_factor,+ start_politeness_factor: live.start_politeness_factor,+ min_politeness_factor: live.min_politeness_factor,+ max_politeness_factor: live.max_politeness_factor,min_crawl_delay_ms: live.min_crawl_delay_ms,max_crawl_delay_ms: live.max_crawl_delay_ms,- max_politeness_factor: live.max_politeness_factor,max_url_slowdown_retry: live.max_url_slowdown_retry,max_redirects: live.max_redirects,timeout_seconds: live.timeout_seconds,diff --git a/frontend/src/routes/webmasters/README.md b/frontend/src/routes/webmasters/README.mdindex 664e87e9..4a12f379 100644--- a/frontend/src/routes/webmasters/README.md+++ b/frontend/src/routes/webmasters/README.md@@ -7,11 +7,32 @@ The crawler uses the user agent `Mozilla/5.0 (compatible; StractBot/0.2; open so## PolitenessStractBot is a polite crawler. It respects the [robots.txt](https://en.wikipedia.org/wiki/Robots.txt) file of the website it is crawling and tries to not overload the server.-It does this by waiting a certain amount of time between requests to the same domain. The waiting time is calculated by _min(politeness \* max(fetchtime, 5 sec), 60 sec)_ where _fetchtime_ is the time it took to fetch the page. The crawler will wait at least 5 sec between requests and at most 60 seconds.-This dynamic waiting time tries to prevent us from disrupting servers that cannot handle the load, while not giving unnecessary politeness to servers that can. The politeness factor starts at 1 and is doubled every time the crawler gets a 429 response from the server (to at most 2048).-The crawler looks for the token `StractBot` in the robots.txt file to determine which pages (if any) the crawler is allowed to crawl.-The robots.txt file is cached for 1 hour, so changes to the file should be respected quite quickly.+### Waiting Time Calculation++The crawler waits a certain amount of time between requests to the same domain. The waiting time is calculated by:+_min((2^politeness) \* max(fetchtime, 5 sec), 180 sec)_++Where:++- _politeness_ is the politeness factor (starting at 2)+- _fetchtime_ is the time it took to fetch the previous page++The crawler will wait at least 5 seconds between requests and at most 180 seconds.++### 429 Response Handling++If the crawler receives a 429 (Too Many Requests) response from the server:++1. The politeness factor is increased, doubling the time between each request.+2. This increased politeness factor persists for the duration of the crawl for that specific domain.+3. The crawler will never decrease the politeness factor for a domain that has returned a 429 response.++If the crawler hasn't received any 429 responses from a server, it may gradually decrease the politeness factor over time, but never below 0.++### Robots.txt++The crawler looks for the token StractBot in the robots.txt file to determine which pages (if any) it is allowed to crawl. The robots.txt file is cached for 1 hour, so changes to the file should be respected quite quickly.## Contact us