Skip to content

web search done right

start politeness factor at 2, decrease iff. we don't receive any 429 responses. also increase max delay to 180 seconds

Mikkel Denker8d0ad573a724
diff --git a/crates/core/src/config/defaults.rs b/crates/core/src/config/defaults.rs
index d506be8d..8c1211e7 100644
--- a/crates/core/src/config/defaults.rs
+++ b/crates/core/src/config/defaults.rs
@@ -101,8 +101,16 @@ impl Crawler {
60 * 60
}
- pub fn politeness_factor() -> f32 {
- 1.0
+ pub fn min_politeness_factor() -> u32 {
+ 0
+ }
+
+ pub fn start_politeness_factor() -> u32 {
+ 2
+ }
+
+ pub fn max_politeness_factor() -> u32 {
+ 11
}
pub fn min_crawl_delay_ms() -> u64 {
@@ -110,11 +118,7 @@ impl Crawler {
}
pub fn max_crawl_delay_ms() -> u64 {
- 60_000
- }
-
- pub fn max_politeness_factor() -> f32 {
- 2048.0
+ 180_000
}
pub fn max_url_slowdown_retry() -> u8 {
diff --git a/crates/core/src/config/mod.rs b/crates/core/src/config/mod.rs
index ffebf2ad..7295ba60 100644
--- a/crates/core/src/config/mod.rs
+++ b/crates/core/src/config/mod.rs
@@ -345,8 +345,11 @@ pub struct CrawlerConfig {
#[serde(default = "defaults::Crawler::robots_txt_cache_sec")]
pub robots_txt_cache_sec: u64,
- #[serde(default = "defaults::Crawler::politeness_factor")]
- pub politeness_factor: f32,
+ #[serde(default = "defaults::Crawler::min_politeness_factor")]
+ pub min_politeness_factor: u32,
+
+ #[serde(default = "defaults::Crawler::start_politeness_factor")]
+ pub start_politeness_factor: u32,
#[serde(default = "defaults::Crawler::min_crawl_delay_ms")]
pub min_crawl_delay_ms: u64,
@@ -355,7 +358,7 @@ pub struct CrawlerConfig {
pub max_crawl_delay_ms: u64,
#[serde(default = "defaults::Crawler::max_politeness_factor")]
- pub max_politeness_factor: f32,
+ pub max_politeness_factor: u32,
#[serde(default = "defaults::Crawler::max_url_slowdown_retry")]
pub max_url_slowdown_retry: u8,
@@ -471,14 +474,16 @@ pub struct LiveIndexConfig {
pub user_agent: UserAgent,
#[serde(default = "defaults::Crawler::robots_txt_cache_sec")]
pub robots_txt_cache_sec: u64,
- #[serde(default = "defaults::Crawler::politeness_factor")]
- pub politeness_factor: f32,
+ #[serde(default = "defaults::Crawler::min_politeness_factor")]
+ pub min_politeness_factor: u32,
+ #[serde(default = "defaults::Crawler::start_politeness_factor")]
+ pub start_politeness_factor: u32,
#[serde(default = "defaults::Crawler::min_crawl_delay_ms")]
pub min_crawl_delay_ms: u64,
#[serde(default = "defaults::Crawler::max_crawl_delay_ms")]
pub max_crawl_delay_ms: u64,
#[serde(default = "defaults::Crawler::max_politeness_factor")]
- pub max_politeness_factor: f32,
+ pub max_politeness_factor: u32,
#[serde(default = "defaults::Crawler::max_url_slowdown_retry")]
pub max_url_slowdown_retry: u8,
#[serde(default = "defaults::Crawler::max_redirects")]
diff --git a/crates/core/src/crawler/worker.rs b/crates/core/src/crawler/worker.rs
index a07eaea4..82b28255 100644
--- a/crates/core/src/crawler/worker.rs
+++ b/crates/core/src/crawler/worker.rs
@@ -129,7 +129,8 @@ impl WorkerThread {
pub struct JobExecutor<S: DatumStream> {
writer: Arc<S>,
client: reqwest::Client,
- politeness_factor: f32,
+ has_gotten_429_response: bool,
+ politeness_factor: u32,
robotstxt: RobotsTxtManager,
crawled_urls: HashSet<Url>,
crawled_sitemaps: HashSet<Site>,
@@ -137,7 +138,8 @@ pub struct JobExecutor<S: DatumStream> {
min_crawl_delay: Duration,
max_crawl_delay: Duration,
max_url_slowdown_retry: u8,
- max_politeness_factor: f32,
+ min_politeness_factor: u32,
+ max_politeness_factor: u32,
wander_prioritiser: WanderPrioritiser,
wandered_urls: u64,
job: WorkerJob,
@@ -152,7 +154,8 @@ impl<S: DatumStream> JobExecutor<S> {
) -> Self {
Self {
writer,
- politeness_factor: config.politeness_factor,
+ politeness_factor: config.start_politeness_factor,
+ min_politeness_factor: config.min_politeness_factor,
robotstxt: RobotsTxtManager::new(&config),
client,
crawled_urls: HashSet::new(),
@@ -164,6 +167,7 @@ impl<S: DatumStream> JobExecutor<S> {
max_url_slowdown_retry: config.max_url_slowdown_retry,
max_politeness_factor: config.max_politeness_factor,
wander_prioritiser: WanderPrioritiser::new(),
+ has_gotten_429_response: false,
job,
}
}
@@ -307,6 +311,10 @@ impl<S: DatumStream> JobExecutor<S> {
match res {
Ok(res) => {
+ if !self.has_gotten_429_response {
+ self.decrease_politeness();
+ }
+
let weight = retryable_url.weighted_url.weight;
for new_url in res.new_urls {
@@ -325,6 +333,8 @@ impl<S: DatumStream> JobExecutor<S> {
status_code,
headers,
}) if status_code == 429 => {
+ self.has_gotten_429_response = true;
+
if headers.contains_key("retry-after") {
let retry_after = headers["retry-after"]
.to_str()
@@ -352,13 +362,26 @@ impl<S: DatumStream> JobExecutor<S> {
}
fn increase_politeness(&mut self) {
- self.politeness_factor *= 2.0;
+ self.politeness_factor += 1;
if self.politeness_factor > self.max_politeness_factor {
self.politeness_factor = self.max_politeness_factor;
}
- tracing::warn!("politeness factor increased to {}", self.politeness_factor);
+ tracing::warn!(
+ "politeness factor increased to {} for {}",
+ self.politeness_factor,
+ self.job.domain.as_str()
+ );
+ }
+
+ fn decrease_politeness(&mut self) {
+ if self.min_politeness_factor >= self.politeness_factor {
+ self.politeness_factor = self.min_politeness_factor;
+ return;
+ }
+
+ self.politeness_factor = self.politeness_factor.saturating_sub(1);
}
fn new_urls(&self, html: &Html) -> Vec<Url> {
@@ -431,7 +454,7 @@ impl<S: DatumStream> JobExecutor<S> {
fn delay_duration(&self) -> Duration {
let mut delay = self.min_crawl_delay;
- delay = delay.mul_f32(self.politeness_factor);
+ delay = delay.mul_f64(2.0_f64.powi(self.politeness_factor as i32));
if delay > self.max_crawl_delay {
delay = self.max_crawl_delay;
@@ -447,7 +470,7 @@ impl<S: DatumStream> JobExecutor<S> {
delay = self.min_crawl_delay;
}
- delay = delay.mul_f32(self.politeness_factor);
+ delay = delay.mul_f64(2.0_f64.powi(self.politeness_factor as i32));
if delay > self.max_crawl_delay {
delay = self.max_crawl_delay;
diff --git a/crates/core/src/live_index/mod.rs b/crates/core/src/live_index/mod.rs
index b8df3d39..98ec42e8 100644
--- a/crates/core/src/live_index/mod.rs
+++ b/crates/core/src/live_index/mod.rs
@@ -63,10 +63,11 @@ impl From<&LiveIndexConfig> for CrawlerConfig {
num_worker_threads: 1, // no impact
user_agent: live.user_agent.clone(),
robots_txt_cache_sec: live.robots_txt_cache_sec,
- politeness_factor: live.politeness_factor,
+ start_politeness_factor: live.start_politeness_factor,
+ min_politeness_factor: live.min_politeness_factor,
+ max_politeness_factor: live.max_politeness_factor,
min_crawl_delay_ms: live.min_crawl_delay_ms,
max_crawl_delay_ms: live.max_crawl_delay_ms,
- max_politeness_factor: live.max_politeness_factor,
max_url_slowdown_retry: live.max_url_slowdown_retry,
max_redirects: live.max_redirects,
timeout_seconds: live.timeout_seconds,
diff --git a/frontend/src/routes/webmasters/README.md b/frontend/src/routes/webmasters/README.md
index 664e87e9..4a12f379 100644
--- a/frontend/src/routes/webmasters/README.md
+++ b/frontend/src/routes/webmasters/README.md
@@ -7,11 +7,32 @@ The crawler uses the user agent `Mozilla/5.0 (compatible; StractBot/0.2; open so
## Politeness
StractBot is a polite crawler. It respects the [robots.txt](https://en.wikipedia.org/wiki/Robots.txt) file of the website it is crawling and tries to not overload the server.
-It does this by waiting a certain amount of time between requests to the same domain. The waiting time is calculated by _min(politeness \* max(fetchtime, 5 sec), 60 sec)_ where _fetchtime_ is the time it took to fetch the page. The crawler will wait at least 5 sec between requests and at most 60 seconds.
-This dynamic waiting time tries to prevent us from disrupting servers that cannot handle the load, while not giving unnecessary politeness to servers that can. The politeness factor starts at 1 and is doubled every time the crawler gets a 429 response from the server (to at most 2048).
-The crawler looks for the token `StractBot` in the robots.txt file to determine which pages (if any) the crawler is allowed to crawl.
-The robots.txt file is cached for 1 hour, so changes to the file should be respected quite quickly.
+### Waiting Time Calculation
+
+The crawler waits a certain amount of time between requests to the same domain. The waiting time is calculated by:
+_min((2^politeness) \* max(fetchtime, 5 sec), 180 sec)_
+
+Where:
+
+- _politeness_ is the politeness factor (starting at 2)
+- _fetchtime_ is the time it took to fetch the previous page
+
+The crawler will wait at least 5 seconds between requests and at most 180 seconds.
+
+### 429 Response Handling
+
+If the crawler receives a 429 (Too Many Requests) response from the server:
+
+1. The politeness factor is increased, doubling the time between each request.
+2. This increased politeness factor persists for the duration of the crawl for that specific domain.
+3. The crawler will never decrease the politeness factor for a domain that has returned a 429 response.
+
+If the crawler hasn't received any 429 responses from a server, it may gradually decrease the politeness factor over time, but never below 0.
+
+### Robots.txt
+
+The crawler looks for the token StractBot in the robots.txt file to determine which pages (if any) it is allowed to crawl. The robots.txt file is cached for 1 hour, so changes to the file should be respected quite quickly.
## Contact us