Skip to content

web search done right

[crawler] some sites seem to only host robots.txt files on the 'www.' subdomain without redirects

Mikkel Denker2376f33e19a0
diff --git a/crates/core/src/crawler/robots_txt.rs b/crates/core/src/crawler/robots_txt.rs
index fe6863f6..675f982d 100644
--- a/crates/core/src/crawler/robots_txt.rs
+++ b/crates/core/src/crawler/robots_txt.rs
@@ -55,23 +55,14 @@ impl RobotsTxtManager {
}
}
- async fn fetch_robots_txt(&self, site: &Site) -> Result<RobotsTxt> {
- let mut res = self
+ async fn fetch_robots_txt_from_url(&self, url: &str) -> Result<RobotsTxt> {
+ let res = self
.client
- .get(&format!("http://{}/robots.txt", site.0))
+ .get(url)
.timeout(Duration::from_secs(60))
.send()
.await;
- if res.is_err() {
- res = self
- .client
- .get(&format!("https://{}/robots.txt", site.0))
- .timeout(Duration::from_secs(60))
- .send()
- .await;
- }
-
let res = res?;
if res.status() != reqwest::StatusCode::OK {
@@ -95,6 +86,29 @@ impl RobotsTxtManager {
}
}
+ async fn fetch_robots_txt(&self, site: &Site) -> Result<RobotsTxt> {
+ if let Ok(robots_txt) = self
+ .fetch_robots_txt_from_url(&format!("http://{}/robots.txt", site.0))
+ .await
+ {
+ return Ok(robots_txt);
+ }
+
+ match self
+ .fetch_robots_txt_from_url(&format!("https://{}/robots.txt", site.0))
+ .await
+ {
+ Ok(robots_txt) => Ok(robots_txt),
+ _ if !site.0.starts_with("www.")
+ && site.0.chars().filter(|&c| c == '.').count() == 1 =>
+ {
+ self.fetch_robots_txt_from_url(&format!("https://www.{}/robots.txt", &site.0))
+ .await
+ }
+ Err(err) => Err(err),
+ }
+ }
+
fn maybe_prune(&mut self) {
if self.last_prune.elapsed() < Duration::from_secs(60) {
return;