web search done right
[crawler] some sites seem to only host robots.txt files on the 'www.' subdomain without redirects
Mikkel Denker2376f33e19a0
diff --git a/crates/core/src/crawler/robots_txt.rs b/crates/core/src/crawler/robots_txt.rsindex fe6863f6..675f982d 100644--- a/crates/core/src/crawler/robots_txt.rs+++ b/crates/core/src/crawler/robots_txt.rs@@ -55,23 +55,14 @@ impl RobotsTxtManager {}}- async fn fetch_robots_txt(&self, site: &Site) -> Result<RobotsTxt> {- let mut res = self+ async fn fetch_robots_txt_from_url(&self, url: &str) -> Result<RobotsTxt> {+ let res = self.client- .get(&format!("http://{}/robots.txt", site.0))+ .get(url).timeout(Duration::from_secs(60)).send().await;- if res.is_err() {- res = self- .client- .get(&format!("https://{}/robots.txt", site.0))- .timeout(Duration::from_secs(60))- .send()- .await;- }-let res = res?;if res.status() != reqwest::StatusCode::OK {@@ -95,6 +86,29 @@ impl RobotsTxtManager {}}+ async fn fetch_robots_txt(&self, site: &Site) -> Result<RobotsTxt> {+ if let Ok(robots_txt) = self+ .fetch_robots_txt_from_url(&format!("http://{}/robots.txt", site.0))+ .await+ {+ return Ok(robots_txt);+ }++ match self+ .fetch_robots_txt_from_url(&format!("https://{}/robots.txt", site.0))+ .await+ {+ Ok(robots_txt) => Ok(robots_txt),+ _ if !site.0.starts_with("www.")+ && site.0.chars().filter(|&c| c == '.').count() == 1 =>+ {+ self.fetch_robots_txt_from_url(&format!("https://www.{}/robots.txt", &site.0))+ .await+ }+ Err(err) => Err(err),+ }+ }+fn maybe_prune(&mut self) {if self.last_prune.elapsed() < Duration::from_secs(60) {return;