web search done right
longer timeout when fetching robots.txt
Mikkel Denkerdffc149b101e
diff --git a/core/src/crawler/robots_txt.rs b/core/src/crawler/robots_txt.rsindex 1b164aa8..0a85c141 100644--- a/core/src/crawler/robots_txt.rs+++ b/core/src/crawler/robots_txt.rs@@ -57,6 +57,7 @@ impl RobotsTxtManager {let mut res = self.client.get(&format!("http://{}/robots.txt", site.0))+ .timeout(Duration::from_secs(30)).send().await;@@ -64,6 +65,7 @@ impl RobotsTxtManager {res = self.client.get(&format!("https://{}/robots.txt", site.0))+ .timeout(Duration::from_secs(30)).send().await;}