Skip to content

web search done right

longer timeout when fetching robots.txt

Mikkel Denkerdffc149b101e
diff --git a/core/src/crawler/robots_txt.rs b/core/src/crawler/robots_txt.rs
index 1b164aa8..0a85c141 100644
--- a/core/src/crawler/robots_txt.rs
+++ b/core/src/crawler/robots_txt.rs
@@ -57,6 +57,7 @@ impl RobotsTxtManager {
let mut res = self
.client
.get(&format!("http://{}/robots.txt", site.0))
+ .timeout(Duration::from_secs(30))
.send()
.await;
@@ -64,6 +65,7 @@ impl RobotsTxtManager {
res = self
.client
.get(&format!("https://{}/robots.txt", site.0))
+ .timeout(Duration::from_secs(30))
.send()
.await;
}