web search done right
wrap reqwest requestbuilder so .send() errors in tests
Mikkel Denkereb41e3c62d46
diff --git a/crates/core/src/crawler/robot_client.rs b/crates/core/src/crawler/robot_client.rsindex 8540e0b3..5fbf3ad6 100644--- a/crates/core/src/crawler/robot_client.rs+++ b/crates/core/src/crawler/robot_client.rs@@ -63,12 +63,41 @@ impl RobotClient {&self.robots_txt_manager}- pub async fn get(&self, url: Url) -> Result<reqwest::RequestBuilder> {+ pub async fn get(&self, url: Url) -> Result<RequestBuilder> {if !self.robots_txt_manager.is_allowed(&url).await {return Err(Error::DisallowedPath);}- Ok(self.client.get(url))+ Ok(RequestBuilder::new(self.client.get(url)))+ }+}++#[allow(dead_code)] // not used in tests, but required for public API+pub struct RequestBuilder(reqwest::RequestBuilder);++impl From<reqwest::RequestBuilder> for RequestBuilder {+ fn from(builder: reqwest::RequestBuilder) -> Self {+ Self::new(builder)+ }+}++impl RequestBuilder {+ pub fn new(builder: reqwest::RequestBuilder) -> Self {+ Self(builder)+ }++ #[cfg(not(test))]+ pub async fn send(self) -> Result<reqwest::Response> {+ self.0.send().await.map_err(|e| Error::from(anyhow!(e)))+ }++ #[cfg(test)]+ pub async fn send(&self) -> Result<reqwest::Response> {+ Err(Error::from(anyhow!("Not allowed in tests")))+ }++ pub fn timeout(self, timeout: Duration) -> Self {+ Self(self.0.timeout(timeout))}}diff --git a/crates/core/src/crawler/robots_txt.rs b/crates/core/src/crawler/robots_txt.rsindex 2dc94777..5eb43415 100644--- a/crates/core/src/crawler/robots_txt.rs+++ b/crates/core/src/crawler/robots_txt.rs@@ -118,9 +118,7 @@ impl InnerRobotsTxtManager {}async fn fetch_robots_txt_from_url(&self, url: &str) -> Lookup<robotstxt::Robots> {- let res = match self- .client- .get(url)+ let res = match super::robot_client::RequestBuilder::new(self.client.get(url)).timeout(Duration::from_secs(60)).send().await