web search done right
reuse RobotClient across sitemap checkers
Mikkel Denker1188235612ea
diff --git a/crates/core/src/live_index/crawler/checker/sitemap.rs b/crates/core/src/live_index/crawler/checker/sitemap.rsindex 574cb1c3..ff139bb4 100644--- a/crates/core/src/live_index/crawler/checker/sitemap.rs+++ b/crates/core/src/live_index/crawler/checker/sitemap.rs@@ -18,7 +18,6 @@ use std::time::Duration;use url::Url;-use crate::config::CrawlerConfig;use crate::crawler::robot_client::RobotClient;use crate::dated_url::DatedUrl;use crate::sitemap::{parse_sitemap, SitemapEntry};@@ -37,13 +36,13 @@ pub struct Sitemap {}impl Sitemap {- pub fn new(site: &site_stats::Site, config: &CrawlerConfig) -> Result<Self> {+ pub fn new(site: &site_stats::Site, client: RobotClient) -> Result<Self> {let robots_url = Url::robust_parse(&format!("{}/robots.txt", site.as_str()))?;Ok(Self {robots_url,+ client,last_check: std::time::Instant::now(),- client: RobotClient::new(config)?,})}diff --git a/crates/core/src/live_index/crawler/crawlable_site.rs b/crates/core/src/live_index/crawler/crawlable_site.rsindex d09613c8..9ccddb34 100644--- a/crates/core/src/live_index/crawler/crawlable_site.rs+++ b/crates/core/src/live_index/crawler/crawlable_site.rs@@ -69,7 +69,7 @@ impl InnerCrawlableSite {.collect(),client.reqwest(),),- sitemap: Sitemap::new(site.site(), client.crawler_config())?,+ sitemap: Sitemap::new(site.site(), client.reqwest())?,frontpage: Frontpage::new(site.site(), client.reqwest())?,last_drip: Instant::now(),drip_rate,diff --git a/crates/core/src/live_index/crawler/mod.rs b/crates/core/src/live_index/crawler/mod.rsindex 3d51f1c8..1e7a302b 100644--- a/crates/core/src/live_index/crawler/mod.rs+++ b/crates/core/src/live_index/crawler/mod.rs@@ -58,7 +58,6 @@ struct Client {live_index: Mutex<ReusableShardedClient<live_index::LiveIndexService>>,search: Mutex<ReusableShardedClient<search_server::SearchService>>,client: RobotClient,- crawler_config: CrawlerConfig,}impl Client {@@ -71,7 +70,6 @@ impl Client {client,live_index,search,- crawler_config: crawler_config.clone(),})}@@ -79,10 +77,6 @@ impl Client {self.client.clone()}- pub fn crawler_config(&self) -> &CrawlerConfig {- &self.crawler_config- }-async fn live_conn(&self) -> Arc<ShardedClient<live_index::LiveIndexService, ShardId>> {self.live_index.lock().await.conn().await}