Skip to content

web search done right

reuse RobotClient across sitemap checkers

Mikkel Denker1188235612ea
diff --git a/crates/core/src/live_index/crawler/checker/sitemap.rs b/crates/core/src/live_index/crawler/checker/sitemap.rs
index 574cb1c3..ff139bb4 100644
--- a/crates/core/src/live_index/crawler/checker/sitemap.rs
+++ b/crates/core/src/live_index/crawler/checker/sitemap.rs
@@ -18,7 +18,6 @@ use std::time::Duration;
use url::Url;
-use crate::config::CrawlerConfig;
use crate::crawler::robot_client::RobotClient;
use crate::dated_url::DatedUrl;
use crate::sitemap::{parse_sitemap, SitemapEntry};
@@ -37,13 +36,13 @@ pub struct Sitemap {
}
impl Sitemap {
- pub fn new(site: &site_stats::Site, config: &CrawlerConfig) -> Result<Self> {
+ pub fn new(site: &site_stats::Site, client: RobotClient) -> Result<Self> {
let robots_url = Url::robust_parse(&format!("{}/robots.txt", site.as_str()))?;
Ok(Self {
robots_url,
+ client,
last_check: std::time::Instant::now(),
- client: RobotClient::new(config)?,
})
}
diff --git a/crates/core/src/live_index/crawler/crawlable_site.rs b/crates/core/src/live_index/crawler/crawlable_site.rs
index d09613c8..9ccddb34 100644
--- a/crates/core/src/live_index/crawler/crawlable_site.rs
+++ b/crates/core/src/live_index/crawler/crawlable_site.rs
@@ -69,7 +69,7 @@ impl InnerCrawlableSite {
.collect(),
client.reqwest(),
),
- sitemap: Sitemap::new(site.site(), client.crawler_config())?,
+ sitemap: Sitemap::new(site.site(), client.reqwest())?,
frontpage: Frontpage::new(site.site(), client.reqwest())?,
last_drip: Instant::now(),
drip_rate,
diff --git a/crates/core/src/live_index/crawler/mod.rs b/crates/core/src/live_index/crawler/mod.rs
index 3d51f1c8..1e7a302b 100644
--- a/crates/core/src/live_index/crawler/mod.rs
+++ b/crates/core/src/live_index/crawler/mod.rs
@@ -58,7 +58,6 @@ struct Client {
live_index: Mutex<ReusableShardedClient<live_index::LiveIndexService>>,
search: Mutex<ReusableShardedClient<search_server::SearchService>>,
client: RobotClient,
- crawler_config: CrawlerConfig,
}
impl Client {
@@ -71,7 +70,6 @@ impl Client {
client,
live_index,
search,
- crawler_config: crawler_config.clone(),
})
}
@@ -79,10 +77,6 @@ impl Client {
self.client.clone()
}
- pub fn crawler_config(&self) -> &CrawlerConfig {
- &self.crawler_config
- }
-
async fn live_conn(&self) -> Arc<ShardedClient<live_index::LiveIndexService, ShardId>> {
self.live_index.lock().await.conn().await
}