web search done right
[live_crawler] make sure frontpage check is skipped if disallowed in robots.txt
Mikkel Denkerc662fe2855af
diff --git a/crates/core/src/config/mod.rs b/crates/core/src/config/mod.rsindex 8e85f168..ac420011 100644--- a/crates/core/src/config/mod.rs+++ b/crates/core/src/config/mod.rs@@ -397,6 +397,35 @@ pub struct CrawlerConfig {pub router_hosts: Vec<String>,}+#[cfg(test)]+impl CrawlerConfig {+ pub fn for_tests() -> Self {+ Self {+ num_worker_threads: 1,+ user_agent: UserAgent {+ full: "test".to_string(),+ token: "test".to_string(),+ },+ robots_txt_cache_sec: defaults::Crawler::robots_txt_cache_sec(),+ min_politeness_factor: defaults::Crawler::min_politeness_factor(),+ start_politeness_factor: defaults::Crawler::start_politeness_factor(),+ min_crawl_delay_ms: defaults::Crawler::min_crawl_delay_ms(),+ max_crawl_delay_ms: defaults::Crawler::max_crawl_delay_ms(),+ max_politeness_factor: defaults::Crawler::max_politeness_factor(),+ max_url_slowdown_retry: defaults::Crawler::max_url_slowdown_retry(),+ timeout_seconds: defaults::Crawler::timeout_seconds(),+ s3: S3Config {+ bucket: "".to_string(),+ folder: "".to_string(),+ access_key: "".to_string(),+ secret_key: "".to_string(),+ endpoint: "".to_string(),+ },+ router_hosts: Vec::new(),+ }+ }+}+#[derive(Debug, Clone, serde::Deserialize, serde::Serialize)]pub struct DailyLiveIndexCrawlerBudget {#[serde(default = "defaults::LiveCrawler::blogs_budget")]diff --git a/crates/core/src/crawler/mod.rs b/crates/core/src/crawler/mod.rsindex d7d41c23..69a25e78 100644--- a/crates/core/src/crawler/mod.rs+++ b/crates/core/src/crawler/mod.rs@@ -1,5 +1,5 @@// Stract is an open source web search engine.-// Copyright (C) 2023 Stract ApS+// Copyright (C) 2024 Stract ApS//// This program is free software: you can redistribute it and/or modify// it under the terms of the GNU Affero General Public License as@@ -14,12 +14,13 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.-use std::{collections::VecDeque, future::Future, net::SocketAddr, sync::Arc, time::Duration};+use std::{collections::VecDeque, future::Future, net::SocketAddr, sync::Arc};type HashMap<K, V> = std::collections::HashMap<K, V, ahash::RandomState>;use anyhow::anyhow;use futures::StreamExt;+use robot_client::RobotClient;use url::Url;use crate::{config::CrawlerConfig, warc, webpage::url_ext::UrlExt};@@ -33,6 +34,7 @@ pub mod router;pub use router::Router;mod file_queue;pub mod planner;+pub mod robot_client;mod wander_prirotiser;mod warc_writer;mod worker;@@ -70,6 +72,9 @@ pub enum Error {#[error("couldn't parse html")]InvalidHtml,+ #[error("request path is disallowed by robots.txt")]+ DisallowedPath,+#[error("an error occurred: {0}")]Anyhow(#[from] anyhow::Error),}@@ -265,6 +270,7 @@ impl Crawler {let writer = Arc::new(WarcWriter::new(config.s3.clone()));let mut handles = Vec::new();let mut router_hosts = Vec::new();+ let client = RobotClient::new(&config)?;for host in &config.router_hosts {router_hosts.push(@@ -274,8 +280,12 @@ impl Crawler {}for _ in 0..config.num_worker_threads {- let worker =- WorkerThread::new(Arc::clone(&writer), config.clone(), router_hosts.clone())?;+ let worker = WorkerThread::new(+ Arc::clone(&writer),+ client.clone(),+ config.clone(),+ router_hosts.clone(),+ )?;handles.push(tokio::spawn(async move {worker.run().await;@@ -299,30 +309,6 @@ pub trait DatumStream: Send + Sync {fn finish(&self) -> impl Future<Output = Result<()>> + Send;}-pub fn reqwest_client(config: &CrawlerConfig) -> Result<reqwest::Client> {- let timeout = Duration::from_secs(config.timeout_seconds);-- let mut headers = reqwest::header::HeaderMap::default();- headers.insert(- reqwest::header::ACCEPT,- reqwest::header::HeaderValue::from_static("text/html"),- );- headers.insert(- reqwest::header::ACCEPT_LANGUAGE,- reqwest::header::HeaderValue::from_static("en-US,en;q=0.9,*;q=0.8"),- );-- reqwest::Client::builder()- .timeout(timeout)- .connect_timeout(timeout)- .http2_keep_alive_interval(None)- .default_headers(headers)- .redirect(reqwest::redirect::Policy::limited(0))- .user_agent(&config.user_agent.full)- .build()- .map_err(|e| Error::from(anyhow!(e)))-}-pub async fn encoded_body(res: reqwest::Response) -> Result<String> {let content_type = res.headers()diff --git a/crates/core/src/crawler/robot_client.rs b/crates/core/src/crawler/robot_client.rsnew file mode 100644index 00000000..8540e0b3--- /dev/null+++ b/crates/core/src/crawler/robot_client.rs@@ -0,0 +1,98 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use super::{Error, Result};+use crate::config::CrawlerConfig;+use crate::crawler::robots_txt::RobotsTxtManager;+use anyhow::anyhow;+use std::time::Duration;+use url::Url;++pub(super) fn reqwest_client(config: &CrawlerConfig) -> Result<reqwest::Client> {+ let timeout = Duration::from_secs(config.timeout_seconds);++ let mut headers = reqwest::header::HeaderMap::default();+ headers.insert(+ reqwest::header::ACCEPT,+ reqwest::header::HeaderValue::from_static("text/html"),+ );+ headers.insert(+ reqwest::header::ACCEPT_LANGUAGE,+ reqwest::header::HeaderValue::from_static("en-US,en;q=0.9,*;q=0.8"),+ );++ reqwest::Client::builder()+ .timeout(timeout)+ .connect_timeout(timeout)+ .http2_keep_alive_interval(None)+ .default_headers(headers)+ .redirect(reqwest::redirect::Policy::limited(0))+ .user_agent(&config.user_agent.full)+ .build()+ .map_err(|e| Error::from(anyhow!(e)))+}++#[derive(Clone)]+pub struct RobotClient {+ robots_txt_manager: RobotsTxtManager,+ client: reqwest::Client,+}++impl RobotClient {+ pub fn new(config: &CrawlerConfig) -> Result<Self> {+ Ok(Self {+ client: reqwest_client(config)?,+ robots_txt_manager: RobotsTxtManager::new(config),+ })+ }++ pub fn robots_txt_manager(&self) -> &RobotsTxtManager {+ &self.robots_txt_manager+ }++ pub async fn get(&self, url: Url) -> Result<reqwest::RequestBuilder> {+ if !self.robots_txt_manager.is_allowed(&url).await {+ return Err(Error::DisallowedPath);+ }++ Ok(self.client.get(url))+ }+}++#[cfg(test)]+mod tests {+ use super::*;++ #[tokio::test]+ async fn test_errs_disallowed_path() {+ let config = CrawlerConfig::for_tests();+ let client = RobotClient::new(&config).unwrap();++ let robots_txt =+ robotstxt::Robots::parse("TestBot", "User-agent: *\nDisallow: /test\nAllow: /example")+ .unwrap();++ client+ .robots_txt_manager()+ .insert("example.com".to_string(), robots_txt);++ let url = Url::parse("http://example.com/test").unwrap();+ assert!(client.get(url).await.is_err());++ let url = Url::parse("http://example.com/example").unwrap();+ assert!(client.get(url).await.is_ok());+ }+}diff --git a/crates/core/src/crawler/robots_txt.rs b/crates/core/src/crawler/robots_txt.rsindex 06afc9b1..2dc94777 100644--- a/crates/core/src/crawler/robots_txt.rs+++ b/crates/core/src/crawler/robots_txt.rs@@ -14,8 +14,14 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.-use std::{collections::BTreeMap, panic, time::Duration};-+use std::{+ ops::Deref,+ panic,+ sync::{Arc, Mutex},+ time::Duration,+};++use dashmap::DashMap;use url::Url;use crate::{config::CrawlerConfig, crawler};@@ -56,9 +62,9 @@ impl CheckedRobotsTxt {}}-pub struct RobotsTxtManager {- cache: BTreeMap<Site, CheckedRobotsTxt>,- last_prune: std::time::Instant,+struct InnerRobotsTxtManager {+ cache: DashMap<Site, CheckedRobotsTxt>,+ last_prune: Arc<Mutex<std::time::Instant>>,client: reqwest::Client,cache_expiration: Duration,user_agent: String,@@ -66,9 +72,9 @@ pub struct RobotsTxtManager {max_crawl_delay: Duration,}-impl RobotsTxtManager {- pub fn new(config: &CrawlerConfig) -> Self {- let client = crawler::reqwest_client(config).unwrap();+impl InnerRobotsTxtManager {+ fn new(config: &CrawlerConfig) -> Self {+ let client = crawler::robot_client::reqwest_client(config).unwrap();let cache_expiration = Duration::from_secs(config.robots_txt_cache_sec);let user_agent = config.user_agent.token.clone();let min_crawl_delay = Duration::from_millis(config.min_crawl_delay_ms);@@ -77,31 +83,31 @@ impl RobotsTxtManager {Self {client,cache_expiration,- last_prune: std::time::Instant::now(),- cache: BTreeMap::new(),+ last_prune: Arc::new(Mutex::new(std::time::Instant::now())),+ cache: DashMap::new(),user_agent: user_agent.to_string(),min_crawl_delay,max_crawl_delay,}}- pub async fn is_allowed(&mut self, url: &Url) -> bool {- match self.get_mut(url).await {+ async fn is_allowed(&self, url: &Url) -> bool {+ match &self.get(url).await.robots {Lookup::Found(robots_txt) => robots_txt.is_allowed(url),Lookup::Unavailable => true,Lookup::Unreachable => false,}}- pub async fn crawl_delay(&mut self, url: &Url) -> Option<Duration> {- match self.get_mut(url).await {+ async fn crawl_delay(&self, url: &Url) -> Option<Duration> {+ match &self.get(url).await.robots {Lookup::Found(robots_txt) => robots_txt.crawl_delay(),Lookup::Unavailable | Lookup::Unreachable => None,}}- pub async fn sitemaps(&mut self, url: &Url) -> Vec<Url> {- match self.get_mut(url).await {+ async fn sitemaps(&self, url: &Url) -> Vec<Url> {+ match &self.get(url).await.robots {Lookup::Found(robots_txt) => robots_txt.sitemaps().iter()@@ -197,18 +203,18 @@ impl RobotsTxtManager {}}- fn maybe_prune(&mut self) {- if self.last_prune.elapsed() < Duration::from_secs(60) {+ fn maybe_prune(&self) {+ if self.last_prune.lock().unwrap().elapsed() < Duration::from_secs(60) {return;}self.cache.retain(|_, v| !v.is_expired(&self.cache_expiration));- self.last_prune = std::time::Instant::now();+ *self.last_prune.lock().unwrap() = std::time::Instant::now();}- async fn get_mut(&mut self, url: &Url) -> &mut Lookup<robotstxt::Robots> {+ async fn get(&self, url: &Url) -> impl Deref<Target = CheckedRobotsTxt> + '_ {self.maybe_prune();let site = Site(url.host_str().unwrap_or_default().to_string());@@ -223,7 +229,51 @@ impl RobotsTxtManager {.insert(site.clone(), self.fetch_robots_txt(&site).await);}- &mut self.cache.get_mut(&site).unwrap().robots+ self.cache.get(&site).unwrap()+ }++ #[cfg(test)]+ fn insert(&self, site: String, robots_txt: robotstxt::Robots) {+ self.cache+ .insert(Site(site), CheckedRobotsTxt::new(Lookup::Found(robots_txt)));+ }+}+pub struct RobotsTxtManager {+ inner: Arc<InnerRobotsTxtManager>,+}++impl Clone for RobotsTxtManager {+ fn clone(&self) -> Self {+ Self {+ inner: self.inner.clone(),+ }+ }+}++impl RobotsTxtManager {+ pub fn new(config: &CrawlerConfig) -> Self {+ Self {+ inner: Arc::new(InnerRobotsTxtManager::new(config)),+ }+ }+}++impl RobotsTxtManager {+ pub async fn is_allowed(&self, url: &Url) -> bool {+ self.inner.is_allowed(url).await+ }++ pub async fn crawl_delay(&self, url: &Url) -> Option<Duration> {+ self.inner.crawl_delay(url).await+ }++ pub async fn sitemaps(&self, url: &Url) -> Vec<Url> {+ self.inner.sitemaps(url).await+ }++ #[cfg(test)]+ pub fn insert(&self, site: String, robots_txt: robotstxt::Robots) {+ self.inner.insert(site, robots_txt);}}diff --git a/crates/core/src/crawler/worker.rs b/crates/core/src/crawler/worker.rsindex 3ed5945b..f50525a8 100644--- a/crates/core/src/crawler/worker.rs+++ b/crates/core/src/crawler/worker.rs@@ -38,10 +38,9 @@ use crate::{};use super::{- encoded_body, reqwest_client, robots_txt::RobotsTxtManager,- wander_prirotiser::WanderPrioritiser, CrawlDatum, DatumStream, Domain, Error, Result,- RetrieableUrl, Site, WarcWriter, WeightedUrl, WorkerJob, MAX_CONTENT_LENGTH,- MAX_OUTGOING_URLS_PER_PAGE,+ encoded_body, robot_client::RobotClient, wander_prirotiser::WanderPrioritiser, CrawlDatum,+ DatumStream, Domain, Error, Result, RetrieableUrl, Site, WarcWriter, WeightedUrl, WorkerJob,+ MAX_CONTENT_LENGTH, MAX_OUTGOING_URLS_PER_PAGE,};const IGNORED_EXTENSIONS: [&str; 27] = [@@ -63,19 +62,18 @@ struct ProcessedUrl {pub struct WorkerThread {writer: Arc<WarcWriter>,- client: reqwest::Client,config: Arc<CrawlerConfig>,router_hosts: Vec<SocketAddr>,+ client: RobotClient,}impl WorkerThread {pub fn new(writer: Arc<WarcWriter>,+ client: RobotClient,config: CrawlerConfig,router_hosts: Vec<SocketAddr>,) -> Result<Self> {- let client = reqwest_client(&config)?;-Ok(Self {writer,client,@@ -109,9 +107,9 @@ impl WorkerThread {Ok(Some(job)) => {let executor = JobExecutor::new(job.into(),- self.client.clone(),self.config.clone(),self.writer.clone(),+ self.client.clone(),);executor.run().await;}@@ -129,10 +127,9 @@ impl WorkerThread {pub struct JobExecutor<S: DatumStream> {writer: Arc<S>,- client: reqwest::Client,+ client: RobotClient,has_gotten_429_response: bool,politeness_factor: u32,- robotstxt: RobotsTxtManager,crawled_urls: HashSet<Url>,crawled_sitemaps: HashSet<Site>,sitemap_urls: HashSet<Url>,@@ -149,15 +146,14 @@ pub struct JobExecutor<S: DatumStream> {impl<S: DatumStream> JobExecutor<S> {pub fn new(job: WorkerJob,- client: reqwest::Client,config: Arc<CrawlerConfig>,writer: Arc<S>,+ client: RobotClient,) -> Self {Self {writer,politeness_factor: config.start_politeness_factor,min_politeness_factor: config.min_politeness_factor,- robotstxt: RobotsTxtManager::new(&config),client,crawled_urls: HashSet::new(),crawled_sitemaps: HashSet::new(),@@ -270,7 +266,12 @@ impl<S: DatumStream> JobExecutor<S> {return UrlVisit::Skip;}- if !self.robotstxt.is_allowed(retryable_url.url()).await {+ if !self+ .client+ .robots_txt_manager()+ .is_allowed(retryable_url.url())+ .await+ {return UrlVisit::Skip;}@@ -284,20 +285,19 @@ impl<S: DatumStream> JobExecutor<S> {}async fn crawl_sitemaps(&mut self) {- for url in self.job.urls.iter().map(|url| url.url()) {+ let urls: Vec<_> = self.job.urls.iter().map(|url| url.url()).cloned().collect();++ for url in urls {let site = Site(url.host_str().unwrap_or_default().to_string());if !self.crawled_sitemaps.contains(&site) {self.crawled_sitemaps.insert(site.clone());- let sitemaps = self.robotstxt.sitemaps(url).await;+ let sitemaps = self.client.robots_txt_manager().sitemaps(&url).await;for sitemap in sitemaps {- self.sitemap_urls.extend(- self.urls_from_sitemap(sitemap, 5)- .await- .into_iter()- .map(|dated_url| dated_url.url),- );+ let res = self.urls_from_sitemap(sitemap, 5).await;+ self.sitemap_urls+ .extend(res.into_iter().map(|dated_url| dated_url.url));}}}@@ -313,7 +313,12 @@ impl<S: DatumStream> JobExecutor<S> {continue;}- if let Some(delay) = self.robotstxt.crawl_delay(retryable_url.url()).await {+ if let Some(delay) = self+ .client+ .robots_txt_manager()+ .crawl_delay(retryable_url.url())+ .await+ {if delay > self.min_crawl_delay {self.min_crawl_delay = delay.min(self.max_crawl_delay);}@@ -456,7 +461,8 @@ impl<S: DatumStream> JobExecutor<S> {async fn fetch(&self, url: Url) -> Result<reqwest::Response> {self.client- .get(url.to_string())+ .get(url)+ .await?.send().await.map_err(|e| Error::from(anyhow!(e)))@@ -619,7 +625,7 @@ impl<S: DatumStream> JobExecutor<S> {Ok(datum)}- async fn urls_from_sitemap(&self, sitemap: Url, max_depth: usize) -> Vec<DatedUrl> {+ async fn urls_from_sitemap(&mut self, sitemap: Url, max_depth: usize) -> Vec<DatedUrl> {let mut stack = vec![(sitemap, 0)];let mut urls = vec![];diff --git a/crates/core/src/live_index/crawler/checker/feeds.rs b/crates/core/src/live_index/crawler/checker/feeds.rsindex f4173da1..41ed7071 100644--- a/crates/core/src/live_index/crawler/checker/feeds.rs+++ b/crates/core/src/live_index/crawler/checker/feeds.rs@@ -20,17 +20,18 @@ use crate::feed::{parse, Feed};use crate::Result;use super::{CheckIntervals, Checker, CrawlableUrl};+use crate::crawler::robot_client::RobotClient;const CRAWL_DELAY: Duration = Duration::from_secs(5);pub struct Feeds {feeds: Vec<Feed>,last_check: std::time::Instant,- client: reqwest::Client,+ client: RobotClient,}impl Feeds {- pub fn new(feeds: Vec<Feed>, client: reqwest::Client) -> Self {+ pub fn new(feeds: Vec<Feed>, client: RobotClient) -> Self {Self {feeds,last_check: std::time::Instant::now(),@@ -40,13 +41,21 @@ impl Feeds {}impl Checker for Feeds {- async fn get_urls(&mut self) -> Result<Vec<CrawlableUrl>> {+ async fn get_urls(&self) -> Result<Vec<CrawlableUrl>> {let mut urls = Vec::new();for feed in &self.feeds {- let resp = self.client.get(feed.url.clone()).send().await?;+ let Ok(req) = self.client.get(feed.url.clone()).await else {+ continue;+ };+ let Ok(resp) = req.send().await else {+ continue;+ };+let text = resp.text().await?;- let parsed_feed = parse(&text, feed.kind)?;+ let Ok(parsed_feed) = parse(&text, feed.kind) else {+ continue;+ };for link in parsed_feed.links {urls.push(CrawlableUrl::from(link));diff --git a/crates/core/src/live_index/crawler/checker/frontpage.rs b/crates/core/src/live_index/crawler/checker/frontpage.rsindex 2df9fabc..13b33005 100644--- a/crates/core/src/live_index/crawler/checker/frontpage.rs+++ b/crates/core/src/live_index/crawler/checker/frontpage.rs@@ -17,6 +17,7 @@use url::Url;use crate::config::CheckIntervals;+use crate::crawler::robot_client::RobotClient;use crate::webpage::Html;use crate::Result;use crate::{entrypoint::site_stats, webpage::url_ext::UrlExt};@@ -26,11 +27,11 @@ use super::{Checker, CrawlableUrl};pub struct Frontpage {url: Url,last_check: std::time::Instant,- client: reqwest::Client,+ client: RobotClient,}impl Frontpage {- pub fn new(site: &site_stats::Site, client: reqwest::Client) -> Result<Self> {+ pub fn new(site: &site_stats::Site, client: RobotClient) -> Result<Self> {let url = Url::robust_parse(&format!("https://{}/", site.as_str()))?;Ok(Self {@@ -42,8 +43,8 @@ impl Frontpage {}impl Checker for Frontpage {- async fn get_urls(&mut self) -> Result<Vec<CrawlableUrl>> {- let res = self.client.get(self.url.clone()).send().await?;+ async fn get_urls(&self) -> Result<Vec<CrawlableUrl>> {+ let res = self.client.get(self.url.clone()).await?.send().await?;let body = res.text().await?;let page = Html::parse(&body, self.url.as_str())?;diff --git a/crates/core/src/live_index/crawler/checker/mod.rs b/crates/core/src/live_index/crawler/checker/mod.rsindex 587e91a0..67ed61cd 100644--- a/crates/core/src/live_index/crawler/checker/mod.rs+++ b/crates/core/src/live_index/crawler/checker/mod.rs@@ -55,7 +55,7 @@ impl From<Url> for CrawlableUrl {}pub trait Checker {- async fn get_urls(&mut self) -> Result<Vec<CrawlableUrl>>;+ async fn get_urls(&self) -> Result<Vec<CrawlableUrl>>;fn should_check(&self, interval: &CheckIntervals) -> bool;fn update_last_check(&mut self);diff --git a/crates/core/src/live_index/crawler/checker/sitemap.rs b/crates/core/src/live_index/crawler/checker/sitemap.rsindex fdc366ef..574cb1c3 100644--- a/crates/core/src/live_index/crawler/checker/sitemap.rs+++ b/crates/core/src/live_index/crawler/checker/sitemap.rs@@ -19,7 +19,7 @@ use std::time::Duration;use url::Url;use crate::config::CrawlerConfig;-use crate::crawler::robots_txt::RobotsTxtManager;+use crate::crawler::robot_client::RobotClient;use crate::dated_url::DatedUrl;use crate::sitemap::{parse_sitemap, SitemapEntry};use crate::Result;@@ -32,26 +32,26 @@ const SITEMAP_DELAY: Duration = Duration::from_secs(60);pub struct Sitemap {robots_url: Url,- robots_manager: RobotsTxtManager,last_check: std::time::Instant,- client: reqwest::Client,+ client: RobotClient,}impl Sitemap {pub fn new(site: &site_stats::Site, config: &CrawlerConfig) -> Result<Self> {let robots_url = Url::robust_parse(&format!("{}/robots.txt", site.as_str()))?;- let robots_manager = RobotsTxtManager::new(config);Ok(Self {robots_url,- robots_manager,last_check: std::time::Instant::now(),- client: crate::crawler::reqwest_client(config)?,+ client: RobotClient::new(config)?,})}- async fn sitemap_urls(&mut self) -> Vec<Url> {- self.robots_manager.sitemaps(&self.robots_url).await+ async fn sitemap_urls(&self) -> Vec<Url> {+ self.client+ .robots_txt_manager()+ .sitemaps(&self.robots_url)+ .await}async fn urls_from_sitemap(&self, sitemap: Url) -> Vec<DatedUrl> {@@ -63,7 +63,10 @@ impl Sitemap {continue;}- let res = self.client.get(url).send().await;+ let Ok(req) = self.client.get(url).await else {+ continue;+ };+ let res = req.send().await;tokio::time::sleep(SITEMAP_DELAY).await;if res.is_err() {@@ -103,7 +106,7 @@ impl Sitemap {}impl Checker for Sitemap {- async fn get_urls(&mut self) -> Result<Vec<CrawlableUrl>> {+ async fn get_urls(&self) -> Result<Vec<CrawlableUrl>> {let sitemap_urls = self.sitemap_urls().await;let mut urls = vec![];diff --git a/crates/core/src/live_index/crawler/crawlable_site.rs b/crates/core/src/live_index/crawler/crawlable_site.rsindex c8bae1ed..d09613c8 100644--- a/crates/core/src/live_index/crawler/crawlable_site.rs+++ b/crates/core/src/live_index/crawler/crawlable_site.rs@@ -67,10 +67,10 @@ impl InnerCrawlableSite {.into_iter().map(|feed| feed.into()).collect(),- client.reqwest().clone(),+ client.reqwest(),),sitemap: Sitemap::new(site.site(), client.crawler_config())?,- frontpage: Frontpage::new(site.site(), client.reqwest().clone())?,+ frontpage: Frontpage::new(site.site(), client.reqwest())?,last_drip: Instant::now(),drip_rate,budget: 0,@@ -277,9 +277,9 @@ impl CrawlableSiteGuard {.collect(),wandering_urls: 0,},- client.reqwest().clone(),- self.config.clone(),+ Arc::clone(&self.config),Arc::clone(&crawl_data),+ client.reqwest(),);executor.run().await;diff --git a/crates/core/src/live_index/crawler/mod.rs b/crates/core/src/live_index/crawler/mod.rsindex bccda47f..3d51f1c8 100644--- a/crates/core/src/live_index/crawler/mod.rs+++ b/crates/core/src/live_index/crawler/mod.rs@@ -26,6 +26,7 @@ use std::time::Duration;use crate::ampc::dht::ShardId;use crate::config::{CheckIntervals, CrawlerConfig, LiveCrawlerConfig};+use crate::crawler::robot_client::RobotClient;use crate::distributed::cluster::Cluster;use crate::distributed::sonic::replication::{RandomReplicaSelector, RandomShardSelector, ShardedClient,@@ -33,7 +34,7 @@ use crate::distributed::sonic::replication::{use crate::distributed::streaming_response::StreamingResponse;use crate::entrypoint::search_server;use crate::entrypoint::site_stats::FinalSiteStats;-use crate::{crawler, Result};+use crate::Result;use crate::{distributed::sonic::replication::ReusableShardedClient,entrypoint::{indexer::IndexableWebpage, live_index},@@ -56,7 +57,7 @@ const TICK_INTERVAL: Duration = Duration::from_secs(5);struct Client {live_index: Mutex<ReusableShardedClient<live_index::LiveIndexService>>,search: Mutex<ReusableShardedClient<search_server::SearchService>>,- reqwest: reqwest::Client,+ client: RobotClient,crawler_config: CrawlerConfig,}@@ -64,17 +65,18 @@ impl Client {pub async fn new(cluster: Arc<Cluster>, crawler_config: &CrawlerConfig) -> Result<Self> {let live_index = Mutex::new(ReusableShardedClient::new(cluster.clone()).await);let search = Mutex::new(ReusableShardedClient::new(cluster.clone()).await);+ let client = RobotClient::new(crawler_config)?;Ok(Self {+ client,live_index,search,- reqwest: crawler::reqwest_client(crawler_config)?,crawler_config: crawler_config.clone(),})}- pub fn reqwest(&self) -> &reqwest::Client {- &self.reqwest+ pub fn reqwest(&self) -> RobotClient {+ self.client.clone()}pub fn crawler_config(&self) -> &CrawlerConfig {