Skip to content

web search done right

[live_crawler] make sure frontpage check is skipped if disallowed in robots.txt

Mikkel Denkerc662fe2855af
diff --git a/crates/core/src/config/mod.rs b/crates/core/src/config/mod.rs
index 8e85f168..ac420011 100644
--- a/crates/core/src/config/mod.rs
+++ b/crates/core/src/config/mod.rs
@@ -397,6 +397,35 @@ pub struct CrawlerConfig {
pub router_hosts: Vec<String>,
}
+#[cfg(test)]
+impl CrawlerConfig {
+ pub fn for_tests() -> Self {
+ Self {
+ num_worker_threads: 1,
+ user_agent: UserAgent {
+ full: "test".to_string(),
+ token: "test".to_string(),
+ },
+ robots_txt_cache_sec: defaults::Crawler::robots_txt_cache_sec(),
+ min_politeness_factor: defaults::Crawler::min_politeness_factor(),
+ start_politeness_factor: defaults::Crawler::start_politeness_factor(),
+ min_crawl_delay_ms: defaults::Crawler::min_crawl_delay_ms(),
+ max_crawl_delay_ms: defaults::Crawler::max_crawl_delay_ms(),
+ max_politeness_factor: defaults::Crawler::max_politeness_factor(),
+ max_url_slowdown_retry: defaults::Crawler::max_url_slowdown_retry(),
+ timeout_seconds: defaults::Crawler::timeout_seconds(),
+ s3: S3Config {
+ bucket: "".to_string(),
+ folder: "".to_string(),
+ access_key: "".to_string(),
+ secret_key: "".to_string(),
+ endpoint: "".to_string(),
+ },
+ router_hosts: Vec::new(),
+ }
+ }
+}
+
#[derive(Debug, Clone, serde::Deserialize, serde::Serialize)]
pub struct DailyLiveIndexCrawlerBudget {
#[serde(default = "defaults::LiveCrawler::blogs_budget")]
diff --git a/crates/core/src/crawler/mod.rs b/crates/core/src/crawler/mod.rs
index d7d41c23..69a25e78 100644
--- a/crates/core/src/crawler/mod.rs
+++ b/crates/core/src/crawler/mod.rs
@@ -1,5 +1,5 @@
// Stract is an open source web search engine.
-// Copyright (C) 2023 Stract ApS
+// Copyright (C) 2024 Stract ApS
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
@@ -14,12 +14,13 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
-use std::{collections::VecDeque, future::Future, net::SocketAddr, sync::Arc, time::Duration};
+use std::{collections::VecDeque, future::Future, net::SocketAddr, sync::Arc};
type HashMap<K, V> = std::collections::HashMap<K, V, ahash::RandomState>;
use anyhow::anyhow;
use futures::StreamExt;
+use robot_client::RobotClient;
use url::Url;
use crate::{config::CrawlerConfig, warc, webpage::url_ext::UrlExt};
@@ -33,6 +34,7 @@ pub mod router;
pub use router::Router;
mod file_queue;
pub mod planner;
+pub mod robot_client;
mod wander_prirotiser;
mod warc_writer;
mod worker;
@@ -70,6 +72,9 @@ pub enum Error {
#[error("couldn't parse html")]
InvalidHtml,
+ #[error("request path is disallowed by robots.txt")]
+ DisallowedPath,
+
#[error("an error occurred: {0}")]
Anyhow(#[from] anyhow::Error),
}
@@ -265,6 +270,7 @@ impl Crawler {
let writer = Arc::new(WarcWriter::new(config.s3.clone()));
let mut handles = Vec::new();
let mut router_hosts = Vec::new();
+ let client = RobotClient::new(&config)?;
for host in &config.router_hosts {
router_hosts.push(
@@ -274,8 +280,12 @@ impl Crawler {
}
for _ in 0..config.num_worker_threads {
- let worker =
- WorkerThread::new(Arc::clone(&writer), config.clone(), router_hosts.clone())?;
+ let worker = WorkerThread::new(
+ Arc::clone(&writer),
+ client.clone(),
+ config.clone(),
+ router_hosts.clone(),
+ )?;
handles.push(tokio::spawn(async move {
worker.run().await;
@@ -299,30 +309,6 @@ pub trait DatumStream: Send + Sync {
fn finish(&self) -> impl Future<Output = Result<()>> + Send;
}
-pub fn reqwest_client(config: &CrawlerConfig) -> Result<reqwest::Client> {
- let timeout = Duration::from_secs(config.timeout_seconds);
-
- let mut headers = reqwest::header::HeaderMap::default();
- headers.insert(
- reqwest::header::ACCEPT,
- reqwest::header::HeaderValue::from_static("text/html"),
- );
- headers.insert(
- reqwest::header::ACCEPT_LANGUAGE,
- reqwest::header::HeaderValue::from_static("en-US,en;q=0.9,*;q=0.8"),
- );
-
- reqwest::Client::builder()
- .timeout(timeout)
- .connect_timeout(timeout)
- .http2_keep_alive_interval(None)
- .default_headers(headers)
- .redirect(reqwest::redirect::Policy::limited(0))
- .user_agent(&config.user_agent.full)
- .build()
- .map_err(|e| Error::from(anyhow!(e)))
-}
-
pub async fn encoded_body(res: reqwest::Response) -> Result<String> {
let content_type = res
.headers()
diff --git a/crates/core/src/crawler/robot_client.rs b/crates/core/src/crawler/robot_client.rs
new file mode 100644
index 00000000..8540e0b3
--- /dev/null
+++ b/crates/core/src/crawler/robot_client.rs
@@ -0,0 +1,98 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2024 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use super::{Error, Result};
+use crate::config::CrawlerConfig;
+use crate::crawler::robots_txt::RobotsTxtManager;
+use anyhow::anyhow;
+use std::time::Duration;
+use url::Url;
+
+pub(super) fn reqwest_client(config: &CrawlerConfig) -> Result<reqwest::Client> {
+ let timeout = Duration::from_secs(config.timeout_seconds);
+
+ let mut headers = reqwest::header::HeaderMap::default();
+ headers.insert(
+ reqwest::header::ACCEPT,
+ reqwest::header::HeaderValue::from_static("text/html"),
+ );
+ headers.insert(
+ reqwest::header::ACCEPT_LANGUAGE,
+ reqwest::header::HeaderValue::from_static("en-US,en;q=0.9,*;q=0.8"),
+ );
+
+ reqwest::Client::builder()
+ .timeout(timeout)
+ .connect_timeout(timeout)
+ .http2_keep_alive_interval(None)
+ .default_headers(headers)
+ .redirect(reqwest::redirect::Policy::limited(0))
+ .user_agent(&config.user_agent.full)
+ .build()
+ .map_err(|e| Error::from(anyhow!(e)))
+}
+
+#[derive(Clone)]
+pub struct RobotClient {
+ robots_txt_manager: RobotsTxtManager,
+ client: reqwest::Client,
+}
+
+impl RobotClient {
+ pub fn new(config: &CrawlerConfig) -> Result<Self> {
+ Ok(Self {
+ client: reqwest_client(config)?,
+ robots_txt_manager: RobotsTxtManager::new(config),
+ })
+ }
+
+ pub fn robots_txt_manager(&self) -> &RobotsTxtManager {
+ &self.robots_txt_manager
+ }
+
+ pub async fn get(&self, url: Url) -> Result<reqwest::RequestBuilder> {
+ if !self.robots_txt_manager.is_allowed(&url).await {
+ return Err(Error::DisallowedPath);
+ }
+
+ Ok(self.client.get(url))
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[tokio::test]
+ async fn test_errs_disallowed_path() {
+ let config = CrawlerConfig::for_tests();
+ let client = RobotClient::new(&config).unwrap();
+
+ let robots_txt =
+ robotstxt::Robots::parse("TestBot", "User-agent: *\nDisallow: /test\nAllow: /example")
+ .unwrap();
+
+ client
+ .robots_txt_manager()
+ .insert("example.com".to_string(), robots_txt);
+
+ let url = Url::parse("http://example.com/test").unwrap();
+ assert!(client.get(url).await.is_err());
+
+ let url = Url::parse("http://example.com/example").unwrap();
+ assert!(client.get(url).await.is_ok());
+ }
+}
diff --git a/crates/core/src/crawler/robots_txt.rs b/crates/core/src/crawler/robots_txt.rs
index 06afc9b1..2dc94777 100644
--- a/crates/core/src/crawler/robots_txt.rs
+++ b/crates/core/src/crawler/robots_txt.rs
@@ -14,8 +14,14 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
-use std::{collections::BTreeMap, panic, time::Duration};
-
+use std::{
+ ops::Deref,
+ panic,
+ sync::{Arc, Mutex},
+ time::Duration,
+};
+
+use dashmap::DashMap;
use url::Url;
use crate::{config::CrawlerConfig, crawler};
@@ -56,9 +62,9 @@ impl CheckedRobotsTxt {
}
}
-pub struct RobotsTxtManager {
- cache: BTreeMap<Site, CheckedRobotsTxt>,
- last_prune: std::time::Instant,
+struct InnerRobotsTxtManager {
+ cache: DashMap<Site, CheckedRobotsTxt>,
+ last_prune: Arc<Mutex<std::time::Instant>>,
client: reqwest::Client,
cache_expiration: Duration,
user_agent: String,
@@ -66,9 +72,9 @@ pub struct RobotsTxtManager {
max_crawl_delay: Duration,
}
-impl RobotsTxtManager {
- pub fn new(config: &CrawlerConfig) -> Self {
- let client = crawler::reqwest_client(config).unwrap();
+impl InnerRobotsTxtManager {
+ fn new(config: &CrawlerConfig) -> Self {
+ let client = crawler::robot_client::reqwest_client(config).unwrap();
let cache_expiration = Duration::from_secs(config.robots_txt_cache_sec);
let user_agent = config.user_agent.token.clone();
let min_crawl_delay = Duration::from_millis(config.min_crawl_delay_ms);
@@ -77,31 +83,31 @@ impl RobotsTxtManager {
Self {
client,
cache_expiration,
- last_prune: std::time::Instant::now(),
- cache: BTreeMap::new(),
+ last_prune: Arc::new(Mutex::new(std::time::Instant::now())),
+ cache: DashMap::new(),
user_agent: user_agent.to_string(),
min_crawl_delay,
max_crawl_delay,
}
}
- pub async fn is_allowed(&mut self, url: &Url) -> bool {
- match self.get_mut(url).await {
+ async fn is_allowed(&self, url: &Url) -> bool {
+ match &self.get(url).await.robots {
Lookup::Found(robots_txt) => robots_txt.is_allowed(url),
Lookup::Unavailable => true,
Lookup::Unreachable => false,
}
}
- pub async fn crawl_delay(&mut self, url: &Url) -> Option<Duration> {
- match self.get_mut(url).await {
+ async fn crawl_delay(&self, url: &Url) -> Option<Duration> {
+ match &self.get(url).await.robots {
Lookup::Found(robots_txt) => robots_txt.crawl_delay(),
Lookup::Unavailable | Lookup::Unreachable => None,
}
}
- pub async fn sitemaps(&mut self, url: &Url) -> Vec<Url> {
- match self.get_mut(url).await {
+ async fn sitemaps(&self, url: &Url) -> Vec<Url> {
+ match &self.get(url).await.robots {
Lookup::Found(robots_txt) => robots_txt
.sitemaps()
.iter()
@@ -197,18 +203,18 @@ impl RobotsTxtManager {
}
}
- fn maybe_prune(&mut self) {
- if self.last_prune.elapsed() < Duration::from_secs(60) {
+ fn maybe_prune(&self) {
+ if self.last_prune.lock().unwrap().elapsed() < Duration::from_secs(60) {
return;
}
self.cache
.retain(|_, v| !v.is_expired(&self.cache_expiration));
- self.last_prune = std::time::Instant::now();
+ *self.last_prune.lock().unwrap() = std::time::Instant::now();
}
- async fn get_mut(&mut self, url: &Url) -> &mut Lookup<robotstxt::Robots> {
+ async fn get(&self, url: &Url) -> impl Deref<Target = CheckedRobotsTxt> + '_ {
self.maybe_prune();
let site = Site(url.host_str().unwrap_or_default().to_string());
@@ -223,7 +229,51 @@ impl RobotsTxtManager {
.insert(site.clone(), self.fetch_robots_txt(&site).await);
}
- &mut self.cache.get_mut(&site).unwrap().robots
+ self.cache.get(&site).unwrap()
+ }
+
+ #[cfg(test)]
+ fn insert(&self, site: String, robots_txt: robotstxt::Robots) {
+ self.cache
+ .insert(Site(site), CheckedRobotsTxt::new(Lookup::Found(robots_txt)));
+ }
+}
+pub struct RobotsTxtManager {
+ inner: Arc<InnerRobotsTxtManager>,
+}
+
+impl Clone for RobotsTxtManager {
+ fn clone(&self) -> Self {
+ Self {
+ inner: self.inner.clone(),
+ }
+ }
+}
+
+impl RobotsTxtManager {
+ pub fn new(config: &CrawlerConfig) -> Self {
+ Self {
+ inner: Arc::new(InnerRobotsTxtManager::new(config)),
+ }
+ }
+}
+
+impl RobotsTxtManager {
+ pub async fn is_allowed(&self, url: &Url) -> bool {
+ self.inner.is_allowed(url).await
+ }
+
+ pub async fn crawl_delay(&self, url: &Url) -> Option<Duration> {
+ self.inner.crawl_delay(url).await
+ }
+
+ pub async fn sitemaps(&self, url: &Url) -> Vec<Url> {
+ self.inner.sitemaps(url).await
+ }
+
+ #[cfg(test)]
+ pub fn insert(&self, site: String, robots_txt: robotstxt::Robots) {
+ self.inner.insert(site, robots_txt);
}
}
diff --git a/crates/core/src/crawler/worker.rs b/crates/core/src/crawler/worker.rs
index 3ed5945b..f50525a8 100644
--- a/crates/core/src/crawler/worker.rs
+++ b/crates/core/src/crawler/worker.rs
@@ -38,10 +38,9 @@ use crate::{
};
use super::{
- encoded_body, reqwest_client, robots_txt::RobotsTxtManager,
- wander_prirotiser::WanderPrioritiser, CrawlDatum, DatumStream, Domain, Error, Result,
- RetrieableUrl, Site, WarcWriter, WeightedUrl, WorkerJob, MAX_CONTENT_LENGTH,
- MAX_OUTGOING_URLS_PER_PAGE,
+ encoded_body, robot_client::RobotClient, wander_prirotiser::WanderPrioritiser, CrawlDatum,
+ DatumStream, Domain, Error, Result, RetrieableUrl, Site, WarcWriter, WeightedUrl, WorkerJob,
+ MAX_CONTENT_LENGTH, MAX_OUTGOING_URLS_PER_PAGE,
};
const IGNORED_EXTENSIONS: [&str; 27] = [
@@ -63,19 +62,18 @@ struct ProcessedUrl {
pub struct WorkerThread {
writer: Arc<WarcWriter>,
- client: reqwest::Client,
config: Arc<CrawlerConfig>,
router_hosts: Vec<SocketAddr>,
+ client: RobotClient,
}
impl WorkerThread {
pub fn new(
writer: Arc<WarcWriter>,
+ client: RobotClient,
config: CrawlerConfig,
router_hosts: Vec<SocketAddr>,
) -> Result<Self> {
- let client = reqwest_client(&config)?;
-
Ok(Self {
writer,
client,
@@ -109,9 +107,9 @@ impl WorkerThread {
Ok(Some(job)) => {
let executor = JobExecutor::new(
job.into(),
- self.client.clone(),
self.config.clone(),
self.writer.clone(),
+ self.client.clone(),
);
executor.run().await;
}
@@ -129,10 +127,9 @@ impl WorkerThread {
pub struct JobExecutor<S: DatumStream> {
writer: Arc<S>,
- client: reqwest::Client,
+ client: RobotClient,
has_gotten_429_response: bool,
politeness_factor: u32,
- robotstxt: RobotsTxtManager,
crawled_urls: HashSet<Url>,
crawled_sitemaps: HashSet<Site>,
sitemap_urls: HashSet<Url>,
@@ -149,15 +146,14 @@ pub struct JobExecutor<S: DatumStream> {
impl<S: DatumStream> JobExecutor<S> {
pub fn new(
job: WorkerJob,
- client: reqwest::Client,
config: Arc<CrawlerConfig>,
writer: Arc<S>,
+ client: RobotClient,
) -> Self {
Self {
writer,
politeness_factor: config.start_politeness_factor,
min_politeness_factor: config.min_politeness_factor,
- robotstxt: RobotsTxtManager::new(&config),
client,
crawled_urls: HashSet::new(),
crawled_sitemaps: HashSet::new(),
@@ -270,7 +266,12 @@ impl<S: DatumStream> JobExecutor<S> {
return UrlVisit::Skip;
}
- if !self.robotstxt.is_allowed(retryable_url.url()).await {
+ if !self
+ .client
+ .robots_txt_manager()
+ .is_allowed(retryable_url.url())
+ .await
+ {
return UrlVisit::Skip;
}
@@ -284,20 +285,19 @@ impl<S: DatumStream> JobExecutor<S> {
}
async fn crawl_sitemaps(&mut self) {
- for url in self.job.urls.iter().map(|url| url.url()) {
+ let urls: Vec<_> = self.job.urls.iter().map(|url| url.url()).cloned().collect();
+
+ for url in urls {
let site = Site(url.host_str().unwrap_or_default().to_string());
if !self.crawled_sitemaps.contains(&site) {
self.crawled_sitemaps.insert(site.clone());
- let sitemaps = self.robotstxt.sitemaps(url).await;
+ let sitemaps = self.client.robots_txt_manager().sitemaps(&url).await;
for sitemap in sitemaps {
- self.sitemap_urls.extend(
- self.urls_from_sitemap(sitemap, 5)
- .await
- .into_iter()
- .map(|dated_url| dated_url.url),
- );
+ let res = self.urls_from_sitemap(sitemap, 5).await;
+ self.sitemap_urls
+ .extend(res.into_iter().map(|dated_url| dated_url.url));
}
}
}
@@ -313,7 +313,12 @@ impl<S: DatumStream> JobExecutor<S> {
continue;
}
- if let Some(delay) = self.robotstxt.crawl_delay(retryable_url.url()).await {
+ if let Some(delay) = self
+ .client
+ .robots_txt_manager()
+ .crawl_delay(retryable_url.url())
+ .await
+ {
if delay > self.min_crawl_delay {
self.min_crawl_delay = delay.min(self.max_crawl_delay);
}
@@ -456,7 +461,8 @@ impl<S: DatumStream> JobExecutor<S> {
async fn fetch(&self, url: Url) -> Result<reqwest::Response> {
self.client
- .get(url.to_string())
+ .get(url)
+ .await?
.send()
.await
.map_err(|e| Error::from(anyhow!(e)))
@@ -619,7 +625,7 @@ impl<S: DatumStream> JobExecutor<S> {
Ok(datum)
}
- async fn urls_from_sitemap(&self, sitemap: Url, max_depth: usize) -> Vec<DatedUrl> {
+ async fn urls_from_sitemap(&mut self, sitemap: Url, max_depth: usize) -> Vec<DatedUrl> {
let mut stack = vec![(sitemap, 0)];
let mut urls = vec![];
diff --git a/crates/core/src/live_index/crawler/checker/feeds.rs b/crates/core/src/live_index/crawler/checker/feeds.rs
index f4173da1..41ed7071 100644
--- a/crates/core/src/live_index/crawler/checker/feeds.rs
+++ b/crates/core/src/live_index/crawler/checker/feeds.rs
@@ -20,17 +20,18 @@ use crate::feed::{parse, Feed};
use crate::Result;
use super::{CheckIntervals, Checker, CrawlableUrl};
+use crate::crawler::robot_client::RobotClient;
const CRAWL_DELAY: Duration = Duration::from_secs(5);
pub struct Feeds {
feeds: Vec<Feed>,
last_check: std::time::Instant,
- client: reqwest::Client,
+ client: RobotClient,
}
impl Feeds {
- pub fn new(feeds: Vec<Feed>, client: reqwest::Client) -> Self {
+ pub fn new(feeds: Vec<Feed>, client: RobotClient) -> Self {
Self {
feeds,
last_check: std::time::Instant::now(),
@@ -40,13 +41,21 @@ impl Feeds {
}
impl Checker for Feeds {
- async fn get_urls(&mut self) -> Result<Vec<CrawlableUrl>> {
+ async fn get_urls(&self) -> Result<Vec<CrawlableUrl>> {
let mut urls = Vec::new();
for feed in &self.feeds {
- let resp = self.client.get(feed.url.clone()).send().await?;
+ let Ok(req) = self.client.get(feed.url.clone()).await else {
+ continue;
+ };
+ let Ok(resp) = req.send().await else {
+ continue;
+ };
+
let text = resp.text().await?;
- let parsed_feed = parse(&text, feed.kind)?;
+ let Ok(parsed_feed) = parse(&text, feed.kind) else {
+ continue;
+ };
for link in parsed_feed.links {
urls.push(CrawlableUrl::from(link));
diff --git a/crates/core/src/live_index/crawler/checker/frontpage.rs b/crates/core/src/live_index/crawler/checker/frontpage.rs
index 2df9fabc..13b33005 100644
--- a/crates/core/src/live_index/crawler/checker/frontpage.rs
+++ b/crates/core/src/live_index/crawler/checker/frontpage.rs
@@ -17,6 +17,7 @@
use url::Url;
use crate::config::CheckIntervals;
+use crate::crawler::robot_client::RobotClient;
use crate::webpage::Html;
use crate::Result;
use crate::{entrypoint::site_stats, webpage::url_ext::UrlExt};
@@ -26,11 +27,11 @@ use super::{Checker, CrawlableUrl};
pub struct Frontpage {
url: Url,
last_check: std::time::Instant,
- client: reqwest::Client,
+ client: RobotClient,
}
impl Frontpage {
- pub fn new(site: &site_stats::Site, client: reqwest::Client) -> Result<Self> {
+ pub fn new(site: &site_stats::Site, client: RobotClient) -> Result<Self> {
let url = Url::robust_parse(&format!("https://{}/", site.as_str()))?;
Ok(Self {
@@ -42,8 +43,8 @@ impl Frontpage {
}
impl Checker for Frontpage {
- async fn get_urls(&mut self) -> Result<Vec<CrawlableUrl>> {
- let res = self.client.get(self.url.clone()).send().await?;
+ async fn get_urls(&self) -> Result<Vec<CrawlableUrl>> {
+ let res = self.client.get(self.url.clone()).await?.send().await?;
let body = res.text().await?;
let page = Html::parse(&body, self.url.as_str())?;
diff --git a/crates/core/src/live_index/crawler/checker/mod.rs b/crates/core/src/live_index/crawler/checker/mod.rs
index 587e91a0..67ed61cd 100644
--- a/crates/core/src/live_index/crawler/checker/mod.rs
+++ b/crates/core/src/live_index/crawler/checker/mod.rs
@@ -55,7 +55,7 @@ impl From<Url> for CrawlableUrl {
}
pub trait Checker {
- async fn get_urls(&mut self) -> Result<Vec<CrawlableUrl>>;
+ async fn get_urls(&self) -> Result<Vec<CrawlableUrl>>;
fn should_check(&self, interval: &CheckIntervals) -> bool;
fn update_last_check(&mut self);
diff --git a/crates/core/src/live_index/crawler/checker/sitemap.rs b/crates/core/src/live_index/crawler/checker/sitemap.rs
index fdc366ef..574cb1c3 100644
--- a/crates/core/src/live_index/crawler/checker/sitemap.rs
+++ b/crates/core/src/live_index/crawler/checker/sitemap.rs
@@ -19,7 +19,7 @@ use std::time::Duration;
use url::Url;
use crate::config::CrawlerConfig;
-use crate::crawler::robots_txt::RobotsTxtManager;
+use crate::crawler::robot_client::RobotClient;
use crate::dated_url::DatedUrl;
use crate::sitemap::{parse_sitemap, SitemapEntry};
use crate::Result;
@@ -32,26 +32,26 @@ const SITEMAP_DELAY: Duration = Duration::from_secs(60);
pub struct Sitemap {
robots_url: Url,
- robots_manager: RobotsTxtManager,
last_check: std::time::Instant,
- client: reqwest::Client,
+ client: RobotClient,
}
impl Sitemap {
pub fn new(site: &site_stats::Site, config: &CrawlerConfig) -> Result<Self> {
let robots_url = Url::robust_parse(&format!("{}/robots.txt", site.as_str()))?;
- let robots_manager = RobotsTxtManager::new(config);
Ok(Self {
robots_url,
- robots_manager,
last_check: std::time::Instant::now(),
- client: crate::crawler::reqwest_client(config)?,
+ client: RobotClient::new(config)?,
})
}
- async fn sitemap_urls(&mut self) -> Vec<Url> {
- self.robots_manager.sitemaps(&self.robots_url).await
+ async fn sitemap_urls(&self) -> Vec<Url> {
+ self.client
+ .robots_txt_manager()
+ .sitemaps(&self.robots_url)
+ .await
}
async fn urls_from_sitemap(&self, sitemap: Url) -> Vec<DatedUrl> {
@@ -63,7 +63,10 @@ impl Sitemap {
continue;
}
- let res = self.client.get(url).send().await;
+ let Ok(req) = self.client.get(url).await else {
+ continue;
+ };
+ let res = req.send().await;
tokio::time::sleep(SITEMAP_DELAY).await;
if res.is_err() {
@@ -103,7 +106,7 @@ impl Sitemap {
}
impl Checker for Sitemap {
- async fn get_urls(&mut self) -> Result<Vec<CrawlableUrl>> {
+ async fn get_urls(&self) -> Result<Vec<CrawlableUrl>> {
let sitemap_urls = self.sitemap_urls().await;
let mut urls = vec![];
diff --git a/crates/core/src/live_index/crawler/crawlable_site.rs b/crates/core/src/live_index/crawler/crawlable_site.rs
index c8bae1ed..d09613c8 100644
--- a/crates/core/src/live_index/crawler/crawlable_site.rs
+++ b/crates/core/src/live_index/crawler/crawlable_site.rs
@@ -67,10 +67,10 @@ impl InnerCrawlableSite {
.into_iter()
.map(|feed| feed.into())
.collect(),
- client.reqwest().clone(),
+ client.reqwest(),
),
sitemap: Sitemap::new(site.site(), client.crawler_config())?,
- frontpage: Frontpage::new(site.site(), client.reqwest().clone())?,
+ frontpage: Frontpage::new(site.site(), client.reqwest())?,
last_drip: Instant::now(),
drip_rate,
budget: 0,
@@ -277,9 +277,9 @@ impl CrawlableSiteGuard {
.collect(),
wandering_urls: 0,
},
- client.reqwest().clone(),
- self.config.clone(),
+ Arc::clone(&self.config),
Arc::clone(&crawl_data),
+ client.reqwest(),
);
executor.run().await;
diff --git a/crates/core/src/live_index/crawler/mod.rs b/crates/core/src/live_index/crawler/mod.rs
index bccda47f..3d51f1c8 100644
--- a/crates/core/src/live_index/crawler/mod.rs
+++ b/crates/core/src/live_index/crawler/mod.rs
@@ -26,6 +26,7 @@ use std::time::Duration;
use crate::ampc::dht::ShardId;
use crate::config::{CheckIntervals, CrawlerConfig, LiveCrawlerConfig};
+use crate::crawler::robot_client::RobotClient;
use crate::distributed::cluster::Cluster;
use crate::distributed::sonic::replication::{
RandomReplicaSelector, RandomShardSelector, ShardedClient,
@@ -33,7 +34,7 @@ use crate::distributed::sonic::replication::{
use crate::distributed::streaming_response::StreamingResponse;
use crate::entrypoint::search_server;
use crate::entrypoint::site_stats::FinalSiteStats;
-use crate::{crawler, Result};
+use crate::Result;
use crate::{
distributed::sonic::replication::ReusableShardedClient,
entrypoint::{indexer::IndexableWebpage, live_index},
@@ -56,7 +57,7 @@ const TICK_INTERVAL: Duration = Duration::from_secs(5);
struct Client {
live_index: Mutex<ReusableShardedClient<live_index::LiveIndexService>>,
search: Mutex<ReusableShardedClient<search_server::SearchService>>,
- reqwest: reqwest::Client,
+ client: RobotClient,
crawler_config: CrawlerConfig,
}
@@ -64,17 +65,18 @@ impl Client {
pub async fn new(cluster: Arc<Cluster>, crawler_config: &CrawlerConfig) -> Result<Self> {
let live_index = Mutex::new(ReusableShardedClient::new(cluster.clone()).await);
let search = Mutex::new(ReusableShardedClient::new(cluster.clone()).await);
+ let client = RobotClient::new(crawler_config)?;
Ok(Self {
+ client,
live_index,
search,
- reqwest: crawler::reqwest_client(crawler_config)?,
crawler_config: crawler_config.clone(),
})
}
- pub fn reqwest(&self) -> &reqwest::Client {
- &self.reqwest
+ pub fn reqwest(&self) -> RobotClient {
+ self.client.clone()
}
pub fn crawler_config(&self) -> &CrawlerConfig {