Skip to content

web search done right

[live_crawler] make sure unreachable robots.txt files aren't re-fetched

Mikkel Denker34230b59160a
diff --git a/crates/core/src/crawler/mod.rs b/crates/core/src/crawler/mod.rs
index 531f8409..d7d41c23 100644
--- a/crates/core/src/crawler/mod.rs
+++ b/crates/core/src/crawler/mod.rs
@@ -28,7 +28,7 @@ use self::{warc_writer::WarcWriter, worker::WorkerThread};
pub use worker::JobExecutor;
pub mod coordinator;
-mod robots_txt;
+pub mod robots_txt;
pub mod router;
pub use router::Router;
mod file_queue;
diff --git a/crates/core/src/crawler/robots_txt.rs b/crates/core/src/crawler/robots_txt.rs
index 88033c11..06afc9b1 100644
--- a/crates/core/src/crawler/robots_txt.rs
+++ b/crates/core/src/crawler/robots_txt.rs
@@ -20,7 +20,7 @@ use url::Url;
use crate::{config::CrawlerConfig, crawler};
-use super::{encoded_body, Result, Site};
+use super::{encoded_body, Site};
const RETRY_ROBOTSTXT_UNREACHABLE: bool = false;
@@ -29,12 +29,35 @@ enum Lookup<T> {
Found(T),
/// 404
Unavailable,
- /// 5xx
+ /// 5xx (and other network errors)
Unreachable,
}
+struct CheckedRobotsTxt {
+ robots: Lookup<robotstxt::Robots>,
+ last_check: std::time::Instant,
+}
+
+impl CheckedRobotsTxt {
+ fn new(robots: Lookup<robotstxt::Robots>) -> Self {
+ Self {
+ robots,
+ last_check: std::time::Instant::now(),
+ }
+ }
+}
+
+impl CheckedRobotsTxt {
+ fn is_expired(&self, expiration: &Duration) -> bool {
+ match self.robots {
+ Lookup::Found(_) | Lookup::Unavailable => self.last_check.elapsed() >= *expiration,
+ Lookup::Unreachable => false,
+ }
+ }
+}
+
pub struct RobotsTxtManager {
- cache: BTreeMap<Site, Lookup<RobotsTxt>>,
+ cache: BTreeMap<Site, CheckedRobotsTxt>,
last_prune: std::time::Instant,
client: reqwest::Client,
cache_expiration: Duration,
@@ -72,12 +95,23 @@ impl RobotsTxtManager {
pub async fn crawl_delay(&mut self, url: &Url) -> Option<Duration> {
match self.get_mut(url).await {
- Lookup::Found(robots_txt) => robots_txt.robots.crawl_delay(),
+ Lookup::Found(robots_txt) => robots_txt.crawl_delay(),
Lookup::Unavailable | Lookup::Unreachable => None,
}
}
- async fn fetch_robots_txt_from_url(&self, url: &str) -> Lookup<RobotsTxt> {
+ pub async fn sitemaps(&mut self, url: &Url) -> Vec<Url> {
+ match self.get_mut(url).await {
+ Lookup::Found(robots_txt) => robots_txt
+ .sitemaps()
+ .iter()
+ .filter_map(|s| Url::parse(s).ok())
+ .collect(),
+ Lookup::Unavailable | Lookup::Unreachable => vec![],
+ }
+ }
+
+ async fn fetch_robots_txt_from_url(&self, url: &str) -> Lookup<robotstxt::Robots> {
let res = match self
.client
.get(url)
@@ -99,7 +133,7 @@ impl RobotsTxtManager {
};
let self_user_agent = self.user_agent.clone();
- match panic::catch_unwind(|| RobotsTxt::new(&self_user_agent, body)) {
+ match panic::catch_unwind(|| robotstxt::Robots::parse(&self_user_agent, &body)) {
Ok(Ok(r)) => Lookup::Found(r),
_ => Lookup::Unreachable,
}
@@ -112,7 +146,7 @@ impl RobotsTxtManager {
res
}
- async fn fetch_robots_txt_without_retry(&self, site: &Site) -> Lookup<RobotsTxt> {
+ async fn fetch_robots_txt_without_retry(&self, site: &Site) -> Lookup<robotstxt::Robots> {
match self
.fetch_robots_txt_from_url(&format!("https://{}/robots.txt", site.0))
.await
@@ -141,11 +175,7 @@ impl RobotsTxtManager {
}
}
- async fn fetch_robots_txt(&self, site: &Site) -> Lookup<RobotsTxt> {
- if !RETRY_ROBOTSTXT_UNREACHABLE {
- return self.fetch_robots_txt_without_retry(site).await;
- }
-
+ async fn fetch_robots_txt_with_retry(&self, site: &Site) -> Lookup<robotstxt::Robots> {
for _ in 0..3 {
match self.fetch_robots_txt_without_retry(site).await {
Lookup::Found(robots_txt) => return Lookup::Found(robots_txt),
@@ -159,74 +189,41 @@ impl RobotsTxtManager {
Lookup::Unreachable
}
+ async fn fetch_robots_txt(&self, site: &Site) -> CheckedRobotsTxt {
+ if RETRY_ROBOTSTXT_UNREACHABLE {
+ CheckedRobotsTxt::new(self.fetch_robots_txt_with_retry(site).await)
+ } else {
+ CheckedRobotsTxt::new(self.fetch_robots_txt_without_retry(site).await)
+ }
+ }
+
fn maybe_prune(&mut self) {
if self.last_prune.elapsed() < Duration::from_secs(60) {
return;
}
- self.cache.retain(|_, v| match v {
- Lookup::Found(robots_txt) => !robots_txt.is_expired(&self.cache_expiration),
- _ => true,
- });
+ self.cache
+ .retain(|_, v| !v.is_expired(&self.cache_expiration));
self.last_prune = std::time::Instant::now();
}
- async fn get_mut(&mut self, url: &Url) -> &mut Lookup<RobotsTxt> {
+ async fn get_mut(&mut self, url: &Url) -> &mut Lookup<robotstxt::Robots> {
self.maybe_prune();
let site = Site(url.host_str().unwrap_or_default().to_string());
- let cache_should_update = match self.cache.get_mut(&site) {
- Some(Lookup::Found(robots_txt)) => robots_txt.is_expired(&self.cache_expiration),
- Some(Lookup::Unavailable) | Some(Lookup::Unreachable) => false,
- None => true,
- };
+ let cache_should_update = self
+ .cache
+ .get_mut(&site)
+ .map(|v| v.is_expired(&self.cache_expiration))
+ .unwrap_or(true);
if cache_should_update {
self.cache
.insert(site.clone(), self.fetch_robots_txt(&site).await);
}
- self.cache.get_mut(&site).unwrap()
- }
-
- pub async fn sitemaps(&mut self, url: &Url) -> Vec<Url> {
- match self.get_mut(url).await {
- Lookup::Found(robotstxt) => robotstxt
- .sitemaps()
- .iter()
- .filter_map(|s| Url::parse(s).ok())
- .collect(),
- Lookup::Unavailable => vec![],
- Lookup::Unreachable => vec![],
- }
- }
-}
-
-#[derive(Debug)]
-struct RobotsTxt {
- download_time: std::time::Instant,
- robots: robotstxt::Robots,
-}
-
-impl RobotsTxt {
- fn new(user_agent: &str, body: String) -> Result<Self> {
- Ok(Self {
- robots: robotstxt::Robots::parse(user_agent, &body)?,
- download_time: std::time::Instant::now(),
- })
- }
-
- fn is_expired(&self, expiration: &Duration) -> bool {
- self.download_time.elapsed() > *expiration
- }
-
- fn is_allowed(&self, url: &Url) -> bool {
- self.robots.is_allowed(url)
- }
-
- fn sitemaps(&self) -> &[String] {
- self.robots.sitemaps()
+ &mut self.cache.get_mut(&site).unwrap().robots
}
}
@@ -234,14 +231,15 @@ impl RobotsTxt {
mod tests {
use super::*;
+ type RobotsTxt = robotstxt::Robots;
+
#[test]
fn simple() {
let ua_token = "StractBot";
- let robots_txt = RobotsTxt::new(
+ let robots_txt = RobotsTxt::parse(
ua_token,
r#"User-agent: StractBot
- Disallow: /test"#
- .to_string(),
+ Disallow: /test"#,
)
.unwrap();
@@ -252,11 +250,10 @@ mod tests {
#[test]
fn lowercase() {
let ua_token = "StractBot";
- let robots_txt = RobotsTxt::new(
+ let robots_txt = RobotsTxt::parse(
ua_token,
r#"User-agent: stractbot
- Disallow: /test"#
- .to_string(),
+ Disallow: /test"#,
)
.unwrap();
@@ -267,13 +264,12 @@ mod tests {
#[test]
fn test_extra_newline() {
let ua_token = "StractBot";
- let robots_txt = RobotsTxt::new(
+ let robots_txt = RobotsTxt::parse(
ua_token,
r#"User-agent: StractBot
- Disallow: /test"#
- .to_string(),
+ Disallow: /test"#,
)
.unwrap();
@@ -285,15 +281,14 @@ mod tests {
fn test_multiple_agents() {
let ua_token = "StractBot";
- let robots_txt = RobotsTxt::new(
+ let robots_txt = RobotsTxt::parse(
ua_token,
r#"User-Agent: GoogleBot
User-Agent: StractBot
Disallow: /
User-Agent: *
-Allow: /"#
- .to_string(),
+Allow: /"#,
)
.unwrap();
@@ -301,14 +296,13 @@ Allow: /"#
let ua_token = "StractBot";
- let robots_txt = RobotsTxt::new(
+ let robots_txt = RobotsTxt::parse(
ua_token,
r#"User-Agent: GoogleBot, StractBot
Disallow: /
User-Agent: *
-Allow: /"#
- .to_string(),
+Allow: /"#,
)
.unwrap();
@@ -318,25 +312,23 @@ Allow: /"#
#[test]
fn test_sitemap() {
let ua_token = "StractBot";
- let robots_txt = RobotsTxt::new(
+ let robots_txt = RobotsTxt::parse(
ua_token,
r#"User-agent: *
Disallow: /test
-Sitemap: http://example.com/sitemap.xml"#
- .to_string(),
+Sitemap: http://example.com/sitemap.xml"#,
)
.unwrap();
assert_eq!(robots_txt.sitemaps(), &["http://example.com/sitemap.xml"]);
- let robots_txt = RobotsTxt::new(
+ let robots_txt = RobotsTxt::parse(
ua_token,
r#"User-agent: *
Disallow: /test
-SiTeMaP: http://example.com/sitemap.xml"#
- .to_string(),
+SiTeMaP: http://example.com/sitemap.xml"#,
)
.unwrap();
@@ -347,12 +339,11 @@ SiTeMaP: http://example.com/sitemap.xml"#
fn wildcard() {
let ua_token = "StractBot";
- let robots_txt = RobotsTxt::new(
+ let robots_txt = RobotsTxt::parse(
ua_token,
r#"User-agent: StractBot
Disallow: /test/*
-"#
- .to_string(),
+"#,
)
.unwrap();
@@ -362,12 +353,11 @@ Disallow: /test/*
assert!(robots_txt.is_allowed(&Url::parse("http://example.com/test").unwrap()));
assert!(robots_txt.is_allowed(&Url::parse("http://example.com/testfoo").unwrap()));
- let robots_txt = RobotsTxt::new(
+ let robots_txt = RobotsTxt::parse(
ua_token,
r#"User-agent: StractBot
Disallow: /test/*/bar
- "#
- .to_string(),
+ "#,
)
.unwrap();
@@ -378,4 +368,11 @@ Disallow: /test/*
assert!(robots_txt.is_allowed(&Url::parse("http://example.com/test").unwrap()));
assert!(robots_txt.is_allowed(&Url::parse("http://example.com/testfoo").unwrap()));
}
+
+ #[test]
+ fn test_unreachable_robots_never_updated() {
+ let checked = CheckedRobotsTxt::new(Lookup::Unreachable);
+ std::thread::sleep(std::time::Duration::from_millis(100));
+ assert!(!checked.is_expired(&Duration::from_millis(10)));
+ }
}
diff --git a/crates/core/src/live_index/crawler/checker/feeds.rs b/crates/core/src/live_index/crawler/checker/feeds.rs
index 1e394de5..f4173da1 100644
--- a/crates/core/src/live_index/crawler/checker/feeds.rs
+++ b/crates/core/src/live_index/crawler/checker/feeds.rs
@@ -55,11 +55,13 @@ impl Checker for Feeds {
tokio::time::sleep(CRAWL_DELAY).await;
}
- self.last_check = std::time::Instant::now();
-
Ok(urls)
}
+ fn update_last_check(&mut self) {
+ self.last_check = std::time::Instant::now();
+ }
+
fn should_check(&self, interval: &CheckIntervals) -> bool {
self.last_check.elapsed() > interval.feeds
}
diff --git a/crates/core/src/live_index/crawler/checker/frontpage.rs b/crates/core/src/live_index/crawler/checker/frontpage.rs
index bf4d92f9..2df9fabc 100644
--- a/crates/core/src/live_index/crawler/checker/frontpage.rs
+++ b/crates/core/src/live_index/crawler/checker/frontpage.rs
@@ -54,11 +54,13 @@ impl Checker for Frontpage {
.map(|link| CrawlableUrl::from(link.destination))
.collect::<Vec<_>>();
- self.last_check = std::time::Instant::now();
-
Ok(urls)
}
+ fn update_last_check(&mut self) {
+ self.last_check = std::time::Instant::now();
+ }
+
fn should_check(&self, interval: &CheckIntervals) -> bool {
self.last_check.elapsed() > interval.frontpage
}
diff --git a/crates/core/src/live_index/crawler/checker/mod.rs b/crates/core/src/live_index/crawler/checker/mod.rs
index ec349c80..587e91a0 100644
--- a/crates/core/src/live_index/crawler/checker/mod.rs
+++ b/crates/core/src/live_index/crawler/checker/mod.rs
@@ -57,4 +57,10 @@ impl From<Url> for CrawlableUrl {
pub trait Checker {
async fn get_urls(&mut self) -> Result<Vec<CrawlableUrl>>;
fn should_check(&self, interval: &CheckIntervals) -> bool;
+ fn update_last_check(&mut self);
+
+ async fn get_urls_and_update_last_check(&mut self) -> Result<Vec<CrawlableUrl>> {
+ self.update_last_check();
+ self.get_urls().await
+ }
}
diff --git a/crates/core/src/live_index/crawler/checker/sitemap.rs b/crates/core/src/live_index/crawler/checker/sitemap.rs
index eeb598a2..fdc366ef 100644
--- a/crates/core/src/live_index/crawler/checker/sitemap.rs
+++ b/crates/core/src/live_index/crawler/checker/sitemap.rs
@@ -18,6 +18,8 @@ use std::time::Duration;
use url::Url;
+use crate::config::CrawlerConfig;
+use crate::crawler::robots_txt::RobotsTxtManager;
use crate::dated_url::DatedUrl;
use crate::sitemap::{parse_sitemap, SitemapEntry};
use crate::Result;
@@ -26,37 +28,30 @@ use crate::{entrypoint::site_stats, webpage::url_ext::UrlExt};
use super::{CheckIntervals, Checker, CrawlableUrl};
const MAX_SITEMAP_DEPTH: usize = 10;
-const SITEMAP_DELAY: Duration = Duration::from_secs(10);
+const SITEMAP_DELAY: Duration = Duration::from_secs(60);
pub struct Sitemap {
- robots_txt: Url,
+ robots_url: Url,
+ robots_manager: RobotsTxtManager,
last_check: std::time::Instant,
client: reqwest::Client,
}
impl Sitemap {
- pub fn new(site: &site_stats::Site, client: reqwest::Client) -> Result<Self> {
- let robots_txt = Url::robust_parse(&format!("{}/robots.txt", site.as_str()))?;
+ pub fn new(site: &site_stats::Site, config: &CrawlerConfig) -> Result<Self> {
+ let robots_url = Url::robust_parse(&format!("{}/robots.txt", site.as_str()))?;
+ let robots_manager = RobotsTxtManager::new(config);
Ok(Self {
- robots_txt,
+ robots_url,
+ robots_manager,
last_check: std::time::Instant::now(),
- client,
+ client: crate::crawler::reqwest_client(config)?,
})
}
- async fn sitemap_urls(&self) -> Result<Vec<Url>> {
- let res = self.client.get(self.robots_txt.clone()).send().await?;
- let body = res.text().await?;
-
- // wildcard useragent is okay as we only use it to check for sitemap directive
- let robots = robotstxt::Robots::parse("*", &body)?;
-
- Ok(robots
- .sitemaps()
- .iter()
- .filter_map(|s| Url::parse(s).ok())
- .collect())
+ async fn sitemap_urls(&mut self) -> Vec<Url> {
+ self.robots_manager.sitemaps(&self.robots_url).await
}
async fn urls_from_sitemap(&self, sitemap: Url) -> Vec<DatedUrl> {
@@ -109,7 +104,7 @@ impl Sitemap {
impl Checker for Sitemap {
async fn get_urls(&mut self) -> Result<Vec<CrawlableUrl>> {
- let sitemap_urls = self.sitemap_urls().await?;
+ let sitemap_urls = self.sitemap_urls().await;
let mut urls = vec![];
for sitemap_url in sitemap_urls {
@@ -121,11 +116,13 @@ impl Checker for Sitemap {
);
}
- self.last_check = std::time::Instant::now();
-
Ok(urls)
}
+ fn update_last_check(&mut self) {
+ self.last_check = std::time::Instant::now();
+ }
+
fn should_check(&self, interval: &CheckIntervals) -> bool {
self.last_check.elapsed() > interval.sitemap
}
diff --git a/crates/core/src/live_index/crawler/crawlable_site.rs b/crates/core/src/live_index/crawler/crawlable_site.rs
index f901be20..c8bae1ed 100644
--- a/crates/core/src/live_index/crawler/crawlable_site.rs
+++ b/crates/core/src/live_index/crawler/crawlable_site.rs
@@ -69,7 +69,7 @@ impl InnerCrawlableSite {
.collect(),
client.reqwest().clone(),
),
- sitemap: Sitemap::new(site.site(), client.reqwest().clone())?,
+ sitemap: Sitemap::new(site.site(), client.crawler_config())?,
frontpage: Frontpage::new(site.site(), client.reqwest().clone())?,
last_drip: Instant::now(),
drip_rate,
@@ -195,15 +195,30 @@ impl CrawlableSiteGuard {
let mut urls = Vec::new();
if site.feeds.should_check(interval) {
- urls.extend(site.feeds.get_urls().await.unwrap_or_default());
+ urls.extend(
+ site.feeds
+ .get_urls_and_update_last_check()
+ .await
+ .unwrap_or_default(),
+ );
}
if site.sitemap.should_check(interval) {
- urls.extend(site.sitemap.get_urls().await.unwrap_or_default());
+ urls.extend(
+ site.sitemap
+ .get_urls_and_update_last_check()
+ .await
+ .unwrap_or_default(),
+ );
}
if site.frontpage.should_check(interval) {
- urls.extend(site.frontpage.get_urls().await.unwrap_or_default());
+ urls.extend(
+ site.frontpage
+ .get_urls_and_update_last_check()
+ .await
+ .unwrap_or_default(),
+ );
}
let url = site.site.url()?;
diff --git a/crates/core/src/live_index/crawler/mod.rs b/crates/core/src/live_index/crawler/mod.rs
index 3233ded4..bccda47f 100644
--- a/crates/core/src/live_index/crawler/mod.rs
+++ b/crates/core/src/live_index/crawler/mod.rs
@@ -57,6 +57,7 @@ struct Client {
live_index: Mutex<ReusableShardedClient<live_index::LiveIndexService>>,
search: Mutex<ReusableShardedClient<search_server::SearchService>>,
reqwest: reqwest::Client,
+ crawler_config: CrawlerConfig,
}
impl Client {
@@ -68,6 +69,7 @@ impl Client {
live_index,
search,
reqwest: crawler::reqwest_client(crawler_config)?,
+ crawler_config: crawler_config.clone(),
})
}
@@ -75,6 +77,10 @@ impl Client {
&self.reqwest
}
+ pub fn crawler_config(&self) -> &CrawlerConfig {
+ &self.crawler_config
+ }
+
async fn live_conn(&self) -> Arc<ShardedClient<live_index::LiveIndexService, ShardId>> {
self.live_index.lock().await.conn().await
}