web search done right
[live_crawler] make sure unreachable robots.txt files aren't re-fetched
Mikkel Denker34230b59160a
diff --git a/crates/core/src/crawler/mod.rs b/crates/core/src/crawler/mod.rsindex 531f8409..d7d41c23 100644--- a/crates/core/src/crawler/mod.rs+++ b/crates/core/src/crawler/mod.rs@@ -28,7 +28,7 @@ use self::{warc_writer::WarcWriter, worker::WorkerThread};pub use worker::JobExecutor;pub mod coordinator;-mod robots_txt;+pub mod robots_txt;pub mod router;pub use router::Router;mod file_queue;diff --git a/crates/core/src/crawler/robots_txt.rs b/crates/core/src/crawler/robots_txt.rsindex 88033c11..06afc9b1 100644--- a/crates/core/src/crawler/robots_txt.rs+++ b/crates/core/src/crawler/robots_txt.rs@@ -20,7 +20,7 @@ use url::Url;use crate::{config::CrawlerConfig, crawler};-use super::{encoded_body, Result, Site};+use super::{encoded_body, Site};const RETRY_ROBOTSTXT_UNREACHABLE: bool = false;@@ -29,12 +29,35 @@ enum Lookup<T> {Found(T),/// 404Unavailable,- /// 5xx+ /// 5xx (and other network errors)Unreachable,}+struct CheckedRobotsTxt {+ robots: Lookup<robotstxt::Robots>,+ last_check: std::time::Instant,+}++impl CheckedRobotsTxt {+ fn new(robots: Lookup<robotstxt::Robots>) -> Self {+ Self {+ robots,+ last_check: std::time::Instant::now(),+ }+ }+}++impl CheckedRobotsTxt {+ fn is_expired(&self, expiration: &Duration) -> bool {+ match self.robots {+ Lookup::Found(_) | Lookup::Unavailable => self.last_check.elapsed() >= *expiration,+ Lookup::Unreachable => false,+ }+ }+}+pub struct RobotsTxtManager {- cache: BTreeMap<Site, Lookup<RobotsTxt>>,+ cache: BTreeMap<Site, CheckedRobotsTxt>,last_prune: std::time::Instant,client: reqwest::Client,cache_expiration: Duration,@@ -72,12 +95,23 @@ impl RobotsTxtManager {pub async fn crawl_delay(&mut self, url: &Url) -> Option<Duration> {match self.get_mut(url).await {- Lookup::Found(robots_txt) => robots_txt.robots.crawl_delay(),+ Lookup::Found(robots_txt) => robots_txt.crawl_delay(),Lookup::Unavailable | Lookup::Unreachable => None,}}- async fn fetch_robots_txt_from_url(&self, url: &str) -> Lookup<RobotsTxt> {+ pub async fn sitemaps(&mut self, url: &Url) -> Vec<Url> {+ match self.get_mut(url).await {+ Lookup::Found(robots_txt) => robots_txt+ .sitemaps()+ .iter()+ .filter_map(|s| Url::parse(s).ok())+ .collect(),+ Lookup::Unavailable | Lookup::Unreachable => vec![],+ }+ }++ async fn fetch_robots_txt_from_url(&self, url: &str) -> Lookup<robotstxt::Robots> {let res = match self.client.get(url)@@ -99,7 +133,7 @@ impl RobotsTxtManager {};let self_user_agent = self.user_agent.clone();- match panic::catch_unwind(|| RobotsTxt::new(&self_user_agent, body)) {+ match panic::catch_unwind(|| robotstxt::Robots::parse(&self_user_agent, &body)) {Ok(Ok(r)) => Lookup::Found(r),_ => Lookup::Unreachable,}@@ -112,7 +146,7 @@ impl RobotsTxtManager {res}- async fn fetch_robots_txt_without_retry(&self, site: &Site) -> Lookup<RobotsTxt> {+ async fn fetch_robots_txt_without_retry(&self, site: &Site) -> Lookup<robotstxt::Robots> {match self.fetch_robots_txt_from_url(&format!("https://{}/robots.txt", site.0)).await@@ -141,11 +175,7 @@ impl RobotsTxtManager {}}- async fn fetch_robots_txt(&self, site: &Site) -> Lookup<RobotsTxt> {- if !RETRY_ROBOTSTXT_UNREACHABLE {- return self.fetch_robots_txt_without_retry(site).await;- }-+ async fn fetch_robots_txt_with_retry(&self, site: &Site) -> Lookup<robotstxt::Robots> {for _ in 0..3 {match self.fetch_robots_txt_without_retry(site).await {Lookup::Found(robots_txt) => return Lookup::Found(robots_txt),@@ -159,74 +189,41 @@ impl RobotsTxtManager {Lookup::Unreachable}+ async fn fetch_robots_txt(&self, site: &Site) -> CheckedRobotsTxt {+ if RETRY_ROBOTSTXT_UNREACHABLE {+ CheckedRobotsTxt::new(self.fetch_robots_txt_with_retry(site).await)+ } else {+ CheckedRobotsTxt::new(self.fetch_robots_txt_without_retry(site).await)+ }+ }+fn maybe_prune(&mut self) {if self.last_prune.elapsed() < Duration::from_secs(60) {return;}- self.cache.retain(|_, v| match v {- Lookup::Found(robots_txt) => !robots_txt.is_expired(&self.cache_expiration),- _ => true,- });+ self.cache+ .retain(|_, v| !v.is_expired(&self.cache_expiration));self.last_prune = std::time::Instant::now();}- async fn get_mut(&mut self, url: &Url) -> &mut Lookup<RobotsTxt> {+ async fn get_mut(&mut self, url: &Url) -> &mut Lookup<robotstxt::Robots> {self.maybe_prune();let site = Site(url.host_str().unwrap_or_default().to_string());- let cache_should_update = match self.cache.get_mut(&site) {- Some(Lookup::Found(robots_txt)) => robots_txt.is_expired(&self.cache_expiration),- Some(Lookup::Unavailable) | Some(Lookup::Unreachable) => false,- None => true,- };+ let cache_should_update = self+ .cache+ .get_mut(&site)+ .map(|v| v.is_expired(&self.cache_expiration))+ .unwrap_or(true);if cache_should_update {self.cache.insert(site.clone(), self.fetch_robots_txt(&site).await);}- self.cache.get_mut(&site).unwrap()- }-- pub async fn sitemaps(&mut self, url: &Url) -> Vec<Url> {- match self.get_mut(url).await {- Lookup::Found(robotstxt) => robotstxt- .sitemaps()- .iter()- .filter_map(|s| Url::parse(s).ok())- .collect(),- Lookup::Unavailable => vec![],- Lookup::Unreachable => vec![],- }- }-}--#[derive(Debug)]-struct RobotsTxt {- download_time: std::time::Instant,- robots: robotstxt::Robots,-}--impl RobotsTxt {- fn new(user_agent: &str, body: String) -> Result<Self> {- Ok(Self {- robots: robotstxt::Robots::parse(user_agent, &body)?,- download_time: std::time::Instant::now(),- })- }-- fn is_expired(&self, expiration: &Duration) -> bool {- self.download_time.elapsed() > *expiration- }-- fn is_allowed(&self, url: &Url) -> bool {- self.robots.is_allowed(url)- }-- fn sitemaps(&self) -> &[String] {- self.robots.sitemaps()+ &mut self.cache.get_mut(&site).unwrap().robots}}@@ -234,14 +231,15 @@ impl RobotsTxt {mod tests {use super::*;+ type RobotsTxt = robotstxt::Robots;+#[test]fn simple() {let ua_token = "StractBot";- let robots_txt = RobotsTxt::new(+ let robots_txt = RobotsTxt::parse(ua_token,r#"User-agent: StractBot- Disallow: /test"#- .to_string(),+ Disallow: /test"#,).unwrap();@@ -252,11 +250,10 @@ mod tests {#[test]fn lowercase() {let ua_token = "StractBot";- let robots_txt = RobotsTxt::new(+ let robots_txt = RobotsTxt::parse(ua_token,r#"User-agent: stractbot- Disallow: /test"#- .to_string(),+ Disallow: /test"#,).unwrap();@@ -267,13 +264,12 @@ mod tests {#[test]fn test_extra_newline() {let ua_token = "StractBot";- let robots_txt = RobotsTxt::new(+ let robots_txt = RobotsTxt::parse(ua_token,r#"User-agent: StractBot- Disallow: /test"#- .to_string(),+ Disallow: /test"#,).unwrap();@@ -285,15 +281,14 @@ mod tests {fn test_multiple_agents() {let ua_token = "StractBot";- let robots_txt = RobotsTxt::new(+ let robots_txt = RobotsTxt::parse(ua_token,r#"User-Agent: GoogleBotUser-Agent: StractBotDisallow: /User-Agent: *-Allow: /"#- .to_string(),+Allow: /"#,).unwrap();@@ -301,14 +296,13 @@ Allow: /"#let ua_token = "StractBot";- let robots_txt = RobotsTxt::new(+ let robots_txt = RobotsTxt::parse(ua_token,r#"User-Agent: GoogleBot, StractBotDisallow: /User-Agent: *-Allow: /"#- .to_string(),+Allow: /"#,).unwrap();@@ -318,25 +312,23 @@ Allow: /"##[test]fn test_sitemap() {let ua_token = "StractBot";- let robots_txt = RobotsTxt::new(+ let robots_txt = RobotsTxt::parse(ua_token,r#"User-agent: *Disallow: /test-Sitemap: http://example.com/sitemap.xml"#- .to_string(),+Sitemap: http://example.com/sitemap.xml"#,).unwrap();assert_eq!(robots_txt.sitemaps(), &["http://example.com/sitemap.xml"]);- let robots_txt = RobotsTxt::new(+ let robots_txt = RobotsTxt::parse(ua_token,r#"User-agent: *Disallow: /test-SiTeMaP: http://example.com/sitemap.xml"#- .to_string(),+SiTeMaP: http://example.com/sitemap.xml"#,).unwrap();@@ -347,12 +339,11 @@ SiTeMaP: http://example.com/sitemap.xml"#fn wildcard() {let ua_token = "StractBot";- let robots_txt = RobotsTxt::new(+ let robots_txt = RobotsTxt::parse(ua_token,r#"User-agent: StractBotDisallow: /test/*-"#- .to_string(),+"#,).unwrap();@@ -362,12 +353,11 @@ Disallow: /test/*assert!(robots_txt.is_allowed(&Url::parse("http://example.com/test").unwrap()));assert!(robots_txt.is_allowed(&Url::parse("http://example.com/testfoo").unwrap()));- let robots_txt = RobotsTxt::new(+ let robots_txt = RobotsTxt::parse(ua_token,r#"User-agent: StractBotDisallow: /test/*/bar- "#- .to_string(),+ "#,).unwrap();@@ -378,4 +368,11 @@ Disallow: /test/*assert!(robots_txt.is_allowed(&Url::parse("http://example.com/test").unwrap()));assert!(robots_txt.is_allowed(&Url::parse("http://example.com/testfoo").unwrap()));}++ #[test]+ fn test_unreachable_robots_never_updated() {+ let checked = CheckedRobotsTxt::new(Lookup::Unreachable);+ std::thread::sleep(std::time::Duration::from_millis(100));+ assert!(!checked.is_expired(&Duration::from_millis(10)));+ }}diff --git a/crates/core/src/live_index/crawler/checker/feeds.rs b/crates/core/src/live_index/crawler/checker/feeds.rsindex 1e394de5..f4173da1 100644--- a/crates/core/src/live_index/crawler/checker/feeds.rs+++ b/crates/core/src/live_index/crawler/checker/feeds.rs@@ -55,11 +55,13 @@ impl Checker for Feeds {tokio::time::sleep(CRAWL_DELAY).await;}- self.last_check = std::time::Instant::now();-Ok(urls)}+ fn update_last_check(&mut self) {+ self.last_check = std::time::Instant::now();+ }+fn should_check(&self, interval: &CheckIntervals) -> bool {self.last_check.elapsed() > interval.feeds}diff --git a/crates/core/src/live_index/crawler/checker/frontpage.rs b/crates/core/src/live_index/crawler/checker/frontpage.rsindex bf4d92f9..2df9fabc 100644--- a/crates/core/src/live_index/crawler/checker/frontpage.rs+++ b/crates/core/src/live_index/crawler/checker/frontpage.rs@@ -54,11 +54,13 @@ impl Checker for Frontpage {.map(|link| CrawlableUrl::from(link.destination)).collect::<Vec<_>>();- self.last_check = std::time::Instant::now();-Ok(urls)}+ fn update_last_check(&mut self) {+ self.last_check = std::time::Instant::now();+ }+fn should_check(&self, interval: &CheckIntervals) -> bool {self.last_check.elapsed() > interval.frontpage}diff --git a/crates/core/src/live_index/crawler/checker/mod.rs b/crates/core/src/live_index/crawler/checker/mod.rsindex ec349c80..587e91a0 100644--- a/crates/core/src/live_index/crawler/checker/mod.rs+++ b/crates/core/src/live_index/crawler/checker/mod.rs@@ -57,4 +57,10 @@ impl From<Url> for CrawlableUrl {pub trait Checker {async fn get_urls(&mut self) -> Result<Vec<CrawlableUrl>>;fn should_check(&self, interval: &CheckIntervals) -> bool;+ fn update_last_check(&mut self);++ async fn get_urls_and_update_last_check(&mut self) -> Result<Vec<CrawlableUrl>> {+ self.update_last_check();+ self.get_urls().await+ }}diff --git a/crates/core/src/live_index/crawler/checker/sitemap.rs b/crates/core/src/live_index/crawler/checker/sitemap.rsindex eeb598a2..fdc366ef 100644--- a/crates/core/src/live_index/crawler/checker/sitemap.rs+++ b/crates/core/src/live_index/crawler/checker/sitemap.rs@@ -18,6 +18,8 @@ use std::time::Duration;use url::Url;+use crate::config::CrawlerConfig;+use crate::crawler::robots_txt::RobotsTxtManager;use crate::dated_url::DatedUrl;use crate::sitemap::{parse_sitemap, SitemapEntry};use crate::Result;@@ -26,37 +28,30 @@ use crate::{entrypoint::site_stats, webpage::url_ext::UrlExt};use super::{CheckIntervals, Checker, CrawlableUrl};const MAX_SITEMAP_DEPTH: usize = 10;-const SITEMAP_DELAY: Duration = Duration::from_secs(10);+const SITEMAP_DELAY: Duration = Duration::from_secs(60);pub struct Sitemap {- robots_txt: Url,+ robots_url: Url,+ robots_manager: RobotsTxtManager,last_check: std::time::Instant,client: reqwest::Client,}impl Sitemap {- pub fn new(site: &site_stats::Site, client: reqwest::Client) -> Result<Self> {- let robots_txt = Url::robust_parse(&format!("{}/robots.txt", site.as_str()))?;+ pub fn new(site: &site_stats::Site, config: &CrawlerConfig) -> Result<Self> {+ let robots_url = Url::robust_parse(&format!("{}/robots.txt", site.as_str()))?;+ let robots_manager = RobotsTxtManager::new(config);Ok(Self {- robots_txt,+ robots_url,+ robots_manager,last_check: std::time::Instant::now(),- client,+ client: crate::crawler::reqwest_client(config)?,})}- async fn sitemap_urls(&self) -> Result<Vec<Url>> {- let res = self.client.get(self.robots_txt.clone()).send().await?;- let body = res.text().await?;-- // wildcard useragent is okay as we only use it to check for sitemap directive- let robots = robotstxt::Robots::parse("*", &body)?;-- Ok(robots- .sitemaps()- .iter()- .filter_map(|s| Url::parse(s).ok())- .collect())+ async fn sitemap_urls(&mut self) -> Vec<Url> {+ self.robots_manager.sitemaps(&self.robots_url).await}async fn urls_from_sitemap(&self, sitemap: Url) -> Vec<DatedUrl> {@@ -109,7 +104,7 @@ impl Sitemap {impl Checker for Sitemap {async fn get_urls(&mut self) -> Result<Vec<CrawlableUrl>> {- let sitemap_urls = self.sitemap_urls().await?;+ let sitemap_urls = self.sitemap_urls().await;let mut urls = vec![];for sitemap_url in sitemap_urls {@@ -121,11 +116,13 @@ impl Checker for Sitemap {);}- self.last_check = std::time::Instant::now();-Ok(urls)}+ fn update_last_check(&mut self) {+ self.last_check = std::time::Instant::now();+ }+fn should_check(&self, interval: &CheckIntervals) -> bool {self.last_check.elapsed() > interval.sitemap}diff --git a/crates/core/src/live_index/crawler/crawlable_site.rs b/crates/core/src/live_index/crawler/crawlable_site.rsindex f901be20..c8bae1ed 100644--- a/crates/core/src/live_index/crawler/crawlable_site.rs+++ b/crates/core/src/live_index/crawler/crawlable_site.rs@@ -69,7 +69,7 @@ impl InnerCrawlableSite {.collect(),client.reqwest().clone(),),- sitemap: Sitemap::new(site.site(), client.reqwest().clone())?,+ sitemap: Sitemap::new(site.site(), client.crawler_config())?,frontpage: Frontpage::new(site.site(), client.reqwest().clone())?,last_drip: Instant::now(),drip_rate,@@ -195,15 +195,30 @@ impl CrawlableSiteGuard {let mut urls = Vec::new();if site.feeds.should_check(interval) {- urls.extend(site.feeds.get_urls().await.unwrap_or_default());+ urls.extend(+ site.feeds+ .get_urls_and_update_last_check()+ .await+ .unwrap_or_default(),+ );}if site.sitemap.should_check(interval) {- urls.extend(site.sitemap.get_urls().await.unwrap_or_default());+ urls.extend(+ site.sitemap+ .get_urls_and_update_last_check()+ .await+ .unwrap_or_default(),+ );}if site.frontpage.should_check(interval) {- urls.extend(site.frontpage.get_urls().await.unwrap_or_default());+ urls.extend(+ site.frontpage+ .get_urls_and_update_last_check()+ .await+ .unwrap_or_default(),+ );}let url = site.site.url()?;diff --git a/crates/core/src/live_index/crawler/mod.rs b/crates/core/src/live_index/crawler/mod.rsindex 3233ded4..bccda47f 100644--- a/crates/core/src/live_index/crawler/mod.rs+++ b/crates/core/src/live_index/crawler/mod.rs@@ -57,6 +57,7 @@ struct Client {live_index: Mutex<ReusableShardedClient<live_index::LiveIndexService>>,search: Mutex<ReusableShardedClient<search_server::SearchService>>,reqwest: reqwest::Client,+ crawler_config: CrawlerConfig,}impl Client {@@ -68,6 +69,7 @@ impl Client {live_index,search,reqwest: crawler::reqwest_client(crawler_config)?,+ crawler_config: crawler_config.clone(),})}@@ -75,6 +77,10 @@ impl Client {&self.reqwest}+ pub fn crawler_config(&self) -> &CrawlerConfig {+ &self.crawler_config+ }+async fn live_conn(&self) -> Arc<ShardedClient<live_index::LiveIndexService, ShardId>> {self.live_index.lock().await.conn().await}