Skip to content

web search done right

limit cache of robots.txt files

Mikkel Denkerb323fa9e4be0
diff --git a/configs/crawler/crawler.toml b/configs/crawler/crawler.toml
index 57141a18..e0e5ff62 100644
--- a/configs/crawler/crawler.toml
+++ b/configs/crawler/crawler.toml
@@ -4,12 +4,12 @@ timeout_seconds = 30
coordinator_host = "0.0.0.0:8080"
[user_agent]
-token = "StractSearch"
-full = "Mozilla/5.0 (compatible; StractSearch/0.1; +https://trystract.com/webmasters)"
+token = "StractBot"
+full = "Mozilla/5.0 (compatible; StractBot/0.1; +https://trystract.com/webmasters)"
[s3]
bucket = "crawl"
-folder = "test1"
+folder = "test"
access_key = "<access_key>"
secret_key = "<secret_key>"
endpoint = "http://s3.trystract.com"
\ No newline at end of file
diff --git a/core/src/config/defaults.rs b/core/src/config/defaults.rs
index fce55059..7d36501a 100644
--- a/core/src/config/defaults.rs
+++ b/core/src/config/defaults.rs
@@ -65,6 +65,10 @@ impl Snippet {
pub struct Crawler;
impl Crawler {
+ pub fn robots_txt_cache_sec() -> u64 {
+ 60 * 60
+ }
+
pub fn politeness_factor() -> f32 {
1.0
}
diff --git a/core/src/config/mod.rs b/core/src/config/mod.rs
index 65281ffd..9b14ef0a 100644
--- a/core/src/config/mod.rs
+++ b/core/src/config/mod.rs
@@ -272,6 +272,9 @@ pub struct CrawlerConfig {
pub num_workers: usize,
pub user_agent: UserAgent,
+ #[serde(default = "defaults::Crawler::robots_txt_cache_sec")]
+ pub robots_txt_cache_sec: u64,
+
#[serde(default = "defaults::Crawler::politeness_factor")]
pub politeness_factor: f32,
diff --git a/core/src/crawler/coordinator.rs b/core/src/crawler/coordinator.rs
index 7adf75ac..29a20d54 100644
--- a/core/src/crawler/coordinator.rs
+++ b/core/src/crawler/coordinator.rs
@@ -76,7 +76,12 @@ impl CrawlCoordinator {
pub fn add_responses(&self, responses: &[JobResponse]) -> Result<()> {
let start = Instant::now();
- self.log_crawls_per_second(responses.iter().map(|res| res.url_responses.len()).sum());
+ let num_crawled_urls = responses.iter().map(|res| res.url_responses.len()).sum();
+
+ self.log_crawls_per_second(num_crawled_urls);
+ self.num_crawled_urls
+ .fetch_add(num_crawled_urls as u64, Ordering::SeqCst);
+
let mut db = self.db.lock().unwrap();
db.insert_urls(responses).unwrap();
diff --git a/core/src/crawler/robots_txt.rs b/core/src/crawler/robots_txt.rs
index 69e769cb..7f7cb8f2 100644
--- a/core/src/crawler/robots_txt.rs
+++ b/core/src/crawler/robots_txt.rs
@@ -14,7 +14,7 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
-use std::{collections::HashMap, panic};
+use std::{collections::HashMap, panic, time::Duration};
use robotstxt_with_cache::matcher::{
CachingRobotsMatcher, LongestMatchRobotsMatchStrategy, RobotsMatcher,
@@ -24,22 +24,29 @@ use url::Url;
use super::{Error, Result, Site};
+enum Lookup<T> {
+ Found(T),
+ NotFound,
+}
+
pub struct RobotsTxtManager {
- cache: HashMap<Site, Option<RobotsTxt>>, // None if robots.txt does not exist
+ cache: HashMap<Site, Lookup<RobotsTxt>>,
client: reqwest::Client,
+ cache_expiration: Duration,
}
impl RobotsTxtManager {
- pub fn new(client: reqwest::Client) -> Self {
+ pub fn new(client: reqwest::Client, cache_expiration: Duration) -> Self {
Self {
client,
+ cache_expiration,
cache: HashMap::new(),
}
}
pub async fn is_allowed(&mut self, url: &Url, user_agent: &str) -> bool {
match self.get_mut(url).await {
- Ok(Some(robots_txt)) => robots_txt
+ Lookup::Found(robots_txt) => robots_txt
.matcher
.one_agent_allowed_by_robots(user_agent, url.as_str()),
_ => true,
@@ -65,61 +72,79 @@ impl RobotsTxtManager {
}
}
- async fn get_mut(&mut self, url: &Url) -> Result<Option<&mut RobotsTxt>> {
+ async fn get_mut(&mut self, url: &Url) -> &mut Lookup<RobotsTxt> {
let site = Site(url.host_str().unwrap_or_default().to_string());
- if self.cache.get(&site).is_none() {
+ let cache_should_update = match self.cache.get_mut(&site) {
+ Some(Lookup::Found(robots_txt)) => robots_txt.is_expired(&self.cache_expiration),
+ Some(Lookup::NotFound) => false,
+ _ => true,
+ };
+
+ if cache_should_update {
match self.fetch_robots_txt(&site).await {
Ok(robots_txt) => {
- self.cache.insert(site.clone(), Some(robots_txt));
+ self.cache.insert(site.clone(), Lookup::Found(robots_txt));
}
Err(err) => match err.downcast_ref() {
Some(Error::FetchFailed(status))
if *status == reqwest::StatusCode::IM_A_TEAPOT =>
{
- self.cache.insert(site.clone(), None);
+ self.cache.insert(site.clone(), Lookup::NotFound);
}
_ => {
- self.cache.insert(site.clone(), None);
+ self.cache.insert(site.clone(), Lookup::NotFound);
tracing::warn!("failed to fetch robots.txt for {}: {}", site.0, err);
}
},
}
}
- match self.cache.get_mut(&site) {
- Some(Some(robot)) => Ok(Some(robot)),
- _ => Ok(None),
- }
+ self.cache.get_mut(&site).unwrap()
}
- pub async fn sitemap(&mut self, url: &Url) -> Result<Option<Url>> {
- Ok(self
- .get_mut(url)
- .await?
- .and_then(|robots_txt| robots_txt.sitemap.clone()))
+ pub async fn sitemap(&mut self, url: &Url) -> Option<Url> {
+ match self.get_mut(url).await {
+ Lookup::Found(robotstxt) => robotstxt.sitemap.clone(),
+ Lookup::NotFound => None,
+ }
}
}
struct RobotsTxt {
+ download_time: std::time::Instant,
matcher: CachingRobotsMatcher<LongestMatchRobotsMatchStrategy>,
sitemap: Option<Url>,
}
impl RobotsTxt {
fn new(body: String) -> Self {
- let mut matcher = CachingRobotsMatcher::new(RobotsMatcher::default());
+ let mut s = Self {
+ matcher: CachingRobotsMatcher::new(RobotsMatcher::default()),
+ sitemap: None,
+ download_time: std::time::Instant::now(),
+ };
+
+ s.update(body);
+
+ s
+ }
+
+ fn is_expired(&self, expiration: &Duration) -> bool {
+ self.download_time.elapsed() > *expiration
+ }
- matcher.parse(&body);
+ fn update(&mut self, body: String) {
+ self.matcher.parse(&body);
- let sitemap = body
+ self.sitemap = body
.to_ascii_lowercase()
.lines()
.find(|line| line.starts_with("sitemap:"))
.map(|line| line.split(':').nth(1).unwrap().trim())
.and_then(|s| Url::parse(s).ok());
- Self { matcher, sitemap }
+ self.download_time = std::time::Instant::now();
}
}
diff --git a/core/src/crawler/worker.rs b/core/src/crawler/worker.rs
index 6fba83d5..58a4a1be 100644
--- a/core/src/crawler/worker.rs
+++ b/core/src/crawler/worker.rs
@@ -71,7 +71,10 @@ impl Worker {
.user_agent(&config.user_agent.full)
.build()?;
- let robotstxt = RobotsTxtManager::new(client.clone());
+ let robotstxt = RobotsTxtManager::new(
+ client.clone(),
+ Duration::from_secs(config.robots_txt_cache_sec),
+ );
Ok(Self {
writer,
@@ -189,7 +192,7 @@ impl Worker {
if job.fetch_sitemap && !crawled_sitemaps.contains(&site) {
crawled_sitemaps.insert(site.clone());
- if let Ok(Some(sitemap)) = self.robotstxt.sitemap(&retryable_url.url).await {
+ if let Some(sitemap) = self.robotstxt.sitemap(&retryable_url.url).await {
let sitemap_urls = self.urls_from_sitemap(sitemap, 0, 5).await;
discovered_urls.extend(sitemap_urls);
}
diff --git a/frontend/src/pages/crawler.mdx b/frontend/src/pages/crawler.mdx
index f53b9ec8..0c036f00 100644
--- a/frontend/src/pages/crawler.mdx
+++ b/frontend/src/pages/crawler.mdx
@@ -14,6 +14,7 @@ It does this by waiting a certain amount of time between requests to the same do
This dynamic waiting time tries to prevent us from disrupting servers that cannot handle the load, while not giving unnecessary politeness to servers that can. The politeness factor starts at 1 and is doubled every time the crawler gets a 429 response from the server (to at most 2048).
The crawler looks for the token `StractBot` in the robots.txt file to determine which pages (if any) the crawler is allowed to crawl.
+The robots.txt file is cached for 1 hour, so changes to the file should be respected quite quickly.
## Contact us
If you have any concerns or bad experiences with our crawler, please don't hesitate to reach out to us at [[email protected]](mailto:[email protected]). Chances are that others experience the same problems and we would love to fix them.
\ No newline at end of file