web search done right
fix /webmasters formatting
Mikkel Denkera1f5a18628c3
diff --git a/core/src/crawler/crawl_db.rs b/core/src/crawler/crawl_db.rsindex 618f141b..2f657859 100644--- a/core/src/crawler/crawl_db.rs+++ b/core/src/crawler/crawl_db.rs@@ -343,19 +343,28 @@ impl UrlStateDb {}}- pub fn put_batch(&mut self, batch: &[(Domain, Vec<(UrlString, UrlState)>)]) -> Result<()> {- let last_shard = self.shards.last_mut().unwrap();+ fn new_shard(&mut self) -> Result<()> {+ let shard_id =+ chrono::Utc::now().to_rfc3339() + "_" + uuid::Uuid::new_v4().to_string().as_str();+ let shard_path = self.path.as_path().join(shard_id);- if last_shard.approximate_size_bytes()? > MAX_URL_DB_SIZE_BYTES {- let shard_id =- chrono::Utc::now().to_rfc3339() + "_" + uuid::Uuid::new_v4().to_string().as_str();- let shard_path = self.path.as_path().join(shard_id);+ std::fs::create_dir_all(&shard_path)?;- std::fs::create_dir_all(&shard_path)?;+ let shard = UrlStateDbShard::open(&shard_path)?;- let shard = UrlStateDbShard::open(&shard_path)?;+ self.shards.push(shard);++ Ok(())+ }- self.shards.push(shard);+ pub fn put_batch(&mut self, batch: &[(Domain, Vec<(UrlString, UrlState)>)]) -> Result<()> {+ if self.shards.is_empty() {+ self.new_shard()?;+ }+ let last_shard = self.shards.last_mut().unwrap();++ if last_shard.approximate_size_bytes()? > MAX_URL_DB_SIZE_BYTES {+ self.new_shard()?;}self.shards.last_mut().unwrap().put_batch(batch)?;@@ -419,10 +428,11 @@ impl DomainStateDb {fn multi_get(&self, domains: &[Domain]) -> Result<HashMap<Domain, DomainState>> {let mut res = HashMap::new();- let domain_bytes: Vec<_> = domains- .iter()- .map(|domain| bincode::serialize(domain).unwrap())- .collect();+ let mut domain_bytes = Vec::with_capacity(domains.len());++ for domain in domains {+ domain_bytes.push(bincode::serialize(domain)?);+ }for (val, domain) in self.db@@ -487,9 +497,10 @@ impl From<Url> for UrlString {}}-impl From<&UrlString> for Url {- fn from(url: &UrlString) -> Self {- Url::parse(&url.0).unwrap()+impl TryFrom<&UrlString> for Url {+ type Error = anyhow::Error;+ fn try_from(url: &UrlString) -> Result<Self, Self::Error> {+ Ok(Url::parse(&url.0)?)}}@@ -665,7 +676,10 @@ impl CrawlDb {let job = Job {domain: domain.clone(),fetch_sitemap: false, // todo: fetch for new sites- urls: sampled.iter().map(|(url, _)| (*url).into()).collect(),+ urls: sampled+ .iter()+ .filter_map(|(url, _)| Url::try_from(*url).ok())+ .collect(),weight_budget: domain_state.weight,};diff --git a/frontend/src/routes/webmasters/README.md b/frontend/src/routes/webmasters/README.mdindex 22a57435..dfb831a4 100644--- a/frontend/src/routes/webmasters/README.md+++ b/frontend/src/routes/webmasters/README.md@@ -7,7 +7,7 @@ The crawler uses the user agent `Mozilla/5.0 (compatible; StractBot/0.1; open so## PolitenessStractBot is a polite crawler. It respects the [robots.txt](https://en.wikipedia.org/wiki/Robots.txt) file of the website it is crawling and tries to not overload the server.-It does this by waiting a certain amount of time between requests to the same domain. The waiting time is calculated by _min(max(fetchtime _ politeness, 1 sec), 60 sec)* where *fetchtime\* is the time it took to fetch the page. The crawler will wait at least 1 sec between requests and at most 60 seconds.+It does this by waiting a certain amount of time between requests to the same domain. The waiting time is calculated by *min(max(fetchtime \* politeness, 1 sec), 60 sec)* where *fetchtime* is the time it took to fetch the page. The crawler will wait at least 1 sec between requests and at most 60 seconds.This dynamic waiting time tries to prevent us from disrupting servers that cannot handle the load, while not giving unnecessary politeness to servers that can. The politeness factor starts at 1 and is doubled every time the crawler gets a 429 response from the server (to at most 2048).The crawler looks for the token `StractBot` in the robots.txt file to determine which pages (if any) the crawler is allowed to crawl.