web search done right
hopefully made crawler more stable?
Mikkel Denkerd25288cb14d9
diff --git a/core/src/crawler/coordinator.rs b/core/src/crawler/coordinator.rsindex 2bdcd2ea..96e48a9d 100644--- a/core/src/crawler/coordinator.rs+++ b/core/src/crawler/coordinator.rs@@ -62,7 +62,7 @@ impl CrawlCoordinator {db,num_urls_to_crawl,num_crawled_urls: AtomicU64::new(0),- call_counter: Mutex::new(CallCounter::new(Duration::from_secs(60))),+ call_counter: Mutex::new(CallCounter::new(Duration::from_secs(10))),})}diff --git a/core/src/crawler/robots_txt.rs b/core/src/crawler/robots_txt.rsindex 77246918..8786c566 100644--- a/core/src/crawler/robots_txt.rs+++ b/core/src/crawler/robots_txt.rs@@ -74,6 +74,7 @@ impl RobotsTxtManager {self.cache.insert(site.clone(), None);}Err(err) => {+ self.cache.insert(site.clone(), None);tracing::warn!("failed to fetch robots.txt for {}: {}", site.0, err);}}diff --git a/core/src/crawler/worker.rs b/core/src/crawler/worker.rsindex 83a3aadc..d65fd961 100644--- a/core/src/crawler/worker.rs+++ b/core/src/crawler/worker.rs@@ -162,6 +162,10 @@ impl Worker {let num_urls = job.job.urls.len();for (i, url) in job.job.urls.into_iter().enumerate() {+ if url.is_empty() || url.site().is_empty() {+ continue;+ }+if !job.robotstxt.is_allowed(&url, &self.user_agent).await {continue;}@@ -207,7 +211,7 @@ impl Worker {let _ = conn.send_with_timeout::<_, Response>(&Request::CrawlResult { job_response },- Duration::from_secs(60),+ Duration::from_secs(60 * 60),).await.expect("Failed to send crawl result to coordinator");diff --git a/core/src/webpage/url.rs b/core/src/webpage/url.rsindex 3b0dec3b..c9f3070d 100644--- a/core/src/webpage/url.rs+++ b/core/src/webpage/url.rs@@ -283,6 +283,10 @@ impl Url {self.without_query().ends_with(ending)}}++ pub fn is_empty(&self) -> bool {+ self.0.is_empty()+ }}#[cfg(test)]