Skip to content

web search done right

hopefully made crawler more stable?

Mikkel Denkerd25288cb14d9
diff --git a/core/src/crawler/coordinator.rs b/core/src/crawler/coordinator.rs
index 2bdcd2ea..96e48a9d 100644
--- a/core/src/crawler/coordinator.rs
+++ b/core/src/crawler/coordinator.rs
@@ -62,7 +62,7 @@ impl CrawlCoordinator {
db,
num_urls_to_crawl,
num_crawled_urls: AtomicU64::new(0),
- call_counter: Mutex::new(CallCounter::new(Duration::from_secs(60))),
+ call_counter: Mutex::new(CallCounter::new(Duration::from_secs(10))),
})
}
diff --git a/core/src/crawler/robots_txt.rs b/core/src/crawler/robots_txt.rs
index 77246918..8786c566 100644
--- a/core/src/crawler/robots_txt.rs
+++ b/core/src/crawler/robots_txt.rs
@@ -74,6 +74,7 @@ impl RobotsTxtManager {
self.cache.insert(site.clone(), None);
}
Err(err) => {
+ self.cache.insert(site.clone(), None);
tracing::warn!("failed to fetch robots.txt for {}: {}", site.0, err);
}
}
diff --git a/core/src/crawler/worker.rs b/core/src/crawler/worker.rs
index 83a3aadc..d65fd961 100644
--- a/core/src/crawler/worker.rs
+++ b/core/src/crawler/worker.rs
@@ -162,6 +162,10 @@ impl Worker {
let num_urls = job.job.urls.len();
for (i, url) in job.job.urls.into_iter().enumerate() {
+ if url.is_empty() || url.site().is_empty() {
+ continue;
+ }
+
if !job.robotstxt.is_allowed(&url, &self.user_agent).await {
continue;
}
@@ -207,7 +211,7 @@ impl Worker {
let _ = conn
.send_with_timeout::<_, Response>(
&Request::CrawlResult { job_response },
- Duration::from_secs(60),
+ Duration::from_secs(60 * 60),
)
.await
.expect("Failed to send crawl result to coordinator");
diff --git a/core/src/webpage/url.rs b/core/src/webpage/url.rs
index 3b0dec3b..c9f3070d 100644
--- a/core/src/webpage/url.rs
+++ b/core/src/webpage/url.rs
@@ -283,6 +283,10 @@ impl Url {
self.without_query().ends_with(ending)
}
}
+
+ pub fn is_empty(&self) -> bool {
+ self.0.is_empty()
+ }
}
#[cfg(test)]