Skip to content

web search done right

have some initial wander steps during crawl

Mikkel Denker1acbafce4c25
diff --git a/crates/core/src/crawler/worker.rs b/crates/core/src/crawler/worker.rs
index d4673f04..a07eaea4 100644
--- a/crates/core/src/crawler/worker.rs
+++ b/crates/core/src/crawler/worker.rs
@@ -49,6 +49,8 @@ const IGNORED_EXTENSIONS: [&str; 27] = [
".flac", ".aac", ".ogg", ".m4a", ".m4v",
];
+const INITIAL_WANDER_STEPS: u64 = 4;
+
enum UrlVisit {
Skip,
CanCrawl,
@@ -168,7 +170,28 @@ impl<S: DatumStream> JobExecutor<S> {
pub async fn run(mut self) {
tracing::info!("Processing job: {:?}", self.job.domain);
+ for site in self
+ .job
+ .urls
+ .iter()
+ .filter_map(|url| url.url().normalized_host())
+ {
+ if let Ok(url) = Url::parse(format!("https://{}", site).as_str()) {
+ self.wander_prioritiser.inc(url, 1.0);
+ }
+ }
+
+ let mut wander_steps = 0;
+ while self.wandered_urls < self.job.wandering_urls
+ && self.wander_prioritiser.known_urls() > 0
+ && wander_steps < INITIAL_WANDER_STEPS
+ {
+ self.wander().await;
+ wander_steps += 1;
+ }
+
self.scheduled_urls().await;
+ self.crawl_sitemaps().await;
while self.wandered_urls < self.job.wandering_urls
&& self.wander_prioritiser.known_urls() > 0
@@ -179,7 +202,7 @@ impl<S: DatumStream> JobExecutor<S> {
async fn scheduled_urls(&mut self) {
let urls = self.job.urls.drain(..).collect();
- self.process_urls(urls, true).await;
+ self.process_urls(urls).await;
}
async fn wander(&mut self) {
@@ -217,7 +240,7 @@ impl<S: DatumStream> JobExecutor<S> {
self.wandered_urls += urls.len() as u64;
- self.process_urls(urls, false).await;
+ self.process_urls(urls).await;
}
async fn verify_url(&mut self, retryable_url: &RetrieableUrl) -> UrlVisit {
@@ -252,7 +275,23 @@ impl<S: DatumStream> JobExecutor<S> {
UrlVisit::CanCrawl
}
- async fn process_urls(&mut self, mut urls: VecDeque<RetrieableUrl>, fetch_sitemap: bool) {
+ async fn crawl_sitemaps(&mut self) {
+ for url in self.job.urls.iter().map(|url| url.url()) {
+ let site = Site(url.host_str().unwrap_or_default().to_string());
+ if !self.crawled_sitemaps.contains(&site) {
+ self.crawled_sitemaps.insert(site.clone());
+
+ let sitemaps = self.robotstxt.sitemaps(url).await;
+
+ for sitemap in sitemaps {
+ self.sitemap_urls
+ .extend(self.urls_from_sitemap(sitemap, 5).await);
+ }
+ }
+ }
+ }
+
+ async fn process_urls(&mut self, mut urls: VecDeque<RetrieableUrl>) {
while let Some(retryable_url) = urls.pop_front() {
if let UrlVisit::Skip = self.verify_url(&retryable_url).await {
continue;
@@ -264,24 +303,6 @@ impl<S: DatumStream> JobExecutor<S> {
}
}
- let site = Site(
- retryable_url
- .url()
- .host_str()
- .unwrap_or_default()
- .to_string(),
- );
- if fetch_sitemap && !self.crawled_sitemaps.contains(&site) {
- self.crawled_sitemaps.insert(site.clone());
-
- let sitemaps = self.robotstxt.sitemaps(retryable_url.url()).await;
-
- for sitemap in sitemaps {
- self.sitemap_urls
- .extend(self.urls_from_sitemap(sitemap, 5).await);
- }
- }
-
let res = self.process_url(retryable_url.url().clone()).await;
match res {