web search done right
[live_crawler] shuffle sites between each tick to make site prioritisation more fair
Mikkel Denker0b1f6eaff143
diff --git a/crates/core/src/live_index/crawler/mod.rs b/crates/core/src/live_index/crawler/mod.rsindex 9747a3fb..396e2794 100644--- a/crates/core/src/live_index/crawler/mod.rs+++ b/crates/core/src/live_index/crawler/mod.rs@@ -42,6 +42,7 @@ use crate::{use crawlable_site::{CrawlableSite, CrawlableSiteGuard};use crawled_db::ShardedCrawledDb;use futures::StreamExt;+use rand::seq::SliceRandom;use site_url_stream::SiteUrlStream;use tokio::sync::Mutex;use tokio::sync::Semaphore;@@ -270,6 +271,7 @@ impl Crawler {interval.tick().await;tracing::debug!("Tick");+ self.sites.shuffle(&mut rand::thread_rng());for site in &mut self.sites {if site.currently_crawling() {tracing::debug!("Site {} is currently crawling", site.site().as_str());