Skip to content

web search done right

[live_crawler] shuffle sites between each tick to make site prioritisation more fair

Mikkel Denker0b1f6eaff143
diff --git a/crates/core/src/live_index/crawler/mod.rs b/crates/core/src/live_index/crawler/mod.rs
index 9747a3fb..396e2794 100644
--- a/crates/core/src/live_index/crawler/mod.rs
+++ b/crates/core/src/live_index/crawler/mod.rs
@@ -42,6 +42,7 @@ use crate::{
use crawlable_site::{CrawlableSite, CrawlableSiteGuard};
use crawled_db::ShardedCrawledDb;
use futures::StreamExt;
+use rand::seq::SliceRandom;
use site_url_stream::SiteUrlStream;
use tokio::sync::Mutex;
use tokio::sync::Semaphore;
@@ -270,6 +271,7 @@ impl Crawler {
interval.tick().await;
tracing::debug!("Tick");
+ self.sites.shuffle(&mut rand::thread_rng());
for site in &mut self.sites {
if site.currently_crawling() {
tracing::debug!("Site {} is currently crawling", site.site().as_str());