web search done right
optionally skip db init
Mikkel Denker240009db93cb
diff --git a/crates/core/src/config/defaults.rs b/crates/core/src/config/defaults.rsindex 47feb601..7b278d3a 100644--- a/crates/core/src/config/defaults.rs+++ b/crates/core/src/config/defaults.rs@@ -241,3 +241,17 @@ impl LiveIndex {200_000}}++pub struct LiveCrawler;+impl LiveCrawler {+ pub fn init_crawl_db() -> bool {+ true+ }+}++pub struct HarmonicNearestSeed;+impl HarmonicNearestSeed {+ pub fn discount_factor() -> f64 {+ 0.5+ }+}diff --git a/crates/core/src/config/mod.rs b/crates/core/src/config/mod.rsindex 87a18e57..914e086a 100644--- a/crates/core/src/config/mod.rs+++ b/crates/core/src/config/mod.rs@@ -458,6 +458,8 @@ pub struct LiveCrawlerConfig {pub check_intervals: CheckIntervals,#[serde(default)]pub daily_budget: DailyLiveIndexCrawlerBudget,+ #[serde(default = "defaults::LiveCrawler::init_crawl_db")]+ pub init_crawl_db: bool,}#[derive(Debug, serde::Serialize, serde::Deserialize, Clone)]@@ -690,3 +692,12 @@ pub struct CanonicalIndexConfig {pub limit_warc_files: Option<usize>,pub skip_warc_files: Option<usize>,}++#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]+pub struct HarmonicNearestSeedConfig {+ pub gossip: GossipConfig,+ pub original_centrality_path: String,+ pub output_path: String,+ #[serde(default = "defaults::HarmonicNearestSeed::discount_factor")]+ pub discount_factor: f64,+}diff --git a/crates/core/src/live_index/crawler/mod.rs b/crates/core/src/live_index/crawler/mod.rsindex c003f053..0e3457b2 100644--- a/crates/core/src/live_index/crawler/mod.rs+++ b/crates/core/src/live_index/crawler/mod.rs@@ -224,14 +224,18 @@ impl Crawler {)?;let mut crawlable_sites = Vec::new();- tracing::debug!("Initializing crawler db with previously crawled urls");- for site in sites {- for url in client.get_site_urls(site.site().as_str()).await {- if !db.has_crawled(&url)? {- db.insert(&url)?;+ if config.init_crawl_db {+ tracing::debug!("Initializing crawler db with previously crawled urls");+ for site in &sites {+ for url in client.get_site_urls(site.site().as_str()).await {+ if !db.has_crawled(&url)? {+ db.insert(&url)?;+ }}}+ }+ for site in sites {if let Some(drip_rate) = budgets.drip_rate(site.site()) {crawlable_sites.push(Arc::new(CrawlableSite::new(site, &client, drip_rate)?));}@@ -251,6 +255,8 @@ impl Crawler {let mut interval = tokio::time::interval(TICK_INTERVAL);let semaphore = Arc::new(Semaphore::new(self.num_worker_threads));+ tracing::info!("Crawler running with {} threads", self.num_worker_threads);+loop {interval.tick().await;tracing::debug!("Tick");diff --git a/crates/core/src/live_index/crawler/site_url_stream.rs b/crates/core/src/live_index/crawler/site_url_stream.rsindex 3e8d307c..d199327e 100644--- a/crates/core/src/live_index/crawler/site_url_stream.rs+++ b/crates/core/src/live_index/crawler/site_url_stream.rs@@ -59,6 +59,8 @@ impl StreamingResponse for SiteUrlStream<search_server::SearchService> {limit: SITE_URL_BATCH_SIZE as u64,};+ self.offset += SITE_URL_BATCH_SIZE;+let res = self.conn.send(req, &AllShardsSelector, &RandomReplicaSelector)