Skip to content

web search done right

optionally skip db init

Mikkel Denker240009db93cb
diff --git a/crates/core/src/config/defaults.rs b/crates/core/src/config/defaults.rs
index 47feb601..7b278d3a 100644
--- a/crates/core/src/config/defaults.rs
+++ b/crates/core/src/config/defaults.rs
@@ -241,3 +241,17 @@ impl LiveIndex {
200_000
}
}
+
+pub struct LiveCrawler;
+impl LiveCrawler {
+ pub fn init_crawl_db() -> bool {
+ true
+ }
+}
+
+pub struct HarmonicNearestSeed;
+impl HarmonicNearestSeed {
+ pub fn discount_factor() -> f64 {
+ 0.5
+ }
+}
diff --git a/crates/core/src/config/mod.rs b/crates/core/src/config/mod.rs
index 87a18e57..914e086a 100644
--- a/crates/core/src/config/mod.rs
+++ b/crates/core/src/config/mod.rs
@@ -458,6 +458,8 @@ pub struct LiveCrawlerConfig {
pub check_intervals: CheckIntervals,
#[serde(default)]
pub daily_budget: DailyLiveIndexCrawlerBudget,
+ #[serde(default = "defaults::LiveCrawler::init_crawl_db")]
+ pub init_crawl_db: bool,
}
#[derive(Debug, serde::Serialize, serde::Deserialize, Clone)]
@@ -690,3 +692,12 @@ pub struct CanonicalIndexConfig {
pub limit_warc_files: Option<usize>,
pub skip_warc_files: Option<usize>,
}
+
+#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]
+pub struct HarmonicNearestSeedConfig {
+ pub gossip: GossipConfig,
+ pub original_centrality_path: String,
+ pub output_path: String,
+ #[serde(default = "defaults::HarmonicNearestSeed::discount_factor")]
+ pub discount_factor: f64,
+}
diff --git a/crates/core/src/live_index/crawler/mod.rs b/crates/core/src/live_index/crawler/mod.rs
index c003f053..0e3457b2 100644
--- a/crates/core/src/live_index/crawler/mod.rs
+++ b/crates/core/src/live_index/crawler/mod.rs
@@ -224,14 +224,18 @@ impl Crawler {
)?;
let mut crawlable_sites = Vec::new();
- tracing::debug!("Initializing crawler db with previously crawled urls");
- for site in sites {
- for url in client.get_site_urls(site.site().as_str()).await {
- if !db.has_crawled(&url)? {
- db.insert(&url)?;
+ if config.init_crawl_db {
+ tracing::debug!("Initializing crawler db with previously crawled urls");
+ for site in &sites {
+ for url in client.get_site_urls(site.site().as_str()).await {
+ if !db.has_crawled(&url)? {
+ db.insert(&url)?;
+ }
}
}
+ }
+ for site in sites {
if let Some(drip_rate) = budgets.drip_rate(site.site()) {
crawlable_sites.push(Arc::new(CrawlableSite::new(site, &client, drip_rate)?));
}
@@ -251,6 +255,8 @@ impl Crawler {
let mut interval = tokio::time::interval(TICK_INTERVAL);
let semaphore = Arc::new(Semaphore::new(self.num_worker_threads));
+ tracing::info!("Crawler running with {} threads", self.num_worker_threads);
+
loop {
interval.tick().await;
tracing::debug!("Tick");
diff --git a/crates/core/src/live_index/crawler/site_url_stream.rs b/crates/core/src/live_index/crawler/site_url_stream.rs
index 3e8d307c..d199327e 100644
--- a/crates/core/src/live_index/crawler/site_url_stream.rs
+++ b/crates/core/src/live_index/crawler/site_url_stream.rs
@@ -59,6 +59,8 @@ impl StreamingResponse for SiteUrlStream<search_server::SearchService> {
limit: SITE_URL_BATCH_SIZE as u64,
};
+ self.offset += SITE_URL_BATCH_SIZE;
+
let res = self
.conn
.send(req, &AllShardsSelector, &RandomReplicaSelector)