web search done right
remove minimum budget of 1 per day in live index
Mikkel Denker5a1a5a322587
diff --git a/crates/core/src/config/defaults.rs b/crates/core/src/config/defaults.rsindex 7b278d3a..c5b87df3 100644--- a/crates/core/src/config/defaults.rs+++ b/crates/core/src/config/defaults.rs@@ -214,9 +214,8 @@ impl Webgraph {}}-pub struct LiveIndex;--impl LiveIndex {+pub struct LiveCrawler;+impl LiveCrawler {pub fn feeds_crawl_interval() -> Duration {Duration::from_secs(30 * 60) // 30 minutes}@@ -238,12 +237,9 @@ impl LiveIndex {}pub fn remaining_budget() -> u64 {- 200_000+ 50_000}-}-pub struct LiveCrawler;-impl LiveCrawler {pub fn init_crawl_db() -> bool {true}diff --git a/crates/core/src/config/mod.rs b/crates/core/src/config/mod.rsindex 6fc69b0c..8e85f168 100644--- a/crates/core/src/config/mod.rs+++ b/crates/core/src/config/mod.rs@@ -399,11 +399,11 @@ pub struct CrawlerConfig {#[derive(Debug, Clone, serde::Deserialize, serde::Serialize)]pub struct DailyLiveIndexCrawlerBudget {- #[serde(default = "defaults::LiveIndex::blogs_budget")]+ #[serde(default = "defaults::LiveCrawler::blogs_budget")]pub blogs: u64,- #[serde(default = "defaults::LiveIndex::news_budget")]+ #[serde(default = "defaults::LiveCrawler::news_budget")]pub news: u64,- #[serde(default = "defaults::LiveIndex::remaining_budget")]+ #[serde(default = "defaults::LiveCrawler::remaining_budget")]pub remaining: u64,}@@ -411,17 +411,17 @@ pub struct DailyLiveIndexCrawlerBudget {pub struct CheckIntervals {#[serde(deserialize_with = "parse_duration",- default = "defaults::LiveIndex::feeds_crawl_interval"+ default = "defaults::LiveCrawler::feeds_crawl_interval")]pub feeds: Duration,#[serde(deserialize_with = "parse_duration",- default = "defaults::LiveIndex::sitemap_crawl_interval"+ default = "defaults::LiveCrawler::sitemap_crawl_interval")]pub sitemap: Duration,#[serde(deserialize_with = "parse_duration",- default = "defaults::LiveIndex::frontpage_crawl_interval"+ default = "defaults::LiveCrawler::frontpage_crawl_interval")]pub frontpage: Duration,}@@ -429,9 +429,9 @@ pub struct CheckIntervals {impl Default for CheckIntervals {fn default() -> Self {Self {- feeds: defaults::LiveIndex::feeds_crawl_interval(),- sitemap: defaults::LiveIndex::sitemap_crawl_interval(),- frontpage: defaults::LiveIndex::frontpage_crawl_interval(),+ feeds: defaults::LiveCrawler::feeds_crawl_interval(),+ sitemap: defaults::LiveCrawler::sitemap_crawl_interval(),+ frontpage: defaults::LiveCrawler::frontpage_crawl_interval(),}}}@@ -439,9 +439,9 @@ impl Default for CheckIntervals {impl Default for DailyLiveIndexCrawlerBudget {fn default() -> Self {Self {- blogs: defaults::LiveIndex::blogs_budget(),- news: defaults::LiveIndex::news_budget(),- remaining: defaults::LiveIndex::remaining_budget(),+ blogs: defaults::LiveCrawler::blogs_budget(),+ news: defaults::LiveCrawler::news_budget(),+ remaining: defaults::LiveCrawler::remaining_budget(),}}}diff --git a/crates/core/src/live_index/crawler/budgets.rs b/crates/core/src/live_index/crawler/budgets.rsindex a5ac5d54..3ed851c5 100644--- a/crates/core/src/live_index/crawler/budgets.rs+++ b/crates/core/src/live_index/crawler/budgets.rs@@ -138,10 +138,10 @@ impl SiteBudgets {.or_else(|| self.news.get(site)).or_else(|| self.remaining.get(site))?;- let budget = budget as u64;+ let budget = budget.round() as u64;if budget == 0 {- Some(Duration::from_millis(MILLIS_PER_DAY)) // once per day+ None} else {Some(Duration::from_millis(MILLIS_PER_DAY / budget))}