Skip to content

web search done right

remove minimum budget of 1 per day in live index

Mikkel Denker5a1a5a322587
diff --git a/crates/core/src/config/defaults.rs b/crates/core/src/config/defaults.rs
index 7b278d3a..c5b87df3 100644
--- a/crates/core/src/config/defaults.rs
+++ b/crates/core/src/config/defaults.rs
@@ -214,9 +214,8 @@ impl Webgraph {
}
}
-pub struct LiveIndex;
-
-impl LiveIndex {
+pub struct LiveCrawler;
+impl LiveCrawler {
pub fn feeds_crawl_interval() -> Duration {
Duration::from_secs(30 * 60) // 30 minutes
}
@@ -238,12 +237,9 @@ impl LiveIndex {
}
pub fn remaining_budget() -> u64 {
- 200_000
+ 50_000
}
-}
-pub struct LiveCrawler;
-impl LiveCrawler {
pub fn init_crawl_db() -> bool {
true
}
diff --git a/crates/core/src/config/mod.rs b/crates/core/src/config/mod.rs
index 6fc69b0c..8e85f168 100644
--- a/crates/core/src/config/mod.rs
+++ b/crates/core/src/config/mod.rs
@@ -399,11 +399,11 @@ pub struct CrawlerConfig {
#[derive(Debug, Clone, serde::Deserialize, serde::Serialize)]
pub struct DailyLiveIndexCrawlerBudget {
- #[serde(default = "defaults::LiveIndex::blogs_budget")]
+ #[serde(default = "defaults::LiveCrawler::blogs_budget")]
pub blogs: u64,
- #[serde(default = "defaults::LiveIndex::news_budget")]
+ #[serde(default = "defaults::LiveCrawler::news_budget")]
pub news: u64,
- #[serde(default = "defaults::LiveIndex::remaining_budget")]
+ #[serde(default = "defaults::LiveCrawler::remaining_budget")]
pub remaining: u64,
}
@@ -411,17 +411,17 @@ pub struct DailyLiveIndexCrawlerBudget {
pub struct CheckIntervals {
#[serde(
deserialize_with = "parse_duration",
- default = "defaults::LiveIndex::feeds_crawl_interval"
+ default = "defaults::LiveCrawler::feeds_crawl_interval"
)]
pub feeds: Duration,
#[serde(
deserialize_with = "parse_duration",
- default = "defaults::LiveIndex::sitemap_crawl_interval"
+ default = "defaults::LiveCrawler::sitemap_crawl_interval"
)]
pub sitemap: Duration,
#[serde(
deserialize_with = "parse_duration",
- default = "defaults::LiveIndex::frontpage_crawl_interval"
+ default = "defaults::LiveCrawler::frontpage_crawl_interval"
)]
pub frontpage: Duration,
}
@@ -429,9 +429,9 @@ pub struct CheckIntervals {
impl Default for CheckIntervals {
fn default() -> Self {
Self {
- feeds: defaults::LiveIndex::feeds_crawl_interval(),
- sitemap: defaults::LiveIndex::sitemap_crawl_interval(),
- frontpage: defaults::LiveIndex::frontpage_crawl_interval(),
+ feeds: defaults::LiveCrawler::feeds_crawl_interval(),
+ sitemap: defaults::LiveCrawler::sitemap_crawl_interval(),
+ frontpage: defaults::LiveCrawler::frontpage_crawl_interval(),
}
}
}
@@ -439,9 +439,9 @@ impl Default for CheckIntervals {
impl Default for DailyLiveIndexCrawlerBudget {
fn default() -> Self {
Self {
- blogs: defaults::LiveIndex::blogs_budget(),
- news: defaults::LiveIndex::news_budget(),
- remaining: defaults::LiveIndex::remaining_budget(),
+ blogs: defaults::LiveCrawler::blogs_budget(),
+ news: defaults::LiveCrawler::news_budget(),
+ remaining: defaults::LiveCrawler::remaining_budget(),
}
}
}
diff --git a/crates/core/src/live_index/crawler/budgets.rs b/crates/core/src/live_index/crawler/budgets.rs
index a5ac5d54..3ed851c5 100644
--- a/crates/core/src/live_index/crawler/budgets.rs
+++ b/crates/core/src/live_index/crawler/budgets.rs
@@ -138,10 +138,10 @@ impl SiteBudgets {
.or_else(|| self.news.get(site))
.or_else(|| self.remaining.get(site))?;
- let budget = budget as u64;
+ let budget = budget.round() as u64;
if budget == 0 {
- Some(Duration::from_millis(MILLIS_PER_DAY)) // once per day
+ None
} else {
Some(Duration::from_millis(MILLIS_PER_DAY / budget))
}