web search done right
add smoothing factor to timestamp to give it a half life of 3 days
Mikkel Denkerc7ea2fd8ff06
diff --git a/crates/core/src/live_index/crawler/mod.rs b/crates/core/src/live_index/crawler/mod.rsindex 0e3457b2..3233ded4 100644--- a/crates/core/src/live_index/crawler/mod.rs+++ b/crates/core/src/live_index/crawler/mod.rs@@ -84,7 +84,7 @@ impl Client {}pub async fn index(&self, pages: Vec<IndexableWebpage>) -> Result<()> {- let conn = self.live_conn().await;+ let mut conn = self.live_conn().await;let req = live_index::IndexWebpages {pages,@@ -97,6 +97,7 @@ impl Client {{tracing::error!("Failed to index pages: {e}");tokio::time::sleep(Duration::from_millis(1_000)).await;+ conn = self.live_conn().await;}Ok(())diff --git a/crates/core/src/ranking/computer/mod.rs b/crates/core/src/ranking/computer/mod.rsindex 3c08ceac..00dbeae1 100644--- a/crates/core/src/ranking/computer/mod.rs+++ b/crates/core/src/ranking/computer/mod.rs@@ -242,7 +242,9 @@ impl SignalComputer {.collect();let update_time_cache = (0..(3 * 365 * 24))- .map(|hours_since_update| 1.0 / ((hours_since_update as f64 + 1.0).log2()))+ .map(|hours_since_update| {+ super::signals::time_cache_calculation(hours_since_update as f64)+ }).collect();let query = query.as_ref().map(|q| QueryData {diff --git a/crates/core/src/ranking/signals/core/non_text.rs b/crates/core/src/ranking/signals/core/non_text.rsindex 5ec20ec8..ef89c0f7 100644--- a/crates/core/src/ranking/signals/core/non_text.rs+++ b/crates/core/src/ranking/signals/core/non_text.rs@@ -27,8 +27,12 @@ fn score_timestamp(page_timestamp: usize, signal_computer: &SignalComputer) -> freturn 0.0;}- let hours_since_update =- (signal_computer.current_timestamp().unwrap() - page_timestamp).max(1) / 3600;+ let hours_since_update = signal_computer+ .current_timestamp()+ .unwrap()+ .saturating_sub(page_timestamp)+ .max(1)+ / 3600;signal_computer.update_time_cache()@@ -37,6 +41,11 @@ fn score_timestamp(page_timestamp: usize, signal_computer: &SignalComputer) -> f.unwrap_or(0.0)}+pub fn time_cache_calculation(hours_since_update: f64) -> f64 {+ const SMOOTHING_FACTOR: f64 = 24.0 * 3.0; // half life of 3 days+ SMOOTHING_FACTOR / (hours_since_update + SMOOTHING_FACTOR)+}+#[inline]fn score_rank(rank: f64) -> f64 {// 10 groups with log base 8 gives us