Skip to content

web search done right

add smoothing factor to timestamp to give it a half life of 3 days

Mikkel Denkerc7ea2fd8ff06
diff --git a/crates/core/src/live_index/crawler/mod.rs b/crates/core/src/live_index/crawler/mod.rs
index 0e3457b2..3233ded4 100644
--- a/crates/core/src/live_index/crawler/mod.rs
+++ b/crates/core/src/live_index/crawler/mod.rs
@@ -84,7 +84,7 @@ impl Client {
}
pub async fn index(&self, pages: Vec<IndexableWebpage>) -> Result<()> {
- let conn = self.live_conn().await;
+ let mut conn = self.live_conn().await;
let req = live_index::IndexWebpages {
pages,
@@ -97,6 +97,7 @@ impl Client {
{
tracing::error!("Failed to index pages: {e}");
tokio::time::sleep(Duration::from_millis(1_000)).await;
+ conn = self.live_conn().await;
}
Ok(())
diff --git a/crates/core/src/ranking/computer/mod.rs b/crates/core/src/ranking/computer/mod.rs
index 3c08ceac..00dbeae1 100644
--- a/crates/core/src/ranking/computer/mod.rs
+++ b/crates/core/src/ranking/computer/mod.rs
@@ -242,7 +242,9 @@ impl SignalComputer {
.collect();
let update_time_cache = (0..(3 * 365 * 24))
- .map(|hours_since_update| 1.0 / ((hours_since_update as f64 + 1.0).log2()))
+ .map(|hours_since_update| {
+ super::signals::time_cache_calculation(hours_since_update as f64)
+ })
.collect();
let query = query.as_ref().map(|q| QueryData {
diff --git a/crates/core/src/ranking/signals/core/non_text.rs b/crates/core/src/ranking/signals/core/non_text.rs
index 5ec20ec8..ef89c0f7 100644
--- a/crates/core/src/ranking/signals/core/non_text.rs
+++ b/crates/core/src/ranking/signals/core/non_text.rs
@@ -27,8 +27,12 @@ fn score_timestamp(page_timestamp: usize, signal_computer: &SignalComputer) -> f
return 0.0;
}
- let hours_since_update =
- (signal_computer.current_timestamp().unwrap() - page_timestamp).max(1) / 3600;
+ let hours_since_update = signal_computer
+ .current_timestamp()
+ .unwrap()
+ .saturating_sub(page_timestamp)
+ .max(1)
+ / 3600;
signal_computer
.update_time_cache()
@@ -37,6 +41,11 @@ fn score_timestamp(page_timestamp: usize, signal_computer: &SignalComputer) -> f
.unwrap_or(0.0)
}
+pub fn time_cache_calculation(hours_since_update: f64) -> f64 {
+ const SMOOTHING_FACTOR: f64 = 24.0 * 3.0; // half life of 3 days
+ SMOOTHING_FACTOR / (hours_since_update + SMOOTHING_FACTOR)
+}
+
#[inline]
fn score_rank(rank: f64) -> f64 {
// 10 groups with log base 8 gives us