Skip to content

web search done right

scaled test values

Mikkel Denker024ae7cbc339
diff --git a/core/src/ranking/optics.rs b/core/src/ranking/optics.rs
index 7f2124a7..163b8c0e 100644
--- a/core/src/ranking/optics.rs
+++ b/core/src/ranking/optics.rs
@@ -27,7 +27,7 @@ use crate::{
use super::{Signal, SignalAggregator};
-pub const SCALE: f32 = 200.0;
+pub const SCALE: f32 = 10.0;
pub trait CreateAggregator {
fn aggregator(&self, approx: Option<&ApproximatedHarmonicCentrality>) -> SignalAggregator;
@@ -132,7 +132,7 @@ mod tests {
),
backlinks: vec![],
host_centrality: 1.0,
- fetch_time_ms: 5000,
+ fetch_time_ms: 50,
pre_computed_score: 0.0,
page_centrality: 0.0,
primary_image: None,
@@ -169,7 +169,7 @@ mod tests {
),
backlinks: vec![],
host_centrality: 1.0,
- fetch_time_ms: 2000,
+ fetch_time_ms: 49,
page_centrality: 0.0,
pre_computed_score: 0.0,
primary_image: None,
diff --git a/core/src/ranking/pipeline.rs b/core/src/ranking/pipeline.rs
index 2ab40cc7..d732f1db 100644
--- a/core/src/ranking/pipeline.rs
+++ b/core/src/ranking/pipeline.rs
@@ -16,7 +16,6 @@
use std::cmp::Ordering;
-use itertools::Itertools;
use serde::{Deserialize, Serialize};
use crate::{
@@ -29,12 +28,17 @@ use super::SignalAggregator;
pub trait AsRankingWebsite: Clone {
fn as_ranking(&self) -> &RankingWebsite;
+ fn as_mut_ranking(&mut self) -> &mut RankingWebsite;
}
impl AsRankingWebsite for RankingWebsite {
fn as_ranking(&self) -> &RankingWebsite {
self
}
+
+ fn as_mut_ranking(&mut self) -> &mut RankingWebsite {
+ self
+ }
}
#[derive(Clone, Debug, Serialize, Deserialize)]
@@ -112,12 +116,14 @@ impl<T: AsRankingWebsite> Scorer<T> for ReRanker {
}
}
-struct SignalFocusText {}
+struct PrioritizeText {}
-impl<T: AsRankingWebsite> Scorer<T> for SignalFocusText {
+impl<T: AsRankingWebsite> Scorer<T> for PrioritizeText {
fn score(&self, websites: &mut [T]) {
- // TODO: Implement actual scoring
- // todo!();
+ for website in websites {
+ let bm25 = website.as_ranking().pointer.score.bm25 as f64;
+ website.as_mut_ranking().score += bm25;
+ }
}
}
@@ -158,62 +164,38 @@ impl<T: AsRankingWebsite> RankingStage<T> {
}
fn apply(&self, top_n: usize, offset: usize) -> Vec<T> {
- match &self.prev {
- Prev::Initial => {
- let (mut a, mut b) = self.memory.clone().unwrap();
-
- self.scorer.score(&mut a);
- a.sort_by(|a, b| {
- b.as_ranking()
- .score
- .partial_cmp(&a.as_ranking().score)
- .unwrap_or(Ordering::Equal)
- });
-
- self.scorer.score(&mut b);
- b.sort_by(|a, b| {
- b.as_ranking()
- .score
- .partial_cmp(&a.as_ranking().score)
- .unwrap_or(Ordering::Equal)
- });
-
- a.into_iter()
- .chain(b.into_iter())
- .skip(offset % self.top_n)
- .take(top_n)
- .collect()
- }
+ let (mut a, mut b) = match &self.prev {
+ Prev::Initial => self.memory.clone().unwrap(),
Prev::Node(n) => {
let k = offset / self.top_n;
- let (mut a, mut b) = (
+ (
n.apply(self.top_n, k * self.top_n),
n.apply(self.top_n, (k + 1) * self.top_n),
- );
-
- self.scorer.score(&mut a);
- self.scorer.score(&mut b);
-
- a.sort_by(|a, b| {
- b.as_ranking()
- .score
- .partial_cmp(&a.as_ranking().score)
- .unwrap_or(Ordering::Equal)
- });
- b.sort_by(|a, b| {
- b.as_ranking()
- .score
- .partial_cmp(&a.as_ranking().score)
- .unwrap_or(Ordering::Equal)
- });
-
- a.into_iter()
- .chain(b.into_iter())
- .skip(offset % self.top_n)
- .take(top_n)
- .collect()
+ )
}
- }
+ };
+
+ self.scorer.score(&mut a);
+ a.sort_by(|a, b| {
+ b.as_ranking()
+ .score
+ .partial_cmp(&a.as_ranking().score)
+ .unwrap_or(Ordering::Equal)
+ });
+
+ self.scorer.score(&mut b);
+ b.sort_by(|a, b| {
+ b.as_ranking()
+ .score
+ .partial_cmp(&a.as_ranking().score)
+ .unwrap_or(Ordering::Equal)
+ });
+
+ a.into_iter()
+ .chain(b.into_iter())
+ .skip(offset % self.top_n)
+ .take(top_n)
+ .collect()
}
}
@@ -228,7 +210,7 @@ impl<T: AsRankingWebsite> RankingPipeline<T> {
let last_stage = RankingStage {
scorer: Box::new(ReRanker {}),
prev: Prev::Node(Box::new(RankingStage {
- scorer: Box::new(SignalFocusText {}),
+ scorer: Box::new(PrioritizeText {}),
prev: Prev::Initial,
memory: None,
top_n: 100,
@@ -277,6 +259,8 @@ impl<T: AsRankingWebsite> RankingPipeline<T> {
#[cfg(test)]
mod tests {
+ use itertools::Itertools;
+
use crate::{
collector::Hashes, inverted_index::DocAddress, prehashed::Prehashed,
ranking::initial::Score,
diff --git a/core/src/ranking/signal.rs b/core/src/ranking/signal.rs
index b991c20e..83937be1 100644
--- a/core/src/ranking/signal.rs
+++ b/core/src/ranking/signal.rs
@@ -148,7 +148,7 @@ impl Signal {
Signal::HostCentrality => 2500.0,
Signal::PageCentrality => 4500.0,
Signal::TopicCentrality => 2500.0,
- Signal::PersonalCentrality => 4500.0,
+ Signal::PersonalCentrality => 1000.0,
Signal::QueryCentrality => 2500.0,
Signal::IsHomepage => 0.1,
Signal::FetchTimeMs => 0.01,
diff --git a/core/src/searcher/distributed.rs b/core/src/searcher/distributed.rs
index 89471de2..70386ff3 100644
--- a/core/src/searcher/distributed.rs
+++ b/core/src/searcher/distributed.rs
@@ -183,6 +183,10 @@ impl AsRankingWebsite for ScoredWebsitePointer {
fn as_ranking(&self) -> &RankingWebsite {
&self.website
}
+
+ fn as_mut_ranking(&mut self) -> &mut RankingWebsite {
+ &mut self.website
+ }
}
impl collector::Doc for ScoredWebsitePointer {