web search done right
scaled test values
Mikkel Denker024ae7cbc339
diff --git a/core/src/ranking/optics.rs b/core/src/ranking/optics.rsindex 7f2124a7..163b8c0e 100644--- a/core/src/ranking/optics.rs+++ b/core/src/ranking/optics.rs@@ -27,7 +27,7 @@ use crate::{use super::{Signal, SignalAggregator};-pub const SCALE: f32 = 200.0;+pub const SCALE: f32 = 10.0;pub trait CreateAggregator {fn aggregator(&self, approx: Option<&ApproximatedHarmonicCentrality>) -> SignalAggregator;@@ -132,7 +132,7 @@ mod tests {),backlinks: vec![],host_centrality: 1.0,- fetch_time_ms: 5000,+ fetch_time_ms: 50,pre_computed_score: 0.0,page_centrality: 0.0,primary_image: None,@@ -169,7 +169,7 @@ mod tests {),backlinks: vec![],host_centrality: 1.0,- fetch_time_ms: 2000,+ fetch_time_ms: 49,page_centrality: 0.0,pre_computed_score: 0.0,primary_image: None,diff --git a/core/src/ranking/pipeline.rs b/core/src/ranking/pipeline.rsindex 2ab40cc7..d732f1db 100644--- a/core/src/ranking/pipeline.rs+++ b/core/src/ranking/pipeline.rs@@ -16,7 +16,6 @@use std::cmp::Ordering;-use itertools::Itertools;use serde::{Deserialize, Serialize};use crate::{@@ -29,12 +28,17 @@ use super::SignalAggregator;pub trait AsRankingWebsite: Clone {fn as_ranking(&self) -> &RankingWebsite;+ fn as_mut_ranking(&mut self) -> &mut RankingWebsite;}impl AsRankingWebsite for RankingWebsite {fn as_ranking(&self) -> &RankingWebsite {self}++ fn as_mut_ranking(&mut self) -> &mut RankingWebsite {+ self+ }}#[derive(Clone, Debug, Serialize, Deserialize)]@@ -112,12 +116,14 @@ impl<T: AsRankingWebsite> Scorer<T> for ReRanker {}}-struct SignalFocusText {}+struct PrioritizeText {}-impl<T: AsRankingWebsite> Scorer<T> for SignalFocusText {+impl<T: AsRankingWebsite> Scorer<T> for PrioritizeText {fn score(&self, websites: &mut [T]) {- // TODO: Implement actual scoring- // todo!();+ for website in websites {+ let bm25 = website.as_ranking().pointer.score.bm25 as f64;+ website.as_mut_ranking().score += bm25;+ }}}@@ -158,62 +164,38 @@ impl<T: AsRankingWebsite> RankingStage<T> {}fn apply(&self, top_n: usize, offset: usize) -> Vec<T> {- match &self.prev {- Prev::Initial => {- let (mut a, mut b) = self.memory.clone().unwrap();-- self.scorer.score(&mut a);- a.sort_by(|a, b| {- b.as_ranking()- .score- .partial_cmp(&a.as_ranking().score)- .unwrap_or(Ordering::Equal)- });-- self.scorer.score(&mut b);- b.sort_by(|a, b| {- b.as_ranking()- .score- .partial_cmp(&a.as_ranking().score)- .unwrap_or(Ordering::Equal)- });-- a.into_iter()- .chain(b.into_iter())- .skip(offset % self.top_n)- .take(top_n)- .collect()- }+ let (mut a, mut b) = match &self.prev {+ Prev::Initial => self.memory.clone().unwrap(),Prev::Node(n) => {let k = offset / self.top_n;- let (mut a, mut b) = (+ (n.apply(self.top_n, k * self.top_n),n.apply(self.top_n, (k + 1) * self.top_n),- );-- self.scorer.score(&mut a);- self.scorer.score(&mut b);-- a.sort_by(|a, b| {- b.as_ranking()- .score- .partial_cmp(&a.as_ranking().score)- .unwrap_or(Ordering::Equal)- });- b.sort_by(|a, b| {- b.as_ranking()- .score- .partial_cmp(&a.as_ranking().score)- .unwrap_or(Ordering::Equal)- });-- a.into_iter()- .chain(b.into_iter())- .skip(offset % self.top_n)- .take(top_n)- .collect()+ )}- }+ };++ self.scorer.score(&mut a);+ a.sort_by(|a, b| {+ b.as_ranking()+ .score+ .partial_cmp(&a.as_ranking().score)+ .unwrap_or(Ordering::Equal)+ });++ self.scorer.score(&mut b);+ b.sort_by(|a, b| {+ b.as_ranking()+ .score+ .partial_cmp(&a.as_ranking().score)+ .unwrap_or(Ordering::Equal)+ });++ a.into_iter()+ .chain(b.into_iter())+ .skip(offset % self.top_n)+ .take(top_n)+ .collect()}}@@ -228,7 +210,7 @@ impl<T: AsRankingWebsite> RankingPipeline<T> {let last_stage = RankingStage {scorer: Box::new(ReRanker {}),prev: Prev::Node(Box::new(RankingStage {- scorer: Box::new(SignalFocusText {}),+ scorer: Box::new(PrioritizeText {}),prev: Prev::Initial,memory: None,top_n: 100,@@ -277,6 +259,8 @@ impl<T: AsRankingWebsite> RankingPipeline<T> {#[cfg(test)]mod tests {+ use itertools::Itertools;+use crate::{collector::Hashes, inverted_index::DocAddress, prehashed::Prehashed,ranking::initial::Score,diff --git a/core/src/ranking/signal.rs b/core/src/ranking/signal.rsindex b991c20e..83937be1 100644--- a/core/src/ranking/signal.rs+++ b/core/src/ranking/signal.rs@@ -148,7 +148,7 @@ impl Signal {Signal::HostCentrality => 2500.0,Signal::PageCentrality => 4500.0,Signal::TopicCentrality => 2500.0,- Signal::PersonalCentrality => 4500.0,+ Signal::PersonalCentrality => 1000.0,Signal::QueryCentrality => 2500.0,Signal::IsHomepage => 0.1,Signal::FetchTimeMs => 0.01,diff --git a/core/src/searcher/distributed.rs b/core/src/searcher/distributed.rsindex 89471de2..70386ff3 100644--- a/core/src/searcher/distributed.rs+++ b/core/src/searcher/distributed.rs@@ -183,6 +183,10 @@ impl AsRankingWebsite for ScoredWebsitePointer {fn as_ranking(&self) -> &RankingWebsite {&self.website}++ fn as_mut_ranking(&mut self) -> &mut RankingWebsite {+ &mut self.website+ }}impl collector::Doc for ScoredWebsitePointer {