web search done right
change crossencoder scores to be calculated from their ranks. the raw scores from the cross encoder model can be very similar even though site A is way better than site B, so it will be difficult to scale correctly with a coefficient. the models are essentially only trained to optimize the ranks.
Mikkel Denker11ad03be77ec
diff --git a/crates/core/src/ranking/models/cross_encoder.rs b/crates/core/src/ranking/models/cross_encoder.rsindex d13096e8..8cf95843 100644--- a/crates/core/src/ranking/models/cross_encoder.rs+++ b/crates/core/src/ranking/models/cross_encoder.rs@@ -86,10 +86,8 @@ impl CrossEncoderModel {dtype,})}-}-impl CrossEncoder for CrossEncoderModel {- fn run(&self, query: &str, bodies: &[String]) -> Vec<f64> {+ fn scores(&self, query: &str, bodies: &[String]) -> Vec<f64> {if bodies.is_empty() {return Vec::new();}@@ -147,6 +145,27 @@ impl CrossEncoder for CrossEncoderModel {}}+impl CrossEncoder for CrossEncoderModel {+ fn run(&self, query: &str, bodies: &[String]) -> Vec<f64> {+ let mut scores = self+ .scores(query, bodies)+ .into_iter()+ .enumerate()+ .collect::<Vec<_>>();++ scores.sort_by(|a, b| b.1.total_cmp(&a.1));++ let mut ranked_scores = scores+ .into_iter()+ .enumerate()+ .map(|(rank, (i, _))| (i, (1.0 / (rank as f64 + 1.0))))+ .collect::<Vec<_>>();++ ranked_scores.sort_by(|a, b| a.0.cmp(&b.0));+ ranked_scores.into_iter().map(|(_, score)| score).collect()+ }+}+pub trait CrossEncoder: Send + Sync {fn run(&self, query: &str, bodies: &[String]) -> Vec<f64>;}diff --git a/crates/core/src/ranking/optics.rs b/crates/core/src/ranking/optics.rsindex 3f07c6d0..f1169576 100644--- a/crates/core/src/ranking/optics.rs+++ b/crates/core/src/ranking/optics.rs@@ -16,7 +16,10 @@#[cfg(test)]mod tests {- use optics::HostRankings;+ use optics::{+ ast::{RankingCoeff, RankingTarget},+ HostRankings, Optic,+ };use crate::{gen_temp_path,@@ -169,22 +172,12 @@ mod tests {disliked: vec![],blocked: vec![],}),- ..Default::default()- })- .expect("Search failed");-- assert_eq!(result.webpages.len(), 3);- assert_eq!(result.webpages[0].url, "https://www.first.com/");- assert_eq!(result.webpages[1].url, "https://www.second.com/");- assert_eq!(result.webpages[2].url, "https://www.third.com/");-- let result = searcher- .search(&SearchQuery {- query: "test".to_string(),- host_rankings: Some(HostRankings {- liked: vec!["www.first.com".to_string()],- disliked: vec![],- blocked: vec![],+ optic: Some(Optic {+ rankings: vec![RankingCoeff {+ target: RankingTarget::Signal("inbound_similarity".to_string()),+ value: 100_000_000.0,+ }],+ ..Default::default()}),..Default::default()})@@ -203,6 +196,13 @@ mod tests {disliked: vec!["www.second.com".to_string()],blocked: vec!["first.com".to_string()],}),+ optic: Some(Optic {+ rankings: vec![RankingCoeff {+ target: RankingTarget::Signal("inbound_similarity".to_string()),+ value: 100_000_000.0,+ }],+ ..Default::default()+ }),return_ranking_signals: true,..Default::default()})@@ -220,6 +220,13 @@ mod tests {disliked: vec![],blocked: vec!["www.first.com".to_string()],}),+ optic: Some(Optic {+ rankings: vec![RankingCoeff {+ target: RankingTarget::Signal("inbound_similarity".to_string()),+ value: 100_000_000.0,+ }],+ ..Default::default()+ }),return_ranking_signals: true,..Default::default()})@@ -237,6 +244,13 @@ mod tests {disliked: vec![],blocked: vec!["abc.first.com".to_string()],}),+ optic: Some(Optic {+ rankings: vec![RankingCoeff {+ target: RankingTarget::Signal("inbound_similarity".to_string()),+ value: 100_000_000.0,+ }],+ ..Default::default()+ }),return_ranking_signals: true,..Default::default()})