Skip to content

web search done right

change crossencoder scores to be calculated from their ranks. the raw scores from the cross encoder model can be very similar even though site A is way better than site B, so it will be difficult to scale correctly with a coefficient. the models are essentially only trained to optimize the ranks.

Mikkel Denker11ad03be77ec
diff --git a/crates/core/src/ranking/models/cross_encoder.rs b/crates/core/src/ranking/models/cross_encoder.rs
index d13096e8..8cf95843 100644
--- a/crates/core/src/ranking/models/cross_encoder.rs
+++ b/crates/core/src/ranking/models/cross_encoder.rs
@@ -86,10 +86,8 @@ impl CrossEncoderModel {
dtype,
})
}
-}
-impl CrossEncoder for CrossEncoderModel {
- fn run(&self, query: &str, bodies: &[String]) -> Vec<f64> {
+ fn scores(&self, query: &str, bodies: &[String]) -> Vec<f64> {
if bodies.is_empty() {
return Vec::new();
}
@@ -147,6 +145,27 @@ impl CrossEncoder for CrossEncoderModel {
}
}
+impl CrossEncoder for CrossEncoderModel {
+ fn run(&self, query: &str, bodies: &[String]) -> Vec<f64> {
+ let mut scores = self
+ .scores(query, bodies)
+ .into_iter()
+ .enumerate()
+ .collect::<Vec<_>>();
+
+ scores.sort_by(|a, b| b.1.total_cmp(&a.1));
+
+ let mut ranked_scores = scores
+ .into_iter()
+ .enumerate()
+ .map(|(rank, (i, _))| (i, (1.0 / (rank as f64 + 1.0))))
+ .collect::<Vec<_>>();
+
+ ranked_scores.sort_by(|a, b| a.0.cmp(&b.0));
+ ranked_scores.into_iter().map(|(_, score)| score).collect()
+ }
+}
+
pub trait CrossEncoder: Send + Sync {
fn run(&self, query: &str, bodies: &[String]) -> Vec<f64>;
}
diff --git a/crates/core/src/ranking/optics.rs b/crates/core/src/ranking/optics.rs
index 3f07c6d0..f1169576 100644
--- a/crates/core/src/ranking/optics.rs
+++ b/crates/core/src/ranking/optics.rs
@@ -16,7 +16,10 @@
#[cfg(test)]
mod tests {
- use optics::HostRankings;
+ use optics::{
+ ast::{RankingCoeff, RankingTarget},
+ HostRankings, Optic,
+ };
use crate::{
gen_temp_path,
@@ -169,22 +172,12 @@ mod tests {
disliked: vec![],
blocked: vec![],
}),
- ..Default::default()
- })
- .expect("Search failed");
-
- assert_eq!(result.webpages.len(), 3);
- assert_eq!(result.webpages[0].url, "https://www.first.com/");
- assert_eq!(result.webpages[1].url, "https://www.second.com/");
- assert_eq!(result.webpages[2].url, "https://www.third.com/");
-
- let result = searcher
- .search(&SearchQuery {
- query: "test".to_string(),
- host_rankings: Some(HostRankings {
- liked: vec!["www.first.com".to_string()],
- disliked: vec![],
- blocked: vec![],
+ optic: Some(Optic {
+ rankings: vec![RankingCoeff {
+ target: RankingTarget::Signal("inbound_similarity".to_string()),
+ value: 100_000_000.0,
+ }],
+ ..Default::default()
}),
..Default::default()
})
@@ -203,6 +196,13 @@ mod tests {
disliked: vec!["www.second.com".to_string()],
blocked: vec!["first.com".to_string()],
}),
+ optic: Some(Optic {
+ rankings: vec![RankingCoeff {
+ target: RankingTarget::Signal("inbound_similarity".to_string()),
+ value: 100_000_000.0,
+ }],
+ ..Default::default()
+ }),
return_ranking_signals: true,
..Default::default()
})
@@ -220,6 +220,13 @@ mod tests {
disliked: vec![],
blocked: vec!["www.first.com".to_string()],
}),
+ optic: Some(Optic {
+ rankings: vec![RankingCoeff {
+ target: RankingTarget::Signal("inbound_similarity".to_string()),
+ value: 100_000_000.0,
+ }],
+ ..Default::default()
+ }),
return_ranking_signals: true,
..Default::default()
})
@@ -237,6 +244,13 @@ mod tests {
disliked: vec![],
blocked: vec!["abc.first.com".to_string()],
}),
+ optic: Some(Optic {
+ rankings: vec![RankingCoeff {
+ target: RankingTarget::Signal("inbound_similarity".to_string()),
+ value: 100_000_000.0,
+ }],
+ ..Default::default()
+ }),
return_ranking_signals: true,
..Default::default()
})