web search done right
giant ranking pipeline refactor to separate ranking stages from sorting/offset logic
Mikkel Denkerc6119e31d734
diff --git a/crates/core/src/api/search.rs b/crates/core/src/api/search.rsindex ab7c2af0..0ae7b9a2 100644--- a/crates/core/src/api/search.rs+++ b/crates/core/src/api/search.rs@@ -17,7 +17,7 @@use crate::{config::defaults,enum_map::EnumMap,- ranking::{SignalCoefficient, SignalEnum, SignalEnumDiscriminants},+ ranking::{SignalCoefficients, SignalEnum, SignalEnumDiscriminants},};use http::StatusCode;use optics::{HostRankings, Optic};@@ -95,7 +95,7 @@ impl TryFrom<ApiSearchQuery> for SearchQuery {None};- let signal_coefficients: Option<SignalCoefficient> =+ let signal_coefficients: Option<SignalCoefficients> =api.signal_coefficients.map(|coefficients| {coefficients.into_iter()diff --git a/crates/core/src/collector/approx_count.rs b/crates/core/src/collector/approx_count.rsindex 85d95b51..36e3ce6c 100644--- a/crates/core/src/collector/approx_count.rs+++ b/crates/core/src/collector/approx_count.rs@@ -66,6 +66,12 @@ impl Count {| (Count::Approximate(a), Count::Approximate(b)) => Count::Approximate(a + b),}}++ pub fn as_u64(&self) -> u64 {+ match self {+ Count::Exact(count) | Count::Approximate(count) => *count,+ }+ }}impl Add for Count {diff --git a/crates/core/src/collector/top_docs.rs b/crates/core/src/collector/top_docs.rsindex 886ab995..c1cff34a 100644--- a/crates/core/src/collector/top_docs.rs+++ b/crates/core/src/collector/top_docs.rs@@ -426,10 +426,12 @@ where}}- Ok(collector- .into_sorted_vec(self.top_docs.de_rank_similar)+ let docs = collector.into_sorted_vec(self.top_docs.de_rank_similar);++ Ok(docs.into_iter().skip(self.top_docs.offset)+ .take(self.top_docs.top_n).map(|doc| WebpagePointer {score: doc.score,hashes: doc.hashes,diff --git a/crates/core/src/inverted_index/mod.rs b/crates/core/src/inverted_index/mod.rsindex 1fef548c..3eec73f6 100644--- a/crates/core/src/inverted_index/mod.rs+++ b/crates/core/src/inverted_index/mod.rs@@ -237,7 +237,7 @@ mod tests {collector::MainCollector,config::CollectorConfig,query::Query,- ranking::{Ranker, SignalComputer},+ ranking::{LocalRanker, SignalComputer},search_ctx::Ctx,searcher::SearchQuery,webgraph::{Edge, NodeDatum},@@ -282,7 +282,7 @@ mod tests {).expect("Failed to parse query");- let ranker = Ranker::new(+ let ranker = LocalRanker::new(SignalComputer::new(Some(&query)),ctx.columnfield_reader.clone(),CollectorConfig::default(),@@ -314,7 +314,7 @@ mod tests {index.commit().expect("failed to commit index");let ctx = index.local_search_ctx();- let ranker = Ranker::new(+ let ranker = LocalRanker::new(SignalComputer::new(Some(&query)),ctx.columnfield_reader.clone(),CollectorConfig::default(),@@ -363,7 +363,7 @@ mod tests {).expect("Failed to parse query");- let ranker = Ranker::new(+ let ranker = LocalRanker::new(SignalComputer::new(Some(&query)),ctx.columnfield_reader.clone(),CollectorConfig::default(),@@ -410,7 +410,7 @@ mod tests {&index,).expect("Failed to parse query");- let ranker = Ranker::new(+ let ranker = LocalRanker::new(SignalComputer::new(Some(&query)),ctx.columnfield_reader.clone(),CollectorConfig::default(),@@ -458,7 +458,7 @@ mod tests {&index,).expect("Failed to parse query");- let ranker = Ranker::new(+ let ranker = LocalRanker::new(SignalComputer::new(Some(&query)),ctx.columnfield_reader.clone(),CollectorConfig::default(),@@ -536,7 +536,7 @@ mod tests {&index,).expect("Failed to parse query");- let ranker = Ranker::new(+ let ranker = LocalRanker::new(SignalComputer::new(Some(&query)),ctx.columnfield_reader.clone(),CollectorConfig::default(),@@ -594,7 +594,7 @@ mod tests {&index,).expect("Failed to parse query");- let ranker = Ranker::new(+ let ranker = LocalRanker::new(SignalComputer::new(Some(&query)),ctx.columnfield_reader.clone(),CollectorConfig::default(),@@ -641,7 +641,7 @@ mod tests {&index,).expect("Failed to parse query");- let ranker = Ranker::new(+ let ranker = LocalRanker::new(SignalComputer::new(Some(&query)),ctx.columnfield_reader.clone(),CollectorConfig::default(),@@ -717,7 +717,7 @@ mod tests {&index,).expect("Failed to parse query");- let ranker = Ranker::new(+ let ranker = LocalRanker::new(SignalComputer::new(Some(&query)),ctx.columnfield_reader.clone(),CollectorConfig::default(),@@ -744,7 +744,7 @@ mod tests {&index,).expect("Failed to parse query");- let ranker = Ranker::new(+ let ranker = LocalRanker::new(SignalComputer::new(Some(&query)),ctx.columnfield_reader.clone(),CollectorConfig::default(),@@ -777,7 +777,7 @@ mod tests {index.commit().expect("failed to commit index");let ctx = index.local_search_ctx();- let ranker = Ranker::new(+ let ranker = LocalRanker::new(SignalComputer::new(Some(&query)),ctx.columnfield_reader.clone(),CollectorConfig::default(),@@ -824,7 +824,7 @@ mod tests {&index,).expect("Failed to parse query");- let ranker = Ranker::new(+ let ranker = LocalRanker::new(SignalComputer::new(Some(&query)),ctx.columnfield_reader.clone(),CollectorConfig::default(),@@ -875,7 +875,7 @@ mod tests {&index,).expect("Failed to parse query");- let ranker = Ranker::new(+ let ranker = LocalRanker::new(SignalComputer::new(Some(&query)),ctx.columnfield_reader.clone(),CollectorConfig::default(),@@ -1048,7 +1048,7 @@ mod tests {).expect("Failed to parse query");- let ranker = Ranker::new(+ let ranker = LocalRanker::new(SignalComputer::new(Some(&query)),ctx.columnfield_reader.clone(),CollectorConfig::default(),@@ -1119,7 +1119,7 @@ mod tests {..Default::default()};- let ranker = Ranker::new(+ let ranker = LocalRanker::new(SignalComputer::new(Some(&query)),ctx.columnfield_reader.clone(),collector_config,@@ -1170,7 +1170,7 @@ mod tests {).expect("Failed to parse query");- let ranker = Ranker::new(+ let ranker = LocalRanker::new(SignalComputer::new(Some(&query)),ctx.columnfield_reader.clone(),CollectorConfig::default(),@@ -1228,7 +1228,7 @@ mod tests {).expect("Failed to parse query");- let ranker = Ranker::new(+ let ranker = LocalRanker::new(SignalComputer::new(Some(&query)),ctx.columnfield_reader.clone(),CollectorConfig::default(),diff --git a/crates/core/src/query/mod.rs b/crates/core/src/query/mod.rsindex 3a857146..ea81f1e9 100644--- a/crates/core/src/query/mod.rs+++ b/crates/core/src/query/mod.rs@@ -17,7 +17,7 @@use crate::{inverted_index::InvertedIndex,query::parser::TermCompound,- ranking::SignalCoefficient,+ ranking::SignalCoefficients,schema::text_field,search_ctx::Ctx,searcher::SearchQuery,@@ -53,7 +53,7 @@ pub struct Query {optics: Vec<Optic>,top_n: usize,count_results_exact: bool,- signal_coefficients: SignalCoefficient,+ signal_coefficients: SignalCoefficients,lang: Option<whatlang::Lang>,}@@ -182,7 +182,7 @@ impl Query {&self.host_rankings}- pub fn signal_coefficients(&self) -> SignalCoefficient {+ pub fn signal_coefficients(&self) -> SignalCoefficients {self.signal_coefficients.clone()}diff --git a/crates/core/src/ranking/core/computer/mod.rs b/crates/core/src/ranking/core/computer/mod.rsindex 7ac030db..ebaed70c 100644--- a/crates/core/src/ranking/core/computer/mod.rs+++ b/crates/core/src/ranking/core/computer/mod.rs@@ -40,7 +40,7 @@ use crate::webpage::region::RegionCount;use crate::ranking::bm25::MultiBm25Weight;use crate::ranking::models::linear::LinearRegression;-use super::{ComputedSignal, Signal, SignalCoefficient, SignalEnum};+use super::{ComputedSignal, Signal, SignalCoefficients, SignalEnum};mod order;pub use order::SignalComputeOrder;@@ -192,7 +192,7 @@ impl QueryData {pub struct SignalComputer {query_data: Option<QueryData>,- query_signal_coefficients: Option<SignalCoefficient>,+ query_signal_coefficients: Option<SignalCoefficients>,segment_reader: Option<RefCell<SegmentReader>>,fetch_time_ms_cache: Vec<f64>,update_time_cache: Vec<f64>,@@ -474,9 +474,9 @@ impl SignalComputer {.filter_map(|signal| {signal.precompute(webpage, self)- .map(|score| ComputedSignal { signal, score })+ .map(|calc| ComputedSignal { signal, calc })})- .map(|computed| self.coefficient(&computed.signal) * computed.score)+ .map(|computed| self.coefficient(&computed.signal) * computed.calc.score).sum()}diff --git a/crates/core/src/ranking/core/computer/order.rs b/crates/core/src/ranking/core/computer/order.rsindex 6b167257..53c93578 100644--- a/crates/core/src/ranking/core/computer/order.rs+++ b/crates/core/src/ranking/core/computer/order.rs@@ -18,7 +18,7 @@ use tantivy::DocId;use crate::{enum_map::EnumMap,- ranking::{ComputedSignal, Signal, SignalEnum},+ ranking::{ComputedSignal, Signal, SignalCalculation, SignalEnum},schema::{text_field::TextField, TextFieldEnum},};@@ -71,13 +71,16 @@ impl SignalComputeOrder {.chain(self.other_signals.iter().map(move |signal| {signal.compute(doc, signal_computer)- .map(|score| ComputedSignal {+ .map(|calc| ComputedSignal {signal: *signal,- score,+ calc,}).unwrap_or_else(|| ComputedSignal {signal: *signal,- score: 0.0,+ calc: SignalCalculation {+ score: 0.0,+ value: 0.0,+ },})}))}@@ -112,14 +115,14 @@ impl NGramComputeOrder {self.signals.iter().map(|(_, s)| s).map(move |signal| {signal.compute(doc, signal_computer)- .map(|score| ComputedSignal {+ .map(|calc| ComputedSignal {signal: *signal,- score,+ calc,}).map(|mut c| {- c.score *= NGRAM_DAMPENING.powi(hits);+ c.calc.score *= NGRAM_DAMPENING.powi(hits);- if c.score > 0.0 {+ if c.calc.score > 0.0 {hits += 1;}@@ -127,7 +130,10 @@ impl NGramComputeOrder {}).unwrap_or_else(|| ComputedSignal {signal: *signal,- score: 0.0,+ calc: SignalCalculation {+ value: 0.0,+ score: 0.0,+ },})})}diff --git a/crates/core/src/ranking/core/mod.rs b/crates/core/src/ranking/core/mod.rsindex 3ca4eb8c..9d818502 100644--- a/crates/core/src/ranking/core/mod.rs+++ b/crates/core/src/ranking/core/mod.rs@@ -49,23 +49,23 @@ impl FromStr for SignalEnumDiscriminants {}#[derive(Debug, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, Clone)]-pub struct SignalCoefficient {+pub struct SignalCoefficients {map: EnumMap<SignalEnum, f64>,}-impl Default for SignalCoefficient {+impl Default for SignalCoefficients {fn default() -> Self {Self::new(SignalEnum::all().map(|signal| (signal, signal.default_coefficient())))}}-impl From<EnumMap<SignalEnum, f64>> for SignalCoefficient {+impl From<EnumMap<SignalEnum, f64>> for SignalCoefficients {fn from(map: EnumMap<SignalEnum, f64>) -> Self {Self { map }}}-impl SignalCoefficient {+impl SignalCoefficients {pub fn get(&self, signal: &SignalEnum) -> f64 {self.map.get(*signal)@@ -83,7 +83,7 @@ impl SignalCoefficient {Self { map }}- pub fn merge_add(&mut self, coeffs: SignalCoefficient) {+ pub fn merge_add(&mut self, coeffs: SignalCoefficients) {for signal in SignalEnum::all() {if let Some(coeff) = coeffs.map.get(signal).copied() {match self.map.get_mut(signal) {@@ -96,7 +96,7 @@ impl SignalCoefficient {}}- pub fn merge_overwrite(&mut self, coeffs: SignalCoefficient) {+ pub fn merge_overwrite(&mut self, coeffs: SignalCoefficients) {for signal in SignalEnum::all() {if let Some(coeff) = coeffs.map.get(signal).copied() {match self.map.get_mut(signal) {@@ -110,10 +110,25 @@ impl SignalCoefficient {}}+#[derive(Debug, Clone, Copy, bincode::Encode, bincode::Decode)]+pub struct SignalCalculation {+ pub value: f64,+ pub score: f64,+}++impl SignalCalculation {+ pub fn new_symmetrical(val: f64) -> Self {+ Self {+ value: val,+ score: val,+ }+ }+}+#[derive(Debug, Clone, Copy)]pub struct ComputedSignal {pub signal: SignalEnum,- pub score: f64,+ pub calc: SignalCalculation,}#[derive(diff --git a/crates/core/src/ranking/core/non_text.rs b/crates/core/src/ranking/core/non_text.rsindex ec52a6db..70cd55e6 100644--- a/crates/core/src/ranking/core/non_text.rs+++ b/crates/core/src/ranking/core/non_text.rs@@ -16,7 +16,7 @@use tantivy::DocId;-use super::{Signal, SignalComputer};+use super::{Signal, SignalCalculation, SignalComputer};use crate::{schema::{self, Field},webpage::Webpage,@@ -117,11 +117,11 @@ impl Signal for HostCentrality {))}- fn precompute(self, webpage: &Webpage, _: &SignalComputer) -> Option<f64> {- Some(webpage.host_centrality)+ fn precompute(self, webpage: &Webpage, _: &SignalComputer) -> Option<SignalCalculation> {+ Some(SignalCalculation::new_symmetrical(webpage.host_centrality))}- fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {+ fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<SignalCalculation> {let seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();let numericalfield_reader = seg_reader.numericalfield_reader().get_field_reader(doc);@@ -129,7 +129,7 @@ impl Signal for HostCentrality {.get(self.as_numericalfield().unwrap()).and_then(|v| v.as_f64()).unwrap();- Some(val)+ Some(SignalCalculation::new_symmetrical(val))}}@@ -157,11 +157,14 @@ impl Signal for HostCentralityRank {))}- fn precompute(self, webpage: &Webpage, _: &SignalComputer) -> Option<f64> {- Some(score_rank(webpage.host_centrality_rank as f64))+ fn precompute(self, webpage: &Webpage, _: &SignalComputer) -> Option<SignalCalculation> {+ Some(SignalCalculation {+ value: webpage.host_centrality_rank as f64,+ score: score_rank(webpage.host_centrality_rank as f64),+ })}- fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {+ fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<SignalCalculation> {let seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();let numericalfield_reader = seg_reader.numericalfield_reader().get_field_reader(doc);@@ -169,7 +172,10 @@ impl Signal for HostCentralityRank {.get(self.as_numericalfield().unwrap()).and_then(|v| v.as_u64()).unwrap();- Some(score_rank(val as f64))+ Some(SignalCalculation {+ value: val as f64,+ score: score_rank(val as f64),+ })}}@@ -197,11 +203,11 @@ impl Signal for PageCentrality {))}- fn precompute(self, webpage: &Webpage, _: &SignalComputer) -> Option<f64> {- Some(webpage.page_centrality)+ fn precompute(self, webpage: &Webpage, _: &SignalComputer) -> Option<SignalCalculation> {+ Some(SignalCalculation::new_symmetrical(webpage.page_centrality))}- fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {+ fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<SignalCalculation> {let seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();let numericalfield_reader = seg_reader.numericalfield_reader().get_field_reader(doc);@@ -209,7 +215,7 @@ impl Signal for PageCentrality {.get(self.as_numericalfield().unwrap()).and_then(|v| v.as_f64()).unwrap();- Some(val)+ Some(SignalCalculation::new_symmetrical(val))}}@@ -237,11 +243,14 @@ impl Signal for PageCentralityRank {))}- fn precompute(self, webpage: &Webpage, _: &SignalComputer) -> Option<f64> {- Some(score_rank(webpage.page_centrality_rank as f64))+ fn precompute(self, webpage: &Webpage, _: &SignalComputer) -> Option<SignalCalculation> {+ Some(SignalCalculation {+ value: webpage.page_centrality_rank as f64,+ score: score_rank(webpage.page_centrality_rank as f64),+ })}- fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {+ fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<SignalCalculation> {let seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();let numericalfield_reader = seg_reader.numericalfield_reader().get_field_reader(doc);@@ -249,7 +258,10 @@ impl Signal for PageCentralityRank {.get(self.as_numericalfield().unwrap()).and_then(|v| v.as_u64()).unwrap();- Some(score_rank(val as f64))+ Some(SignalCalculation {+ value: val as f64,+ score: score_rank(val as f64),+ })}}@@ -275,11 +287,13 @@ impl Signal for IsHomepage {Some(Field::Numerical(schema::numerical_field::IsHomepage.into()))}- fn precompute(self, webpage: &Webpage, _: &SignalComputer) -> Option<f64> {- Some(webpage.html.is_homepage().into())+ fn precompute(self, webpage: &Webpage, _: &SignalComputer) -> Option<SignalCalculation> {+ Some(SignalCalculation::new_symmetrical(+ webpage.html.is_homepage().into(),+ ))}- fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {+ fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<SignalCalculation> {let seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();let numericalfield_reader = seg_reader.numericalfield_reader().get_field_reader(doc);@@ -289,9 +303,9 @@ impl Signal for IsHomepage {.unwrap();if val {- Some(1.0)+ Some(SignalCalculation::new_symmetrical(1.0))} else {- Some(0.0)+ Some(SignalCalculation::new_symmetrical(0.0))}}}@@ -320,16 +334,25 @@ impl Signal for FetchTimeMs {))}- fn precompute(self, webpage: &Webpage, signal_computer: &SignalComputer) -> Option<f64> {+ fn precompute(+ self,+ webpage: &Webpage,+ signal_computer: &SignalComputer,+ ) -> Option<SignalCalculation> {let fetch_time_ms = webpage.fetch_time_ms as usize;- if fetch_time_ms >= signal_computer.fetch_time_ms_cache().len() {- Some(0.0)+ let score = if fetch_time_ms >= signal_computer.fetch_time_ms_cache().len() {+ 0.0} else {- Some(signal_computer.fetch_time_ms_cache()[fetch_time_ms])- }+ signal_computer.fetch_time_ms_cache()[fetch_time_ms]+ };++ Some(SignalCalculation {+ value: fetch_time_ms as f64,+ score,+ })}- fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {+ fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<SignalCalculation> {let seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();let numericalfield_reader = seg_reader.numericalfield_reader().get_field_reader(doc);@@ -338,11 +361,16 @@ impl Signal for FetchTimeMs {.and_then(|v| v.as_u64()).unwrap() as usize;- if fetch_time_ms >= signal_computer.fetch_time_ms_cache().len() {- Some(0.0)+ let score = if fetch_time_ms >= signal_computer.fetch_time_ms_cache().len() {+ 0.0} else {- Some(signal_computer.fetch_time_ms_cache()[fetch_time_ms])- }+ signal_computer.fetch_time_ms_cache()[fetch_time_ms]+ };++ Some(SignalCalculation {+ value: fetch_time_ms as f64,+ score,+ })}}@@ -370,17 +398,26 @@ impl Signal for UpdateTimestamp {))}- fn precompute(self, webpage: &Webpage, signal_computer: &SignalComputer) -> Option<f64> {+ fn precompute(+ self,+ webpage: &Webpage,+ signal_computer: &SignalComputer,+ ) -> Option<SignalCalculation> {let update_timestamp = webpage.html.updated_time().map(|date| date.timestamp().max(0)).unwrap_or(0) as usize;- Some(score_timestamp(update_timestamp, signal_computer))+ let score = score_timestamp(update_timestamp, signal_computer);++ Some(SignalCalculation {+ value: update_timestamp as f64,+ score,+ })}- fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {+ fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<SignalCalculation> {let seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();let numericalfield_reader = seg_reader.numericalfield_reader().get_field_reader(doc);@@ -389,7 +426,12 @@ impl Signal for UpdateTimestamp {.and_then(|v| v.as_u64()).unwrap() as usize;- Some(score_timestamp(val, signal_computer))+ let score = score_timestamp(val, signal_computer);++ Some(SignalCalculation {+ value: val as f64,+ score,+ })}}@@ -417,12 +459,17 @@ impl Signal for TrackerScore {))}- fn precompute(self, webpage: &Webpage, _: &SignalComputer) -> Option<f64> {+ fn precompute(self, webpage: &Webpage, _: &SignalComputer) -> Option<SignalCalculation> {let num_trackers = webpage.html.trackers().len() as f64;- Some(score_trackers(num_trackers))+ let score = score_trackers(num_trackers);++ Some(SignalCalculation {+ value: num_trackers,+ score,+ })}- fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {+ fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<SignalCalculation> {let seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();let numericalfield_reader = seg_reader.numericalfield_reader().get_field_reader(doc);@@ -430,7 +477,12 @@ impl Signal for TrackerScore {.get(self.as_numericalfield().unwrap()).and_then(|v| v.as_u64()).unwrap();- Some(score_trackers(val as f64))+ let score = score_trackers(val as f64);++ Some(SignalCalculation {+ value: val as f64,+ score,+ })}}@@ -456,13 +508,22 @@ impl Signal for Region {Some(Field::Numerical(schema::numerical_field::Region.into()))}- fn precompute(self, webpage: &Webpage, signal_computer: &SignalComputer) -> Option<f64> {+ fn precompute(+ self,+ webpage: &Webpage,+ signal_computer: &SignalComputer,+ ) -> Option<SignalCalculation> {let region =crate::webpage::Region::guess_from(webpage).unwrap_or(crate::webpage::Region::All);- Some(score_region(region, signal_computer))+ let score = score_region(region, signal_computer);++ Some(SignalCalculation {+ value: region.id() as f64,+ score,+ })}- fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {+ fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<SignalCalculation> {let seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();let numericalfield_reader = seg_reader.numericalfield_reader().get_field_reader(doc);@@ -471,7 +532,12 @@ impl Signal for Region {.and_then(|v| v.as_u64()).unwrap();let region = crate::webpage::Region::from_id(val);- Some(score_region(region, signal_computer))+ let score = score_region(region, signal_computer);++ Some(SignalCalculation {+ value: val as f64,+ score,+ })}}@@ -497,7 +563,7 @@ impl Signal for QueryCentrality {None}- fn compute(&self, _doc: DocId, _signal_computer: &SignalComputer) -> Option<f64> {+ fn compute(&self, _doc: DocId, _signal_computer: &SignalComputer) -> Option<SignalCalculation> {// unimplemented!()None}@@ -525,7 +591,7 @@ impl Signal for InboundSimilarity {None}- fn compute(&self, _doc: DocId, _signal_computer: &SignalComputer) -> Option<f64> {+ fn compute(&self, _doc: DocId, _signal_computer: &SignalComputer) -> Option<SignalCalculation> {None // computed in later ranking stage}}@@ -552,7 +618,7 @@ impl Signal for LambdaMart {None}- fn compute(&self, _: DocId, _: &SignalComputer) -> Option<f64> {+ fn compute(&self, _: DocId, _: &SignalComputer) -> Option<SignalCalculation> {None // computed in later ranking stage}}@@ -581,7 +647,7 @@ impl Signal for UrlDigits {))}- fn precompute(self, webpage: &Webpage, _: &SignalComputer) -> Option<f64> {+ fn precompute(self, webpage: &Webpage, _: &SignalComputer) -> Option<SignalCalculation> {let num_digits = (webpage.html.url()@@ -598,18 +664,25 @@ impl Signal for UrlDigits {.filter(|c| c.is_ascii_digit()).count()) as f64;- Some(score_digits(num_digits))+ let score = score_digits(num_digits);++ Some(SignalCalculation {+ value: num_digits,+ score,+ })}- fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {+ fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<SignalCalculation> {let seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();let numericalfield_reader = seg_reader.numericalfield_reader().get_field_reader(doc);let val = numericalfield_reader.get(self.as_numericalfield().unwrap()).and_then(|v| v.as_u64())- .unwrap();- Some(score_digits(val as f64))+ .unwrap() as f64;+ let score = score_digits(val);++ Some(SignalCalculation { value: val, score })}}@@ -637,7 +710,7 @@ impl Signal for UrlSlashes {))}- fn precompute(self, webpage: &Webpage, _: &SignalComputer) -> Option<f64> {+ fn precompute(self, webpage: &Webpage, _: &SignalComputer) -> Option<SignalCalculation> {let num_slashes = webpage.html.url()@@ -645,18 +718,25 @@ impl Signal for UrlSlashes {.chars().filter(|c| c == &'/').count() as f64;- Some(score_slashes(num_slashes))+ let score = score_slashes(num_slashes);++ Some(SignalCalculation {+ value: num_slashes,+ score,+ })}- fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {+ fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<SignalCalculation> {let seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();let numericalfield_reader = seg_reader.numericalfield_reader().get_field_reader(doc);let val = numericalfield_reader.get(self.as_numericalfield().unwrap()).and_then(|v| v.as_u64())- .unwrap();- Some(score_slashes(val as f64))+ .unwrap() as f64;+ let score = score_slashes(val);++ Some(SignalCalculation { value: val, score })}}@@ -684,12 +764,17 @@ impl Signal for LinkDensity {))}- fn precompute(self, webpage: &Webpage, _: &SignalComputer) -> Option<f64> {+ fn precompute(self, webpage: &Webpage, _: &SignalComputer) -> Option<SignalCalculation> {let link_density = webpage.html.link_density();- Some(score_link_density(link_density))+ let score = score_link_density(link_density);++ Some(SignalCalculation {+ value: link_density,+ score,+ })}- fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {+ fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<SignalCalculation> {let seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();let numericalfield_reader = seg_reader.numericalfield_reader().get_field_reader(doc);@@ -697,7 +782,9 @@ impl Signal for LinkDensity {.get(self.as_numericalfield().unwrap()).and_then(|v| v.as_f64()).unwrap();- Some(score_link_density(val))+ let score = score_link_density(val);++ Some(SignalCalculation { value: val, score })}}@@ -725,15 +812,15 @@ impl Signal for HasAds {))}- fn precompute(self, webpage: &Webpage, _: &SignalComputer) -> Option<f64> {- if !webpage.html.likely_has_ads() {- Some(1.0)- } else {- Some(0.0)- }+ fn precompute(self, webpage: &Webpage, _: &SignalComputer) -> Option<SignalCalculation> {+ let has_ads = webpage.html.likely_has_ads();+ let value: f64 = has_ads.into();+ let score = if !has_ads { 1.0 } else { 0.0 };++ Some(SignalCalculation { value, score })}- fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {+ fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<SignalCalculation> {let seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();let numericalfield_reader = seg_reader.numericalfield_reader().get_field_reader(doc);@@ -742,10 +829,9 @@ impl Signal for HasAds {.and_then(|v| v.as_bool()).unwrap();- if !has_ads {- Some(1.0)- } else {- Some(0.0)- }+ let value: f64 = has_ads.into();+ let score = if !has_ads { 1.0 } else { 0.0 };++ Some(SignalCalculation { value, score })}}diff --git a/crates/core/src/ranking/core/prelude.rs b/crates/core/src/ranking/core/prelude.rsindex 164048a4..a5164e64 100644--- a/crates/core/src/ranking/core/prelude.rs+++ b/crates/core/src/ranking/core/prelude.rs@@ -26,10 +26,10 @@ use enum_dispatch::enum_dispatch;use strum::{EnumDiscriminants, VariantArray};-use super::non_text::*;use super::query::*;use super::text::*;use super::SignalComputer;+use super::{non_text::*, SignalCalculation};use tantivy::DocId;#[enum_dispatch]@@ -38,9 +38,13 @@ pub trait Signal:{fn default_coefficient(&self) -> f64;fn as_field(&self) -> Option<Field>;- fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64>;+ fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<SignalCalculation>;- fn precompute(self, _webpage: &Webpage, _signal_computer: &SignalComputer) -> Option<f64> {+ fn precompute(+ self,+ _webpage: &Webpage,+ _signal_computer: &SignalComputer,+ ) -> Option<SignalCalculation> {None}diff --git a/crates/core/src/ranking/core/query.rs b/crates/core/src/ranking/core/query.rsindex 3af796bb..ca1b5f23 100644--- a/crates/core/src/ranking/core/query.rs+++ b/crates/core/src/ranking/core/query.rs@@ -18,7 +18,7 @@ use crate::schema::Field;// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.-use super::{Signal, SignalComputer};+use super::{Signal, SignalCalculation, SignalComputer};#[derive(Debug,@@ -42,9 +42,10 @@ impl Signal for NumQueryTerms {None}- fn compute(&self, _: DocId, signal_computer: &SignalComputer) -> Option<f64> {+ fn compute(&self, _: DocId, signal_computer: &SignalComputer) -> Option<SignalCalculation> {signal_computer.query_data().map(|d| d.simple_terms().len() as f64)+ .map(SignalCalculation::new_symmetrical)}}diff --git a/crates/core/src/ranking/core/text.rs b/crates/core/src/ranking/core/text.rsindex 216b9aa7..da405f5d 100644--- a/crates/core/src/ranking/core/text.rs+++ b/crates/core/src/ranking/core/text.rs@@ -18,7 +18,7 @@ use crate::schema::{self, Field};use tantivy::DocId;-use super::{Signal, SignalComputer};+use super::{Signal, SignalCalculation, SignalComputer};#[derive(Debug,@@ -42,16 +42,16 @@ impl Signal for Bm25F {None}- fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {+ fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<SignalCalculation> {let mut seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();- Some(+ Some(SignalCalculation::new_symmetrical(seg_reader.text_fields_mut().values_mut().map(|field| field.bm25f(doc)).sum(),- )+ ))}}@@ -81,13 +81,14 @@ impl Signal for Bm25Title {true}- fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {+ fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<SignalCalculation> {let mut seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();seg_reader.text_fields_mut().get_mut(self.as_textfield().unwrap()).map(|field| field.bm25(doc))+ .map(SignalCalculation::new_symmetrical)}}@@ -113,13 +114,13 @@ impl Signal for TitleCoverage {Some(Field::Text(schema::text_field::Title.into()))}- fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {+ fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<SignalCalculation> {let mut seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();seg_reader.text_fields_mut().get_mut(self.as_textfield().unwrap())- .map(|field| field.coverage(doc))+ .map(|field| SignalCalculation::new_symmetrical(field.coverage(doc)))}}@@ -149,13 +150,14 @@ impl Signal for Bm25TitleBigrams {Some(Field::Text(schema::text_field::TitleBigrams.into()))}- fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {+ fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<SignalCalculation> {let mut seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();seg_reader.text_fields_mut().get_mut(self.as_textfield().unwrap()).map(|field| field.bm25(doc))+ .map(SignalCalculation::new_symmetrical)}}@@ -185,13 +187,14 @@ impl Signal for Bm25TitleTrigrams {Some(Field::Text(schema::text_field::TitleTrigrams.into()))}- fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {+ fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<SignalCalculation> {let mut seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();seg_reader.text_fields_mut().get_mut(self.as_textfield().unwrap()).map(|field| field.bm25(doc))+ .map(SignalCalculation::new_symmetrical)}}@@ -221,13 +224,14 @@ impl Signal for Bm25CleanBody {Some(Field::Text(schema::text_field::CleanBody.into()))}- fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {+ fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<SignalCalculation> {let mut seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();seg_reader.text_fields_mut().get_mut(self.as_textfield().unwrap()).map(|field| field.bm25(doc))+ .map(SignalCalculation::new_symmetrical)}}@@ -253,13 +257,14 @@ impl Signal for CleanBodyCoverage {Some(Field::Text(schema::text_field::CleanBody.into()))}- fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {+ fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<SignalCalculation> {let mut seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();seg_reader.text_fields_mut().get_mut(self.as_textfield().unwrap()).map(|field| field.coverage(doc))+ .map(SignalCalculation::new_symmetrical)}}@@ -289,13 +294,14 @@ impl Signal for Bm25CleanBodyBigrams {Some(Field::Text(schema::text_field::CleanBodyBigrams.into()))}- fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {+ fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<SignalCalculation> {let mut seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();seg_reader.text_fields_mut().get_mut(self.as_textfield().unwrap()).map(|field| field.bm25(doc))+ .map(SignalCalculation::new_symmetrical)}}@@ -325,13 +331,14 @@ impl Signal for Bm25CleanBodyTrigrams {Some(Field::Text(schema::text_field::CleanBodyTrigrams.into()))}- fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {+ fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<SignalCalculation> {let mut seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();seg_reader.text_fields_mut().get_mut(self.as_textfield().unwrap()).map(|field| field.bm25(doc))+ .map(SignalCalculation::new_symmetrical)}}@@ -357,13 +364,14 @@ impl Signal for Bm25StemmedTitle {Some(Field::Text(schema::text_field::StemmedTitle.into()))}- fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {+ fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<SignalCalculation> {let mut seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();seg_reader.text_fields_mut().get_mut(self.as_textfield().unwrap()).map(|field| field.bm25(doc))+ .map(SignalCalculation::new_symmetrical)}}@@ -389,13 +397,14 @@ impl Signal for Bm25StemmedCleanBody {Some(Field::Text(schema::text_field::StemmedCleanBody.into()))}- fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {+ fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<SignalCalculation> {let mut seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();seg_reader.text_fields_mut().get_mut(self.as_textfield().unwrap()).map(|field| field.bm25(doc))+ .map(SignalCalculation::new_symmetrical)}}@@ -421,13 +430,14 @@ impl Signal for Bm25AllBody {Some(Field::Text(schema::text_field::AllBody.into()))}- fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {+ fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<SignalCalculation> {let mut seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();seg_reader.text_fields_mut().get_mut(self.as_textfield().unwrap()).map(|field| field.bm25(doc))+ .map(SignalCalculation::new_symmetrical)}}@@ -453,13 +463,14 @@ impl Signal for Bm25Keywords {Some(Field::Text(schema::text_field::Keywords.into()))}- fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {+ fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<SignalCalculation> {let mut seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();seg_reader.text_fields_mut().get_mut(self.as_textfield().unwrap()).map(|field| field.bm25(doc))+ .map(SignalCalculation::new_symmetrical)}}@@ -485,13 +496,14 @@ impl Signal for Bm25BacklinkText {Some(Field::Text(schema::text_field::BacklinkText.into()))}- fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {+ fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<SignalCalculation> {let mut seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();seg_reader.text_fields_mut().get_mut(self.as_textfield().unwrap()).map(|field| field.bm25(doc))+ .map(SignalCalculation::new_symmetrical)}}@@ -517,13 +529,14 @@ impl Signal for IdfSumUrl {Some(Field::Text(schema::text_field::Url.into()))}- fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {+ fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<SignalCalculation> {let mut seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();seg_reader.text_fields_mut().get_mut(self.as_textfield().unwrap()).map(|field| field.idf_sum(doc))+ .map(SignalCalculation::new_symmetrical)}}@@ -549,13 +562,14 @@ impl Signal for IdfSumSite {Some(Field::Text(schema::text_field::SiteWithout.into()))}- fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {+ fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<SignalCalculation> {let mut seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();seg_reader.text_fields_mut().get_mut(self.as_textfield().unwrap()).map(|field| field.idf_sum(doc))+ .map(SignalCalculation::new_symmetrical)}}@@ -581,13 +595,14 @@ impl Signal for IdfSumDomain {Some(Field::Text(schema::text_field::Domain.into()))}- fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {+ fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<SignalCalculation> {let mut seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();seg_reader.text_fields_mut().get_mut(self.as_textfield().unwrap()).map(|field| field.idf_sum(doc))+ .map(SignalCalculation::new_symmetrical)}}@@ -613,13 +628,14 @@ impl Signal for IdfSumSiteNoTokenizer {Some(Field::Text(schema::text_field::SiteNoTokenizer.into()))}- fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {+ fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<SignalCalculation> {let mut seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();seg_reader.text_fields_mut().get_mut(self.as_textfield().unwrap()).map(|field| field.idf_sum(doc))+ .map(SignalCalculation::new_symmetrical)}}@@ -645,13 +661,14 @@ impl Signal for IdfSumDomainNoTokenizer {Some(Field::Text(schema::text_field::DomainNoTokenizer.into()))}- fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {+ fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<SignalCalculation> {let mut seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();seg_reader.text_fields_mut().get_mut(self.as_textfield().unwrap()).map(|field| field.idf_sum(doc))+ .map(SignalCalculation::new_symmetrical)}}@@ -679,13 +696,14 @@ impl Signal for IdfSumDomainNameNoTokenizer {))}- fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {+ fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<SignalCalculation> {let mut seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();seg_reader.text_fields_mut().get_mut(self.as_textfield().unwrap()).map(|field| field.idf_sum(doc))+ .map(SignalCalculation::new_symmetrical)}}@@ -711,13 +729,14 @@ impl Signal for IdfSumDomainIfHomepage {Some(Field::Text(schema::text_field::DomainIfHomepage.into()))}- fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {+ fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<SignalCalculation> {let mut seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();seg_reader.text_fields_mut().get_mut(self.as_textfield().unwrap()).map(|field| field.idf_sum(doc))+ .map(SignalCalculation::new_symmetrical)}}@@ -745,13 +764,14 @@ impl Signal for IdfSumDomainNameIfHomepageNoTokenizer {))}- fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {+ fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<SignalCalculation> {let mut seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();seg_reader.text_fields_mut().get_mut(self.as_textfield().unwrap()).map(|field| field.idf_sum(doc))+ .map(SignalCalculation::new_symmetrical)}}@@ -779,13 +799,14 @@ impl Signal for IdfSumDomainIfHomepageNoTokenizer {))}- fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {+ fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<SignalCalculation> {let mut seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();seg_reader.text_fields_mut().get_mut(self.as_textfield().unwrap()).map(|field| field.idf_sum(doc))+ .map(SignalCalculation::new_symmetrical)}}@@ -811,13 +832,14 @@ impl Signal for IdfSumTitleIfHomepage {Some(Field::Text(schema::text_field::TitleIfHomepage.into()))}- fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {+ fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<SignalCalculation> {let mut seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();seg_reader.text_fields_mut().get_mut(self.as_textfield().unwrap()).map(|field| field.idf_sum(doc))+ .map(SignalCalculation::new_symmetrical)}}@@ -843,7 +865,7 @@ impl Signal for CrossEncoderSnippet {None}- fn compute(&self, _: DocId, _: &SignalComputer) -> Option<f64> {+ fn compute(&self, _: DocId, _: &SignalComputer) -> Option<SignalCalculation> {None // computed in later ranking stage}}@@ -870,7 +892,7 @@ impl Signal for CrossEncoderTitle {None}- fn compute(&self, _: DocId, _: &SignalComputer) -> Option<f64> {+ fn compute(&self, _: DocId, _: &SignalComputer) -> Option<SignalCalculation> {None // computed in later ranking stage}}@@ -899,7 +921,7 @@ impl Signal for TitleEmbeddingSimilarity {))}- fn compute(&self, _: DocId, _: &SignalComputer) -> Option<f64> {+ fn compute(&self, _: DocId, _: &SignalComputer) -> Option<SignalCalculation> {None // computed in later ranking stage}}@@ -928,7 +950,7 @@ impl Signal for KeywordEmbeddingSimilarity {))}- fn compute(&self, _: DocId, _: &SignalComputer) -> Option<f64> {+ fn compute(&self, _: DocId, _: &SignalComputer) -> Option<SignalCalculation> {None // computed in later ranking stage}}@@ -955,7 +977,7 @@ impl Signal for MinTitleSlop {Some(Field::Text(schema::text_field::Title.into()))}- fn compute(&self, _: DocId, _: &SignalComputer) -> Option<f64> {+ fn compute(&self, _: DocId, _: &SignalComputer) -> Option<SignalCalculation> {None // computed in later ranking stage}}@@ -982,7 +1004,7 @@ impl Signal for MinCleanBodySlop {Some(Field::Text(schema::text_field::CleanBody.into()))}- fn compute(&self, _: DocId, _: &SignalComputer) -> Option<f64> {+ fn compute(&self, _: DocId, _: &SignalComputer) -> Option<SignalCalculation> {None // computed in later ranking stage}}diff --git a/crates/core/src/ranking/initial.rs b/crates/core/src/ranking/initial.rsindex a32dbf38..4ee87479 100644--- a/crates/core/src/ranking/initial.rs+++ b/crates/core/src/ranking/initial.rs@@ -81,7 +81,7 @@ impl ScoreSegmentTweaker<Score> for InitialSegmentScoreTweaker {let mut total = self.computer.compute_signals(doc)- .map(|computed| self.computer.coefficient(&computed.signal) * computed.score)+ .map(|computed| self.computer.coefficient(&computed.signal) * computed.calc.score).sum();if let Some(boost) = self.computer.boosts(doc) {diff --git a/crates/core/src/ranking/mod.rs b/crates/core/src/ranking/mod.rsindex 1d126e37..a67c8c77 100644--- a/crates/core/src/ranking/mod.rs+++ b/crates/core/src/ranking/mod.rs@@ -37,7 +37,7 @@ use crate::{pub use self::core::*;#[derive(Clone)]-pub struct Ranker {+pub struct LocalRanker {max_docs: Option<MaxDocsConsidered>,offset: Option<usize>,computer: SignalComputer,@@ -47,13 +47,13 @@ pub struct Ranker {collector_config: CollectorConfig,}-impl Ranker {+impl LocalRanker {pub fn new(computer: SignalComputer,columnfield_reader: NumericalFieldReader,collector_config: CollectorConfig,) -> Self {- Ranker {+ LocalRanker {offset: None,computer,max_docs: None,diff --git a/crates/core/src/ranking/models/lambdamart.rs b/crates/core/src/ranking/models/lambdamart.rsindex 1632fc8d..5d1ee789 100644--- a/crates/core/src/ranking/models/lambdamart.rs+++ b/crates/core/src/ranking/models/lambdamart.rs@@ -20,7 +20,7 @@ use core::SignalEnumDiscriminants;use crate::{enum_map::EnumMap,- ranking::{core, SignalEnum},+ ranking::{core, SignalCalculation, SignalEnum},};type Result<T> = std::result::Result<T, Error>;@@ -64,6 +64,12 @@ impl AsValue for f64 {}}+impl AsValue for SignalCalculation {+ fn as_value(&self) -> f64 {+ self.score+ }+}+#[derive(Debug)]struct Node {threshold: f64,diff --git a/crates/core/src/ranking/models/mod.rs b/crates/core/src/ranking/models/mod.rsindex 41e9906e..756d2067 100644--- a/crates/core/src/ranking/models/mod.rs+++ b/crates/core/src/ranking/models/mod.rs@@ -1,3 +1,6 @@pub mod cross_encoder;pub mod lambdamart;pub mod linear;++pub use cross_encoder::CrossEncoder;+pub use lambdamart::LambdaMART;diff --git a/crates/core/src/ranking/pipeline/mod.rs b/crates/core/src/ranking/pipeline/mod.rsindex 367c98d2..4de438fb 100644--- a/crates/core/src/ranking/pipeline/mod.rs+++ b/crates/core/src/ranking/pipeline/mod.rs@@ -14,43 +14,37 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.-use std::{ops::Add, sync::Arc};-use crate::{- collector::{self, BucketCollector},- config::CollectorConfig,+ collector::{self},enum_map::EnumMap,searcher::SearchQuery,};use super::{- models::lambdamart::{self, LambdaMART},- InboundSimilarity, SignalCoefficient, SignalEnum, SignalScore,+ models::lambdamart::{self},+ SignalCalculation, SignalEnum, SignalScore,};+mod modifiers;mod scorers;mod stages;-pub use scorers::{ReRanker, Recall, Scorer};+use modifiers::FullModifier;+pub use scorers::{FullRankingStage, ReRanker};pub use stages::{LocalRecallRankingWebpage, PrecisionRankingWebpage, RecallRankingWebpage};-const INBOUND_SIMILARITY_SMOOTHING: f64 = 8.0;-pub trait RankableWebpage: collector::Doc + Send + Sync {- fn set_score(&mut self, score: f64);- fn boost(&self) -> Option<f64>;- fn signals(&self) -> &EnumMap<SignalEnum, f64>;- fn signals_mut(&mut self) -> &mut EnumMap<SignalEnum, f64>;+ fn set_raw_score(&mut self, score: f64);+ fn unboosted_score(&self) -> f64;+ fn boost(&self) -> f64;+ fn set_boost(&mut self, boost: f64);+ fn signals(&self) -> &EnumMap<SignalEnum, SignalCalculation>;+ fn signals_mut(&mut self) -> &mut EnumMap<SignalEnum, SignalCalculation>;fn as_local_recall(&self) -> &LocalRecallRankingWebpage;- fn boost_score(&mut self) {- if let Some(boost) = self.boost() {- if boost != 0.0 {- let score = self.score() * boost;- self.set_score(score);- }- }+ fn score(&self) -> f64 {+ self.boost() * self.unboosted_score()}}@@ -60,136 +54,136 @@ impl lambdamart::AsValue for SignalScore {}}-struct RankingStage<T> {- scorer: Box<dyn Scorer<T>>,- stage_top_n: usize,- derank_similar: bool,- model: Option<Arc<LambdaMART>>,- coefficients: SignalCoefficient,+enum StageOrModifier<T> {+ Stage(Box<dyn FullRankingStage<Webpage = T>>),+ Modifier(Box<dyn FullModifier<Webpage = T>>),}-impl<T: RankableWebpage> RankingStage<T> {- fn apply(- &self,- websites: Vec<T>,- top_n: usize,- offset: usize,- collector_config: CollectorConfig,- ) -> Vec<T> {- let mut websites = websites- .into_iter()- .skip(offset)- .take(self.stage_top_n.max(top_n))- .collect::<Vec<_>>();-- self.scorer.score(&mut websites);-- let mut collector =- BucketCollector::new(self.stage_top_n.max(top_n) + offset, collector_config);+pub enum Top {+ Unlimited,+ Limit(usize),+}- for mut website in websites {- website.set_score(self.calculate_score(website.signals()));- website.boost_score();- collector.insert(website);+impl<T> StageOrModifier<T>+where+ T: RankableWebpage + Send + Sync,+{+ fn top_n(&self) -> Top {+ match self {+ StageOrModifier::Stage(stage) => stage.top_n(),+ StageOrModifier::Modifier(modifier) => modifier.top_n(),}-- collector- .into_sorted_vec(self.derank_similar)- .into_iter()- .take(top_n)- .collect()}- fn calculate_score(&self, signals: &EnumMap<SignalEnum, f64>) -> f64 {- match self.model.as_ref() {- Some(model) => {- let coeff = self.coefficients.get(&super::core::LambdaMart.into());- if coeff == 0.0 {- signals- .iter()- .map(|(signal, score)| self.coefficients.get(&signal) * score)- .sum()- } else {- signals- .get(InboundSimilarity.into())- .copied()- .unwrap_or(0.0)- .add(INBOUND_SIMILARITY_SMOOTHING)- * coeff- * model.predict(signals)- }- }- None => signals- .iter()- .map(|(signal, score)| self.coefficients.get(&signal) * score)- .sum(),+ fn compute(&self, webpages: &mut [T]) {+ match self {+ StageOrModifier::Stage(stage) => stage.compute(webpages),+ StageOrModifier::Modifier(modifier) => modifier.update_boosts(webpages),}}- fn set_query_info(&mut self, query: &SearchQuery) {- self.scorer.set_query_info(query);+ fn update_scores(&self, webpages: &mut [T], coefficients: &super::SignalCoefficients) {+ match self {+ StageOrModifier::Stage(stage) => stage.update_scores(webpages, coefficients),+ StageOrModifier::Modifier(_) => {}+ }+ }- self.coefficients = query.signal_coefficients();+ fn rank(&self, webpages: &mut [T]) {+ match self {+ StageOrModifier::Stage(stage) => stage.rank(webpages),+ StageOrModifier::Modifier(modifier) => modifier.rank(webpages),+ }}}pub struct RankingPipeline<T> {- stage: RankingStage<T>,- page: usize,- pub top_n: usize,- collector_config: CollectorConfig,+ stages_or_modifiers: Vec<StageOrModifier<T>>,}-impl<T: RankableWebpage> RankingPipeline<T> {- fn set_query_info(&mut self, query: &mut SearchQuery) {- self.stage.set_query_info(query);- self.page = query.page;- self.top_n = query.num_results;-- query.num_results = self.collector_top_n();- query.page = 0;+impl<T> RankingPipeline<T>+where+ T: RankableWebpage,+{+ fn new() -> Self {+ Self {+ stages_or_modifiers: Vec::new(),+ }}- pub fn offset(&self) -> usize {- self.top_n * self.page+ pub fn add_stage<R>(mut self, stage: R) -> Self+ where+ R: FullRankingStage<Webpage = T> + 'static,+ {+ self.stages_or_modifiers.push(StageOrModifier::Stage(+ Box::new(stage) as Box<dyn FullRankingStage<Webpage = T>>+ ));++ self}- pub fn apply(self, websites: Vec<T>) -> Vec<T> {- if websites.len() <= 1 {- return websites;- }+ pub fn add_modifier<R>(mut self, modifier: R) -> Self+ where+ R: FullModifier<Webpage = T> + 'static,+ {+ self.stages_or_modifiers.push(StageOrModifier::Modifier(+ Box::new(modifier) as Box<dyn FullModifier<Webpage = T>>+ ));- self.stage.apply(- websites,- self.top_n,- self.offset(),- self.collector_config.clone(),- )+ self}- pub fn collector_top_n(&self) -> usize {- (self.initial_top_n().max(self.top_n) + self.top_n * self.page) + 1- }+ pub fn apply(&self, webpages: Vec<T>, query: &SearchQuery) -> Vec<T> {+ let mut webpages = webpages;+ let num_pages = webpages.len();+ let coefficients = query.signal_coefficients();- pub fn initial_top_n(&self) -> usize {- self.stage.stage_top_n.max(self.top_n)+ for stage_or_modifier in self.stages_or_modifiers.iter() {+ let webpages = if let Top::Limit(top_n) = stage_or_modifier.top_n() {+ if query.offset() > top_n {+ continue;+ }++ &mut webpages[..top_n.min(num_pages)]+ } else {+ &mut webpages+ };++ stage_or_modifier.compute(webpages);+ stage_or_modifier.update_scores(webpages, &coefficients);+ stage_or_modifier.rank(webpages);+ }++ webpages+ .into_iter()+ .skip(query.offset())+ .take(query.num_results())+ .collect()}}#[cfg(test)]mod tests {- use itertools::Itertools;+ use scorers::term_distance;use crate::{+ ampc::dht::ShardId,collector::Hashes,inverted_index::{DocAddress, WebpagePointer},prehashed::Prehashed,- ranking::{self, initial::Score},+ ranking::{self, bitvec_similarity::BitVec, initial::Score},+ searcher::api,};use super::*;- fn sample_websites(n: usize) -> Vec<LocalRecallRankingWebpage> {+ fn pipeline() -> RankingPipeline<api::ScoredWebpagePointer> {+ RankingPipeline::new()+ .add_stage(term_distance::TitleDistanceScorer)+ .add_stage(term_distance::BodyDistanceScorer)+ }++ fn sample_websites(n: usize) -> Vec<api::ScoredWebpagePointer> {(0..n).map(|i| -> LocalRecallRankingWebpage {let pointer = WebpagePointer {@@ -208,118 +202,49 @@ mod tests {};let mut signals = EnumMap::new();- signals.insert(ranking::core::HostCentrality.into(), 1.0 / i as f64);- LocalRecallRankingWebpage::new_testing(pointer, signals, 1.0 / i as f64)+ let score = 1.0 / i as f64;+ let calc = ranking::SignalCalculation {+ value: i as f64,+ score,+ };+ signals.insert(ranking::core::HostCentrality.into(), calc);+ LocalRecallRankingWebpage::new_testing(pointer, signals, calc.score)+ })+ .map(|local| {+ api::ScoredWebpagePointer::Normal(+ crate::searcher::distributed::ScoredWebpagePointer {+ website: RecallRankingWebpage::new(local, BitVec::new(vec![])),+ shard: ShardId::new(0),+ },+ )}).collect()}#[test]fn simple() {- let pipeline = RankingPipeline::<LocalRecallRankingWebpage>::recall_stage(- &mut SearchQuery {- ..Default::default()- },- None,- None,- CollectorConfig::default(),- 20,- );- assert_eq!(pipeline.collector_top_n(), 20 + 1);-- let sample = sample_websites(pipeline.collector_top_n());+ let pipeline = pipeline();++ let sample = sample_websites(20);let res: Vec<_> = pipeline- .apply(sample)+ .apply(+ sample,+ &SearchQuery {+ page: 0,+ num_results: 20,+ ..Default::default()+ },+ ).into_iter()- .map(|w| w.pointer().address)+ .map(|w| w.as_ranking().pointer().address).collect();let expected: Vec<_> = sample_websites(100).into_iter().take(20)- .map(|w| w.pointer().address)- .collect();-- assert_eq!(res, expected);- }-- #[test]- fn top_n() {- let num_results = 100;- let pipeline = RankingPipeline::<LocalRecallRankingWebpage>::recall_stage(- &mut SearchQuery {- num_results,- ..Default::default()- },- None,- None,- CollectorConfig::default(),- num_results,- );-- let sample: Vec<_> = sample_websites(pipeline.collector_top_n());-- let expected: Vec<_> = sample- .clone()- .into_iter()- .take(num_results)- .map(|w| w.pointer().address)+ .map(|w| w.as_ranking().pointer().address).collect();- let res = pipeline- .apply(sample)- .into_iter()- .map(|w| w.pointer().address)- .collect_vec();-- assert_eq!(res.len(), num_results);assert_eq!(res, expected);}-- #[test]- fn offsets() {- let num_results = 20;- let pipeline = RankingPipeline::<LocalRecallRankingWebpage>::recall_stage(- &mut SearchQuery {- page: 0,- num_results,- ..Default::default()- },- None,- None,- CollectorConfig::default(),- num_results,- );-- let sample: Vec<_> = sample_websites(pipeline.collector_top_n());- let mut prev: Vec<_> = pipeline.apply(sample);- for p in 1..1_000 {- let pipeline = RankingPipeline::<LocalRecallRankingWebpage>::recall_stage(- &mut SearchQuery {- page: p,- ..Default::default()- },- None,- None,- CollectorConfig::default(),- num_results,- );-- let sample: Vec<_> = sample_websites(pipeline.collector_top_n());- let res: Vec<_> = pipeline.apply(sample).into_iter().collect();-- assert_eq!(- res.len(),- num_results,- "Every page should have {num_results} results"- );-- assert!(!prev- .iter()- .zip_eq(res.iter())- .any(|(p, r)| p.pointer().address.doc_id == r.pointer().address.doc_id));-- prev = res;- }- }}diff --git a/crates/core/src/ranking/pipeline/modifiers/inbound_similarity.rs b/crates/core/src/ranking/pipeline/modifiers/inbound_similarity.rsnew file mode 100644index 00000000..09874545--- /dev/null+++ b/crates/core/src/ranking/pipeline/modifiers/inbound_similarity.rs@@ -0,0 +1,40 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use crate::{+ ranking::{self, pipeline::RankableWebpage},+ searcher::api,+};++use super::Modifier;++const INBOUND_SIMILARITY_SMOOTHING: f64 = 8.0;++pub struct InboundSimilarity;++impl Modifier for InboundSimilarity {+ type Webpage = api::ScoredWebpagePointer;++ fn boost(&self, webpage: &Self::Webpage) -> f64 {+ webpage+ .as_ranking()+ .signals()+ .get(ranking::InboundSimilarity.into())+ .map(|calc| calc.value)+ .unwrap_or(0.0)+ + INBOUND_SIMILARITY_SMOOTHING+ }+}diff --git a/crates/core/src/ranking/pipeline/modifiers/mod.rs b/crates/core/src/ranking/pipeline/modifiers/mod.rsnew file mode 100644index 00000000..fa481e3c--- /dev/null+++ b/crates/core/src/ranking/pipeline/modifiers/mod.rs@@ -0,0 +1,60 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++mod inbound_similarity;++use super::{RankableWebpage, Top};+pub use inbound_similarity::InboundSimilarity;++pub trait FullModifier: Send + Sync {+ type Webpage: RankableWebpage;+ fn update_boosts(&self, webpages: &mut [Self::Webpage]);++ fn rank(&self, webpages: &mut [Self::Webpage]) {+ webpages.sort_by(|a, b| b.score().partial_cmp(&a.score()).unwrap());+ }++ fn top_n(&self) -> Top {+ Top::Unlimited+ }+}++pub trait Modifier: Send + Sync {+ type Webpage: RankableWebpage;+ fn boost(&self, webpage: &Self::Webpage) -> f64;++ fn top(&self) -> Top {+ Top::Unlimited+ }+}++impl<T> FullModifier for T+where+ T: Modifier,+{+ type Webpage = <T as Modifier>::Webpage;++ fn update_boosts(&self, webpages: &mut [Self::Webpage]) {+ for webpage in webpages {+ let cur_boost = webpage.boost();+ webpage.set_boost(cur_boost * self.boost(webpage));+ }+ }++ fn top_n(&self) -> Top {+ Modifier::top(self)+ }+}diff --git a/crates/core/src/ranking/pipeline/scorers/embedding.rs b/crates/core/src/ranking/pipeline/scorers/embedding.rsindex 73c834ca..bba0fb5c 100644--- a/crates/core/src/ranking/pipeline/scorers/embedding.rs+++ b/crates/core/src/ranking/pipeline/scorers/embedding.rs@@ -22,14 +22,14 @@ use crate::{models::dual_encoder::DualEncoder,ranking::{self,- pipeline::{stages::StoredEmbeddings, LocalRecallRankingWebpage, RankableWebpage},+ pipeline::{stages::StoredEmbeddings, RankableWebpage},SignalEnum,},- searcher::{api::ScoredWebpagePointer, SearchQuery},+ searcher::api::ScoredWebpagePointer,Result,};-use super::Scorer;+use super::FullRankingStage;pub struct Embedding(Tensor);@@ -72,15 +72,15 @@ impl Embedding {pub struct EmbeddingScorer<W, E: EmbeddingSignal<W>> {dual_encoder: Option<Arc<DualEncoder>>,- query: Option<String>,+ query: String,_marker: std::marker::PhantomData<(E, W)>,}impl<W, E: EmbeddingSignal<W>> EmbeddingScorer<W, E> {- pub fn new(dual_encoder: Option<Arc<DualEncoder>>) -> Self {+ pub fn new(query: String, dual_encoder: Option<Arc<DualEncoder>>) -> Self {Self {dual_encoder,- query: None,+ query,_marker: std::marker::PhantomData,}}@@ -89,31 +89,15 @@ impl<W, E: EmbeddingSignal<W>> EmbeddingScorer<W, E> {impl<W, E: EmbeddingSignal<W>> EmbeddingScorer<W, E> {pub fn query_emb(&self) -> Option<Embedding> {self.dual_encoder.as_ref().and_then(|dual_encoder| {- self.query.as_ref().and_then(|query| {- dual_encoder- .embed(&[query.to_string()])- .ok()- .and_then(|d| d.squeeze(0).ok())- .map(Embedding)- })+ dual_encoder+ .embed(&[self.query.clone()])+ .ok()+ .and_then(|d| d.squeeze(0).ok())+ .map(Embedding)})}}-impl LocalRecallRankingWebpage {- fn title_emb(&self, hidden_size: usize) -> Option<Embedding> {- self.title_embedding()- .as_ref()- .and_then(|s| Embedding::new(s, hidden_size).ok())- }-- fn keyword_emb(&self, hidden_size: usize) -> Option<Embedding> {- self.keyword_embedding()- .as_ref()- .and_then(|s| Embedding::new(s, hidden_size).ok())- }-}-impl ScoredWebpagePointer {fn title_emb(&self, hidden_size: usize) -> Option<Embedding> {self.as_ranking()@@ -130,8 +114,12 @@ impl ScoredWebpagePointer {}}-impl<W: RankableWebpage, E: EmbeddingSignal<W>> Scorer<W> for EmbeddingScorer<W, E> {- fn score(&self, webpages: &mut [W]) {+impl<E: EmbeddingSignal<ScoredWebpagePointer>> FullRankingStage+ for EmbeddingScorer<ScoredWebpagePointer, E>+{+ type Webpage = ScoredWebpagePointer;++ fn compute(&self, webpages: &mut [Self::Webpage]) {if !webpages.iter().any(E::has_embedding) {return;}@@ -146,10 +134,6 @@ impl<W: RankableWebpage, E: EmbeddingSignal<W>> Scorer<W> for EmbeddingScorer<W,}}}-- fn set_query_info(&mut self, query: &SearchQuery) {- self.query = Some(query.query.clone());- }}pub struct TitleEmbeddings;@@ -177,27 +161,10 @@ impl EmbeddingSignal<ScoredWebpagePointer> for TitleEmbeddings {fn insert_signal(webpage: &mut ScoredWebpagePointer, score: f64) {let sig = <TitleEmbeddings as EmbeddingSignal<ScoredWebpagePointer>>::signal();- webpage.as_ranking_mut().signals_mut().insert(sig, score);- }-}--impl EmbeddingSignal<LocalRecallRankingWebpage> for TitleEmbeddings {- fn signal() -> SignalEnum {- ranking::core::TitleEmbeddingSimilarity.into()- }-- fn has_embedding(webpage: &LocalRecallRankingWebpage) -> bool {- webpage.title_embedding().is_some()- }-- fn embedding(webpage: &LocalRecallRankingWebpage, hidden_size: usize) -> Option<Embedding> {- webpage.title_emb(hidden_size)- }-- fn insert_signal(webpage: &mut LocalRecallRankingWebpage, score: f64) {- let sig = <TitleEmbeddings as EmbeddingSignal<LocalRecallRankingWebpage>>::signal();-- webpage.signals_mut().insert(sig, score);+ webpage+ .as_ranking_mut()+ .signals_mut()+ .insert(sig, ranking::SignalCalculation::new_symmetrical(score));}}@@ -216,26 +183,10 @@ impl EmbeddingSignal<ScoredWebpagePointer> for KeywordEmbeddings {fn insert_signal(webpage: &mut ScoredWebpagePointer, score: f64) {let sig = <KeywordEmbeddings as EmbeddingSignal<ScoredWebpagePointer>>::signal();- webpage.as_ranking_mut().signals_mut().insert(sig, score);- }-}--impl EmbeddingSignal<LocalRecallRankingWebpage> for KeywordEmbeddings {- fn signal() -> SignalEnum {- ranking::core::KeywordEmbeddingSimilarity.into()- }-- fn has_embedding(webpage: &LocalRecallRankingWebpage) -> bool {- webpage.keyword_embedding().is_some()- }-- fn embedding(webpage: &LocalRecallRankingWebpage, hidden_size: usize) -> Option<Embedding> {- webpage.keyword_emb(hidden_size)- }-- fn insert_signal(webpage: &mut LocalRecallRankingWebpage, score: f64) {- let sig = <KeywordEmbeddings as EmbeddingSignal<LocalRecallRankingWebpage>>::signal();- webpage.signals_mut().insert(sig, score);+ webpage+ .as_ranking_mut()+ .signals_mut()+ .insert(sig, ranking::SignalCalculation::new_symmetrical(score));}}diff --git a/crates/core/src/ranking/pipeline/scorers/inbound_similarity.rs b/crates/core/src/ranking/pipeline/scorers/inbound_similarity.rsindex 0ad1e977..e5bdf645 100644--- a/crates/core/src/ranking/pipeline/scorers/inbound_similarity.rs+++ b/crates/core/src/ranking/pipeline/scorers/inbound_similarity.rs@@ -17,11 +17,11 @@use std::sync::Mutex;use crate::{- ranking::{core, inbound_similarity},+ ranking::{self, core, inbound_similarity, pipeline::RankableWebpage},searcher::api::ScoredWebpagePointer,};-use super::Scorer;+use super::FullRankingStage;pub struct InboundScorer {scorer: Mutex<inbound_similarity::Scorer>,@@ -35,8 +35,10 @@ impl InboundScorer {}}-impl Scorer<ScoredWebpagePointer> for InboundScorer {- fn score(&self, webpages: &mut [ScoredWebpagePointer]) {+impl FullRankingStage for InboundScorer {+ type Webpage = ScoredWebpagePointer;++ fn compute(&self, webpages: &mut [Self::Webpage]) {let mut scorer = self.scorer.lock().unwrap();for webpage in webpages {@@ -44,10 +46,10 @@ impl Scorer<ScoredWebpagePointer> for InboundScorer {webpage.as_ranking().host_id(),webpage.as_ranking().inbound_edges(),);- webpage- .as_ranking_mut()- .signals_mut()- .insert(core::InboundSimilarity.into(), score);+ webpage.as_ranking_mut().signals_mut().insert(+ core::InboundSimilarity.into(),+ ranking::SignalCalculation::new_symmetrical(score),+ );}}}diff --git a/crates/core/src/ranking/pipeline/scorers/lambdamart.rs b/crates/core/src/ranking/pipeline/scorers/lambdamart.rsnew file mode 100644index 00000000..9dfc00c9--- /dev/null+++ b/crates/core/src/ranking/pipeline/scorers/lambdamart.rs@@ -0,0 +1,70 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use crate::{+ ranking::{+ self, models,+ pipeline::{PrecisionRankingWebpage, RankableWebpage, Top},+ SignalCalculation, SignalEnum,+ },+ searcher::api::ScoredWebpagePointer,+};+use std::sync::Arc;++use super::RankingStage;++impl RankingStage for Arc<models::LambdaMART> {+ type Webpage = ScoredWebpagePointer;++ fn compute(&self, webpages: &Self::Webpage) -> (SignalEnum, SignalCalculation) {+ let signals = webpages+ .signals()+ .iter()+ .map(|(signal, calc)| (signal, calc.value))+ .collect();+ (+ ranking::core::LambdaMart.into(),+ SignalCalculation::new_symmetrical(self.predict(&signals)),+ )+ }++ fn top_n(&self) -> Top {+ Top::Limit(20)+ }+}++pub struct PrecisionLambda(Arc<models::LambdaMART>);++impl From<Arc<models::LambdaMART>> for PrecisionLambda {+ fn from(model: Arc<models::LambdaMART>) -> Self {+ Self(model)+ }+}++impl RankingStage for PrecisionLambda {+ type Webpage = PrecisionRankingWebpage;++ fn compute(&self, webpage: &Self::Webpage) -> (SignalEnum, SignalCalculation) {+ (+ ranking::core::LambdaMart.into(),+ SignalCalculation::new_symmetrical(self.0.predict(webpage.ranking().signals())),+ )+ }++ fn top_n(&self) -> Top {+ Top::Limit(20)+ }+}diff --git a/crates/core/src/ranking/pipeline/scorers/mod.rs b/crates/core/src/ranking/pipeline/scorers/mod.rsindex 242389d7..bc561e52 100644--- a/crates/core/src/ranking/pipeline/scorers/mod.rs+++ b/crates/core/src/ranking/pipeline/scorers/mod.rs@@ -16,48 +16,60 @@pub mod embedding;pub mod inbound_similarity;-pub mod recall;+pub mod lambdamart;pub mod reranker;pub mod term_distance;-pub use recall::Recall;pub use reranker::ReRanker;-use crate::searcher::SearchQuery;+use crate::ranking::{SignalCalculation, SignalCoefficients, SignalEnum};-use super::RankableWebpage;+use super::{RankableWebpage, Top};-pub trait Scorer<T: RankableWebpage>: Send + Sync {- fn score(&self, webpages: &mut [T]);- fn set_query_info(&mut self, _query: &SearchQuery) {}-}+pub trait FullRankingStage: Send + Sync {+ type Webpage: RankableWebpage;-pub struct IdentityScorer;+ fn compute(&self, webpages: &mut [Self::Webpage]);+ fn top_n(&self) -> Top {+ Top::Unlimited+ }-impl<T: RankableWebpage> Scorer<T> for IdentityScorer {- fn score(&self, _webpages: &mut [T]) {}-}+ fn update_scores(&self, webpages: &mut [Self::Webpage], coefficients: &SignalCoefficients) {+ for webpage in webpages.iter_mut() {+ webpage.set_raw_score(webpage.signals().iter().fold(0.0, |acc, (signal, calc)| {+ acc + calc.score * coefficients.get(&signal)+ }));+ }+ }-pub struct MultiScorer<T: RankableWebpage> {- scorers: Vec<Box<dyn Scorer<T>>>,+ fn rank(&self, webpages: &mut [Self::Webpage]) {+ webpages.sort_by(|a, b| b.score().partial_cmp(&a.score()).unwrap());+ }}-impl<T: RankableWebpage> MultiScorer<T> {- pub fn new(scorers: Vec<Box<dyn Scorer<T>>>) -> Self {- Self { scorers }+pub trait RankingStage: Send + Sync {+ type Webpage: RankableWebpage;++ fn compute(&self, webpage: &Self::Webpage) -> (SignalEnum, SignalCalculation);+ fn top_n(&self) -> Top {+ Top::Unlimited}}-impl<T: RankableWebpage> Scorer<T> for MultiScorer<T> {- fn score(&self, webpages: &mut [T]) {- for scorer in &self.scorers {- scorer.score(webpages);+impl<T> FullRankingStage for T+where+ T: RankingStage,+{+ type Webpage = <T as RankingStage>::Webpage;++ fn compute(&self, webpages: &mut [Self::Webpage]) {+ for webpage in webpages.iter_mut() {+ let (signal, signal_calculation) = self.compute(webpage);+ webpage.signals_mut().insert(signal, signal_calculation);}}- fn set_query_info(&mut self, query: &SearchQuery) {- for scorer in &mut self.scorers {- scorer.set_query_info(query);- }+ fn top_n(&self) -> Top {+ self.top_n()}}diff --git a/crates/core/src/ranking/pipeline/scorers/recall.rs b/crates/core/src/ranking/pipeline/scorers/recall.rsdeleted file mode 100644index b7a0820d..00000000--- a/crates/core/src/ranking/pipeline/scorers/recall.rs+++ /dev/null@@ -1,97 +0,0 @@-// Stract is an open source web search engine.-// Copyright (C) 2024 Stract ApS-//-// This program is free software: you can redistribute it and/or modify-// it under the terms of the GNU Affero General Public License as-// published by the Free Software Foundation, either version 3 of the-// License, or (at your option) any later version.-//-// This program is distributed in the hope that it will be useful,-// but WITHOUT ANY WARRANTY; without even the implied warranty of-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the-// GNU Affero General Public License for more details.-//-// You should have received a copy of the GNU Affero General Public License-// along with this program. If not, see <https://www.gnu.org/licenses/>.--use std::sync::Arc;--use crate::{- models::dual_encoder::DualEncoder,- ranking::{- inbound_similarity,- pipeline::{LocalRecallRankingWebpage, RankableWebpage},- },- searcher::{- api::{self},- SearchQuery,- },-};--use super::{- embedding::{EmbeddingScorer, KeywordEmbeddings, TitleEmbeddings},- inbound_similarity::InboundScorer,- term_distance, MultiScorer, Scorer,-};--pub struct Recall<T: RankableWebpage> {- scorer: MultiScorer<T>,-}--impl Recall<api::ScoredWebpagePointer> {- pub fn new(- inbound: inbound_similarity::Scorer,- dual_encoder: Option<Arc<DualEncoder>>,- ) -> Self {- Self {- scorer: MultiScorer::new(vec![- Box::new(- EmbeddingScorer::<api::ScoredWebpagePointer, TitleEmbeddings>::new(- dual_encoder.clone(),- ),- ),- Box::new(EmbeddingScorer::<- api::ScoredWebpagePointer,- KeywordEmbeddings,- >::new(dual_encoder)),- Box::new(InboundScorer::new(inbound)),- ]),- }- }-}--impl Scorer<api::ScoredWebpagePointer> for Recall<api::ScoredWebpagePointer> {- fn score(&self, webpages: &mut [api::ScoredWebpagePointer]) {- self.scorer.score(webpages);- }-}--impl Recall<LocalRecallRankingWebpage> {- pub fn new(dual_encoder: Option<Arc<DualEncoder>>) -> Self {- Self {- scorer: MultiScorer::new(vec![- Box::new(term_distance::TitleDistanceScorer),- Box::new(term_distance::BodyDistanceScorer),- Box::new(- EmbeddingScorer::<LocalRecallRankingWebpage, TitleEmbeddings>::new(- dual_encoder.clone(),- ),- ),- Box::new(EmbeddingScorer::<- LocalRecallRankingWebpage,- KeywordEmbeddings,- >::new(dual_encoder)),- ]),- }- }-}--impl Scorer<LocalRecallRankingWebpage> for Recall<LocalRecallRankingWebpage> {- fn score(&self, webpages: &mut [LocalRecallRankingWebpage]) {- self.scorer.score(webpages);- }-- fn set_query_info(&mut self, query: &SearchQuery) {- self.scorer.set_query_info(query);- }-}diff --git a/crates/core/src/ranking/pipeline/scorers/reranker.rs b/crates/core/src/ranking/pipeline/scorers/reranker.rsindex 5eefd5ca..59aa754b 100644--- a/crates/core/src/ranking/pipeline/scorers/reranker.rs+++ b/crates/core/src/ranking/pipeline/scorers/reranker.rs@@ -16,23 +16,20 @@use std::sync::Arc;-use crate::{- ranking::{self, models::cross_encoder::CrossEncoder},- searcher::SearchQuery,-};+use crate::ranking::{self, models::cross_encoder::CrossEncoder, pipeline::RankableWebpage};-use crate::ranking::pipeline::{PrecisionRankingWebpage, Scorer};+use crate::ranking::pipeline::{FullRankingStage, PrecisionRankingWebpage, Top};pub struct ReRanker<M: CrossEncoder> {crossencoder: Arc<M>,- query: Option<SearchQuery>,+ query: String,}impl<M: CrossEncoder> ReRanker<M> {- pub fn new(crossencoder: Arc<M>) -> Self {+ pub fn new(query: String, crossencoder: Arc<M>) -> Self {Self {crossencoder,- query: None,+ query,}}@@ -45,31 +42,33 @@ impl<M: CrossEncoder> ReRanker<M> {snippets.push(webpage.retrieved_webpage().snippet.unhighlighted_string());}- let query = &self.query.as_ref().unwrap().query;+ let query = &self.query;let snippet_scores = self.crossencoder.run(query, &snippets);let title_scores = self.crossencoder.run(query, &titles);for ((webpage, snippet), title) in webpage.iter_mut().zip(snippet_scores).zip(title_scores){- webpage- .ranking_mut()- .signals_mut()- .insert(ranking::core::CrossEncoderSnippet.into(), snippet);+ webpage.ranking_mut().signals_mut().insert(+ ranking::core::CrossEncoderSnippet.into(),+ ranking::SignalCalculation::new_symmetrical(snippet),+ );- webpage- .ranking_mut()- .signals_mut()- .insert(ranking::core::CrossEncoderTitle.into(), title);+ webpage.ranking_mut().signals_mut().insert(+ ranking::core::CrossEncoderTitle.into(),+ ranking::SignalCalculation::new_symmetrical(title),+ );}}}-impl<M: CrossEncoder> Scorer<PrecisionRankingWebpage> for ReRanker<M> {- fn score(&self, webpages: &mut [PrecisionRankingWebpage]) {+impl<M: CrossEncoder> FullRankingStage for ReRanker<M> {+ type Webpage = PrecisionRankingWebpage;++ fn compute(&self, webpages: &mut [Self::Webpage]) {self.crossencoder_score_webpages(webpages);}- fn set_query_info(&mut self, query: &SearchQuery) {- self.query = Some(query.clone());+ fn top_n(&self) -> Top {+ Top::Limit(20)}}diff --git a/crates/core/src/ranking/pipeline/scorers/term_distance.rs b/crates/core/src/ranking/pipeline/scorers/term_distance.rsindex fe8eb0e7..856e1687 100644--- a/crates/core/src/ranking/pipeline/scorers/term_distance.rs+++ b/crates/core/src/ranking/pipeline/scorers/term_distance.rs@@ -16,8 +16,9 @@use itertools::Itertools;-use crate::ranking;use crate::ranking::pipeline::RankableWebpage;+use crate::ranking::{self, SignalCalculation, SignalEnum};+use crate::searcher::api;fn min_slop_two_positions(pos_a: &[u32], pos_b: &[u32]) -> u32 {let mut cur_min = u32::MAX;@@ -52,35 +53,47 @@ fn min_slop<'a>(positions: impl Iterator<Item = &'a [u32]>) -> u32 {.unwrap_or(u32::MAX)}+fn score_slop(slop: f64) -> f64 {+ 1.0 / (slop + 1.0)+}+#[derive(Debug, Default)]pub struct TitleDistanceScorer;-impl<W: RankableWebpage> super::Scorer<W> for TitleDistanceScorer {- fn score(&self, webpages: &mut [W]) {- for webpage in webpages {- let min_slop = min_slop(webpage.as_local_recall().iter_title_positions());+impl super::RankingStage for TitleDistanceScorer {+ type Webpage = api::ScoredWebpagePointer;- webpage.signals_mut().insert(- ranking::core::MinTitleSlop.into(),- 1.0 / (min_slop as f64 + 1.0),- );- }+ fn compute(&self, webpage: &Self::Webpage) -> (SignalEnum, SignalCalculation) {+ let min_slop = min_slop(webpage.as_local_recall().iter_title_positions()) as f64;+ let score = score_slop(min_slop);++ (+ ranking::core::MinTitleSlop.into(),+ ranking::SignalCalculation {+ value: min_slop,+ score,+ },+ )}}#[derive(Debug, Default)]pub struct BodyDistanceScorer;-impl<W: RankableWebpage> super::Scorer<W> for BodyDistanceScorer {- fn score(&self, webpages: &mut [W]) {- for webpage in webpages {- let min_slop = min_slop(webpage.as_local_recall().iter_clean_body_positions());+impl super::RankingStage for BodyDistanceScorer {+ type Webpage = api::ScoredWebpagePointer;- webpage.signals_mut().insert(- ranking::core::MinCleanBodySlop.into(),- 1.0 / (min_slop as f64 + 1.0),- );- }+ fn compute(&self, webpage: &Self::Webpage) -> (SignalEnum, SignalCalculation) {+ let min_slop = min_slop(webpage.as_local_recall().iter_clean_body_positions()) as f64;+ let score = score_slop(min_slop);++ (+ ranking::core::MinCleanBodySlop.into(),+ ranking::SignalCalculation {+ value: min_slop,+ score,+ },+ )}}diff --git a/crates/core/src/ranking/pipeline/stages/precision.rs b/crates/core/src/ranking/pipeline/stages/precision.rsindex b84f1cad..59847eaa 100644--- a/crates/core/src/ranking/pipeline/stages/precision.rs+++ b/crates/core/src/ranking/pipeline/stages/precision.rs@@ -18,24 +18,21 @@ use std::sync::Arc;use crate::{collector,- config::CollectorConfig,enum_map::EnumMap,inverted_index::RetrievedWebpage,ranking::{- models::{cross_encoder::CrossEncoder, lambdamart::LambdaMART},+ models::{self, cross_encoder::CrossEncoder},pipeline::{- scorers::IdentityScorer, RankableWebpage, RankingPipeline, RankingStage, ReRanker,- Scorer,+ scorers::lambdamart::PrecisionLambda, RankableWebpage, RankingPipeline, ReRanker,},- SignalEnum,+ SignalCalculation, SignalEnum,},searcher::SearchQuery,- Result,};use super::RecallRankingWebpage;-#[derive(Clone, Debug, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]+#[derive(Clone, Debug, bincode::Encode, bincode::Decode)]pub struct PrecisionRankingWebpage {retrieved_webpage: RetrievedWebpage,ranking: RecallRankingWebpage,@@ -57,7 +54,7 @@ impl PrecisionRankingWebpage {impl collector::Doc for PrecisionRankingWebpage {fn score(&self) -> f64 {- self.ranking.score()+ RankableWebpage::score(self)}fn hashes(&self) -> collector::Hashes {@@ -66,19 +63,27 @@ impl collector::Doc for PrecisionRankingWebpage {}impl RankableWebpage for PrecisionRankingWebpage {- fn set_score(&mut self, score: f64) {- self.ranking.set_score(score);+ fn set_raw_score(&mut self, score: f64) {+ self.ranking.set_raw_score(score);}- fn boost(&self) -> Option<f64> {+ fn unboosted_score(&self) -> f64 {+ self.ranking.unboosted_score()+ }++ fn boost(&self) -> f64 {self.ranking.boost()}- fn signals(&self) -> &EnumMap<SignalEnum, f64> {+ fn set_boost(&mut self, boost: f64) {+ self.ranking.set_boost(boost)+ }++ fn signals(&self) -> &EnumMap<SignalEnum, SignalCalculation> {self.ranking.signals()}- fn signals_mut(&mut self) -> &mut EnumMap<SignalEnum, f64> {+ fn signals_mut(&mut self) -> &mut EnumMap<SignalEnum, SignalCalculation> {self.ranking.signals_mut()}@@ -101,46 +106,18 @@ impl PrecisionRankingWebpage {}impl RankingPipeline<PrecisionRankingWebpage> {- fn create_reranking_stage<M: CrossEncoder + 'static>(- crossencoder: Option<Arc<M>>,- lambda: Option<Arc<LambdaMART>>,- collector_config: CollectorConfig,- top_n_considered: usize,- ) -> Result<Self> {- let scorer = match crossencoder {- Some(cross_encoder) => {- Box::new(ReRanker::new(cross_encoder)) as Box<dyn Scorer<PrecisionRankingWebpage>>- }- None => Box::new(IdentityScorer) as Box<dyn Scorer<PrecisionRankingWebpage>>,- };-- let stage = RankingStage {- scorer,- stage_top_n: top_n_considered,- derank_similar: true,- model: lambda,- coefficients: Default::default(),- };-- Ok(Self {- stage,- page: 0,- top_n: 0,- collector_config,- })- }-pub fn reranker<M: CrossEncoder + 'static>(- query: &mut SearchQuery,- crossencoder: Option<Arc<M>>,- lambda: Option<Arc<LambdaMART>>,- collector_config: CollectorConfig,- top_n_considered: usize,- ) -> Result<Self> {- let mut pipeline =- Self::create_reranking_stage(crossencoder, lambda, collector_config, top_n_considered)?;- pipeline.set_query_info(query);-- Ok(pipeline)+ query: &SearchQuery,+ crossencoder: Arc<M>,+ lambda: Option<Arc<models::LambdaMART>>,+ ) -> Self {+ let mut s = Self::new().add_stage(ReRanker::new(query.text().to_string(), crossencoder));++ if let Some(lambda) = lambda {+ let lambda = PrecisionLambda::from(lambda);+ s = s.add_stage(lambda);+ }++ s}}diff --git a/crates/core/src/ranking/pipeline/stages/recall.rs b/crates/core/src/ranking/pipeline/stages/recall.rsindex fd7f0507..3122e5ca 100644--- a/crates/core/src/ranking/pipeline/stages/recall.rs+++ b/crates/core/src/ranking/pipeline/stages/recall.rs@@ -18,7 +18,6 @@ use std::sync::Arc;use crate::{collector,- config::CollectorConfig,enum_map::EnumMap,inverted_index::WebpagePointer,models::dual_encoder::DualEncoder,@@ -26,8 +25,16 @@ use crate::{ranking::{bitvec_similarity, inbound_similarity,models::lambdamart::LambdaMART,- pipeline::{RankableWebpage, RankingPipeline, RankingStage, Recall, Scorer},- SignalComputer, SignalEnum,+ pipeline::{+ modifiers,+ scorers::{+ embedding::{EmbeddingScorer, KeywordEmbeddings, TitleEmbeddings},+ inbound_similarity::InboundScorer,+ term_distance,+ },+ RankableWebpage, RankingPipeline,+ },+ SignalCalculation, SignalComputer, SignalEnum,},schema::{numerical_field, text_field},searcher::{api, SearchQuery},@@ -42,7 +49,7 @@ impl StoredEmbeddings {&self.0}}-#[derive(Clone, Debug, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]+#[derive(Clone, Debug, bincode::Encode, bincode::Decode)]pub struct RecallRankingWebpage {local: LocalRecallRankingWebpage,inbound_edges: bitvec_similarity::BitVec,@@ -68,26 +75,6 @@ impl RecallRankingWebpage {self.local.keyword_embedding()}- pub fn score(&self) -> f64 {- self.local.score()- }-- pub fn signals(&self) -> &EnumMap<SignalEnum, f64> {- self.local.signals()- }-- pub fn signals_mut(&mut self) -> &mut EnumMap<SignalEnum, f64> {- self.local.signals_mut()- }-- pub fn boost(&self) -> Option<f64> {- self.local.boost()- }-- pub fn set_score(&mut self, score: f64) {- self.local.set_score(score)- }-pub fn inbound_edges(&self) -> &bitvec_similarity::BitVec {&self.inbound_edges}@@ -108,19 +95,27 @@ impl collector::Doc for RecallRankingWebpage {}impl RankableWebpage for RecallRankingWebpage {- fn set_score(&mut self, score: f64) {+ fn set_raw_score(&mut self, score: f64) {self.local.set_score(score);}- fn boost(&self) -> Option<f64> {+ fn unboosted_score(&self) -> f64 {+ self.local.unboosted_score()+ }++ fn boost(&self) -> f64 {self.local.boost()}- fn signals(&self) -> &EnumMap<SignalEnum, f64> {+ fn set_boost(&mut self, boost: f64) {+ self.local.set_boost(boost)+ }++ fn signals(&self) -> &EnumMap<SignalEnum, SignalCalculation> {self.local.signals()}- fn signals_mut(&mut self) -> &mut EnumMap<SignalEnum, f64> {+ fn signals_mut(&mut self) -> &mut EnumMap<SignalEnum, SignalCalculation> {self.local.signals_mut()}@@ -129,13 +124,13 @@ impl RankableWebpage for RecallRankingWebpage {}}-#[derive(Clone, Debug, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]+#[derive(Clone, Debug, bincode::Encode, bincode::Decode)]pub struct LocalRecallRankingWebpage {pointer: WebpagePointer,- signals: EnumMap<SignalEnum, f64>,+ signals: EnumMap<SignalEnum, SignalCalculation>,title_positions: Vec<Vec<u32>>,clean_body_positions: Vec<Vec<u32>>,- optic_boost: Option<f64>,+ boost: f64,title_embedding: Option<StoredEmbeddings>,keyword_embedding: Option<StoredEmbeddings>,score: f64,@@ -146,7 +141,7 @@ impl LocalRecallRankingWebpage {#[cfg(test)]pub fn new_testing(pointer: WebpagePointer,- signals: EnumMap<SignalEnum, f64>,+ signals: EnumMap<SignalEnum, SignalCalculation>,score: f64,) -> Self {LocalRecallRankingWebpage {@@ -154,7 +149,7 @@ impl LocalRecallRankingWebpage {signals,title_positions: Vec::new(),clean_body_positions: Vec::new(),- optic_boost: None,+ boost: 1.0,title_embedding: None,keyword_embedding: None,score,@@ -198,7 +193,7 @@ impl LocalRecallRankingWebpage {let mut res = LocalRecallRankingWebpage {signals: EnumMap::new(),score: pointer.score.total,- optic_boost: None,+ boost: 1.0,pointer: pointer.clone(),title_embedding: title_embedding.map(StoredEmbeddings),keyword_embedding: keyword_embedding.map(StoredEmbeddings),@@ -209,11 +204,11 @@ impl LocalRecallRankingWebpage {for computed_signal in computer.compute_signals(pointer.address.doc_id) {res.signals- .insert(computed_signal.signal, computed_signal.score);+ .insert(computed_signal.signal, computed_signal.calc);}if let Some(boost) = computer.boosts(pointer.address.doc_id) {- res.optic_boost = Some(boost);+ res.boost *= boost;}res@@ -232,21 +227,17 @@ impl LocalRecallRankingWebpage {}pub fn score(&self) -> f64 {- self.score+ self.score * self.boost}- pub fn signals(&self) -> &EnumMap<SignalEnum, f64> {+ pub fn signals(&self) -> &EnumMap<SignalEnum, SignalCalculation> {&self.signals}- pub fn signals_mut(&mut self) -> &mut EnumMap<SignalEnum, f64> {+ pub fn signals_mut(&mut self) -> &mut EnumMap<SignalEnum, SignalCalculation> {&mut self.signals}- pub fn boost(&self) -> Option<f64> {- self.optic_boost- }-pub fn set_score(&mut self, score: f64) {self.score = score;}@@ -265,19 +256,27 @@ impl LocalRecallRankingWebpage {}impl RankableWebpage for LocalRecallRankingWebpage {- fn set_score(&mut self, score: f64) {+ fn set_raw_score(&mut self, score: f64) {self.score = score;}- fn boost(&self) -> Option<f64> {- self.optic_boost+ fn unboosted_score(&self) -> f64 {+ self.score+ }++ fn boost(&self) -> f64 {+ self.boost}- fn signals(&self) -> &EnumMap<SignalEnum, f64> {+ fn set_boost(&mut self, boost: f64) {+ self.boost = boost;+ }++ fn signals(&self) -> &EnumMap<SignalEnum, SignalCalculation> {&self.signals}- fn signals_mut(&mut self) -> &mut EnumMap<SignalEnum, f64> {+ fn signals_mut(&mut self) -> &mut EnumMap<SignalEnum, SignalCalculation> {&mut self.signals}@@ -288,7 +287,7 @@ impl RankableWebpage for LocalRecallRankingWebpage {impl collector::Doc for LocalRecallRankingWebpage {fn score(&self) -> f64 {- self.score+ RankableWebpage::score(self)}fn hashes(&self) -> collector::Hashes {@@ -296,89 +295,33 @@ impl collector::Doc for LocalRecallRankingWebpage {}}-impl RankingPipeline<LocalRecallRankingWebpage> {- fn create_recall_stage(- lambdamart: Option<Arc<LambdaMART>>,- dual_encoder: Option<Arc<DualEncoder>>,- collector_config: CollectorConfig,- stage_top_n: usize,- ) -> Self {- let last_stage = RankingStage {- scorer: Box::new(Recall::<LocalRecallRankingWebpage>::new(dual_encoder))- as Box<dyn Scorer<LocalRecallRankingWebpage>>,- stage_top_n,- derank_similar: true,- model: lambdamart,- coefficients: Default::default(),- };-- Self {- stage: last_stage,- page: 0,- top_n: 0,- collector_config,- }- }-- pub fn recall_stage(- query: &mut SearchQuery,- lambdamart: Option<Arc<LambdaMART>>,- dual_encoder: Option<Arc<DualEncoder>>,- collector_config: CollectorConfig,- top_n_considered: usize,- ) -> Self {- let mut pipeline =- Self::create_recall_stage(lambdamart, dual_encoder, collector_config, top_n_considered);- pipeline.set_query_info(query);-- pipeline- }-}-impl RankingPipeline<api::ScoredWebpagePointer> {- fn create_recall_stage(+ pub fn recall_stage(+ query: &SearchQuery,inbound: inbound_similarity::Scorer,lambdamart: Option<Arc<LambdaMART>>,dual_encoder: Option<Arc<DualEncoder>>,- collector_config: CollectorConfig,- stage_top_n: usize,) -> Self {- let last_stage = RankingStage {- scorer: Box::new(Recall::<api::ScoredWebpagePointer>::new(- inbound,- dual_encoder,- )),- stage_top_n,- derank_similar: true,- model: lambdamart,- coefficients: Default::default(),- };-- Self {- stage: last_stage,- page: 0,- top_n: 0,- collector_config,+ let mut s = Self::new()+ .add_stage(term_distance::TitleDistanceScorer)+ .add_stage(term_distance::BodyDistanceScorer)+ .add_stage(+ EmbeddingScorer::<api::ScoredWebpagePointer, TitleEmbeddings>::new(+ query.text().to_string(),+ dual_encoder.clone(),+ ),+ )+ .add_stage(EmbeddingScorer::<+ api::ScoredWebpagePointer,+ KeywordEmbeddings,+ >::new(query.text().to_string(), dual_encoder))+ .add_stage(InboundScorer::new(inbound))+ .add_modifier(modifiers::InboundSimilarity);++ if let Some(lambda) = lambdamart {+ s = s.add_stage(lambda);}- }- pub fn recall_stage(- query: &mut SearchQuery,- inbound: inbound_similarity::Scorer,- lambdamart: Option<Arc<LambdaMART>>,- dual_encoder: Option<Arc<DualEncoder>>,- collector_config: CollectorConfig,- top_n_considered: usize,- ) -> Self {- let mut pipeline = Self::create_recall_stage(- inbound,- lambdamart,- dual_encoder,- collector_config,- top_n_considered,- );- pipeline.set_query_info(query);-- pipeline+ s}}diff --git a/crates/core/src/search_prettifier/mod.rs b/crates/core/src/search_prettifier/mod.rsindex 4e27ec8f..966fe333 100644--- a/crates/core/src/search_prettifier/mod.rs+++ b/crates/core/src/search_prettifier/mod.rs@@ -197,7 +197,6 @@ pub struct DisplayedWebpage {pub rich_snippet: Option<RichSnippet>,pub ranking_signals: Option<HashMap<SignalEnumDiscriminants, SignalScore>>,pub structured_data: Option<Vec<StructuredData>>,- pub score: Option<f64>,pub likely_has_ads: bool,pub likely_has_paywall: bool,}@@ -251,7 +250,6 @@ impl DisplayedWebpage {#[cfg(feature = "return_body")]body,ranking_signals: None,- score: None,likely_has_ads: webpage.likely_has_ads,likely_has_paywall: webpage.likely_has_paywall,rich_snippet,diff --git a/crates/core/src/searcher/api/mod.rs b/crates/core/src/searcher/api/mod.rsindex 1a9b5192..d6df9aa2 100644--- a/crates/core/src/searcher/api/mod.rs+++ b/crates/core/src/searcher/api/mod.rs@@ -27,7 +27,7 @@ use url::Url;use ahash::AHashMap as HashMap;use crate::bangs::{Bang, BangHit};-use crate::collector::{self, approx_count, Doc};+use crate::collector::{self, approx_count};use crate::config::{ApiConfig, ApiSpellCheck, ApiThresholds, CollectorConfig, WidgetsConfig};use crate::enum_map::EnumMap;use crate::image_store::Image;@@ -36,7 +36,8 @@ use crate::models::dual_encoder::DualEncoder;use crate::ranking::models::cross_encoder::CrossEncoderModel;use crate::ranking::pipeline::{PrecisionRankingWebpage, RankableWebpage, RecallRankingWebpage};use crate::ranking::{- bitvec_similarity, inbound_similarity, SignalCoefficient, SignalEnum, SignalScore,+ bitvec_similarity, inbound_similarity, SignalCalculation, SignalCoefficients, SignalEnum,+ SignalScore,};use crate::search_prettifier::{DisplayedSidebar, DisplayedWebpage, HighlightedSpellCorrection};use crate::web_spell::SpellChecker;@@ -55,6 +56,8 @@ use self::widget::WidgetManager;use super::{distributed, live, SearchQuery, SearchResult, WebsitesResult};+const NUM_PIPELINE_RANKING_RESULTS: usize = 300;+#[derive(Clone)]pub enum ScoredWebpagePointer {Normal(distributed::ScoredWebpagePointer),@@ -78,26 +81,34 @@ impl ScoredWebpagePointer {}impl RankableWebpage for ScoredWebpagePointer {- fn set_score(&mut self, score: f64) {- self.as_ranking_mut().set_score(score);+ fn set_raw_score(&mut self, score: f64) {+ self.as_ranking_mut().set_raw_score(score);+ }++ fn unboosted_score(&self) -> f64 {+ self.as_ranking().unboosted_score()}- fn boost(&self) -> Option<f64> {+ fn boost(&self) -> f64 {self.as_ranking().boost()}+ fn set_boost(&mut self, boost: f64) {+ self.as_ranking_mut().set_boost(boost)+ }+fn as_local_recall(&self) -> &crate::ranking::pipeline::LocalRecallRankingWebpage {self.as_ranking().as_local_recall()}- fn signals(&self) -> &EnumMap<SignalEnum, f64> {+ fn signals(&self) -> &EnumMap<SignalEnum, SignalCalculation> {match self {ScoredWebpagePointer::Normal(p) => p.website.signals(),ScoredWebpagePointer::Live(p) => p.website.signals(),}}- fn signals_mut(&mut self) -> &mut EnumMap<SignalEnum, f64> {+ fn signals_mut(&mut self) -> &mut EnumMap<SignalEnum, SignalCalculation> {match self {ScoredWebpagePointer::Normal(p) => p.website.signals_mut(),ScoredWebpagePointer::Live(p) => p.website.signals_mut(),@@ -107,7 +118,7 @@ impl RankableWebpage for ScoredWebpagePointer {impl collector::Doc for ScoredWebpagePointer {fn score(&self) -> f64 {- self.as_ranking().score()+ RankableWebpage::score(self)}fn hashes(&self) -> collector::Hashes {@@ -118,7 +129,7 @@ impl collector::Doc for ScoredWebpagePointer {pub fn add_ranking_signals(websites: &mut [DisplayedWebpage],pointers: &[ScoredWebpagePointer],- coeffs: &SignalCoefficient,+ coeffs: &SignalCoefficients,) {for (website, pointer) in websites.iter_mut().zip(pointers.iter()) {let mut signals = std::collections::HashMap::new();@@ -128,7 +139,7 @@ pub fn add_ranking_signals(signals.insert(signal.into(),SignalScore {- value: *signal_value,+ value: signal_value.score,coefficient: coeffs.get(&signal),},);@@ -469,12 +480,12 @@ whereasync fn combine_results(&self,- collector_config: CollectorConfig,+ query: &SearchQuery,initial_results: Vec<distributed::InitialSearchResultShard>,live_results: Vec<live::InitialSearchResultSplit>,- pipeline: RankingPipeline<ScoredWebpagePointer>,) -> (Vec<ScoredWebpagePointer>, bool) {- let mut collector = BucketCollector::new(pipeline.collector_top_n(), collector_config);+ let mut collector =+ BucketCollector::new(NUM_PIPELINE_RANKING_RESULTS, self.collector_config.clone());let initial_host_nodes = initial_results.iter()@@ -502,12 +513,10 @@ where.map(|(v, n)| (n, v)).collect::<HashMap<_, _>>();- let mut has_more = false;- for result in initial_results {- if result.local_result.has_more {- has_more = true;- }+ let mut num_results = 0;+ for result in initial_results {+ num_results += result.local_result.websites.len();for website in result.local_result.websites {let inbound = host_nodes.get(website.host_id())@@ -525,10 +534,7 @@ where}for result in live_results {- if result.local_result.has_more {- has_more = true;- }-+ num_results += result.local_result.websites.len();for website in result.local_result.websites {let inbound = host_nodes.get(website.host_id())@@ -545,14 +551,14 @@ where}}- let top_websites = collector+ let has_more = query.offset() + query.num_results() < num_results;++ let res = collector.into_sorted_vec(true).into_iter()- .take(pipeline.collector_top_n())+ .take(NUM_PIPELINE_RANKING_RESULTS).collect::<Vec<_>>();- let res = pipeline.apply(top_websites);-(res, has_more)}@@ -578,6 +584,57 @@ where}}+ async fn search_websites_approx_offsets(&self, query: &SearchQuery) -> Result<WebsitesResult> {+ let start = Instant::now();++ let search_query = SearchQuery {+ num_results: query.num_results + 1,+ ..query.clone()+ };++ let results = self+ .distributed_searcher+ .search_initial(&search_query)+ .await;++ let has_more_results = results+ .iter()+ .any(|res| res.local_result.websites.len() > query.num_results());++ let num_docs = results+ .iter()+ .map(|result| result.local_result.num_websites)+ .fold(approx_count::Count::Exact(0), |acc, count| acc + count);++ let (combined, _) = self.combine_results(query, results, vec![]).await;+ let combined: Vec<_> = combined.into_iter().take(query.num_results).collect();++ let mut retrieved_webpages: Vec<_> = self+ .retrieve_webpages(&query.query, &combined)+ .await+ .into_iter()+ .map(|webpage| webpage.into_retrieved_webpage())+ .map(|webpage| DisplayedWebpage::new(webpage, query))+ .collect();++ if query.return_ranking_signals {+ add_ranking_signals(+ &mut retrieved_webpages,+ &combined,+ &query.signal_coefficients(),+ );+ }++ let search_duration_ms = start.elapsed().as_millis();++ Ok(WebsitesResult {+ num_hits: num_docs,+ webpages: retrieved_webpages,+ search_duration_ms,+ has_more_results,+ })+ }+async fn search_websites(&self, query: &SearchQuery) -> Result<WebsitesResult> {let start = Instant::now();@@ -585,22 +642,17 @@ wherereturn Err(distributed::Error::EmptyQuery.into());}- let mut search_query = query.clone();- let inbound_scorer = self.inbound_scorer(&search_query).await;-- let top_n = search_query.num_results;+ if query.offset() + query.num_results() > NUM_PIPELINE_RANKING_RESULTS {+ // this is most likely a bot+ // let's not spend too much time correctly offsetting+ranking results+ return self.search_websites_approx_offsets(query).await;+ }- // This pipeline should be created before the first search is performed- // so the query knows how many results to fetch from the indices- let recall_pipeline: RankingPipeline<ScoredWebpagePointer> =- RankingPipeline::<ScoredWebpagePointer>::recall_stage(- &mut search_query,- inbound_scorer,- self.lambda_model.clone(),- self.dual_encoder.clone(),- self.collector_config.clone(),- top_n,- );+ let search_query = SearchQuery {+ num_results: NUM_PIPELINE_RANKING_RESULTS,+ page: 0,+ ..query.clone()+ };let (initial_results, live_results) = tokio::join!(self.distributed_searcher.search_initial(&search_query),@@ -613,38 +665,45 @@ where.fold(approx_count::Count::Exact(0), |acc, count| acc + count);let (top_websites, has_more_results) = self- .combine_results(- self.collector_config.clone(),- initial_results,- live_results.unwrap_or_default(),- recall_pipeline,- )+ .combine_results(query, initial_results, live_results.unwrap_or_default()).await;- let retrieved_webpages = self- .retrieve_webpages(&search_query.query, &top_websites)- .await;-- let mut search_query = SearchQuery {- page: 0,- ..query.clone()- };+ let inbound_scorer = self.inbound_scorer(query).await;- let reranking_pipeline: RankingPipeline<PrecisionRankingWebpage> =- RankingPipeline::<PrecisionRankingWebpage>::reranker(- &mut search_query,- self.cross_encoder.clone(),+ let pipeline: RankingPipeline<ScoredWebpagePointer> =+ RankingPipeline::<ScoredWebpagePointer>::recall_stage(+ query,+ inbound_scorer,self.lambda_model.clone(),- self.collector_config.clone(),- query.num_results,- )?;+ self.dual_encoder.clone(),+ );- let retrieved_webpages = reranking_pipeline.apply(retrieved_webpages);+ let top_websites = pipeline.apply(top_websites, query);++ let mut retrieved_webpages = self.retrieve_webpages(&query.query, &top_websites).await;++ if let Some(cross_encoder) = self.cross_encoder.clone() {+ if query.page < 2 {+ let query = SearchQuery {+ page: 1,+ ..query.clone()+ };++ let reranking_pipeline: RankingPipeline<PrecisionRankingWebpage> =+ RankingPipeline::<PrecisionRankingWebpage>::reranker(+ &query,+ cross_encoder,+ self.lambda_model.clone(),+ );++ retrieved_webpages = reranking_pipeline.apply(retrieved_webpages, &query);+ }+ }let mut retrieved_webpages: Vec<_> = retrieved_webpages.into_iter().map(|webpage| webpage.into_retrieved_webpage())- .map(|webpage| DisplayedWebpage::new(webpage, &search_query))+ .map(|webpage| DisplayedWebpage::new(webpage, query)).collect();if retrieved_webpages.len() != top_websites.len() {@@ -655,14 +714,10 @@ whereadd_ranking_signals(&mut retrieved_webpages,&top_websites,- &search_query.signal_coefficients(),+ &query.signal_coefficients(),);}- for (website, pointer) in retrieved_webpages.iter_mut().zip(top_websites.iter()) {- website.score = Some(pointer.score());- }-let search_duration_ms = start.elapsed().as_millis();Ok(WebsitesResult {diff --git a/crates/core/src/searcher/local.rs b/crates/core/src/searcher/local.rsindex aa73952c..21ccaa38 100644--- a/crates/core/src/searcher/local.rs+++ b/crates/core/src/searcher/local.rs@@ -29,9 +29,9 @@ use crate::query::Query;use crate::ranking::models::lambdamart::LambdaMART;use crate::ranking::models::linear::LinearRegression;use crate::ranking::pipeline::{- LocalRecallRankingWebpage, PrecisionRankingWebpage, RankingPipeline, RecallRankingWebpage,+ LocalRecallRankingWebpage, PrecisionRankingWebpage, RankableWebpage, RecallRankingWebpage,};-use crate::ranking::{Ranker, SignalComputer, SignalEnum, SignalScore};+use crate::ranking::{LocalRanker, SignalComputer, SignalEnum, SignalScore};use crate::search_ctx::Ctx;use crate::search_prettifier::DisplayedWebpage;use crate::{inverted_index, live_index, Result};@@ -121,7 +121,6 @@ wherestruct InvertedIndexResult {webpages: Vec<LocalRecallRankingWebpage>,num_hits: approx_count::Count,- has_more: bool,}impl<I> LocalSearcher<I>@@ -173,8 +172,8 @@ whereguard: &G,de_rank_similar: bool,computer: SignalComputer,- ) -> Result<Ranker> {- let mut ranker = Ranker::new(+ ) -> Result<LocalRanker> {+ let mut ranker = LocalRanker::new(computer,guard.inverted_index().columnfield_reader(),self.collector_config.clone(),@@ -198,16 +197,7 @@ wherequery: &SearchQuery,de_rank_similar: bool,) -> Result<InvertedIndexResult> {- let mut query = query.clone();- let pipeline: RankingPipeline<LocalRecallRankingWebpage> =- RankingPipeline::<LocalRecallRankingWebpage>::recall_stage(- &mut query,- self.lambda_model.clone(),- self.dual_encoder.clone(),- self.collector_config.clone(),- 100,- );- let parsed_query = self.parse_query(ctx, guard, &query)?;+ let parsed_query = self.parse_query(ctx, guard, query)?;let mut computer = SignalComputer::new(Some(&parsed_query));@@ -241,15 +231,9 @@ where&columnfield_reader,)?;- let pipe_top_n = pipeline.top_n;- let has_more = ranking_websites.len() > pipe_top_n;-- let ranking_websites = pipeline.apply(ranking_websites);-Ok(InvertedIndexResult {webpages: ranking_websites,num_hits: res.num_websites,- has_more,})}@@ -270,7 +254,6 @@ whereOk(InitialWebsiteResult {websites: inverted_index_result.webpages,num_websites: inverted_index_result.num_hits,- has_more: inverted_index_result.has_more,})}@@ -295,27 +278,7 @@ whereuse std::time::Instant;let start = Instant::now();- let mut search_query = query.clone();-- let pipeline = {- use crate::ranking::models::cross_encoder::CrossEncoderModel;- match CrossEncoderModel::open("data/cross_encoder") {- Ok(model) => RankingPipeline::reranker::<CrossEncoderModel>(- &mut search_query,- Some(Arc::new(model)),- None,- self.collector_config.clone(),- query.num_results,- )?,- Err(_) => RankingPipeline::reranker::<CrossEncoderModel>(- &mut search_query,- None,- None,- self.collector_config.clone(),- query.num_results,- )?,- }- };+ let search_query = query.clone();let search_result = self.search_initial(&search_query, true)?;@@ -335,13 +298,7 @@ where}).collect();- let search_len = websites.len();-- let top_websites = pipeline.apply(websites);-- let has_more_results = search_len != top_websites.len();-- let pointers: Vec<_> = top_websites+ let pointers: Vec<_> = websites.iter().map(|website| website.ranking().pointer().clone()).collect();@@ -355,15 +312,15 @@ where.map(|webpage| DisplayedWebpage::new(webpage, query)).collect();- for (webpage, ranking) in webpages.iter_mut().zip(top_websites) {+ for (webpage, ranking) in webpages.iter_mut().zip(websites) {let mut ranking_signals = HashMap::new();for signal in SignalEnum::all() {- if let Some(score) = ranking.ranking().signals().get(signal) {+ if let Some(calc) = ranking.ranking().signals().get(signal) {ranking_signals.insert(signal.into(),SignalScore {- value: *score,+ value: calc.score,coefficient: coefficients.get(&signal),},);@@ -377,7 +334,8 @@ wherenum_hits: search_result.num_websites,webpages,search_duration_ms: start.elapsed().as_millis(),- has_more_results,+ has_more_results: (search_result.num_websites.as_u64() as usize)+ > query.offset() + query.num_results(),})}diff --git a/crates/core/src/searcher/mod.rs b/crates/core/src/searcher/mod.rsindex d0f34814..ad6ec8d1 100644--- a/crates/core/src/searcher/mod.rs+++ b/crates/core/src/searcher/mod.rs@@ -31,7 +31,7 @@ use crate::{bangs::BangHit,collector::approx_count::Count,config::defaults,- ranking::{pipeline::LocalRecallRankingWebpage, SignalCoefficient},+ ranking::{pipeline::LocalRecallRankingWebpage, SignalCoefficients},search_prettifier::DisplayedWebpage,webpage::region::Region,};@@ -80,14 +80,13 @@ pub struct SearchQuery {pub return_body: Option<ReturnBody>,pub return_structured_data: bool,- pub signal_coefficients: SignalCoefficient,+ pub signal_coefficients: SignalCoefficients,}-#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]+#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]pub struct InitialWebsiteResult {pub num_websites: Count,pub websites: Vec<LocalRecallRankingWebpage>,- pub has_more: bool,}impl Default for SearchQuery {@@ -117,7 +116,7 @@ impl SearchQuery {self.query.is_empty()}- pub fn signal_coefficients(&self) -> SignalCoefficient {+ pub fn signal_coefficients(&self) -> SignalCoefficients {self.signal_coefficients.clone()}@@ -134,4 +133,16 @@ impl SearchQuery {rankings}++ pub fn text(&self) -> &str {+ &self.query+ }++ pub fn offset(&self) -> usize {+ self.page * self.num_results+ }++ pub fn num_results(&self) -> usize {+ self.num_results+ }}diff --git a/crates/core/src/webgraph/shortest_path.rs b/crates/core/src/webgraph/shortest_path.rsindex 16071736..536fcc3b 100644--- a/crates/core/src/webgraph/shortest_path.rs+++ b/crates/core/src/webgraph/shortest_path.rs@@ -25,6 +25,8 @@ pub trait ShortestPaths {fn distances(&self, source: Node) -> BTreeMap<Node, u8>;fn raw_distances(&self, source: NodeID) -> BTreeMap<NodeID, u8>;fn raw_distances_with_max(&self, source: NodeID, max_dist: u8) -> BTreeMap<NodeID, u8>;+ fn raw_reversed_distances_with_max(&self, source: NodeID, max_dist: u8)+ -> BTreeMap<NodeID, u8>;fn raw_reversed_distances(&self, source: NodeID) -> BTreeMap<NodeID, u8>;fn reversed_distances(&self, source: Node) -> BTreeMap<Node, u8>;}@@ -113,6 +115,19 @@ impl ShortestPaths for Webgraph {)}+ fn raw_reversed_distances_with_max(+ &self,+ source: NodeID,+ max_dist: u8,+ ) -> BTreeMap<NodeID, u8> {+ dijkstra_multi(+ &[source],+ |node| self.raw_ingoing_edges(&node, super::EdgeLimit::Unlimited),+ |edge| edge.from.node(),+ Some(max_dist),+ )+ }+fn reversed_distances(&self, source: Node) -> BTreeMap<Node, u8> {self.raw_reversed_distances(source.id()).into_iter()diff --git a/crates/optics-lsp/Cargo.lock b/crates/optics-lsp/Cargo.lockindex 72f30d53..e7f00e93 100644--- a/crates/optics-lsp/Cargo.lock+++ b/crates/optics-lsp/Cargo.lock@@ -456,7 +456,6 @@ dependencies = ["lalrpop","lalrpop-util","logos",- "once_cell","serde","thiserror","utoipa",diff --git a/frontend/src/lib/api/index.ts b/frontend/src/lib/api/index.tsindex 979a3893..a11a8944 100644--- a/frontend/src/lib/api/index.ts+++ b/frontend/src/lib/api/index.ts@@ -272,7 +272,6 @@ export type DisplayedWebpage = {prettyUrl: string;rankingSignals?: {};richSnippet?: RichSnippet;- score?: number;site: string;snippet: Snippet;structuredData?: StructuredData[];@@ -373,9 +372,11 @@ export type SidebarQuery = {export type SignalEnumDiscriminants =| 'bm25_f'| 'bm25_title'+ | 'title_coverage'| 'bm25_title_bigrams'| 'bm25_title_trigrams'| 'bm25_clean_body'+ | 'clean_body_coverage'| 'bm25_clean_body_bigrams'| 'bm25_clean_body_trigrams'| 'bm25_stemmed_title'@@ -411,13 +412,19 @@ export type SignalEnumDiscriminants =| 'url_slashes'| 'link_density'| 'title_embedding_similarity'- | 'keyword_embedding_similarity';+ | 'keyword_embedding_similarity'+ | 'has_ads'+ | 'num_query_terms'+ | 'min_title_slop'+ | 'min_clean_body_slop';export const SIGNAL_ENUM_DISCRIMINANTS = ['bm25_f','bm25_title',+ 'title_coverage','bm25_title_bigrams','bm25_title_trigrams','bm25_clean_body',+ 'clean_body_coverage','bm25_clean_body_bigrams','bm25_clean_body_trigrams','bm25_stemmed_title',@@ -454,6 +461,10 @@ export const SIGNAL_ENUM_DISCRIMINANTS = ['link_density','title_embedding_similarity','keyword_embedding_similarity',+ 'has_ads',+ 'num_query_terms',+ 'min_title_slop',+ 'min_clean_body_slop',] satisfies SignalEnumDiscriminants[];export type SignalScore = {coefficient: number;diff --git a/frontend/src/routes/search/Discussions.svelte b/frontend/src/routes/search/Discussions.svelteindex 856e1d54..1d685a41 100644--- a/frontend/src/routes/search/Discussions.svelte+++ b/frontend/src/routes/search/Discussions.svelte@@ -10,17 +10,17 @@let showMore = false;$: {- let medianScore = 0;- if (discussions) {- const scores = discussions.map((d) => d.score || 0);- scores.sort((a, b) => a - b);- const mid = Math.floor(scores.length / 2);- medianScore = scores.length % 2 !== 0 ? scores[mid] : (scores[mid - 1] + scores[mid]) / 2;- }+ // let medianScore = 0;+ // if (discussions) {+ // const scores = discussions.map((d) => d.score || 0);+ // scores.sort((a, b) => a - b);+ // const mid = Math.floor(scores.length / 2);+ // medianScore = scores.length % 2 !== 0 ? scores[mid] : (scores[mid - 1] + scores[mid]) / 2;+ // }- if (medianScore < 0.1) {- discussions = undefined;- }+ // if (medianScore < 0.1) {+ // discussions = undefined;+ // }let numDiscussions = discussions?.length || 0;if (numDiscussions < 5) {diff --git a/frontend/src/routes/search/Modal.svelte b/frontend/src/routes/search/Modal.svelteindex 29e3b621..a85ac60e 100644--- a/frontend/src/routes/search/Modal.svelte+++ b/frontend/src/routes/search/Modal.svelte@@ -78,16 +78,22 @@.with('fetch_time_ms', () => 'fetchTime' as const).with('update_timestamp', () => 'updateTimestamp' as const).with('tracker_score', () => 'trackerScore' as const)+ .with('has_ads', () => 'trackerScore' as const).with('region', () => 'region' as const).with('query_centrality', () => 'queryCentrality' as const).with('inbound_similarity', () => 'inboundSimilarity' as const).with('lambda_mart', () => undefined)+ .with('num_query_terms', () => undefined).with('url_digits', () => 'urlSymbols' as const).with('url_slashes', () => 'urlSymbols' as const).with('link_density', () => 'linkDensity' as const).with('title_embedding_similarity', () => 'title' as const).with('keyword_embedding_similarity', () => 'keywords' as const).with('bm25_f', () => 'combinedText' as const)+ .with('min_clean_body_slop', () => 'body' as const)+ .with('min_title_slop', () => 'title' as const)+ .with('title_coverage', () => 'title' as const)+ .with('clean_body_coverage', () => 'body' as const).exhaustive();};