Skip to content

web search done right

giant ranking pipeline refactor to separate ranking stages from sorting/offset logic

Mikkel Denkerc6119e31d734
diff --git a/crates/core/src/api/search.rs b/crates/core/src/api/search.rs
index ab7c2af0..0ae7b9a2 100644
--- a/crates/core/src/api/search.rs
+++ b/crates/core/src/api/search.rs
@@ -17,7 +17,7 @@
use crate::{
config::defaults,
enum_map::EnumMap,
- ranking::{SignalCoefficient, SignalEnum, SignalEnumDiscriminants},
+ ranking::{SignalCoefficients, SignalEnum, SignalEnumDiscriminants},
};
use http::StatusCode;
use optics::{HostRankings, Optic};
@@ -95,7 +95,7 @@ impl TryFrom<ApiSearchQuery> for SearchQuery {
None
};
- let signal_coefficients: Option<SignalCoefficient> =
+ let signal_coefficients: Option<SignalCoefficients> =
api.signal_coefficients.map(|coefficients| {
coefficients
.into_iter()
diff --git a/crates/core/src/collector/approx_count.rs b/crates/core/src/collector/approx_count.rs
index 85d95b51..36e3ce6c 100644
--- a/crates/core/src/collector/approx_count.rs
+++ b/crates/core/src/collector/approx_count.rs
@@ -66,6 +66,12 @@ impl Count {
| (Count::Approximate(a), Count::Approximate(b)) => Count::Approximate(a + b),
}
}
+
+ pub fn as_u64(&self) -> u64 {
+ match self {
+ Count::Exact(count) | Count::Approximate(count) => *count,
+ }
+ }
}
impl Add for Count {
diff --git a/crates/core/src/collector/top_docs.rs b/crates/core/src/collector/top_docs.rs
index 886ab995..c1cff34a 100644
--- a/crates/core/src/collector/top_docs.rs
+++ b/crates/core/src/collector/top_docs.rs
@@ -426,10 +426,12 @@ where
}
}
- Ok(collector
- .into_sorted_vec(self.top_docs.de_rank_similar)
+ let docs = collector.into_sorted_vec(self.top_docs.de_rank_similar);
+
+ Ok(docs
.into_iter()
.skip(self.top_docs.offset)
+ .take(self.top_docs.top_n)
.map(|doc| WebpagePointer {
score: doc.score,
hashes: doc.hashes,
diff --git a/crates/core/src/inverted_index/mod.rs b/crates/core/src/inverted_index/mod.rs
index 1fef548c..3eec73f6 100644
--- a/crates/core/src/inverted_index/mod.rs
+++ b/crates/core/src/inverted_index/mod.rs
@@ -237,7 +237,7 @@ mod tests {
collector::MainCollector,
config::CollectorConfig,
query::Query,
- ranking::{Ranker, SignalComputer},
+ ranking::{LocalRanker, SignalComputer},
search_ctx::Ctx,
searcher::SearchQuery,
webgraph::{Edge, NodeDatum},
@@ -282,7 +282,7 @@ mod tests {
)
.expect("Failed to parse query");
- let ranker = Ranker::new(
+ let ranker = LocalRanker::new(
SignalComputer::new(Some(&query)),
ctx.columnfield_reader.clone(),
CollectorConfig::default(),
@@ -314,7 +314,7 @@ mod tests {
index.commit().expect("failed to commit index");
let ctx = index.local_search_ctx();
- let ranker = Ranker::new(
+ let ranker = LocalRanker::new(
SignalComputer::new(Some(&query)),
ctx.columnfield_reader.clone(),
CollectorConfig::default(),
@@ -363,7 +363,7 @@ mod tests {
)
.expect("Failed to parse query");
- let ranker = Ranker::new(
+ let ranker = LocalRanker::new(
SignalComputer::new(Some(&query)),
ctx.columnfield_reader.clone(),
CollectorConfig::default(),
@@ -410,7 +410,7 @@ mod tests {
&index,
)
.expect("Failed to parse query");
- let ranker = Ranker::new(
+ let ranker = LocalRanker::new(
SignalComputer::new(Some(&query)),
ctx.columnfield_reader.clone(),
CollectorConfig::default(),
@@ -458,7 +458,7 @@ mod tests {
&index,
)
.expect("Failed to parse query");
- let ranker = Ranker::new(
+ let ranker = LocalRanker::new(
SignalComputer::new(Some(&query)),
ctx.columnfield_reader.clone(),
CollectorConfig::default(),
@@ -536,7 +536,7 @@ mod tests {
&index,
)
.expect("Failed to parse query");
- let ranker = Ranker::new(
+ let ranker = LocalRanker::new(
SignalComputer::new(Some(&query)),
ctx.columnfield_reader.clone(),
CollectorConfig::default(),
@@ -594,7 +594,7 @@ mod tests {
&index,
)
.expect("Failed to parse query");
- let ranker = Ranker::new(
+ let ranker = LocalRanker::new(
SignalComputer::new(Some(&query)),
ctx.columnfield_reader.clone(),
CollectorConfig::default(),
@@ -641,7 +641,7 @@ mod tests {
&index,
)
.expect("Failed to parse query");
- let ranker = Ranker::new(
+ let ranker = LocalRanker::new(
SignalComputer::new(Some(&query)),
ctx.columnfield_reader.clone(),
CollectorConfig::default(),
@@ -717,7 +717,7 @@ mod tests {
&index,
)
.expect("Failed to parse query");
- let ranker = Ranker::new(
+ let ranker = LocalRanker::new(
SignalComputer::new(Some(&query)),
ctx.columnfield_reader.clone(),
CollectorConfig::default(),
@@ -744,7 +744,7 @@ mod tests {
&index,
)
.expect("Failed to parse query");
- let ranker = Ranker::new(
+ let ranker = LocalRanker::new(
SignalComputer::new(Some(&query)),
ctx.columnfield_reader.clone(),
CollectorConfig::default(),
@@ -777,7 +777,7 @@ mod tests {
index.commit().expect("failed to commit index");
let ctx = index.local_search_ctx();
- let ranker = Ranker::new(
+ let ranker = LocalRanker::new(
SignalComputer::new(Some(&query)),
ctx.columnfield_reader.clone(),
CollectorConfig::default(),
@@ -824,7 +824,7 @@ mod tests {
&index,
)
.expect("Failed to parse query");
- let ranker = Ranker::new(
+ let ranker = LocalRanker::new(
SignalComputer::new(Some(&query)),
ctx.columnfield_reader.clone(),
CollectorConfig::default(),
@@ -875,7 +875,7 @@ mod tests {
&index,
)
.expect("Failed to parse query");
- let ranker = Ranker::new(
+ let ranker = LocalRanker::new(
SignalComputer::new(Some(&query)),
ctx.columnfield_reader.clone(),
CollectorConfig::default(),
@@ -1048,7 +1048,7 @@ mod tests {
)
.expect("Failed to parse query");
- let ranker = Ranker::new(
+ let ranker = LocalRanker::new(
SignalComputer::new(Some(&query)),
ctx.columnfield_reader.clone(),
CollectorConfig::default(),
@@ -1119,7 +1119,7 @@ mod tests {
..Default::default()
};
- let ranker = Ranker::new(
+ let ranker = LocalRanker::new(
SignalComputer::new(Some(&query)),
ctx.columnfield_reader.clone(),
collector_config,
@@ -1170,7 +1170,7 @@ mod tests {
)
.expect("Failed to parse query");
- let ranker = Ranker::new(
+ let ranker = LocalRanker::new(
SignalComputer::new(Some(&query)),
ctx.columnfield_reader.clone(),
CollectorConfig::default(),
@@ -1228,7 +1228,7 @@ mod tests {
)
.expect("Failed to parse query");
- let ranker = Ranker::new(
+ let ranker = LocalRanker::new(
SignalComputer::new(Some(&query)),
ctx.columnfield_reader.clone(),
CollectorConfig::default(),
diff --git a/crates/core/src/query/mod.rs b/crates/core/src/query/mod.rs
index 3a857146..ea81f1e9 100644
--- a/crates/core/src/query/mod.rs
+++ b/crates/core/src/query/mod.rs
@@ -17,7 +17,7 @@
use crate::{
inverted_index::InvertedIndex,
query::parser::TermCompound,
- ranking::SignalCoefficient,
+ ranking::SignalCoefficients,
schema::text_field,
search_ctx::Ctx,
searcher::SearchQuery,
@@ -53,7 +53,7 @@ pub struct Query {
optics: Vec<Optic>,
top_n: usize,
count_results_exact: bool,
- signal_coefficients: SignalCoefficient,
+ signal_coefficients: SignalCoefficients,
lang: Option<whatlang::Lang>,
}
@@ -182,7 +182,7 @@ impl Query {
&self.host_rankings
}
- pub fn signal_coefficients(&self) -> SignalCoefficient {
+ pub fn signal_coefficients(&self) -> SignalCoefficients {
self.signal_coefficients.clone()
}
diff --git a/crates/core/src/ranking/core/computer/mod.rs b/crates/core/src/ranking/core/computer/mod.rs
index 7ac030db..ebaed70c 100644
--- a/crates/core/src/ranking/core/computer/mod.rs
+++ b/crates/core/src/ranking/core/computer/mod.rs
@@ -40,7 +40,7 @@ use crate::webpage::region::RegionCount;
use crate::ranking::bm25::MultiBm25Weight;
use crate::ranking::models::linear::LinearRegression;
-use super::{ComputedSignal, Signal, SignalCoefficient, SignalEnum};
+use super::{ComputedSignal, Signal, SignalCoefficients, SignalEnum};
mod order;
pub use order::SignalComputeOrder;
@@ -192,7 +192,7 @@ impl QueryData {
pub struct SignalComputer {
query_data: Option<QueryData>,
- query_signal_coefficients: Option<SignalCoefficient>,
+ query_signal_coefficients: Option<SignalCoefficients>,
segment_reader: Option<RefCell<SegmentReader>>,
fetch_time_ms_cache: Vec<f64>,
update_time_cache: Vec<f64>,
@@ -474,9 +474,9 @@ impl SignalComputer {
.filter_map(|signal| {
signal
.precompute(webpage, self)
- .map(|score| ComputedSignal { signal, score })
+ .map(|calc| ComputedSignal { signal, calc })
})
- .map(|computed| self.coefficient(&computed.signal) * computed.score)
+ .map(|computed| self.coefficient(&computed.signal) * computed.calc.score)
.sum()
}
diff --git a/crates/core/src/ranking/core/computer/order.rs b/crates/core/src/ranking/core/computer/order.rs
index 6b167257..53c93578 100644
--- a/crates/core/src/ranking/core/computer/order.rs
+++ b/crates/core/src/ranking/core/computer/order.rs
@@ -18,7 +18,7 @@ use tantivy::DocId;
use crate::{
enum_map::EnumMap,
- ranking::{ComputedSignal, Signal, SignalEnum},
+ ranking::{ComputedSignal, Signal, SignalCalculation, SignalEnum},
schema::{text_field::TextField, TextFieldEnum},
};
@@ -71,13 +71,16 @@ impl SignalComputeOrder {
.chain(self.other_signals.iter().map(move |signal| {
signal
.compute(doc, signal_computer)
- .map(|score| ComputedSignal {
+ .map(|calc| ComputedSignal {
signal: *signal,
- score,
+ calc,
})
.unwrap_or_else(|| ComputedSignal {
signal: *signal,
- score: 0.0,
+ calc: SignalCalculation {
+ score: 0.0,
+ value: 0.0,
+ },
})
}))
}
@@ -112,14 +115,14 @@ impl NGramComputeOrder {
self.signals.iter().map(|(_, s)| s).map(move |signal| {
signal
.compute(doc, signal_computer)
- .map(|score| ComputedSignal {
+ .map(|calc| ComputedSignal {
signal: *signal,
- score,
+ calc,
})
.map(|mut c| {
- c.score *= NGRAM_DAMPENING.powi(hits);
+ c.calc.score *= NGRAM_DAMPENING.powi(hits);
- if c.score > 0.0 {
+ if c.calc.score > 0.0 {
hits += 1;
}
@@ -127,7 +130,10 @@ impl NGramComputeOrder {
})
.unwrap_or_else(|| ComputedSignal {
signal: *signal,
- score: 0.0,
+ calc: SignalCalculation {
+ value: 0.0,
+ score: 0.0,
+ },
})
})
}
diff --git a/crates/core/src/ranking/core/mod.rs b/crates/core/src/ranking/core/mod.rs
index 3ca4eb8c..9d818502 100644
--- a/crates/core/src/ranking/core/mod.rs
+++ b/crates/core/src/ranking/core/mod.rs
@@ -49,23 +49,23 @@ impl FromStr for SignalEnumDiscriminants {
}
#[derive(Debug, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, Clone)]
-pub struct SignalCoefficient {
+pub struct SignalCoefficients {
map: EnumMap<SignalEnum, f64>,
}
-impl Default for SignalCoefficient {
+impl Default for SignalCoefficients {
fn default() -> Self {
Self::new(SignalEnum::all().map(|signal| (signal, signal.default_coefficient())))
}
}
-impl From<EnumMap<SignalEnum, f64>> for SignalCoefficient {
+impl From<EnumMap<SignalEnum, f64>> for SignalCoefficients {
fn from(map: EnumMap<SignalEnum, f64>) -> Self {
Self { map }
}
}
-impl SignalCoefficient {
+impl SignalCoefficients {
pub fn get(&self, signal: &SignalEnum) -> f64 {
self.map
.get(*signal)
@@ -83,7 +83,7 @@ impl SignalCoefficient {
Self { map }
}
- pub fn merge_add(&mut self, coeffs: SignalCoefficient) {
+ pub fn merge_add(&mut self, coeffs: SignalCoefficients) {
for signal in SignalEnum::all() {
if let Some(coeff) = coeffs.map.get(signal).copied() {
match self.map.get_mut(signal) {
@@ -96,7 +96,7 @@ impl SignalCoefficient {
}
}
- pub fn merge_overwrite(&mut self, coeffs: SignalCoefficient) {
+ pub fn merge_overwrite(&mut self, coeffs: SignalCoefficients) {
for signal in SignalEnum::all() {
if let Some(coeff) = coeffs.map.get(signal).copied() {
match self.map.get_mut(signal) {
@@ -110,10 +110,25 @@ impl SignalCoefficient {
}
}
+#[derive(Debug, Clone, Copy, bincode::Encode, bincode::Decode)]
+pub struct SignalCalculation {
+ pub value: f64,
+ pub score: f64,
+}
+
+impl SignalCalculation {
+ pub fn new_symmetrical(val: f64) -> Self {
+ Self {
+ value: val,
+ score: val,
+ }
+ }
+}
+
#[derive(Debug, Clone, Copy)]
pub struct ComputedSignal {
pub signal: SignalEnum,
- pub score: f64,
+ pub calc: SignalCalculation,
}
#[derive(
diff --git a/crates/core/src/ranking/core/non_text.rs b/crates/core/src/ranking/core/non_text.rs
index ec52a6db..70cd55e6 100644
--- a/crates/core/src/ranking/core/non_text.rs
+++ b/crates/core/src/ranking/core/non_text.rs
@@ -16,7 +16,7 @@
use tantivy::DocId;
-use super::{Signal, SignalComputer};
+use super::{Signal, SignalCalculation, SignalComputer};
use crate::{
schema::{self, Field},
webpage::Webpage,
@@ -117,11 +117,11 @@ impl Signal for HostCentrality {
))
}
- fn precompute(self, webpage: &Webpage, _: &SignalComputer) -> Option<f64> {
- Some(webpage.host_centrality)
+ fn precompute(self, webpage: &Webpage, _: &SignalComputer) -> Option<SignalCalculation> {
+ Some(SignalCalculation::new_symmetrical(webpage.host_centrality))
}
- fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {
+ fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<SignalCalculation> {
let seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();
let numericalfield_reader = seg_reader.numericalfield_reader().get_field_reader(doc);
@@ -129,7 +129,7 @@ impl Signal for HostCentrality {
.get(self.as_numericalfield().unwrap())
.and_then(|v| v.as_f64())
.unwrap();
- Some(val)
+ Some(SignalCalculation::new_symmetrical(val))
}
}
@@ -157,11 +157,14 @@ impl Signal for HostCentralityRank {
))
}
- fn precompute(self, webpage: &Webpage, _: &SignalComputer) -> Option<f64> {
- Some(score_rank(webpage.host_centrality_rank as f64))
+ fn precompute(self, webpage: &Webpage, _: &SignalComputer) -> Option<SignalCalculation> {
+ Some(SignalCalculation {
+ value: webpage.host_centrality_rank as f64,
+ score: score_rank(webpage.host_centrality_rank as f64),
+ })
}
- fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {
+ fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<SignalCalculation> {
let seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();
let numericalfield_reader = seg_reader.numericalfield_reader().get_field_reader(doc);
@@ -169,7 +172,10 @@ impl Signal for HostCentralityRank {
.get(self.as_numericalfield().unwrap())
.and_then(|v| v.as_u64())
.unwrap();
- Some(score_rank(val as f64))
+ Some(SignalCalculation {
+ value: val as f64,
+ score: score_rank(val as f64),
+ })
}
}
@@ -197,11 +203,11 @@ impl Signal for PageCentrality {
))
}
- fn precompute(self, webpage: &Webpage, _: &SignalComputer) -> Option<f64> {
- Some(webpage.page_centrality)
+ fn precompute(self, webpage: &Webpage, _: &SignalComputer) -> Option<SignalCalculation> {
+ Some(SignalCalculation::new_symmetrical(webpage.page_centrality))
}
- fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {
+ fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<SignalCalculation> {
let seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();
let numericalfield_reader = seg_reader.numericalfield_reader().get_field_reader(doc);
@@ -209,7 +215,7 @@ impl Signal for PageCentrality {
.get(self.as_numericalfield().unwrap())
.and_then(|v| v.as_f64())
.unwrap();
- Some(val)
+ Some(SignalCalculation::new_symmetrical(val))
}
}
@@ -237,11 +243,14 @@ impl Signal for PageCentralityRank {
))
}
- fn precompute(self, webpage: &Webpage, _: &SignalComputer) -> Option<f64> {
- Some(score_rank(webpage.page_centrality_rank as f64))
+ fn precompute(self, webpage: &Webpage, _: &SignalComputer) -> Option<SignalCalculation> {
+ Some(SignalCalculation {
+ value: webpage.page_centrality_rank as f64,
+ score: score_rank(webpage.page_centrality_rank as f64),
+ })
}
- fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {
+ fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<SignalCalculation> {
let seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();
let numericalfield_reader = seg_reader.numericalfield_reader().get_field_reader(doc);
@@ -249,7 +258,10 @@ impl Signal for PageCentralityRank {
.get(self.as_numericalfield().unwrap())
.and_then(|v| v.as_u64())
.unwrap();
- Some(score_rank(val as f64))
+ Some(SignalCalculation {
+ value: val as f64,
+ score: score_rank(val as f64),
+ })
}
}
@@ -275,11 +287,13 @@ impl Signal for IsHomepage {
Some(Field::Numerical(schema::numerical_field::IsHomepage.into()))
}
- fn precompute(self, webpage: &Webpage, _: &SignalComputer) -> Option<f64> {
- Some(webpage.html.is_homepage().into())
+ fn precompute(self, webpage: &Webpage, _: &SignalComputer) -> Option<SignalCalculation> {
+ Some(SignalCalculation::new_symmetrical(
+ webpage.html.is_homepage().into(),
+ ))
}
- fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {
+ fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<SignalCalculation> {
let seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();
let numericalfield_reader = seg_reader.numericalfield_reader().get_field_reader(doc);
@@ -289,9 +303,9 @@ impl Signal for IsHomepage {
.unwrap();
if val {
- Some(1.0)
+ Some(SignalCalculation::new_symmetrical(1.0))
} else {
- Some(0.0)
+ Some(SignalCalculation::new_symmetrical(0.0))
}
}
}
@@ -320,16 +334,25 @@ impl Signal for FetchTimeMs {
))
}
- fn precompute(self, webpage: &Webpage, signal_computer: &SignalComputer) -> Option<f64> {
+ fn precompute(
+ self,
+ webpage: &Webpage,
+ signal_computer: &SignalComputer,
+ ) -> Option<SignalCalculation> {
let fetch_time_ms = webpage.fetch_time_ms as usize;
- if fetch_time_ms >= signal_computer.fetch_time_ms_cache().len() {
- Some(0.0)
+ let score = if fetch_time_ms >= signal_computer.fetch_time_ms_cache().len() {
+ 0.0
} else {
- Some(signal_computer.fetch_time_ms_cache()[fetch_time_ms])
- }
+ signal_computer.fetch_time_ms_cache()[fetch_time_ms]
+ };
+
+ Some(SignalCalculation {
+ value: fetch_time_ms as f64,
+ score,
+ })
}
- fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {
+ fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<SignalCalculation> {
let seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();
let numericalfield_reader = seg_reader.numericalfield_reader().get_field_reader(doc);
@@ -338,11 +361,16 @@ impl Signal for FetchTimeMs {
.and_then(|v| v.as_u64())
.unwrap() as usize;
- if fetch_time_ms >= signal_computer.fetch_time_ms_cache().len() {
- Some(0.0)
+ let score = if fetch_time_ms >= signal_computer.fetch_time_ms_cache().len() {
+ 0.0
} else {
- Some(signal_computer.fetch_time_ms_cache()[fetch_time_ms])
- }
+ signal_computer.fetch_time_ms_cache()[fetch_time_ms]
+ };
+
+ Some(SignalCalculation {
+ value: fetch_time_ms as f64,
+ score,
+ })
}
}
@@ -370,17 +398,26 @@ impl Signal for UpdateTimestamp {
))
}
- fn precompute(self, webpage: &Webpage, signal_computer: &SignalComputer) -> Option<f64> {
+ fn precompute(
+ self,
+ webpage: &Webpage,
+ signal_computer: &SignalComputer,
+ ) -> Option<SignalCalculation> {
let update_timestamp = webpage
.html
.updated_time()
.map(|date| date.timestamp().max(0))
.unwrap_or(0) as usize;
- Some(score_timestamp(update_timestamp, signal_computer))
+ let score = score_timestamp(update_timestamp, signal_computer);
+
+ Some(SignalCalculation {
+ value: update_timestamp as f64,
+ score,
+ })
}
- fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {
+ fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<SignalCalculation> {
let seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();
let numericalfield_reader = seg_reader.numericalfield_reader().get_field_reader(doc);
@@ -389,7 +426,12 @@ impl Signal for UpdateTimestamp {
.and_then(|v| v.as_u64())
.unwrap() as usize;
- Some(score_timestamp(val, signal_computer))
+ let score = score_timestamp(val, signal_computer);
+
+ Some(SignalCalculation {
+ value: val as f64,
+ score,
+ })
}
}
@@ -417,12 +459,17 @@ impl Signal for TrackerScore {
))
}
- fn precompute(self, webpage: &Webpage, _: &SignalComputer) -> Option<f64> {
+ fn precompute(self, webpage: &Webpage, _: &SignalComputer) -> Option<SignalCalculation> {
let num_trackers = webpage.html.trackers().len() as f64;
- Some(score_trackers(num_trackers))
+ let score = score_trackers(num_trackers);
+
+ Some(SignalCalculation {
+ value: num_trackers,
+ score,
+ })
}
- fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {
+ fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<SignalCalculation> {
let seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();
let numericalfield_reader = seg_reader.numericalfield_reader().get_field_reader(doc);
@@ -430,7 +477,12 @@ impl Signal for TrackerScore {
.get(self.as_numericalfield().unwrap())
.and_then(|v| v.as_u64())
.unwrap();
- Some(score_trackers(val as f64))
+ let score = score_trackers(val as f64);
+
+ Some(SignalCalculation {
+ value: val as f64,
+ score,
+ })
}
}
@@ -456,13 +508,22 @@ impl Signal for Region {
Some(Field::Numerical(schema::numerical_field::Region.into()))
}
- fn precompute(self, webpage: &Webpage, signal_computer: &SignalComputer) -> Option<f64> {
+ fn precompute(
+ self,
+ webpage: &Webpage,
+ signal_computer: &SignalComputer,
+ ) -> Option<SignalCalculation> {
let region =
crate::webpage::Region::guess_from(webpage).unwrap_or(crate::webpage::Region::All);
- Some(score_region(region, signal_computer))
+ let score = score_region(region, signal_computer);
+
+ Some(SignalCalculation {
+ value: region.id() as f64,
+ score,
+ })
}
- fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {
+ fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<SignalCalculation> {
let seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();
let numericalfield_reader = seg_reader.numericalfield_reader().get_field_reader(doc);
@@ -471,7 +532,12 @@ impl Signal for Region {
.and_then(|v| v.as_u64())
.unwrap();
let region = crate::webpage::Region::from_id(val);
- Some(score_region(region, signal_computer))
+ let score = score_region(region, signal_computer);
+
+ Some(SignalCalculation {
+ value: val as f64,
+ score,
+ })
}
}
@@ -497,7 +563,7 @@ impl Signal for QueryCentrality {
None
}
- fn compute(&self, _doc: DocId, _signal_computer: &SignalComputer) -> Option<f64> {
+ fn compute(&self, _doc: DocId, _signal_computer: &SignalComputer) -> Option<SignalCalculation> {
// unimplemented!()
None
}
@@ -525,7 +591,7 @@ impl Signal for InboundSimilarity {
None
}
- fn compute(&self, _doc: DocId, _signal_computer: &SignalComputer) -> Option<f64> {
+ fn compute(&self, _doc: DocId, _signal_computer: &SignalComputer) -> Option<SignalCalculation> {
None // computed in later ranking stage
}
}
@@ -552,7 +618,7 @@ impl Signal for LambdaMart {
None
}
- fn compute(&self, _: DocId, _: &SignalComputer) -> Option<f64> {
+ fn compute(&self, _: DocId, _: &SignalComputer) -> Option<SignalCalculation> {
None // computed in later ranking stage
}
}
@@ -581,7 +647,7 @@ impl Signal for UrlDigits {
))
}
- fn precompute(self, webpage: &Webpage, _: &SignalComputer) -> Option<f64> {
+ fn precompute(self, webpage: &Webpage, _: &SignalComputer) -> Option<SignalCalculation> {
let num_digits = (webpage
.html
.url()
@@ -598,18 +664,25 @@ impl Signal for UrlDigits {
.filter(|c| c.is_ascii_digit())
.count()) as f64;
- Some(score_digits(num_digits))
+ let score = score_digits(num_digits);
+
+ Some(SignalCalculation {
+ value: num_digits,
+ score,
+ })
}
- fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {
+ fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<SignalCalculation> {
let seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();
let numericalfield_reader = seg_reader.numericalfield_reader().get_field_reader(doc);
let val = numericalfield_reader
.get(self.as_numericalfield().unwrap())
.and_then(|v| v.as_u64())
- .unwrap();
- Some(score_digits(val as f64))
+ .unwrap() as f64;
+ let score = score_digits(val);
+
+ Some(SignalCalculation { value: val, score })
}
}
@@ -637,7 +710,7 @@ impl Signal for UrlSlashes {
))
}
- fn precompute(self, webpage: &Webpage, _: &SignalComputer) -> Option<f64> {
+ fn precompute(self, webpage: &Webpage, _: &SignalComputer) -> Option<SignalCalculation> {
let num_slashes = webpage
.html
.url()
@@ -645,18 +718,25 @@ impl Signal for UrlSlashes {
.chars()
.filter(|c| c == &'/')
.count() as f64;
- Some(score_slashes(num_slashes))
+ let score = score_slashes(num_slashes);
+
+ Some(SignalCalculation {
+ value: num_slashes,
+ score,
+ })
}
- fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {
+ fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<SignalCalculation> {
let seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();
let numericalfield_reader = seg_reader.numericalfield_reader().get_field_reader(doc);
let val = numericalfield_reader
.get(self.as_numericalfield().unwrap())
.and_then(|v| v.as_u64())
- .unwrap();
- Some(score_slashes(val as f64))
+ .unwrap() as f64;
+ let score = score_slashes(val);
+
+ Some(SignalCalculation { value: val, score })
}
}
@@ -684,12 +764,17 @@ impl Signal for LinkDensity {
))
}
- fn precompute(self, webpage: &Webpage, _: &SignalComputer) -> Option<f64> {
+ fn precompute(self, webpage: &Webpage, _: &SignalComputer) -> Option<SignalCalculation> {
let link_density = webpage.html.link_density();
- Some(score_link_density(link_density))
+ let score = score_link_density(link_density);
+
+ Some(SignalCalculation {
+ value: link_density,
+ score,
+ })
}
- fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {
+ fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<SignalCalculation> {
let seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();
let numericalfield_reader = seg_reader.numericalfield_reader().get_field_reader(doc);
@@ -697,7 +782,9 @@ impl Signal for LinkDensity {
.get(self.as_numericalfield().unwrap())
.and_then(|v| v.as_f64())
.unwrap();
- Some(score_link_density(val))
+ let score = score_link_density(val);
+
+ Some(SignalCalculation { value: val, score })
}
}
@@ -725,15 +812,15 @@ impl Signal for HasAds {
))
}
- fn precompute(self, webpage: &Webpage, _: &SignalComputer) -> Option<f64> {
- if !webpage.html.likely_has_ads() {
- Some(1.0)
- } else {
- Some(0.0)
- }
+ fn precompute(self, webpage: &Webpage, _: &SignalComputer) -> Option<SignalCalculation> {
+ let has_ads = webpage.html.likely_has_ads();
+ let value: f64 = has_ads.into();
+ let score = if !has_ads { 1.0 } else { 0.0 };
+
+ Some(SignalCalculation { value, score })
}
- fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {
+ fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<SignalCalculation> {
let seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();
let numericalfield_reader = seg_reader.numericalfield_reader().get_field_reader(doc);
@@ -742,10 +829,9 @@ impl Signal for HasAds {
.and_then(|v| v.as_bool())
.unwrap();
- if !has_ads {
- Some(1.0)
- } else {
- Some(0.0)
- }
+ let value: f64 = has_ads.into();
+ let score = if !has_ads { 1.0 } else { 0.0 };
+
+ Some(SignalCalculation { value, score })
}
}
diff --git a/crates/core/src/ranking/core/prelude.rs b/crates/core/src/ranking/core/prelude.rs
index 164048a4..a5164e64 100644
--- a/crates/core/src/ranking/core/prelude.rs
+++ b/crates/core/src/ranking/core/prelude.rs
@@ -26,10 +26,10 @@ use enum_dispatch::enum_dispatch;
use strum::{EnumDiscriminants, VariantArray};
-use super::non_text::*;
use super::query::*;
use super::text::*;
use super::SignalComputer;
+use super::{non_text::*, SignalCalculation};
use tantivy::DocId;
#[enum_dispatch]
@@ -38,9 +38,13 @@ pub trait Signal:
{
fn default_coefficient(&self) -> f64;
fn as_field(&self) -> Option<Field>;
- fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64>;
+ fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<SignalCalculation>;
- fn precompute(self, _webpage: &Webpage, _signal_computer: &SignalComputer) -> Option<f64> {
+ fn precompute(
+ self,
+ _webpage: &Webpage,
+ _signal_computer: &SignalComputer,
+ ) -> Option<SignalCalculation> {
None
}
diff --git a/crates/core/src/ranking/core/query.rs b/crates/core/src/ranking/core/query.rs
index 3af796bb..ca1b5f23 100644
--- a/crates/core/src/ranking/core/query.rs
+++ b/crates/core/src/ranking/core/query.rs
@@ -18,7 +18,7 @@ use crate::schema::Field;
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
-use super::{Signal, SignalComputer};
+use super::{Signal, SignalCalculation, SignalComputer};
#[derive(
Debug,
@@ -42,9 +42,10 @@ impl Signal for NumQueryTerms {
None
}
- fn compute(&self, _: DocId, signal_computer: &SignalComputer) -> Option<f64> {
+ fn compute(&self, _: DocId, signal_computer: &SignalComputer) -> Option<SignalCalculation> {
signal_computer
.query_data()
.map(|d| d.simple_terms().len() as f64)
+ .map(SignalCalculation::new_symmetrical)
}
}
diff --git a/crates/core/src/ranking/core/text.rs b/crates/core/src/ranking/core/text.rs
index 216b9aa7..da405f5d 100644
--- a/crates/core/src/ranking/core/text.rs
+++ b/crates/core/src/ranking/core/text.rs
@@ -18,7 +18,7 @@ use crate::schema::{self, Field};
use tantivy::DocId;
-use super::{Signal, SignalComputer};
+use super::{Signal, SignalCalculation, SignalComputer};
#[derive(
Debug,
@@ -42,16 +42,16 @@ impl Signal for Bm25F {
None
}
- fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {
+ fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<SignalCalculation> {
let mut seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();
- Some(
+ Some(SignalCalculation::new_symmetrical(
seg_reader
.text_fields_mut()
.values_mut()
.map(|field| field.bm25f(doc))
.sum(),
- )
+ ))
}
}
@@ -81,13 +81,14 @@ impl Signal for Bm25Title {
true
}
- fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {
+ fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<SignalCalculation> {
let mut seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();
seg_reader
.text_fields_mut()
.get_mut(self.as_textfield().unwrap())
.map(|field| field.bm25(doc))
+ .map(SignalCalculation::new_symmetrical)
}
}
@@ -113,13 +114,13 @@ impl Signal for TitleCoverage {
Some(Field::Text(schema::text_field::Title.into()))
}
- fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {
+ fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<SignalCalculation> {
let mut seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();
seg_reader
.text_fields_mut()
.get_mut(self.as_textfield().unwrap())
- .map(|field| field.coverage(doc))
+ .map(|field| SignalCalculation::new_symmetrical(field.coverage(doc)))
}
}
@@ -149,13 +150,14 @@ impl Signal for Bm25TitleBigrams {
Some(Field::Text(schema::text_field::TitleBigrams.into()))
}
- fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {
+ fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<SignalCalculation> {
let mut seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();
seg_reader
.text_fields_mut()
.get_mut(self.as_textfield().unwrap())
.map(|field| field.bm25(doc))
+ .map(SignalCalculation::new_symmetrical)
}
}
@@ -185,13 +187,14 @@ impl Signal for Bm25TitleTrigrams {
Some(Field::Text(schema::text_field::TitleTrigrams.into()))
}
- fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {
+ fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<SignalCalculation> {
let mut seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();
seg_reader
.text_fields_mut()
.get_mut(self.as_textfield().unwrap())
.map(|field| field.bm25(doc))
+ .map(SignalCalculation::new_symmetrical)
}
}
@@ -221,13 +224,14 @@ impl Signal for Bm25CleanBody {
Some(Field::Text(schema::text_field::CleanBody.into()))
}
- fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {
+ fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<SignalCalculation> {
let mut seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();
seg_reader
.text_fields_mut()
.get_mut(self.as_textfield().unwrap())
.map(|field| field.bm25(doc))
+ .map(SignalCalculation::new_symmetrical)
}
}
@@ -253,13 +257,14 @@ impl Signal for CleanBodyCoverage {
Some(Field::Text(schema::text_field::CleanBody.into()))
}
- fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {
+ fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<SignalCalculation> {
let mut seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();
seg_reader
.text_fields_mut()
.get_mut(self.as_textfield().unwrap())
.map(|field| field.coverage(doc))
+ .map(SignalCalculation::new_symmetrical)
}
}
@@ -289,13 +294,14 @@ impl Signal for Bm25CleanBodyBigrams {
Some(Field::Text(schema::text_field::CleanBodyBigrams.into()))
}
- fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {
+ fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<SignalCalculation> {
let mut seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();
seg_reader
.text_fields_mut()
.get_mut(self.as_textfield().unwrap())
.map(|field| field.bm25(doc))
+ .map(SignalCalculation::new_symmetrical)
}
}
@@ -325,13 +331,14 @@ impl Signal for Bm25CleanBodyTrigrams {
Some(Field::Text(schema::text_field::CleanBodyTrigrams.into()))
}
- fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {
+ fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<SignalCalculation> {
let mut seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();
seg_reader
.text_fields_mut()
.get_mut(self.as_textfield().unwrap())
.map(|field| field.bm25(doc))
+ .map(SignalCalculation::new_symmetrical)
}
}
@@ -357,13 +364,14 @@ impl Signal for Bm25StemmedTitle {
Some(Field::Text(schema::text_field::StemmedTitle.into()))
}
- fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {
+ fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<SignalCalculation> {
let mut seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();
seg_reader
.text_fields_mut()
.get_mut(self.as_textfield().unwrap())
.map(|field| field.bm25(doc))
+ .map(SignalCalculation::new_symmetrical)
}
}
@@ -389,13 +397,14 @@ impl Signal for Bm25StemmedCleanBody {
Some(Field::Text(schema::text_field::StemmedCleanBody.into()))
}
- fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {
+ fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<SignalCalculation> {
let mut seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();
seg_reader
.text_fields_mut()
.get_mut(self.as_textfield().unwrap())
.map(|field| field.bm25(doc))
+ .map(SignalCalculation::new_symmetrical)
}
}
@@ -421,13 +430,14 @@ impl Signal for Bm25AllBody {
Some(Field::Text(schema::text_field::AllBody.into()))
}
- fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {
+ fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<SignalCalculation> {
let mut seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();
seg_reader
.text_fields_mut()
.get_mut(self.as_textfield().unwrap())
.map(|field| field.bm25(doc))
+ .map(SignalCalculation::new_symmetrical)
}
}
@@ -453,13 +463,14 @@ impl Signal for Bm25Keywords {
Some(Field::Text(schema::text_field::Keywords.into()))
}
- fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {
+ fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<SignalCalculation> {
let mut seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();
seg_reader
.text_fields_mut()
.get_mut(self.as_textfield().unwrap())
.map(|field| field.bm25(doc))
+ .map(SignalCalculation::new_symmetrical)
}
}
@@ -485,13 +496,14 @@ impl Signal for Bm25BacklinkText {
Some(Field::Text(schema::text_field::BacklinkText.into()))
}
- fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {
+ fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<SignalCalculation> {
let mut seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();
seg_reader
.text_fields_mut()
.get_mut(self.as_textfield().unwrap())
.map(|field| field.bm25(doc))
+ .map(SignalCalculation::new_symmetrical)
}
}
@@ -517,13 +529,14 @@ impl Signal for IdfSumUrl {
Some(Field::Text(schema::text_field::Url.into()))
}
- fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {
+ fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<SignalCalculation> {
let mut seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();
seg_reader
.text_fields_mut()
.get_mut(self.as_textfield().unwrap())
.map(|field| field.idf_sum(doc))
+ .map(SignalCalculation::new_symmetrical)
}
}
@@ -549,13 +562,14 @@ impl Signal for IdfSumSite {
Some(Field::Text(schema::text_field::SiteWithout.into()))
}
- fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {
+ fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<SignalCalculation> {
let mut seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();
seg_reader
.text_fields_mut()
.get_mut(self.as_textfield().unwrap())
.map(|field| field.idf_sum(doc))
+ .map(SignalCalculation::new_symmetrical)
}
}
@@ -581,13 +595,14 @@ impl Signal for IdfSumDomain {
Some(Field::Text(schema::text_field::Domain.into()))
}
- fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {
+ fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<SignalCalculation> {
let mut seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();
seg_reader
.text_fields_mut()
.get_mut(self.as_textfield().unwrap())
.map(|field| field.idf_sum(doc))
+ .map(SignalCalculation::new_symmetrical)
}
}
@@ -613,13 +628,14 @@ impl Signal for IdfSumSiteNoTokenizer {
Some(Field::Text(schema::text_field::SiteNoTokenizer.into()))
}
- fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {
+ fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<SignalCalculation> {
let mut seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();
seg_reader
.text_fields_mut()
.get_mut(self.as_textfield().unwrap())
.map(|field| field.idf_sum(doc))
+ .map(SignalCalculation::new_symmetrical)
}
}
@@ -645,13 +661,14 @@ impl Signal for IdfSumDomainNoTokenizer {
Some(Field::Text(schema::text_field::DomainNoTokenizer.into()))
}
- fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {
+ fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<SignalCalculation> {
let mut seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();
seg_reader
.text_fields_mut()
.get_mut(self.as_textfield().unwrap())
.map(|field| field.idf_sum(doc))
+ .map(SignalCalculation::new_symmetrical)
}
}
@@ -679,13 +696,14 @@ impl Signal for IdfSumDomainNameNoTokenizer {
))
}
- fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {
+ fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<SignalCalculation> {
let mut seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();
seg_reader
.text_fields_mut()
.get_mut(self.as_textfield().unwrap())
.map(|field| field.idf_sum(doc))
+ .map(SignalCalculation::new_symmetrical)
}
}
@@ -711,13 +729,14 @@ impl Signal for IdfSumDomainIfHomepage {
Some(Field::Text(schema::text_field::DomainIfHomepage.into()))
}
- fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {
+ fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<SignalCalculation> {
let mut seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();
seg_reader
.text_fields_mut()
.get_mut(self.as_textfield().unwrap())
.map(|field| field.idf_sum(doc))
+ .map(SignalCalculation::new_symmetrical)
}
}
@@ -745,13 +764,14 @@ impl Signal for IdfSumDomainNameIfHomepageNoTokenizer {
))
}
- fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {
+ fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<SignalCalculation> {
let mut seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();
seg_reader
.text_fields_mut()
.get_mut(self.as_textfield().unwrap())
.map(|field| field.idf_sum(doc))
+ .map(SignalCalculation::new_symmetrical)
}
}
@@ -779,13 +799,14 @@ impl Signal for IdfSumDomainIfHomepageNoTokenizer {
))
}
- fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {
+ fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<SignalCalculation> {
let mut seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();
seg_reader
.text_fields_mut()
.get_mut(self.as_textfield().unwrap())
.map(|field| field.idf_sum(doc))
+ .map(SignalCalculation::new_symmetrical)
}
}
@@ -811,13 +832,14 @@ impl Signal for IdfSumTitleIfHomepage {
Some(Field::Text(schema::text_field::TitleIfHomepage.into()))
}
- fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {
+ fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<SignalCalculation> {
let mut seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();
seg_reader
.text_fields_mut()
.get_mut(self.as_textfield().unwrap())
.map(|field| field.idf_sum(doc))
+ .map(SignalCalculation::new_symmetrical)
}
}
@@ -843,7 +865,7 @@ impl Signal for CrossEncoderSnippet {
None
}
- fn compute(&self, _: DocId, _: &SignalComputer) -> Option<f64> {
+ fn compute(&self, _: DocId, _: &SignalComputer) -> Option<SignalCalculation> {
None // computed in later ranking stage
}
}
@@ -870,7 +892,7 @@ impl Signal for CrossEncoderTitle {
None
}
- fn compute(&self, _: DocId, _: &SignalComputer) -> Option<f64> {
+ fn compute(&self, _: DocId, _: &SignalComputer) -> Option<SignalCalculation> {
None // computed in later ranking stage
}
}
@@ -899,7 +921,7 @@ impl Signal for TitleEmbeddingSimilarity {
))
}
- fn compute(&self, _: DocId, _: &SignalComputer) -> Option<f64> {
+ fn compute(&self, _: DocId, _: &SignalComputer) -> Option<SignalCalculation> {
None // computed in later ranking stage
}
}
@@ -928,7 +950,7 @@ impl Signal for KeywordEmbeddingSimilarity {
))
}
- fn compute(&self, _: DocId, _: &SignalComputer) -> Option<f64> {
+ fn compute(&self, _: DocId, _: &SignalComputer) -> Option<SignalCalculation> {
None // computed in later ranking stage
}
}
@@ -955,7 +977,7 @@ impl Signal for MinTitleSlop {
Some(Field::Text(schema::text_field::Title.into()))
}
- fn compute(&self, _: DocId, _: &SignalComputer) -> Option<f64> {
+ fn compute(&self, _: DocId, _: &SignalComputer) -> Option<SignalCalculation> {
None // computed in later ranking stage
}
}
@@ -982,7 +1004,7 @@ impl Signal for MinCleanBodySlop {
Some(Field::Text(schema::text_field::CleanBody.into()))
}
- fn compute(&self, _: DocId, _: &SignalComputer) -> Option<f64> {
+ fn compute(&self, _: DocId, _: &SignalComputer) -> Option<SignalCalculation> {
None // computed in later ranking stage
}
}
diff --git a/crates/core/src/ranking/initial.rs b/crates/core/src/ranking/initial.rs
index a32dbf38..4ee87479 100644
--- a/crates/core/src/ranking/initial.rs
+++ b/crates/core/src/ranking/initial.rs
@@ -81,7 +81,7 @@ impl ScoreSegmentTweaker<Score> for InitialSegmentScoreTweaker {
let mut total = self
.computer
.compute_signals(doc)
- .map(|computed| self.computer.coefficient(&computed.signal) * computed.score)
+ .map(|computed| self.computer.coefficient(&computed.signal) * computed.calc.score)
.sum();
if let Some(boost) = self.computer.boosts(doc) {
diff --git a/crates/core/src/ranking/mod.rs b/crates/core/src/ranking/mod.rs
index 1d126e37..a67c8c77 100644
--- a/crates/core/src/ranking/mod.rs
+++ b/crates/core/src/ranking/mod.rs
@@ -37,7 +37,7 @@ use crate::{
pub use self::core::*;
#[derive(Clone)]
-pub struct Ranker {
+pub struct LocalRanker {
max_docs: Option<MaxDocsConsidered>,
offset: Option<usize>,
computer: SignalComputer,
@@ -47,13 +47,13 @@ pub struct Ranker {
collector_config: CollectorConfig,
}
-impl Ranker {
+impl LocalRanker {
pub fn new(
computer: SignalComputer,
columnfield_reader: NumericalFieldReader,
collector_config: CollectorConfig,
) -> Self {
- Ranker {
+ LocalRanker {
offset: None,
computer,
max_docs: None,
diff --git a/crates/core/src/ranking/models/lambdamart.rs b/crates/core/src/ranking/models/lambdamart.rs
index 1632fc8d..5d1ee789 100644
--- a/crates/core/src/ranking/models/lambdamart.rs
+++ b/crates/core/src/ranking/models/lambdamart.rs
@@ -20,7 +20,7 @@ use core::SignalEnumDiscriminants;
use crate::{
enum_map::EnumMap,
- ranking::{core, SignalEnum},
+ ranking::{core, SignalCalculation, SignalEnum},
};
type Result<T> = std::result::Result<T, Error>;
@@ -64,6 +64,12 @@ impl AsValue for f64 {
}
}
+impl AsValue for SignalCalculation {
+ fn as_value(&self) -> f64 {
+ self.score
+ }
+}
+
#[derive(Debug)]
struct Node {
threshold: f64,
diff --git a/crates/core/src/ranking/models/mod.rs b/crates/core/src/ranking/models/mod.rs
index 41e9906e..756d2067 100644
--- a/crates/core/src/ranking/models/mod.rs
+++ b/crates/core/src/ranking/models/mod.rs
@@ -1,3 +1,6 @@
pub mod cross_encoder;
pub mod lambdamart;
pub mod linear;
+
+pub use cross_encoder::CrossEncoder;
+pub use lambdamart::LambdaMART;
diff --git a/crates/core/src/ranking/pipeline/mod.rs b/crates/core/src/ranking/pipeline/mod.rs
index 367c98d2..4de438fb 100644
--- a/crates/core/src/ranking/pipeline/mod.rs
+++ b/crates/core/src/ranking/pipeline/mod.rs
@@ -14,43 +14,37 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
-use std::{ops::Add, sync::Arc};
-
use crate::{
- collector::{self, BucketCollector},
- config::CollectorConfig,
+ collector::{self},
enum_map::EnumMap,
searcher::SearchQuery,
};
use super::{
- models::lambdamart::{self, LambdaMART},
- InboundSimilarity, SignalCoefficient, SignalEnum, SignalScore,
+ models::lambdamart::{self},
+ SignalCalculation, SignalEnum, SignalScore,
};
+mod modifiers;
mod scorers;
mod stages;
-pub use scorers::{ReRanker, Recall, Scorer};
+use modifiers::FullModifier;
+pub use scorers::{FullRankingStage, ReRanker};
pub use stages::{LocalRecallRankingWebpage, PrecisionRankingWebpage, RecallRankingWebpage};
-const INBOUND_SIMILARITY_SMOOTHING: f64 = 8.0;
-
pub trait RankableWebpage: collector::Doc + Send + Sync {
- fn set_score(&mut self, score: f64);
- fn boost(&self) -> Option<f64>;
- fn signals(&self) -> &EnumMap<SignalEnum, f64>;
- fn signals_mut(&mut self) -> &mut EnumMap<SignalEnum, f64>;
+ fn set_raw_score(&mut self, score: f64);
+ fn unboosted_score(&self) -> f64;
+ fn boost(&self) -> f64;
+ fn set_boost(&mut self, boost: f64);
+ fn signals(&self) -> &EnumMap<SignalEnum, SignalCalculation>;
+ fn signals_mut(&mut self) -> &mut EnumMap<SignalEnum, SignalCalculation>;
fn as_local_recall(&self) -> &LocalRecallRankingWebpage;
- fn boost_score(&mut self) {
- if let Some(boost) = self.boost() {
- if boost != 0.0 {
- let score = self.score() * boost;
- self.set_score(score);
- }
- }
+ fn score(&self) -> f64 {
+ self.boost() * self.unboosted_score()
}
}
@@ -60,136 +54,136 @@ impl lambdamart::AsValue for SignalScore {
}
}
-struct RankingStage<T> {
- scorer: Box<dyn Scorer<T>>,
- stage_top_n: usize,
- derank_similar: bool,
- model: Option<Arc<LambdaMART>>,
- coefficients: SignalCoefficient,
+enum StageOrModifier<T> {
+ Stage(Box<dyn FullRankingStage<Webpage = T>>),
+ Modifier(Box<dyn FullModifier<Webpage = T>>),
}
-impl<T: RankableWebpage> RankingStage<T> {
- fn apply(
- &self,
- websites: Vec<T>,
- top_n: usize,
- offset: usize,
- collector_config: CollectorConfig,
- ) -> Vec<T> {
- let mut websites = websites
- .into_iter()
- .skip(offset)
- .take(self.stage_top_n.max(top_n))
- .collect::<Vec<_>>();
-
- self.scorer.score(&mut websites);
-
- let mut collector =
- BucketCollector::new(self.stage_top_n.max(top_n) + offset, collector_config);
+pub enum Top {
+ Unlimited,
+ Limit(usize),
+}
- for mut website in websites {
- website.set_score(self.calculate_score(website.signals()));
- website.boost_score();
- collector.insert(website);
+impl<T> StageOrModifier<T>
+where
+ T: RankableWebpage + Send + Sync,
+{
+ fn top_n(&self) -> Top {
+ match self {
+ StageOrModifier::Stage(stage) => stage.top_n(),
+ StageOrModifier::Modifier(modifier) => modifier.top_n(),
}
-
- collector
- .into_sorted_vec(self.derank_similar)
- .into_iter()
- .take(top_n)
- .collect()
}
- fn calculate_score(&self, signals: &EnumMap<SignalEnum, f64>) -> f64 {
- match self.model.as_ref() {
- Some(model) => {
- let coeff = self.coefficients.get(&super::core::LambdaMart.into());
- if coeff == 0.0 {
- signals
- .iter()
- .map(|(signal, score)| self.coefficients.get(&signal) * score)
- .sum()
- } else {
- signals
- .get(InboundSimilarity.into())
- .copied()
- .unwrap_or(0.0)
- .add(INBOUND_SIMILARITY_SMOOTHING)
- * coeff
- * model.predict(signals)
- }
- }
- None => signals
- .iter()
- .map(|(signal, score)| self.coefficients.get(&signal) * score)
- .sum(),
+ fn compute(&self, webpages: &mut [T]) {
+ match self {
+ StageOrModifier::Stage(stage) => stage.compute(webpages),
+ StageOrModifier::Modifier(modifier) => modifier.update_boosts(webpages),
}
}
- fn set_query_info(&mut self, query: &SearchQuery) {
- self.scorer.set_query_info(query);
+ fn update_scores(&self, webpages: &mut [T], coefficients: &super::SignalCoefficients) {
+ match self {
+ StageOrModifier::Stage(stage) => stage.update_scores(webpages, coefficients),
+ StageOrModifier::Modifier(_) => {}
+ }
+ }
- self.coefficients = query.signal_coefficients();
+ fn rank(&self, webpages: &mut [T]) {
+ match self {
+ StageOrModifier::Stage(stage) => stage.rank(webpages),
+ StageOrModifier::Modifier(modifier) => modifier.rank(webpages),
+ }
}
}
pub struct RankingPipeline<T> {
- stage: RankingStage<T>,
- page: usize,
- pub top_n: usize,
- collector_config: CollectorConfig,
+ stages_or_modifiers: Vec<StageOrModifier<T>>,
}
-impl<T: RankableWebpage> RankingPipeline<T> {
- fn set_query_info(&mut self, query: &mut SearchQuery) {
- self.stage.set_query_info(query);
- self.page = query.page;
- self.top_n = query.num_results;
-
- query.num_results = self.collector_top_n();
- query.page = 0;
+impl<T> RankingPipeline<T>
+where
+ T: RankableWebpage,
+{
+ fn new() -> Self {
+ Self {
+ stages_or_modifiers: Vec::new(),
+ }
}
- pub fn offset(&self) -> usize {
- self.top_n * self.page
+ pub fn add_stage<R>(mut self, stage: R) -> Self
+ where
+ R: FullRankingStage<Webpage = T> + 'static,
+ {
+ self.stages_or_modifiers.push(StageOrModifier::Stage(
+ Box::new(stage) as Box<dyn FullRankingStage<Webpage = T>>
+ ));
+
+ self
}
- pub fn apply(self, websites: Vec<T>) -> Vec<T> {
- if websites.len() <= 1 {
- return websites;
- }
+ pub fn add_modifier<R>(mut self, modifier: R) -> Self
+ where
+ R: FullModifier<Webpage = T> + 'static,
+ {
+ self.stages_or_modifiers.push(StageOrModifier::Modifier(
+ Box::new(modifier) as Box<dyn FullModifier<Webpage = T>>
+ ));
- self.stage.apply(
- websites,
- self.top_n,
- self.offset(),
- self.collector_config.clone(),
- )
+ self
}
- pub fn collector_top_n(&self) -> usize {
- (self.initial_top_n().max(self.top_n) + self.top_n * self.page) + 1
- }
+ pub fn apply(&self, webpages: Vec<T>, query: &SearchQuery) -> Vec<T> {
+ let mut webpages = webpages;
+ let num_pages = webpages.len();
+ let coefficients = query.signal_coefficients();
- pub fn initial_top_n(&self) -> usize {
- self.stage.stage_top_n.max(self.top_n)
+ for stage_or_modifier in self.stages_or_modifiers.iter() {
+ let webpages = if let Top::Limit(top_n) = stage_or_modifier.top_n() {
+ if query.offset() > top_n {
+ continue;
+ }
+
+ &mut webpages[..top_n.min(num_pages)]
+ } else {
+ &mut webpages
+ };
+
+ stage_or_modifier.compute(webpages);
+ stage_or_modifier.update_scores(webpages, &coefficients);
+ stage_or_modifier.rank(webpages);
+ }
+
+ webpages
+ .into_iter()
+ .skip(query.offset())
+ .take(query.num_results())
+ .collect()
}
}
#[cfg(test)]
mod tests {
- use itertools::Itertools;
+ use scorers::term_distance;
use crate::{
+ ampc::dht::ShardId,
collector::Hashes,
inverted_index::{DocAddress, WebpagePointer},
prehashed::Prehashed,
- ranking::{self, initial::Score},
+ ranking::{self, bitvec_similarity::BitVec, initial::Score},
+ searcher::api,
};
use super::*;
- fn sample_websites(n: usize) -> Vec<LocalRecallRankingWebpage> {
+ fn pipeline() -> RankingPipeline<api::ScoredWebpagePointer> {
+ RankingPipeline::new()
+ .add_stage(term_distance::TitleDistanceScorer)
+ .add_stage(term_distance::BodyDistanceScorer)
+ }
+
+ fn sample_websites(n: usize) -> Vec<api::ScoredWebpagePointer> {
(0..n)
.map(|i| -> LocalRecallRankingWebpage {
let pointer = WebpagePointer {
@@ -208,118 +202,49 @@ mod tests {
};
let mut signals = EnumMap::new();
- signals.insert(ranking::core::HostCentrality.into(), 1.0 / i as f64);
- LocalRecallRankingWebpage::new_testing(pointer, signals, 1.0 / i as f64)
+ let score = 1.0 / i as f64;
+ let calc = ranking::SignalCalculation {
+ value: i as f64,
+ score,
+ };
+ signals.insert(ranking::core::HostCentrality.into(), calc);
+ LocalRecallRankingWebpage::new_testing(pointer, signals, calc.score)
+ })
+ .map(|local| {
+ api::ScoredWebpagePointer::Normal(
+ crate::searcher::distributed::ScoredWebpagePointer {
+ website: RecallRankingWebpage::new(local, BitVec::new(vec![])),
+ shard: ShardId::new(0),
+ },
+ )
})
.collect()
}
#[test]
fn simple() {
- let pipeline = RankingPipeline::<LocalRecallRankingWebpage>::recall_stage(
- &mut SearchQuery {
- ..Default::default()
- },
- None,
- None,
- CollectorConfig::default(),
- 20,
- );
- assert_eq!(pipeline.collector_top_n(), 20 + 1);
-
- let sample = sample_websites(pipeline.collector_top_n());
+ let pipeline = pipeline();
+
+ let sample = sample_websites(20);
let res: Vec<_> = pipeline
- .apply(sample)
+ .apply(
+ sample,
+ &SearchQuery {
+ page: 0,
+ num_results: 20,
+ ..Default::default()
+ },
+ )
.into_iter()
- .map(|w| w.pointer().address)
+ .map(|w| w.as_ranking().pointer().address)
.collect();
let expected: Vec<_> = sample_websites(100)
.into_iter()
.take(20)
- .map(|w| w.pointer().address)
- .collect();
-
- assert_eq!(res, expected);
- }
-
- #[test]
- fn top_n() {
- let num_results = 100;
- let pipeline = RankingPipeline::<LocalRecallRankingWebpage>::recall_stage(
- &mut SearchQuery {
- num_results,
- ..Default::default()
- },
- None,
- None,
- CollectorConfig::default(),
- num_results,
- );
-
- let sample: Vec<_> = sample_websites(pipeline.collector_top_n());
-
- let expected: Vec<_> = sample
- .clone()
- .into_iter()
- .take(num_results)
- .map(|w| w.pointer().address)
+ .map(|w| w.as_ranking().pointer().address)
.collect();
- let res = pipeline
- .apply(sample)
- .into_iter()
- .map(|w| w.pointer().address)
- .collect_vec();
-
- assert_eq!(res.len(), num_results);
assert_eq!(res, expected);
}
-
- #[test]
- fn offsets() {
- let num_results = 20;
- let pipeline = RankingPipeline::<LocalRecallRankingWebpage>::recall_stage(
- &mut SearchQuery {
- page: 0,
- num_results,
- ..Default::default()
- },
- None,
- None,
- CollectorConfig::default(),
- num_results,
- );
-
- let sample: Vec<_> = sample_websites(pipeline.collector_top_n());
- let mut prev: Vec<_> = pipeline.apply(sample);
- for p in 1..1_000 {
- let pipeline = RankingPipeline::<LocalRecallRankingWebpage>::recall_stage(
- &mut SearchQuery {
- page: p,
- ..Default::default()
- },
- None,
- None,
- CollectorConfig::default(),
- num_results,
- );
-
- let sample: Vec<_> = sample_websites(pipeline.collector_top_n());
- let res: Vec<_> = pipeline.apply(sample).into_iter().collect();
-
- assert_eq!(
- res.len(),
- num_results,
- "Every page should have {num_results} results"
- );
-
- assert!(!prev
- .iter()
- .zip_eq(res.iter())
- .any(|(p, r)| p.pointer().address.doc_id == r.pointer().address.doc_id));
-
- prev = res;
- }
- }
}
diff --git a/crates/core/src/ranking/pipeline/modifiers/inbound_similarity.rs b/crates/core/src/ranking/pipeline/modifiers/inbound_similarity.rs
new file mode 100644
index 00000000..09874545
--- /dev/null
+++ b/crates/core/src/ranking/pipeline/modifiers/inbound_similarity.rs
@@ -0,0 +1,40 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2024 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use crate::{
+ ranking::{self, pipeline::RankableWebpage},
+ searcher::api,
+};
+
+use super::Modifier;
+
+const INBOUND_SIMILARITY_SMOOTHING: f64 = 8.0;
+
+pub struct InboundSimilarity;
+
+impl Modifier for InboundSimilarity {
+ type Webpage = api::ScoredWebpagePointer;
+
+ fn boost(&self, webpage: &Self::Webpage) -> f64 {
+ webpage
+ .as_ranking()
+ .signals()
+ .get(ranking::InboundSimilarity.into())
+ .map(|calc| calc.value)
+ .unwrap_or(0.0)
+ + INBOUND_SIMILARITY_SMOOTHING
+ }
+}
diff --git a/crates/core/src/ranking/pipeline/modifiers/mod.rs b/crates/core/src/ranking/pipeline/modifiers/mod.rs
new file mode 100644
index 00000000..fa481e3c
--- /dev/null
+++ b/crates/core/src/ranking/pipeline/modifiers/mod.rs
@@ -0,0 +1,60 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2024 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+mod inbound_similarity;
+
+use super::{RankableWebpage, Top};
+pub use inbound_similarity::InboundSimilarity;
+
+pub trait FullModifier: Send + Sync {
+ type Webpage: RankableWebpage;
+ fn update_boosts(&self, webpages: &mut [Self::Webpage]);
+
+ fn rank(&self, webpages: &mut [Self::Webpage]) {
+ webpages.sort_by(|a, b| b.score().partial_cmp(&a.score()).unwrap());
+ }
+
+ fn top_n(&self) -> Top {
+ Top::Unlimited
+ }
+}
+
+pub trait Modifier: Send + Sync {
+ type Webpage: RankableWebpage;
+ fn boost(&self, webpage: &Self::Webpage) -> f64;
+
+ fn top(&self) -> Top {
+ Top::Unlimited
+ }
+}
+
+impl<T> FullModifier for T
+where
+ T: Modifier,
+{
+ type Webpage = <T as Modifier>::Webpage;
+
+ fn update_boosts(&self, webpages: &mut [Self::Webpage]) {
+ for webpage in webpages {
+ let cur_boost = webpage.boost();
+ webpage.set_boost(cur_boost * self.boost(webpage));
+ }
+ }
+
+ fn top_n(&self) -> Top {
+ Modifier::top(self)
+ }
+}
diff --git a/crates/core/src/ranking/pipeline/scorers/embedding.rs b/crates/core/src/ranking/pipeline/scorers/embedding.rs
index 73c834ca..bba0fb5c 100644
--- a/crates/core/src/ranking/pipeline/scorers/embedding.rs
+++ b/crates/core/src/ranking/pipeline/scorers/embedding.rs
@@ -22,14 +22,14 @@ use crate::{
models::dual_encoder::DualEncoder,
ranking::{
self,
- pipeline::{stages::StoredEmbeddings, LocalRecallRankingWebpage, RankableWebpage},
+ pipeline::{stages::StoredEmbeddings, RankableWebpage},
SignalEnum,
},
- searcher::{api::ScoredWebpagePointer, SearchQuery},
+ searcher::api::ScoredWebpagePointer,
Result,
};
-use super::Scorer;
+use super::FullRankingStage;
pub struct Embedding(Tensor);
@@ -72,15 +72,15 @@ impl Embedding {
pub struct EmbeddingScorer<W, E: EmbeddingSignal<W>> {
dual_encoder: Option<Arc<DualEncoder>>,
- query: Option<String>,
+ query: String,
_marker: std::marker::PhantomData<(E, W)>,
}
impl<W, E: EmbeddingSignal<W>> EmbeddingScorer<W, E> {
- pub fn new(dual_encoder: Option<Arc<DualEncoder>>) -> Self {
+ pub fn new(query: String, dual_encoder: Option<Arc<DualEncoder>>) -> Self {
Self {
dual_encoder,
- query: None,
+ query,
_marker: std::marker::PhantomData,
}
}
@@ -89,31 +89,15 @@ impl<W, E: EmbeddingSignal<W>> EmbeddingScorer<W, E> {
impl<W, E: EmbeddingSignal<W>> EmbeddingScorer<W, E> {
pub fn query_emb(&self) -> Option<Embedding> {
self.dual_encoder.as_ref().and_then(|dual_encoder| {
- self.query.as_ref().and_then(|query| {
- dual_encoder
- .embed(&[query.to_string()])
- .ok()
- .and_then(|d| d.squeeze(0).ok())
- .map(Embedding)
- })
+ dual_encoder
+ .embed(&[self.query.clone()])
+ .ok()
+ .and_then(|d| d.squeeze(0).ok())
+ .map(Embedding)
})
}
}
-impl LocalRecallRankingWebpage {
- fn title_emb(&self, hidden_size: usize) -> Option<Embedding> {
- self.title_embedding()
- .as_ref()
- .and_then(|s| Embedding::new(s, hidden_size).ok())
- }
-
- fn keyword_emb(&self, hidden_size: usize) -> Option<Embedding> {
- self.keyword_embedding()
- .as_ref()
- .and_then(|s| Embedding::new(s, hidden_size).ok())
- }
-}
-
impl ScoredWebpagePointer {
fn title_emb(&self, hidden_size: usize) -> Option<Embedding> {
self.as_ranking()
@@ -130,8 +114,12 @@ impl ScoredWebpagePointer {
}
}
-impl<W: RankableWebpage, E: EmbeddingSignal<W>> Scorer<W> for EmbeddingScorer<W, E> {
- fn score(&self, webpages: &mut [W]) {
+impl<E: EmbeddingSignal<ScoredWebpagePointer>> FullRankingStage
+ for EmbeddingScorer<ScoredWebpagePointer, E>
+{
+ type Webpage = ScoredWebpagePointer;
+
+ fn compute(&self, webpages: &mut [Self::Webpage]) {
if !webpages.iter().any(E::has_embedding) {
return;
}
@@ -146,10 +134,6 @@ impl<W: RankableWebpage, E: EmbeddingSignal<W>> Scorer<W> for EmbeddingScorer<W,
}
}
}
-
- fn set_query_info(&mut self, query: &SearchQuery) {
- self.query = Some(query.query.clone());
- }
}
pub struct TitleEmbeddings;
@@ -177,27 +161,10 @@ impl EmbeddingSignal<ScoredWebpagePointer> for TitleEmbeddings {
fn insert_signal(webpage: &mut ScoredWebpagePointer, score: f64) {
let sig = <TitleEmbeddings as EmbeddingSignal<ScoredWebpagePointer>>::signal();
- webpage.as_ranking_mut().signals_mut().insert(sig, score);
- }
-}
-
-impl EmbeddingSignal<LocalRecallRankingWebpage> for TitleEmbeddings {
- fn signal() -> SignalEnum {
- ranking::core::TitleEmbeddingSimilarity.into()
- }
-
- fn has_embedding(webpage: &LocalRecallRankingWebpage) -> bool {
- webpage.title_embedding().is_some()
- }
-
- fn embedding(webpage: &LocalRecallRankingWebpage, hidden_size: usize) -> Option<Embedding> {
- webpage.title_emb(hidden_size)
- }
-
- fn insert_signal(webpage: &mut LocalRecallRankingWebpage, score: f64) {
- let sig = <TitleEmbeddings as EmbeddingSignal<LocalRecallRankingWebpage>>::signal();
-
- webpage.signals_mut().insert(sig, score);
+ webpage
+ .as_ranking_mut()
+ .signals_mut()
+ .insert(sig, ranking::SignalCalculation::new_symmetrical(score));
}
}
@@ -216,26 +183,10 @@ impl EmbeddingSignal<ScoredWebpagePointer> for KeywordEmbeddings {
fn insert_signal(webpage: &mut ScoredWebpagePointer, score: f64) {
let sig = <KeywordEmbeddings as EmbeddingSignal<ScoredWebpagePointer>>::signal();
- webpage.as_ranking_mut().signals_mut().insert(sig, score);
- }
-}
-
-impl EmbeddingSignal<LocalRecallRankingWebpage> for KeywordEmbeddings {
- fn signal() -> SignalEnum {
- ranking::core::KeywordEmbeddingSimilarity.into()
- }
-
- fn has_embedding(webpage: &LocalRecallRankingWebpage) -> bool {
- webpage.keyword_embedding().is_some()
- }
-
- fn embedding(webpage: &LocalRecallRankingWebpage, hidden_size: usize) -> Option<Embedding> {
- webpage.keyword_emb(hidden_size)
- }
-
- fn insert_signal(webpage: &mut LocalRecallRankingWebpage, score: f64) {
- let sig = <KeywordEmbeddings as EmbeddingSignal<LocalRecallRankingWebpage>>::signal();
- webpage.signals_mut().insert(sig, score);
+ webpage
+ .as_ranking_mut()
+ .signals_mut()
+ .insert(sig, ranking::SignalCalculation::new_symmetrical(score));
}
}
diff --git a/crates/core/src/ranking/pipeline/scorers/inbound_similarity.rs b/crates/core/src/ranking/pipeline/scorers/inbound_similarity.rs
index 0ad1e977..e5bdf645 100644
--- a/crates/core/src/ranking/pipeline/scorers/inbound_similarity.rs
+++ b/crates/core/src/ranking/pipeline/scorers/inbound_similarity.rs
@@ -17,11 +17,11 @@
use std::sync::Mutex;
use crate::{
- ranking::{core, inbound_similarity},
+ ranking::{self, core, inbound_similarity, pipeline::RankableWebpage},
searcher::api::ScoredWebpagePointer,
};
-use super::Scorer;
+use super::FullRankingStage;
pub struct InboundScorer {
scorer: Mutex<inbound_similarity::Scorer>,
@@ -35,8 +35,10 @@ impl InboundScorer {
}
}
-impl Scorer<ScoredWebpagePointer> for InboundScorer {
- fn score(&self, webpages: &mut [ScoredWebpagePointer]) {
+impl FullRankingStage for InboundScorer {
+ type Webpage = ScoredWebpagePointer;
+
+ fn compute(&self, webpages: &mut [Self::Webpage]) {
let mut scorer = self.scorer.lock().unwrap();
for webpage in webpages {
@@ -44,10 +46,10 @@ impl Scorer<ScoredWebpagePointer> for InboundScorer {
webpage.as_ranking().host_id(),
webpage.as_ranking().inbound_edges(),
);
- webpage
- .as_ranking_mut()
- .signals_mut()
- .insert(core::InboundSimilarity.into(), score);
+ webpage.as_ranking_mut().signals_mut().insert(
+ core::InboundSimilarity.into(),
+ ranking::SignalCalculation::new_symmetrical(score),
+ );
}
}
}
diff --git a/crates/core/src/ranking/pipeline/scorers/lambdamart.rs b/crates/core/src/ranking/pipeline/scorers/lambdamart.rs
new file mode 100644
index 00000000..9dfc00c9
--- /dev/null
+++ b/crates/core/src/ranking/pipeline/scorers/lambdamart.rs
@@ -0,0 +1,70 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2024 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use crate::{
+ ranking::{
+ self, models,
+ pipeline::{PrecisionRankingWebpage, RankableWebpage, Top},
+ SignalCalculation, SignalEnum,
+ },
+ searcher::api::ScoredWebpagePointer,
+};
+use std::sync::Arc;
+
+use super::RankingStage;
+
+impl RankingStage for Arc<models::LambdaMART> {
+ type Webpage = ScoredWebpagePointer;
+
+ fn compute(&self, webpages: &Self::Webpage) -> (SignalEnum, SignalCalculation) {
+ let signals = webpages
+ .signals()
+ .iter()
+ .map(|(signal, calc)| (signal, calc.value))
+ .collect();
+ (
+ ranking::core::LambdaMart.into(),
+ SignalCalculation::new_symmetrical(self.predict(&signals)),
+ )
+ }
+
+ fn top_n(&self) -> Top {
+ Top::Limit(20)
+ }
+}
+
+pub struct PrecisionLambda(Arc<models::LambdaMART>);
+
+impl From<Arc<models::LambdaMART>> for PrecisionLambda {
+ fn from(model: Arc<models::LambdaMART>) -> Self {
+ Self(model)
+ }
+}
+
+impl RankingStage for PrecisionLambda {
+ type Webpage = PrecisionRankingWebpage;
+
+ fn compute(&self, webpage: &Self::Webpage) -> (SignalEnum, SignalCalculation) {
+ (
+ ranking::core::LambdaMart.into(),
+ SignalCalculation::new_symmetrical(self.0.predict(webpage.ranking().signals())),
+ )
+ }
+
+ fn top_n(&self) -> Top {
+ Top::Limit(20)
+ }
+}
diff --git a/crates/core/src/ranking/pipeline/scorers/mod.rs b/crates/core/src/ranking/pipeline/scorers/mod.rs
index 242389d7..bc561e52 100644
--- a/crates/core/src/ranking/pipeline/scorers/mod.rs
+++ b/crates/core/src/ranking/pipeline/scorers/mod.rs
@@ -16,48 +16,60 @@
pub mod embedding;
pub mod inbound_similarity;
-pub mod recall;
+pub mod lambdamart;
pub mod reranker;
pub mod term_distance;
-pub use recall::Recall;
pub use reranker::ReRanker;
-use crate::searcher::SearchQuery;
+use crate::ranking::{SignalCalculation, SignalCoefficients, SignalEnum};
-use super::RankableWebpage;
+use super::{RankableWebpage, Top};
-pub trait Scorer<T: RankableWebpage>: Send + Sync {
- fn score(&self, webpages: &mut [T]);
- fn set_query_info(&mut self, _query: &SearchQuery) {}
-}
+pub trait FullRankingStage: Send + Sync {
+ type Webpage: RankableWebpage;
-pub struct IdentityScorer;
+ fn compute(&self, webpages: &mut [Self::Webpage]);
+ fn top_n(&self) -> Top {
+ Top::Unlimited
+ }
-impl<T: RankableWebpage> Scorer<T> for IdentityScorer {
- fn score(&self, _webpages: &mut [T]) {}
-}
+ fn update_scores(&self, webpages: &mut [Self::Webpage], coefficients: &SignalCoefficients) {
+ for webpage in webpages.iter_mut() {
+ webpage.set_raw_score(webpage.signals().iter().fold(0.0, |acc, (signal, calc)| {
+ acc + calc.score * coefficients.get(&signal)
+ }));
+ }
+ }
-pub struct MultiScorer<T: RankableWebpage> {
- scorers: Vec<Box<dyn Scorer<T>>>,
+ fn rank(&self, webpages: &mut [Self::Webpage]) {
+ webpages.sort_by(|a, b| b.score().partial_cmp(&a.score()).unwrap());
+ }
}
-impl<T: RankableWebpage> MultiScorer<T> {
- pub fn new(scorers: Vec<Box<dyn Scorer<T>>>) -> Self {
- Self { scorers }
+pub trait RankingStage: Send + Sync {
+ type Webpage: RankableWebpage;
+
+ fn compute(&self, webpage: &Self::Webpage) -> (SignalEnum, SignalCalculation);
+ fn top_n(&self) -> Top {
+ Top::Unlimited
}
}
-impl<T: RankableWebpage> Scorer<T> for MultiScorer<T> {
- fn score(&self, webpages: &mut [T]) {
- for scorer in &self.scorers {
- scorer.score(webpages);
+impl<T> FullRankingStage for T
+where
+ T: RankingStage,
+{
+ type Webpage = <T as RankingStage>::Webpage;
+
+ fn compute(&self, webpages: &mut [Self::Webpage]) {
+ for webpage in webpages.iter_mut() {
+ let (signal, signal_calculation) = self.compute(webpage);
+ webpage.signals_mut().insert(signal, signal_calculation);
}
}
- fn set_query_info(&mut self, query: &SearchQuery) {
- for scorer in &mut self.scorers {
- scorer.set_query_info(query);
- }
+ fn top_n(&self) -> Top {
+ self.top_n()
}
}
diff --git a/crates/core/src/ranking/pipeline/scorers/recall.rs b/crates/core/src/ranking/pipeline/scorers/recall.rs
deleted file mode 100644
index b7a0820d..00000000
--- a/crates/core/src/ranking/pipeline/scorers/recall.rs
+++ /dev/null
@@ -1,97 +0,0 @@
-// Stract is an open source web search engine.
-// Copyright (C) 2024 Stract ApS
-//
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as
-// published by the Free Software Foundation, either version 3 of the
-// License, or (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with this program. If not, see <https://www.gnu.org/licenses/>.
-
-use std::sync::Arc;
-
-use crate::{
- models::dual_encoder::DualEncoder,
- ranking::{
- inbound_similarity,
- pipeline::{LocalRecallRankingWebpage, RankableWebpage},
- },
- searcher::{
- api::{self},
- SearchQuery,
- },
-};
-
-use super::{
- embedding::{EmbeddingScorer, KeywordEmbeddings, TitleEmbeddings},
- inbound_similarity::InboundScorer,
- term_distance, MultiScorer, Scorer,
-};
-
-pub struct Recall<T: RankableWebpage> {
- scorer: MultiScorer<T>,
-}
-
-impl Recall<api::ScoredWebpagePointer> {
- pub fn new(
- inbound: inbound_similarity::Scorer,
- dual_encoder: Option<Arc<DualEncoder>>,
- ) -> Self {
- Self {
- scorer: MultiScorer::new(vec![
- Box::new(
- EmbeddingScorer::<api::ScoredWebpagePointer, TitleEmbeddings>::new(
- dual_encoder.clone(),
- ),
- ),
- Box::new(EmbeddingScorer::<
- api::ScoredWebpagePointer,
- KeywordEmbeddings,
- >::new(dual_encoder)),
- Box::new(InboundScorer::new(inbound)),
- ]),
- }
- }
-}
-
-impl Scorer<api::ScoredWebpagePointer> for Recall<api::ScoredWebpagePointer> {
- fn score(&self, webpages: &mut [api::ScoredWebpagePointer]) {
- self.scorer.score(webpages);
- }
-}
-
-impl Recall<LocalRecallRankingWebpage> {
- pub fn new(dual_encoder: Option<Arc<DualEncoder>>) -> Self {
- Self {
- scorer: MultiScorer::new(vec![
- Box::new(term_distance::TitleDistanceScorer),
- Box::new(term_distance::BodyDistanceScorer),
- Box::new(
- EmbeddingScorer::<LocalRecallRankingWebpage, TitleEmbeddings>::new(
- dual_encoder.clone(),
- ),
- ),
- Box::new(EmbeddingScorer::<
- LocalRecallRankingWebpage,
- KeywordEmbeddings,
- >::new(dual_encoder)),
- ]),
- }
- }
-}
-
-impl Scorer<LocalRecallRankingWebpage> for Recall<LocalRecallRankingWebpage> {
- fn score(&self, webpages: &mut [LocalRecallRankingWebpage]) {
- self.scorer.score(webpages);
- }
-
- fn set_query_info(&mut self, query: &SearchQuery) {
- self.scorer.set_query_info(query);
- }
-}
diff --git a/crates/core/src/ranking/pipeline/scorers/reranker.rs b/crates/core/src/ranking/pipeline/scorers/reranker.rs
index 5eefd5ca..59aa754b 100644
--- a/crates/core/src/ranking/pipeline/scorers/reranker.rs
+++ b/crates/core/src/ranking/pipeline/scorers/reranker.rs
@@ -16,23 +16,20 @@
use std::sync::Arc;
-use crate::{
- ranking::{self, models::cross_encoder::CrossEncoder},
- searcher::SearchQuery,
-};
+use crate::ranking::{self, models::cross_encoder::CrossEncoder, pipeline::RankableWebpage};
-use crate::ranking::pipeline::{PrecisionRankingWebpage, Scorer};
+use crate::ranking::pipeline::{FullRankingStage, PrecisionRankingWebpage, Top};
pub struct ReRanker<M: CrossEncoder> {
crossencoder: Arc<M>,
- query: Option<SearchQuery>,
+ query: String,
}
impl<M: CrossEncoder> ReRanker<M> {
- pub fn new(crossencoder: Arc<M>) -> Self {
+ pub fn new(query: String, crossencoder: Arc<M>) -> Self {
Self {
crossencoder,
- query: None,
+ query,
}
}
@@ -45,31 +42,33 @@ impl<M: CrossEncoder> ReRanker<M> {
snippets.push(webpage.retrieved_webpage().snippet.unhighlighted_string());
}
- let query = &self.query.as_ref().unwrap().query;
+ let query = &self.query;
let snippet_scores = self.crossencoder.run(query, &snippets);
let title_scores = self.crossencoder.run(query, &titles);
for ((webpage, snippet), title) in webpage.iter_mut().zip(snippet_scores).zip(title_scores)
{
- webpage
- .ranking_mut()
- .signals_mut()
- .insert(ranking::core::CrossEncoderSnippet.into(), snippet);
+ webpage.ranking_mut().signals_mut().insert(
+ ranking::core::CrossEncoderSnippet.into(),
+ ranking::SignalCalculation::new_symmetrical(snippet),
+ );
- webpage
- .ranking_mut()
- .signals_mut()
- .insert(ranking::core::CrossEncoderTitle.into(), title);
+ webpage.ranking_mut().signals_mut().insert(
+ ranking::core::CrossEncoderTitle.into(),
+ ranking::SignalCalculation::new_symmetrical(title),
+ );
}
}
}
-impl<M: CrossEncoder> Scorer<PrecisionRankingWebpage> for ReRanker<M> {
- fn score(&self, webpages: &mut [PrecisionRankingWebpage]) {
+impl<M: CrossEncoder> FullRankingStage for ReRanker<M> {
+ type Webpage = PrecisionRankingWebpage;
+
+ fn compute(&self, webpages: &mut [Self::Webpage]) {
self.crossencoder_score_webpages(webpages);
}
- fn set_query_info(&mut self, query: &SearchQuery) {
- self.query = Some(query.clone());
+ fn top_n(&self) -> Top {
+ Top::Limit(20)
}
}
diff --git a/crates/core/src/ranking/pipeline/scorers/term_distance.rs b/crates/core/src/ranking/pipeline/scorers/term_distance.rs
index fe8eb0e7..856e1687 100644
--- a/crates/core/src/ranking/pipeline/scorers/term_distance.rs
+++ b/crates/core/src/ranking/pipeline/scorers/term_distance.rs
@@ -16,8 +16,9 @@
use itertools::Itertools;
-use crate::ranking;
use crate::ranking::pipeline::RankableWebpage;
+use crate::ranking::{self, SignalCalculation, SignalEnum};
+use crate::searcher::api;
fn min_slop_two_positions(pos_a: &[u32], pos_b: &[u32]) -> u32 {
let mut cur_min = u32::MAX;
@@ -52,35 +53,47 @@ fn min_slop<'a>(positions: impl Iterator<Item = &'a [u32]>) -> u32 {
.unwrap_or(u32::MAX)
}
+fn score_slop(slop: f64) -> f64 {
+ 1.0 / (slop + 1.0)
+}
+
#[derive(Debug, Default)]
pub struct TitleDistanceScorer;
-impl<W: RankableWebpage> super::Scorer<W> for TitleDistanceScorer {
- fn score(&self, webpages: &mut [W]) {
- for webpage in webpages {
- let min_slop = min_slop(webpage.as_local_recall().iter_title_positions());
+impl super::RankingStage for TitleDistanceScorer {
+ type Webpage = api::ScoredWebpagePointer;
- webpage.signals_mut().insert(
- ranking::core::MinTitleSlop.into(),
- 1.0 / (min_slop as f64 + 1.0),
- );
- }
+ fn compute(&self, webpage: &Self::Webpage) -> (SignalEnum, SignalCalculation) {
+ let min_slop = min_slop(webpage.as_local_recall().iter_title_positions()) as f64;
+ let score = score_slop(min_slop);
+
+ (
+ ranking::core::MinTitleSlop.into(),
+ ranking::SignalCalculation {
+ value: min_slop,
+ score,
+ },
+ )
}
}
#[derive(Debug, Default)]
pub struct BodyDistanceScorer;
-impl<W: RankableWebpage> super::Scorer<W> for BodyDistanceScorer {
- fn score(&self, webpages: &mut [W]) {
- for webpage in webpages {
- let min_slop = min_slop(webpage.as_local_recall().iter_clean_body_positions());
+impl super::RankingStage for BodyDistanceScorer {
+ type Webpage = api::ScoredWebpagePointer;
- webpage.signals_mut().insert(
- ranking::core::MinCleanBodySlop.into(),
- 1.0 / (min_slop as f64 + 1.0),
- );
- }
+ fn compute(&self, webpage: &Self::Webpage) -> (SignalEnum, SignalCalculation) {
+ let min_slop = min_slop(webpage.as_local_recall().iter_clean_body_positions()) as f64;
+ let score = score_slop(min_slop);
+
+ (
+ ranking::core::MinCleanBodySlop.into(),
+ ranking::SignalCalculation {
+ value: min_slop,
+ score,
+ },
+ )
}
}
diff --git a/crates/core/src/ranking/pipeline/stages/precision.rs b/crates/core/src/ranking/pipeline/stages/precision.rs
index b84f1cad..59847eaa 100644
--- a/crates/core/src/ranking/pipeline/stages/precision.rs
+++ b/crates/core/src/ranking/pipeline/stages/precision.rs
@@ -18,24 +18,21 @@ use std::sync::Arc;
use crate::{
collector,
- config::CollectorConfig,
enum_map::EnumMap,
inverted_index::RetrievedWebpage,
ranking::{
- models::{cross_encoder::CrossEncoder, lambdamart::LambdaMART},
+ models::{self, cross_encoder::CrossEncoder},
pipeline::{
- scorers::IdentityScorer, RankableWebpage, RankingPipeline, RankingStage, ReRanker,
- Scorer,
+ scorers::lambdamart::PrecisionLambda, RankableWebpage, RankingPipeline, ReRanker,
},
- SignalEnum,
+ SignalCalculation, SignalEnum,
},
searcher::SearchQuery,
- Result,
};
use super::RecallRankingWebpage;
-#[derive(Clone, Debug, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]
+#[derive(Clone, Debug, bincode::Encode, bincode::Decode)]
pub struct PrecisionRankingWebpage {
retrieved_webpage: RetrievedWebpage,
ranking: RecallRankingWebpage,
@@ -57,7 +54,7 @@ impl PrecisionRankingWebpage {
impl collector::Doc for PrecisionRankingWebpage {
fn score(&self) -> f64 {
- self.ranking.score()
+ RankableWebpage::score(self)
}
fn hashes(&self) -> collector::Hashes {
@@ -66,19 +63,27 @@ impl collector::Doc for PrecisionRankingWebpage {
}
impl RankableWebpage for PrecisionRankingWebpage {
- fn set_score(&mut self, score: f64) {
- self.ranking.set_score(score);
+ fn set_raw_score(&mut self, score: f64) {
+ self.ranking.set_raw_score(score);
}
- fn boost(&self) -> Option<f64> {
+ fn unboosted_score(&self) -> f64 {
+ self.ranking.unboosted_score()
+ }
+
+ fn boost(&self) -> f64 {
self.ranking.boost()
}
- fn signals(&self) -> &EnumMap<SignalEnum, f64> {
+ fn set_boost(&mut self, boost: f64) {
+ self.ranking.set_boost(boost)
+ }
+
+ fn signals(&self) -> &EnumMap<SignalEnum, SignalCalculation> {
self.ranking.signals()
}
- fn signals_mut(&mut self) -> &mut EnumMap<SignalEnum, f64> {
+ fn signals_mut(&mut self) -> &mut EnumMap<SignalEnum, SignalCalculation> {
self.ranking.signals_mut()
}
@@ -101,46 +106,18 @@ impl PrecisionRankingWebpage {
}
impl RankingPipeline<PrecisionRankingWebpage> {
- fn create_reranking_stage<M: CrossEncoder + 'static>(
- crossencoder: Option<Arc<M>>,
- lambda: Option<Arc<LambdaMART>>,
- collector_config: CollectorConfig,
- top_n_considered: usize,
- ) -> Result<Self> {
- let scorer = match crossencoder {
- Some(cross_encoder) => {
- Box::new(ReRanker::new(cross_encoder)) as Box<dyn Scorer<PrecisionRankingWebpage>>
- }
- None => Box::new(IdentityScorer) as Box<dyn Scorer<PrecisionRankingWebpage>>,
- };
-
- let stage = RankingStage {
- scorer,
- stage_top_n: top_n_considered,
- derank_similar: true,
- model: lambda,
- coefficients: Default::default(),
- };
-
- Ok(Self {
- stage,
- page: 0,
- top_n: 0,
- collector_config,
- })
- }
-
pub fn reranker<M: CrossEncoder + 'static>(
- query: &mut SearchQuery,
- crossencoder: Option<Arc<M>>,
- lambda: Option<Arc<LambdaMART>>,
- collector_config: CollectorConfig,
- top_n_considered: usize,
- ) -> Result<Self> {
- let mut pipeline =
- Self::create_reranking_stage(crossencoder, lambda, collector_config, top_n_considered)?;
- pipeline.set_query_info(query);
-
- Ok(pipeline)
+ query: &SearchQuery,
+ crossencoder: Arc<M>,
+ lambda: Option<Arc<models::LambdaMART>>,
+ ) -> Self {
+ let mut s = Self::new().add_stage(ReRanker::new(query.text().to_string(), crossencoder));
+
+ if let Some(lambda) = lambda {
+ let lambda = PrecisionLambda::from(lambda);
+ s = s.add_stage(lambda);
+ }
+
+ s
}
}
diff --git a/crates/core/src/ranking/pipeline/stages/recall.rs b/crates/core/src/ranking/pipeline/stages/recall.rs
index fd7f0507..3122e5ca 100644
--- a/crates/core/src/ranking/pipeline/stages/recall.rs
+++ b/crates/core/src/ranking/pipeline/stages/recall.rs
@@ -18,7 +18,6 @@ use std::sync::Arc;
use crate::{
collector,
- config::CollectorConfig,
enum_map::EnumMap,
inverted_index::WebpagePointer,
models::dual_encoder::DualEncoder,
@@ -26,8 +25,16 @@ use crate::{
ranking::{
bitvec_similarity, inbound_similarity,
models::lambdamart::LambdaMART,
- pipeline::{RankableWebpage, RankingPipeline, RankingStage, Recall, Scorer},
- SignalComputer, SignalEnum,
+ pipeline::{
+ modifiers,
+ scorers::{
+ embedding::{EmbeddingScorer, KeywordEmbeddings, TitleEmbeddings},
+ inbound_similarity::InboundScorer,
+ term_distance,
+ },
+ RankableWebpage, RankingPipeline,
+ },
+ SignalCalculation, SignalComputer, SignalEnum,
},
schema::{numerical_field, text_field},
searcher::{api, SearchQuery},
@@ -42,7 +49,7 @@ impl StoredEmbeddings {
&self.0
}
}
-#[derive(Clone, Debug, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]
+#[derive(Clone, Debug, bincode::Encode, bincode::Decode)]
pub struct RecallRankingWebpage {
local: LocalRecallRankingWebpage,
inbound_edges: bitvec_similarity::BitVec,
@@ -68,26 +75,6 @@ impl RecallRankingWebpage {
self.local.keyword_embedding()
}
- pub fn score(&self) -> f64 {
- self.local.score()
- }
-
- pub fn signals(&self) -> &EnumMap<SignalEnum, f64> {
- self.local.signals()
- }
-
- pub fn signals_mut(&mut self) -> &mut EnumMap<SignalEnum, f64> {
- self.local.signals_mut()
- }
-
- pub fn boost(&self) -> Option<f64> {
- self.local.boost()
- }
-
- pub fn set_score(&mut self, score: f64) {
- self.local.set_score(score)
- }
-
pub fn inbound_edges(&self) -> &bitvec_similarity::BitVec {
&self.inbound_edges
}
@@ -108,19 +95,27 @@ impl collector::Doc for RecallRankingWebpage {
}
impl RankableWebpage for RecallRankingWebpage {
- fn set_score(&mut self, score: f64) {
+ fn set_raw_score(&mut self, score: f64) {
self.local.set_score(score);
}
- fn boost(&self) -> Option<f64> {
+ fn unboosted_score(&self) -> f64 {
+ self.local.unboosted_score()
+ }
+
+ fn boost(&self) -> f64 {
self.local.boost()
}
- fn signals(&self) -> &EnumMap<SignalEnum, f64> {
+ fn set_boost(&mut self, boost: f64) {
+ self.local.set_boost(boost)
+ }
+
+ fn signals(&self) -> &EnumMap<SignalEnum, SignalCalculation> {
self.local.signals()
}
- fn signals_mut(&mut self) -> &mut EnumMap<SignalEnum, f64> {
+ fn signals_mut(&mut self) -> &mut EnumMap<SignalEnum, SignalCalculation> {
self.local.signals_mut()
}
@@ -129,13 +124,13 @@ impl RankableWebpage for RecallRankingWebpage {
}
}
-#[derive(Clone, Debug, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]
+#[derive(Clone, Debug, bincode::Encode, bincode::Decode)]
pub struct LocalRecallRankingWebpage {
pointer: WebpagePointer,
- signals: EnumMap<SignalEnum, f64>,
+ signals: EnumMap<SignalEnum, SignalCalculation>,
title_positions: Vec<Vec<u32>>,
clean_body_positions: Vec<Vec<u32>>,
- optic_boost: Option<f64>,
+ boost: f64,
title_embedding: Option<StoredEmbeddings>,
keyword_embedding: Option<StoredEmbeddings>,
score: f64,
@@ -146,7 +141,7 @@ impl LocalRecallRankingWebpage {
#[cfg(test)]
pub fn new_testing(
pointer: WebpagePointer,
- signals: EnumMap<SignalEnum, f64>,
+ signals: EnumMap<SignalEnum, SignalCalculation>,
score: f64,
) -> Self {
LocalRecallRankingWebpage {
@@ -154,7 +149,7 @@ impl LocalRecallRankingWebpage {
signals,
title_positions: Vec::new(),
clean_body_positions: Vec::new(),
- optic_boost: None,
+ boost: 1.0,
title_embedding: None,
keyword_embedding: None,
score,
@@ -198,7 +193,7 @@ impl LocalRecallRankingWebpage {
let mut res = LocalRecallRankingWebpage {
signals: EnumMap::new(),
score: pointer.score.total,
- optic_boost: None,
+ boost: 1.0,
pointer: pointer.clone(),
title_embedding: title_embedding.map(StoredEmbeddings),
keyword_embedding: keyword_embedding.map(StoredEmbeddings),
@@ -209,11 +204,11 @@ impl LocalRecallRankingWebpage {
for computed_signal in computer.compute_signals(pointer.address.doc_id) {
res.signals
- .insert(computed_signal.signal, computed_signal.score);
+ .insert(computed_signal.signal, computed_signal.calc);
}
if let Some(boost) = computer.boosts(pointer.address.doc_id) {
- res.optic_boost = Some(boost);
+ res.boost *= boost;
}
res
@@ -232,21 +227,17 @@ impl LocalRecallRankingWebpage {
}
pub fn score(&self) -> f64 {
- self.score
+ self.score * self.boost
}
- pub fn signals(&self) -> &EnumMap<SignalEnum, f64> {
+ pub fn signals(&self) -> &EnumMap<SignalEnum, SignalCalculation> {
&self.signals
}
- pub fn signals_mut(&mut self) -> &mut EnumMap<SignalEnum, f64> {
+ pub fn signals_mut(&mut self) -> &mut EnumMap<SignalEnum, SignalCalculation> {
&mut self.signals
}
- pub fn boost(&self) -> Option<f64> {
- self.optic_boost
- }
-
pub fn set_score(&mut self, score: f64) {
self.score = score;
}
@@ -265,19 +256,27 @@ impl LocalRecallRankingWebpage {
}
impl RankableWebpage for LocalRecallRankingWebpage {
- fn set_score(&mut self, score: f64) {
+ fn set_raw_score(&mut self, score: f64) {
self.score = score;
}
- fn boost(&self) -> Option<f64> {
- self.optic_boost
+ fn unboosted_score(&self) -> f64 {
+ self.score
+ }
+
+ fn boost(&self) -> f64 {
+ self.boost
}
- fn signals(&self) -> &EnumMap<SignalEnum, f64> {
+ fn set_boost(&mut self, boost: f64) {
+ self.boost = boost;
+ }
+
+ fn signals(&self) -> &EnumMap<SignalEnum, SignalCalculation> {
&self.signals
}
- fn signals_mut(&mut self) -> &mut EnumMap<SignalEnum, f64> {
+ fn signals_mut(&mut self) -> &mut EnumMap<SignalEnum, SignalCalculation> {
&mut self.signals
}
@@ -288,7 +287,7 @@ impl RankableWebpage for LocalRecallRankingWebpage {
impl collector::Doc for LocalRecallRankingWebpage {
fn score(&self) -> f64 {
- self.score
+ RankableWebpage::score(self)
}
fn hashes(&self) -> collector::Hashes {
@@ -296,89 +295,33 @@ impl collector::Doc for LocalRecallRankingWebpage {
}
}
-impl RankingPipeline<LocalRecallRankingWebpage> {
- fn create_recall_stage(
- lambdamart: Option<Arc<LambdaMART>>,
- dual_encoder: Option<Arc<DualEncoder>>,
- collector_config: CollectorConfig,
- stage_top_n: usize,
- ) -> Self {
- let last_stage = RankingStage {
- scorer: Box::new(Recall::<LocalRecallRankingWebpage>::new(dual_encoder))
- as Box<dyn Scorer<LocalRecallRankingWebpage>>,
- stage_top_n,
- derank_similar: true,
- model: lambdamart,
- coefficients: Default::default(),
- };
-
- Self {
- stage: last_stage,
- page: 0,
- top_n: 0,
- collector_config,
- }
- }
-
- pub fn recall_stage(
- query: &mut SearchQuery,
- lambdamart: Option<Arc<LambdaMART>>,
- dual_encoder: Option<Arc<DualEncoder>>,
- collector_config: CollectorConfig,
- top_n_considered: usize,
- ) -> Self {
- let mut pipeline =
- Self::create_recall_stage(lambdamart, dual_encoder, collector_config, top_n_considered);
- pipeline.set_query_info(query);
-
- pipeline
- }
-}
-
impl RankingPipeline<api::ScoredWebpagePointer> {
- fn create_recall_stage(
+ pub fn recall_stage(
+ query: &SearchQuery,
inbound: inbound_similarity::Scorer,
lambdamart: Option<Arc<LambdaMART>>,
dual_encoder: Option<Arc<DualEncoder>>,
- collector_config: CollectorConfig,
- stage_top_n: usize,
) -> Self {
- let last_stage = RankingStage {
- scorer: Box::new(Recall::<api::ScoredWebpagePointer>::new(
- inbound,
- dual_encoder,
- )),
- stage_top_n,
- derank_similar: true,
- model: lambdamart,
- coefficients: Default::default(),
- };
-
- Self {
- stage: last_stage,
- page: 0,
- top_n: 0,
- collector_config,
+ let mut s = Self::new()
+ .add_stage(term_distance::TitleDistanceScorer)
+ .add_stage(term_distance::BodyDistanceScorer)
+ .add_stage(
+ EmbeddingScorer::<api::ScoredWebpagePointer, TitleEmbeddings>::new(
+ query.text().to_string(),
+ dual_encoder.clone(),
+ ),
+ )
+ .add_stage(EmbeddingScorer::<
+ api::ScoredWebpagePointer,
+ KeywordEmbeddings,
+ >::new(query.text().to_string(), dual_encoder))
+ .add_stage(InboundScorer::new(inbound))
+ .add_modifier(modifiers::InboundSimilarity);
+
+ if let Some(lambda) = lambdamart {
+ s = s.add_stage(lambda);
}
- }
- pub fn recall_stage(
- query: &mut SearchQuery,
- inbound: inbound_similarity::Scorer,
- lambdamart: Option<Arc<LambdaMART>>,
- dual_encoder: Option<Arc<DualEncoder>>,
- collector_config: CollectorConfig,
- top_n_considered: usize,
- ) -> Self {
- let mut pipeline = Self::create_recall_stage(
- inbound,
- lambdamart,
- dual_encoder,
- collector_config,
- top_n_considered,
- );
- pipeline.set_query_info(query);
-
- pipeline
+ s
}
}
diff --git a/crates/core/src/search_prettifier/mod.rs b/crates/core/src/search_prettifier/mod.rs
index 4e27ec8f..966fe333 100644
--- a/crates/core/src/search_prettifier/mod.rs
+++ b/crates/core/src/search_prettifier/mod.rs
@@ -197,7 +197,6 @@ pub struct DisplayedWebpage {
pub rich_snippet: Option<RichSnippet>,
pub ranking_signals: Option<HashMap<SignalEnumDiscriminants, SignalScore>>,
pub structured_data: Option<Vec<StructuredData>>,
- pub score: Option<f64>,
pub likely_has_ads: bool,
pub likely_has_paywall: bool,
}
@@ -251,7 +250,6 @@ impl DisplayedWebpage {
#[cfg(feature = "return_body")]
body,
ranking_signals: None,
- score: None,
likely_has_ads: webpage.likely_has_ads,
likely_has_paywall: webpage.likely_has_paywall,
rich_snippet,
diff --git a/crates/core/src/searcher/api/mod.rs b/crates/core/src/searcher/api/mod.rs
index 1a9b5192..d6df9aa2 100644
--- a/crates/core/src/searcher/api/mod.rs
+++ b/crates/core/src/searcher/api/mod.rs
@@ -27,7 +27,7 @@ use url::Url;
use ahash::AHashMap as HashMap;
use crate::bangs::{Bang, BangHit};
-use crate::collector::{self, approx_count, Doc};
+use crate::collector::{self, approx_count};
use crate::config::{ApiConfig, ApiSpellCheck, ApiThresholds, CollectorConfig, WidgetsConfig};
use crate::enum_map::EnumMap;
use crate::image_store::Image;
@@ -36,7 +36,8 @@ use crate::models::dual_encoder::DualEncoder;
use crate::ranking::models::cross_encoder::CrossEncoderModel;
use crate::ranking::pipeline::{PrecisionRankingWebpage, RankableWebpage, RecallRankingWebpage};
use crate::ranking::{
- bitvec_similarity, inbound_similarity, SignalCoefficient, SignalEnum, SignalScore,
+ bitvec_similarity, inbound_similarity, SignalCalculation, SignalCoefficients, SignalEnum,
+ SignalScore,
};
use crate::search_prettifier::{DisplayedSidebar, DisplayedWebpage, HighlightedSpellCorrection};
use crate::web_spell::SpellChecker;
@@ -55,6 +56,8 @@ use self::widget::WidgetManager;
use super::{distributed, live, SearchQuery, SearchResult, WebsitesResult};
+const NUM_PIPELINE_RANKING_RESULTS: usize = 300;
+
#[derive(Clone)]
pub enum ScoredWebpagePointer {
Normal(distributed::ScoredWebpagePointer),
@@ -78,26 +81,34 @@ impl ScoredWebpagePointer {
}
impl RankableWebpage for ScoredWebpagePointer {
- fn set_score(&mut self, score: f64) {
- self.as_ranking_mut().set_score(score);
+ fn set_raw_score(&mut self, score: f64) {
+ self.as_ranking_mut().set_raw_score(score);
+ }
+
+ fn unboosted_score(&self) -> f64 {
+ self.as_ranking().unboosted_score()
}
- fn boost(&self) -> Option<f64> {
+ fn boost(&self) -> f64 {
self.as_ranking().boost()
}
+ fn set_boost(&mut self, boost: f64) {
+ self.as_ranking_mut().set_boost(boost)
+ }
+
fn as_local_recall(&self) -> &crate::ranking::pipeline::LocalRecallRankingWebpage {
self.as_ranking().as_local_recall()
}
- fn signals(&self) -> &EnumMap<SignalEnum, f64> {
+ fn signals(&self) -> &EnumMap<SignalEnum, SignalCalculation> {
match self {
ScoredWebpagePointer::Normal(p) => p.website.signals(),
ScoredWebpagePointer::Live(p) => p.website.signals(),
}
}
- fn signals_mut(&mut self) -> &mut EnumMap<SignalEnum, f64> {
+ fn signals_mut(&mut self) -> &mut EnumMap<SignalEnum, SignalCalculation> {
match self {
ScoredWebpagePointer::Normal(p) => p.website.signals_mut(),
ScoredWebpagePointer::Live(p) => p.website.signals_mut(),
@@ -107,7 +118,7 @@ impl RankableWebpage for ScoredWebpagePointer {
impl collector::Doc for ScoredWebpagePointer {
fn score(&self) -> f64 {
- self.as_ranking().score()
+ RankableWebpage::score(self)
}
fn hashes(&self) -> collector::Hashes {
@@ -118,7 +129,7 @@ impl collector::Doc for ScoredWebpagePointer {
pub fn add_ranking_signals(
websites: &mut [DisplayedWebpage],
pointers: &[ScoredWebpagePointer],
- coeffs: &SignalCoefficient,
+ coeffs: &SignalCoefficients,
) {
for (website, pointer) in websites.iter_mut().zip(pointers.iter()) {
let mut signals = std::collections::HashMap::new();
@@ -128,7 +139,7 @@ pub fn add_ranking_signals(
signals.insert(
signal.into(),
SignalScore {
- value: *signal_value,
+ value: signal_value.score,
coefficient: coeffs.get(&signal),
},
);
@@ -469,12 +480,12 @@ where
async fn combine_results(
&self,
- collector_config: CollectorConfig,
+ query: &SearchQuery,
initial_results: Vec<distributed::InitialSearchResultShard>,
live_results: Vec<live::InitialSearchResultSplit>,
- pipeline: RankingPipeline<ScoredWebpagePointer>,
) -> (Vec<ScoredWebpagePointer>, bool) {
- let mut collector = BucketCollector::new(pipeline.collector_top_n(), collector_config);
+ let mut collector =
+ BucketCollector::new(NUM_PIPELINE_RANKING_RESULTS, self.collector_config.clone());
let initial_host_nodes = initial_results
.iter()
@@ -502,12 +513,10 @@ where
.map(|(v, n)| (n, v))
.collect::<HashMap<_, _>>();
- let mut has_more = false;
- for result in initial_results {
- if result.local_result.has_more {
- has_more = true;
- }
+ let mut num_results = 0;
+ for result in initial_results {
+ num_results += result.local_result.websites.len();
for website in result.local_result.websites {
let inbound = host_nodes
.get(website.host_id())
@@ -525,10 +534,7 @@ where
}
for result in live_results {
- if result.local_result.has_more {
- has_more = true;
- }
-
+ num_results += result.local_result.websites.len();
for website in result.local_result.websites {
let inbound = host_nodes
.get(website.host_id())
@@ -545,14 +551,14 @@ where
}
}
- let top_websites = collector
+ let has_more = query.offset() + query.num_results() < num_results;
+
+ let res = collector
.into_sorted_vec(true)
.into_iter()
- .take(pipeline.collector_top_n())
+ .take(NUM_PIPELINE_RANKING_RESULTS)
.collect::<Vec<_>>();
- let res = pipeline.apply(top_websites);
-
(res, has_more)
}
@@ -578,6 +584,57 @@ where
}
}
+ async fn search_websites_approx_offsets(&self, query: &SearchQuery) -> Result<WebsitesResult> {
+ let start = Instant::now();
+
+ let search_query = SearchQuery {
+ num_results: query.num_results + 1,
+ ..query.clone()
+ };
+
+ let results = self
+ .distributed_searcher
+ .search_initial(&search_query)
+ .await;
+
+ let has_more_results = results
+ .iter()
+ .any(|res| res.local_result.websites.len() > query.num_results());
+
+ let num_docs = results
+ .iter()
+ .map(|result| result.local_result.num_websites)
+ .fold(approx_count::Count::Exact(0), |acc, count| acc + count);
+
+ let (combined, _) = self.combine_results(query, results, vec![]).await;
+ let combined: Vec<_> = combined.into_iter().take(query.num_results).collect();
+
+ let mut retrieved_webpages: Vec<_> = self
+ .retrieve_webpages(&query.query, &combined)
+ .await
+ .into_iter()
+ .map(|webpage| webpage.into_retrieved_webpage())
+ .map(|webpage| DisplayedWebpage::new(webpage, query))
+ .collect();
+
+ if query.return_ranking_signals {
+ add_ranking_signals(
+ &mut retrieved_webpages,
+ &combined,
+ &query.signal_coefficients(),
+ );
+ }
+
+ let search_duration_ms = start.elapsed().as_millis();
+
+ Ok(WebsitesResult {
+ num_hits: num_docs,
+ webpages: retrieved_webpages,
+ search_duration_ms,
+ has_more_results,
+ })
+ }
+
async fn search_websites(&self, query: &SearchQuery) -> Result<WebsitesResult> {
let start = Instant::now();
@@ -585,22 +642,17 @@ where
return Err(distributed::Error::EmptyQuery.into());
}
- let mut search_query = query.clone();
- let inbound_scorer = self.inbound_scorer(&search_query).await;
-
- let top_n = search_query.num_results;
+ if query.offset() + query.num_results() > NUM_PIPELINE_RANKING_RESULTS {
+ // this is most likely a bot
+ // let's not spend too much time correctly offsetting+ranking results
+ return self.search_websites_approx_offsets(query).await;
+ }
- // This pipeline should be created before the first search is performed
- // so the query knows how many results to fetch from the indices
- let recall_pipeline: RankingPipeline<ScoredWebpagePointer> =
- RankingPipeline::<ScoredWebpagePointer>::recall_stage(
- &mut search_query,
- inbound_scorer,
- self.lambda_model.clone(),
- self.dual_encoder.clone(),
- self.collector_config.clone(),
- top_n,
- );
+ let search_query = SearchQuery {
+ num_results: NUM_PIPELINE_RANKING_RESULTS,
+ page: 0,
+ ..query.clone()
+ };
let (initial_results, live_results) = tokio::join!(
self.distributed_searcher.search_initial(&search_query),
@@ -613,38 +665,45 @@ where
.fold(approx_count::Count::Exact(0), |acc, count| acc + count);
let (top_websites, has_more_results) = self
- .combine_results(
- self.collector_config.clone(),
- initial_results,
- live_results.unwrap_or_default(),
- recall_pipeline,
- )
+ .combine_results(query, initial_results, live_results.unwrap_or_default())
.await;
- let retrieved_webpages = self
- .retrieve_webpages(&search_query.query, &top_websites)
- .await;
-
- let mut search_query = SearchQuery {
- page: 0,
- ..query.clone()
- };
+ let inbound_scorer = self.inbound_scorer(query).await;
- let reranking_pipeline: RankingPipeline<PrecisionRankingWebpage> =
- RankingPipeline::<PrecisionRankingWebpage>::reranker(
- &mut search_query,
- self.cross_encoder.clone(),
+ let pipeline: RankingPipeline<ScoredWebpagePointer> =
+ RankingPipeline::<ScoredWebpagePointer>::recall_stage(
+ query,
+ inbound_scorer,
self.lambda_model.clone(),
- self.collector_config.clone(),
- query.num_results,
- )?;
+ self.dual_encoder.clone(),
+ );
- let retrieved_webpages = reranking_pipeline.apply(retrieved_webpages);
+ let top_websites = pipeline.apply(top_websites, query);
+
+ let mut retrieved_webpages = self.retrieve_webpages(&query.query, &top_websites).await;
+
+ if let Some(cross_encoder) = self.cross_encoder.clone() {
+ if query.page < 2 {
+ let query = SearchQuery {
+ page: 1,
+ ..query.clone()
+ };
+
+ let reranking_pipeline: RankingPipeline<PrecisionRankingWebpage> =
+ RankingPipeline::<PrecisionRankingWebpage>::reranker(
+ &query,
+ cross_encoder,
+ self.lambda_model.clone(),
+ );
+
+ retrieved_webpages = reranking_pipeline.apply(retrieved_webpages, &query);
+ }
+ }
let mut retrieved_webpages: Vec<_> = retrieved_webpages
.into_iter()
.map(|webpage| webpage.into_retrieved_webpage())
- .map(|webpage| DisplayedWebpage::new(webpage, &search_query))
+ .map(|webpage| DisplayedWebpage::new(webpage, query))
.collect();
if retrieved_webpages.len() != top_websites.len() {
@@ -655,14 +714,10 @@ where
add_ranking_signals(
&mut retrieved_webpages,
&top_websites,
- &search_query.signal_coefficients(),
+ &query.signal_coefficients(),
);
}
- for (website, pointer) in retrieved_webpages.iter_mut().zip(top_websites.iter()) {
- website.score = Some(pointer.score());
- }
-
let search_duration_ms = start.elapsed().as_millis();
Ok(WebsitesResult {
diff --git a/crates/core/src/searcher/local.rs b/crates/core/src/searcher/local.rs
index aa73952c..21ccaa38 100644
--- a/crates/core/src/searcher/local.rs
+++ b/crates/core/src/searcher/local.rs
@@ -29,9 +29,9 @@ use crate::query::Query;
use crate::ranking::models::lambdamart::LambdaMART;
use crate::ranking::models::linear::LinearRegression;
use crate::ranking::pipeline::{
- LocalRecallRankingWebpage, PrecisionRankingWebpage, RankingPipeline, RecallRankingWebpage,
+ LocalRecallRankingWebpage, PrecisionRankingWebpage, RankableWebpage, RecallRankingWebpage,
};
-use crate::ranking::{Ranker, SignalComputer, SignalEnum, SignalScore};
+use crate::ranking::{LocalRanker, SignalComputer, SignalEnum, SignalScore};
use crate::search_ctx::Ctx;
use crate::search_prettifier::DisplayedWebpage;
use crate::{inverted_index, live_index, Result};
@@ -121,7 +121,6 @@ where
struct InvertedIndexResult {
webpages: Vec<LocalRecallRankingWebpage>,
num_hits: approx_count::Count,
- has_more: bool,
}
impl<I> LocalSearcher<I>
@@ -173,8 +172,8 @@ where
guard: &G,
de_rank_similar: bool,
computer: SignalComputer,
- ) -> Result<Ranker> {
- let mut ranker = Ranker::new(
+ ) -> Result<LocalRanker> {
+ let mut ranker = LocalRanker::new(
computer,
guard.inverted_index().columnfield_reader(),
self.collector_config.clone(),
@@ -198,16 +197,7 @@ where
query: &SearchQuery,
de_rank_similar: bool,
) -> Result<InvertedIndexResult> {
- let mut query = query.clone();
- let pipeline: RankingPipeline<LocalRecallRankingWebpage> =
- RankingPipeline::<LocalRecallRankingWebpage>::recall_stage(
- &mut query,
- self.lambda_model.clone(),
- self.dual_encoder.clone(),
- self.collector_config.clone(),
- 100,
- );
- let parsed_query = self.parse_query(ctx, guard, &query)?;
+ let parsed_query = self.parse_query(ctx, guard, query)?;
let mut computer = SignalComputer::new(Some(&parsed_query));
@@ -241,15 +231,9 @@ where
&columnfield_reader,
)?;
- let pipe_top_n = pipeline.top_n;
- let has_more = ranking_websites.len() > pipe_top_n;
-
- let ranking_websites = pipeline.apply(ranking_websites);
-
Ok(InvertedIndexResult {
webpages: ranking_websites,
num_hits: res.num_websites,
- has_more,
})
}
@@ -270,7 +254,6 @@ where
Ok(InitialWebsiteResult {
websites: inverted_index_result.webpages,
num_websites: inverted_index_result.num_hits,
- has_more: inverted_index_result.has_more,
})
}
@@ -295,27 +278,7 @@ where
use std::time::Instant;
let start = Instant::now();
- let mut search_query = query.clone();
-
- let pipeline = {
- use crate::ranking::models::cross_encoder::CrossEncoderModel;
- match CrossEncoderModel::open("data/cross_encoder") {
- Ok(model) => RankingPipeline::reranker::<CrossEncoderModel>(
- &mut search_query,
- Some(Arc::new(model)),
- None,
- self.collector_config.clone(),
- query.num_results,
- )?,
- Err(_) => RankingPipeline::reranker::<CrossEncoderModel>(
- &mut search_query,
- None,
- None,
- self.collector_config.clone(),
- query.num_results,
- )?,
- }
- };
+ let search_query = query.clone();
let search_result = self.search_initial(&search_query, true)?;
@@ -335,13 +298,7 @@ where
})
.collect();
- let search_len = websites.len();
-
- let top_websites = pipeline.apply(websites);
-
- let has_more_results = search_len != top_websites.len();
-
- let pointers: Vec<_> = top_websites
+ let pointers: Vec<_> = websites
.iter()
.map(|website| website.ranking().pointer().clone())
.collect();
@@ -355,15 +312,15 @@ where
.map(|webpage| DisplayedWebpage::new(webpage, query))
.collect();
- for (webpage, ranking) in webpages.iter_mut().zip(top_websites) {
+ for (webpage, ranking) in webpages.iter_mut().zip(websites) {
let mut ranking_signals = HashMap::new();
for signal in SignalEnum::all() {
- if let Some(score) = ranking.ranking().signals().get(signal) {
+ if let Some(calc) = ranking.ranking().signals().get(signal) {
ranking_signals.insert(
signal.into(),
SignalScore {
- value: *score,
+ value: calc.score,
coefficient: coefficients.get(&signal),
},
);
@@ -377,7 +334,8 @@ where
num_hits: search_result.num_websites,
webpages,
search_duration_ms: start.elapsed().as_millis(),
- has_more_results,
+ has_more_results: (search_result.num_websites.as_u64() as usize)
+ > query.offset() + query.num_results(),
})
}
diff --git a/crates/core/src/searcher/mod.rs b/crates/core/src/searcher/mod.rs
index d0f34814..ad6ec8d1 100644
--- a/crates/core/src/searcher/mod.rs
+++ b/crates/core/src/searcher/mod.rs
@@ -31,7 +31,7 @@ use crate::{
bangs::BangHit,
collector::approx_count::Count,
config::defaults,
- ranking::{pipeline::LocalRecallRankingWebpage, SignalCoefficient},
+ ranking::{pipeline::LocalRecallRankingWebpage, SignalCoefficients},
search_prettifier::DisplayedWebpage,
webpage::region::Region,
};
@@ -80,14 +80,13 @@ pub struct SearchQuery {
pub return_body: Option<ReturnBody>,
pub return_structured_data: bool,
- pub signal_coefficients: SignalCoefficient,
+ pub signal_coefficients: SignalCoefficients,
}
-#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]
+#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]
pub struct InitialWebsiteResult {
pub num_websites: Count,
pub websites: Vec<LocalRecallRankingWebpage>,
- pub has_more: bool,
}
impl Default for SearchQuery {
@@ -117,7 +116,7 @@ impl SearchQuery {
self.query.is_empty()
}
- pub fn signal_coefficients(&self) -> SignalCoefficient {
+ pub fn signal_coefficients(&self) -> SignalCoefficients {
self.signal_coefficients.clone()
}
@@ -134,4 +133,16 @@ impl SearchQuery {
rankings
}
+
+ pub fn text(&self) -> &str {
+ &self.query
+ }
+
+ pub fn offset(&self) -> usize {
+ self.page * self.num_results
+ }
+
+ pub fn num_results(&self) -> usize {
+ self.num_results
+ }
}
diff --git a/crates/core/src/webgraph/shortest_path.rs b/crates/core/src/webgraph/shortest_path.rs
index 16071736..536fcc3b 100644
--- a/crates/core/src/webgraph/shortest_path.rs
+++ b/crates/core/src/webgraph/shortest_path.rs
@@ -25,6 +25,8 @@ pub trait ShortestPaths {
fn distances(&self, source: Node) -> BTreeMap<Node, u8>;
fn raw_distances(&self, source: NodeID) -> BTreeMap<NodeID, u8>;
fn raw_distances_with_max(&self, source: NodeID, max_dist: u8) -> BTreeMap<NodeID, u8>;
+ fn raw_reversed_distances_with_max(&self, source: NodeID, max_dist: u8)
+ -> BTreeMap<NodeID, u8>;
fn raw_reversed_distances(&self, source: NodeID) -> BTreeMap<NodeID, u8>;
fn reversed_distances(&self, source: Node) -> BTreeMap<Node, u8>;
}
@@ -113,6 +115,19 @@ impl ShortestPaths for Webgraph {
)
}
+ fn raw_reversed_distances_with_max(
+ &self,
+ source: NodeID,
+ max_dist: u8,
+ ) -> BTreeMap<NodeID, u8> {
+ dijkstra_multi(
+ &[source],
+ |node| self.raw_ingoing_edges(&node, super::EdgeLimit::Unlimited),
+ |edge| edge.from.node(),
+ Some(max_dist),
+ )
+ }
+
fn reversed_distances(&self, source: Node) -> BTreeMap<Node, u8> {
self.raw_reversed_distances(source.id())
.into_iter()
diff --git a/crates/optics-lsp/Cargo.lock b/crates/optics-lsp/Cargo.lock
index 72f30d53..e7f00e93 100644
--- a/crates/optics-lsp/Cargo.lock
+++ b/crates/optics-lsp/Cargo.lock
@@ -456,7 +456,6 @@ dependencies = [
"lalrpop",
"lalrpop-util",
"logos",
- "once_cell",
"serde",
"thiserror",
"utoipa",
diff --git a/frontend/src/lib/api/index.ts b/frontend/src/lib/api/index.ts
index 979a3893..a11a8944 100644
--- a/frontend/src/lib/api/index.ts
+++ b/frontend/src/lib/api/index.ts
@@ -272,7 +272,6 @@ export type DisplayedWebpage = {
prettyUrl: string;
rankingSignals?: {};
richSnippet?: RichSnippet;
- score?: number;
site: string;
snippet: Snippet;
structuredData?: StructuredData[];
@@ -373,9 +372,11 @@ export type SidebarQuery = {
export type SignalEnumDiscriminants =
| 'bm25_f'
| 'bm25_title'
+ | 'title_coverage'
| 'bm25_title_bigrams'
| 'bm25_title_trigrams'
| 'bm25_clean_body'
+ | 'clean_body_coverage'
| 'bm25_clean_body_bigrams'
| 'bm25_clean_body_trigrams'
| 'bm25_stemmed_title'
@@ -411,13 +412,19 @@ export type SignalEnumDiscriminants =
| 'url_slashes'
| 'link_density'
| 'title_embedding_similarity'
- | 'keyword_embedding_similarity';
+ | 'keyword_embedding_similarity'
+ | 'has_ads'
+ | 'num_query_terms'
+ | 'min_title_slop'
+ | 'min_clean_body_slop';
export const SIGNAL_ENUM_DISCRIMINANTS = [
'bm25_f',
'bm25_title',
+ 'title_coverage',
'bm25_title_bigrams',
'bm25_title_trigrams',
'bm25_clean_body',
+ 'clean_body_coverage',
'bm25_clean_body_bigrams',
'bm25_clean_body_trigrams',
'bm25_stemmed_title',
@@ -454,6 +461,10 @@ export const SIGNAL_ENUM_DISCRIMINANTS = [
'link_density',
'title_embedding_similarity',
'keyword_embedding_similarity',
+ 'has_ads',
+ 'num_query_terms',
+ 'min_title_slop',
+ 'min_clean_body_slop',
] satisfies SignalEnumDiscriminants[];
export type SignalScore = {
coefficient: number;
diff --git a/frontend/src/routes/search/Discussions.svelte b/frontend/src/routes/search/Discussions.svelte
index 856e1d54..1d685a41 100644
--- a/frontend/src/routes/search/Discussions.svelte
+++ b/frontend/src/routes/search/Discussions.svelte
@@ -10,17 +10,17 @@
let showMore = false;
$: {
- let medianScore = 0;
- if (discussions) {
- const scores = discussions.map((d) => d.score || 0);
- scores.sort((a, b) => a - b);
- const mid = Math.floor(scores.length / 2);
- medianScore = scores.length % 2 !== 0 ? scores[mid] : (scores[mid - 1] + scores[mid]) / 2;
- }
+ // let medianScore = 0;
+ // if (discussions) {
+ // const scores = discussions.map((d) => d.score || 0);
+ // scores.sort((a, b) => a - b);
+ // const mid = Math.floor(scores.length / 2);
+ // medianScore = scores.length % 2 !== 0 ? scores[mid] : (scores[mid - 1] + scores[mid]) / 2;
+ // }
- if (medianScore < 0.1) {
- discussions = undefined;
- }
+ // if (medianScore < 0.1) {
+ // discussions = undefined;
+ // }
let numDiscussions = discussions?.length || 0;
if (numDiscussions < 5) {
diff --git a/frontend/src/routes/search/Modal.svelte b/frontend/src/routes/search/Modal.svelte
index 29e3b621..a85ac60e 100644
--- a/frontend/src/routes/search/Modal.svelte
+++ b/frontend/src/routes/search/Modal.svelte
@@ -78,16 +78,22 @@
.with('fetch_time_ms', () => 'fetchTime' as const)
.with('update_timestamp', () => 'updateTimestamp' as const)
.with('tracker_score', () => 'trackerScore' as const)
+ .with('has_ads', () => 'trackerScore' as const)
.with('region', () => 'region' as const)
.with('query_centrality', () => 'queryCentrality' as const)
.with('inbound_similarity', () => 'inboundSimilarity' as const)
.with('lambda_mart', () => undefined)
+ .with('num_query_terms', () => undefined)
.with('url_digits', () => 'urlSymbols' as const)
.with('url_slashes', () => 'urlSymbols' as const)
.with('link_density', () => 'linkDensity' as const)
.with('title_embedding_similarity', () => 'title' as const)
.with('keyword_embedding_similarity', () => 'keywords' as const)
.with('bm25_f', () => 'combinedText' as const)
+ .with('min_clean_body_slop', () => 'body' as const)
+ .with('min_title_slop', () => 'title' as const)
+ .with('title_coverage', () => 'title' as const)
+ .with('clean_body_coverage', () => 'body' as const)
.exhaustive();
};