Skip to content

web search done right

split 'Signal' trait into 'CoreSignal' and 'Signal' to distinguish between the signals that are calculated initially and later during ranking

Mikkel Denker9cba6c13fdbd
diff --git a/crates/core/src/api/docs.rs b/crates/core/src/api/docs.rs
index 514e52fc..78ad892a 100644
--- a/crates/core/src/api/docs.rs
+++ b/crates/core/src/api/docs.rs
@@ -77,8 +77,8 @@ use utoipa_swagger_ui::SwaggerUi;
crate::widgets::thesaurus::PartOfSpeech,
crate::widgets::thesaurus::PartOfSpeechMeaning,
- crate::ranking::core::SignalEnumDiscriminants,
- crate::ranking::core::SignalScore,
+ crate::ranking::SignalEnumDiscriminants,
+ crate::ranking::SignalScore,
crate::bangs::BangHit,
crate::bangs::Bang,
diff --git a/crates/core/src/index.rs b/crates/core/src/index.rs
index d39056ba..9a2fc4d2 100644
--- a/crates/core/src/index.rs
+++ b/crates/core/src/index.rs
@@ -246,7 +246,7 @@ mod tests {
.ranking_signals
.as_ref()
.unwrap()
- .get(&crate::ranking::SignalEnum::from(ranking::core::Bm25Title).into())
+ .get(&crate::ranking::SignalEnum::from(ranking::signals::Bm25Title).into())
.unwrap())
.all(|&v| v.value > 0.0));
}
diff --git a/crates/core/src/query/optic.rs b/crates/core/src/query/optic.rs
index 0b9be805..3b607dd6 100644
--- a/crates/core/src/query/optic.rs
+++ b/crates/core/src/query/optic.rs
@@ -352,7 +352,7 @@ mod tests {
.search(&SearchQuery {
query: "website".to_string(),
signal_coefficients: enum_map! {
- crate::ranking::SignalEnum::from(crate::ranking::core::Bm25Title) => 1_000_000.0
+ crate::ranking::SignalEnum::from(crate::ranking::signals::core::Bm25Title) => 1_000_000.0
}
.into(),
@@ -760,7 +760,7 @@ mod tests {
.search(&SearchQuery {
query: "website".to_string(),
signal_coefficients: crate::enum_map! {
- crate::ranking::SignalEnum::from(crate::ranking::core::InboundSimilarity) => 100_000.0
+ crate::ranking::SignalEnum::from(crate::ranking::signals::InboundSimilarity) => 100_000.0
}.into(),
optic: Some(Optic {
diff --git a/crates/core/src/ranking/core/computer/mod.rs b/crates/core/src/ranking/computer/mod.rs
similarity index 97%
rename from crates/core/src/ranking/core/computer/mod.rs
rename to crates/core/src/ranking/computer/mod.rs
index ebaed70c..7f2ef108 100644
--- a/crates/core/src/ranking/core/computer/mod.rs
+++ b/crates/core/src/ranking/computer/mod.rs
@@ -40,7 +40,7 @@ use crate::webpage::region::RegionCount;
use crate::ranking::bm25::MultiBm25Weight;
use crate::ranking::models::linear::LinearRegression;
-use super::{ComputedSignal, Signal, SignalCoefficients, SignalEnum};
+use super::{ComputedSignal, CoreSignal, CoreSignalEnum, Signal, SignalCoefficients, SignalEnum};
mod order;
pub use order::SignalComputeOrder;
@@ -288,7 +288,7 @@ impl SignalComputer {
if let Some(query) = &self.query_data {
if !query.simple_terms.is_empty() {
- for signal in SignalEnum::all() {
+ for signal in CoreSignalEnum::all() {
if let Some((text_field, tv_field)) = signal
.as_textfield()
.and_then(|f| (f.tantivy_field(schema).map(|tv_field| (f, tv_field))))
@@ -350,7 +350,7 @@ impl SignalComputer {
bm25,
bm25f,
fieldnorm_reader,
- signal_coefficient: self.coefficient(&signal),
+ signal_coefficient: self.coefficient(&signal.into()),
num_query_terms: terms.len(),
},
);
@@ -470,11 +470,12 @@ impl SignalComputer {
}
pub fn precompute_score(&self, webpage: &Webpage) -> f64 {
- SignalEnum::all()
+ CoreSignalEnum::all()
.filter_map(|signal| {
- signal
- .precompute(webpage, self)
- .map(|calc| ComputedSignal { signal, calc })
+ signal.precompute(webpage, self).map(|calc| ComputedSignal {
+ signal: signal.into(),
+ calc,
+ })
})
.map(|computed| self.coefficient(&computed.signal) * computed.calc.score)
.sum()
diff --git a/crates/core/src/ranking/core/computer/order.rs b/crates/core/src/ranking/computer/order.rs
similarity index 85%
rename from crates/core/src/ranking/core/computer/order.rs
rename to crates/core/src/ranking/computer/order.rs
index 53c93578..61b33821 100644
--- a/crates/core/src/ranking/core/computer/order.rs
+++ b/crates/core/src/ranking/computer/order.rs
@@ -18,7 +18,7 @@ use tantivy::DocId;
use crate::{
enum_map::EnumMap,
- ranking::{ComputedSignal, Signal, SignalCalculation, SignalEnum},
+ ranking::{ComputedSignal, CoreSignal, CoreSignalEnum, SignalCalculation},
schema::{text_field::TextField, TextFieldEnum},
};
@@ -27,7 +27,7 @@ use super::SignalComputer;
#[derive(Clone)]
pub struct SignalComputeOrder {
text_signals: EnumMap<TextFieldEnum, NGramComputeOrder>,
- other_signals: Vec<SignalEnum>,
+ other_signals: Vec<CoreSignalEnum>,
}
impl SignalComputeOrder {
@@ -35,7 +35,7 @@ impl SignalComputeOrder {
let mut text_signals = EnumMap::new();
let mut other_signals = Vec::new();
- for signal in SignalEnum::all() {
+ for signal in CoreSignalEnum::all() {
if let Some(text_field) = signal.as_textfield() {
if signal.has_sibling_ngrams() {
let mono = text_field.monogram_field();
@@ -68,15 +68,15 @@ impl SignalComputeOrder {
self.text_signals
.values()
.flat_map(move |ngram| ngram.compute(doc, signal_computer))
- .chain(self.other_signals.iter().map(move |signal| {
+ .chain(self.other_signals.iter().map(move |&signal| {
signal
.compute(doc, signal_computer)
.map(|calc| ComputedSignal {
- signal: *signal,
+ signal: signal.into(),
calc,
})
.unwrap_or_else(|| ComputedSignal {
- signal: *signal,
+ signal: signal.into(),
calc: SignalCalculation {
score: 0.0,
value: 0.0,
@@ -86,6 +86,12 @@ impl SignalComputeOrder {
}
}
+impl Default for SignalComputeOrder {
+ fn default() -> Self {
+ Self::new()
+ }
+}
+
/// If an ngram of size n matches the query for a given document in a given field,
/// the score of all ngrams where n' < n is dampened by NGRAM_DAMPENING.
///
@@ -96,11 +102,11 @@ const NGRAM_DAMPENING: f64 = 0.4;
#[derive(Debug, Default, Clone)]
pub struct NGramComputeOrder {
/// ordered by descending ngram size. e.g. [title_bm25_trigram, title_bm25_bigram, title_bm25]
- signals: Vec<(usize, SignalEnum)>,
+ signals: Vec<(usize, CoreSignalEnum)>,
}
impl NGramComputeOrder {
- fn push(&mut self, signal: SignalEnum, ngram: usize) {
+ fn push(&mut self, signal: CoreSignalEnum, ngram: usize) {
self.signals.push((ngram, signal));
self.signals.sort_unstable_by(|(a, _), (b, _)| b.cmp(a));
}
@@ -112,11 +118,11 @@ impl NGramComputeOrder {
) -> impl Iterator<Item = ComputedSignal> + 'a {
let mut hits = 0;
- self.signals.iter().map(|(_, s)| s).map(move |signal| {
+ self.signals.iter().map(|(_, s)| s).map(move |&signal| {
signal
.compute(doc, signal_computer)
.map(|calc| ComputedSignal {
- signal: *signal,
+ signal: signal.into(),
calc,
})
.map(|mut c| {
@@ -129,7 +135,7 @@ impl NGramComputeOrder {
c
})
.unwrap_or_else(|| ComputedSignal {
- signal: *signal,
+ signal: signal.into(),
calc: SignalCalculation {
value: 0.0,
score: 0.0,
diff --git a/crates/core/src/ranking/core/mod.rs b/crates/core/src/ranking/core/mod.rs
deleted file mode 100644
index 9d818502..00000000
--- a/crates/core/src/ranking/core/mod.rs
+++ /dev/null
@@ -1,148 +0,0 @@
-// Stract is an open source web search engine.
-// Copyright (C) 2024 Stract ApS
-//
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as
-// published by the Free Software Foundation, either version 3 of the
-// License, or (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with this program. If not, see <https://www.gnu.org/licenses/>.
-
-use crate::enum_map::EnumMap;
-
-use std::str::FromStr;
-
-use thiserror::Error;
-use utoipa::ToSchema;
-
-mod computer;
-mod non_text;
-mod prelude;
-mod query;
-mod text;
-
-pub use computer::SignalComputer;
-pub use non_text::*;
-pub use prelude::*;
-pub use text::*;
-
-#[derive(Debug, Error)]
-pub enum Error {
- #[error("unknown signal: {0}")]
- UnknownSignal(#[from] serde_json::Error),
-}
-
-impl FromStr for SignalEnumDiscriminants {
- type Err = Error;
-
- fn from_str(name: &str) -> std::result::Result<Self, Self::Err> {
- let s = "\"".to_string() + name + "\"";
- let signal = serde_json::from_str(&s)?;
- Ok(signal)
- }
-}
-
-#[derive(Debug, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, Clone)]
-pub struct SignalCoefficients {
- map: EnumMap<SignalEnum, f64>,
-}
-
-impl Default for SignalCoefficients {
- fn default() -> Self {
- Self::new(SignalEnum::all().map(|signal| (signal, signal.default_coefficient())))
- }
-}
-
-impl From<EnumMap<SignalEnum, f64>> for SignalCoefficients {
- fn from(map: EnumMap<SignalEnum, f64>) -> Self {
- Self { map }
- }
-}
-
-impl SignalCoefficients {
- pub fn get(&self, signal: &SignalEnum) -> f64 {
- self.map
- .get(*signal)
- .copied()
- .unwrap_or(signal.default_coefficient())
- }
-
- pub fn new(coefficients: impl Iterator<Item = (SignalEnum, f64)>) -> Self {
- let mut map = EnumMap::default();
-
- for (signal, coefficient) in coefficients {
- map.insert(signal, coefficient);
- }
-
- Self { map }
- }
-
- pub fn merge_add(&mut self, coeffs: SignalCoefficients) {
- for signal in SignalEnum::all() {
- if let Some(coeff) = coeffs.map.get(signal).copied() {
- match self.map.get_mut(signal) {
- Some(existing_coeff) => *existing_coeff += coeff,
- None => {
- self.map.insert(signal, coeff);
- }
- }
- }
- }
- }
-
- pub fn merge_overwrite(&mut self, coeffs: SignalCoefficients) {
- for signal in SignalEnum::all() {
- if let Some(coeff) = coeffs.map.get(signal).copied() {
- match self.map.get_mut(signal) {
- Some(existing_coeff) => *existing_coeff = coeff,
- None => {
- self.map.insert(signal, coeff);
- }
- }
- }
- }
- }
-}
-
-#[derive(Debug, Clone, Copy, bincode::Encode, bincode::Decode)]
-pub struct SignalCalculation {
- pub value: f64,
- pub score: f64,
-}
-
-impl SignalCalculation {
- pub fn new_symmetrical(val: f64) -> Self {
- Self {
- value: val,
- score: val,
- }
- }
-}
-
-#[derive(Debug, Clone, Copy)]
-pub struct ComputedSignal {
- pub signal: SignalEnum,
- pub calc: SignalCalculation,
-}
-
-#[derive(
- Debug,
- Clone,
- Copy,
- serde::Serialize,
- serde::Deserialize,
- bincode::Encode,
- bincode::Decode,
- ToSchema,
-)]
-#[serde(rename_all = "camelCase")]
-pub struct SignalScore {
- pub coefficient: f64,
- pub value: f64,
-}
diff --git a/crates/core/src/ranking/core/prelude.rs b/crates/core/src/ranking/core/prelude.rs
deleted file mode 100644
index a5164e64..00000000
--- a/crates/core/src/ranking/core/prelude.rs
+++ /dev/null
@@ -1,224 +0,0 @@
-// Stract is an open source web search engine.
-// Copyright (C) 2024 Stract ApS
-//
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as
-// published by the Free Software Foundation, either version 3 of the
-// License, or (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with this program. If not, see <https://www.gnu.org/licenses/>
-
-use crate::enum_dispatch_from_discriminant;
-use crate::enum_map::{GetEnumMapKey, InsertEnumMapKey};
-
-use crate::schema::Field;
-use crate::{
- schema::{NumericalFieldEnum, TextFieldEnum},
- webpage::Webpage,
-};
-use enum_dispatch::enum_dispatch;
-
-use strum::{EnumDiscriminants, VariantArray};
-
-use super::query::*;
-use super::text::*;
-use super::SignalComputer;
-use super::{non_text::*, SignalCalculation};
-use tantivy::DocId;
-
-#[enum_dispatch]
-pub trait Signal:
- Clone + Copy + std::fmt::Debug + PartialEq + Eq + std::hash::Hash + Into<SignalEnum>
-{
- fn default_coefficient(&self) -> f64;
- fn as_field(&self) -> Option<Field>;
- fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<SignalCalculation>;
-
- fn precompute(
- self,
- _webpage: &Webpage,
- _signal_computer: &SignalComputer,
- ) -> Option<SignalCalculation> {
- None
- }
-
- fn as_textfield(&self) -> Option<TextFieldEnum> {
- self.as_field().and_then(|field| field.as_text())
- }
-
- fn has_sibling_ngrams(&self) -> bool {
- false
- }
-
- fn as_numericalfield(&self) -> Option<NumericalFieldEnum> {
- self.as_field().and_then(|field| field.as_numerical())
- }
-}
-
-#[enum_dispatch(Signal)]
-#[derive(
- Debug,
- serde::Serialize,
- serde::Deserialize,
- bincode::Encode,
- bincode::Decode,
- Clone,
- Copy,
- PartialEq,
- Eq,
- Hash,
- EnumDiscriminants,
-)]
-#[strum_discriminants(derive(
- VariantArray,
- serde::Serialize,
- serde::Deserialize,
- bincode::Encode,
- bincode::Decode,
- PartialOrd,
- Ord,
- Hash,
- utoipa::ToSchema,
-))]
-#[strum_discriminants(serde(rename_all = "snake_case"))]
-pub enum SignalEnum {
- Bm25F,
- Bm25Title,
- TitleCoverage,
- Bm25TitleBigrams,
- Bm25TitleTrigrams,
- Bm25CleanBody,
- CleanBodyCoverage,
- Bm25CleanBodyBigrams,
- Bm25CleanBodyTrigrams,
- Bm25StemmedTitle,
- Bm25StemmedCleanBody,
- Bm25AllBody,
- Bm25Keywords,
- Bm25BacklinkText,
- IdfSumUrl,
- IdfSumSite,
- IdfSumDomain,
- IdfSumSiteNoTokenizer,
- IdfSumDomainNoTokenizer,
- IdfSumDomainNameNoTokenizer,
- IdfSumDomainIfHomepage,
- IdfSumDomainNameIfHomepageNoTokenizer,
- IdfSumDomainIfHomepageNoTokenizer,
- IdfSumTitleIfHomepage,
- CrossEncoderSnippet,
- CrossEncoderTitle,
- HostCentrality,
- HostCentralityRank,
- PageCentrality,
- PageCentralityRank,
- IsHomepage,
- FetchTimeMs,
- UpdateTimestamp,
- TrackerScore,
- Region,
- QueryCentrality,
- InboundSimilarity,
- LambdaMart,
- UrlDigits,
- UrlSlashes,
- LinkDensity,
- TitleEmbeddingSimilarity,
- KeywordEmbeddingSimilarity,
- HasAds,
- NumQueryTerms,
- MinTitleSlop,
- MinCleanBodySlop,
-}
-
-enum_dispatch_from_discriminant!(SignalEnumDiscriminants => SignalEnum,
-[
- Bm25F,
- Bm25Title,
- TitleCoverage,
- Bm25TitleBigrams,
- Bm25TitleTrigrams,
- Bm25CleanBody,
- CleanBodyCoverage,
- Bm25CleanBodyBigrams,
- Bm25CleanBodyTrigrams,
- Bm25StemmedTitle,
- Bm25StemmedCleanBody,
- Bm25AllBody,
- Bm25Keywords,
- Bm25BacklinkText,
- IdfSumUrl,
- IdfSumSite,
- IdfSumDomain,
- IdfSumSiteNoTokenizer,
- IdfSumDomainNoTokenizer,
- IdfSumDomainNameNoTokenizer,
- IdfSumDomainIfHomepage,
- IdfSumDomainNameIfHomepageNoTokenizer,
- IdfSumDomainIfHomepageNoTokenizer,
- IdfSumTitleIfHomepage,
- CrossEncoderSnippet,
- CrossEncoderTitle,
- HostCentrality,
- HostCentralityRank,
- PageCentrality,
- PageCentralityRank,
- IsHomepage,
- FetchTimeMs,
- UpdateTimestamp,
- TrackerScore,
- Region,
- QueryCentrality,
- InboundSimilarity,
- LambdaMart,
- UrlDigits,
- UrlSlashes,
- LinkDensity,
- TitleEmbeddingSimilarity,
- KeywordEmbeddingSimilarity,
- HasAds,
- NumQueryTerms,
- MinTitleSlop,
- MinCleanBodySlop,
-]);
-
-impl SignalEnum {
- pub fn num_variants() -> usize {
- SignalEnumDiscriminants::VARIANTS.len()
- }
-
- pub fn all() -> impl Iterator<Item = SignalEnum> {
- SignalEnumDiscriminants::VARIANTS
- .iter()
- .copied()
- .map(|v| v.into())
- }
-
- pub fn get(field_id: usize) -> Option<SignalEnum> {
- SignalEnumDiscriminants::VARIANTS
- .get(field_id)
- .copied()
- .map(SignalEnum::from)
- }
-}
-
-impl InsertEnumMapKey for SignalEnum {
- fn into_usize(self) -> usize {
- SignalEnumDiscriminants::from(self) as usize
- }
-}
-
-impl GetEnumMapKey for SignalEnum {
- fn from_usize(value: usize) -> Option<Self> {
- SignalEnumDiscriminants::VARIANTS
- .get(value)
- .copied()
- .map(SignalEnum::from)
- }
-}
diff --git a/crates/core/src/ranking/mod.rs b/crates/core/src/ranking/mod.rs
index a67c8c77..3c58526c 100644
--- a/crates/core/src/ranking/mod.rs
+++ b/crates/core/src/ranking/mod.rs
@@ -17,15 +17,22 @@
pub mod bitvec_similarity;
pub mod bm25;
pub mod bm25f;
-pub mod core;
+pub mod computer;
pub mod inbound_similarity;
pub mod initial;
pub mod models;
pub mod optics;
pub mod pipeline;
+pub mod signals;
+pub use computer::SignalComputer;
use initial::InitialScoreTweaker;
+pub use signals::{
+ ComputedSignal, CoreSignal, CoreSignalEnum, Signal, SignalCalculation, SignalCoefficients,
+ SignalEnum, SignalEnumDiscriminants, SignalScore,
+};
+
use crate::{
collector::{MainCollector, MaxDocsConsidered, TopDocs},
config::CollectorConfig,
@@ -34,8 +41,6 @@ use crate::{
searcher::NUM_RESULTS_PER_PAGE,
};
-pub use self::core::*;
-
#[derive(Clone)]
pub struct LocalRanker {
max_docs: Option<MaxDocsConsidered>,
@@ -331,7 +336,7 @@ mod tests {
query: "title".to_string(),
return_ranking_signals: true,
signal_coefficients: crate::enum_map! {
- crate::ranking::SignalEnum::from(crate::ranking::core::UpdateTimestamp) => 100_000.0,
+ crate::ranking::SignalEnum::from(crate::ranking::signals::UpdateTimestamp) => 100_000.0,
}.into(),
..Default::default()
})
@@ -556,8 +561,8 @@ mod tests {
query: "example".to_string(),
signal_coefficients: crate::enum_map! {
- crate::ranking::SignalEnum::from(crate::ranking::core::Bm25Title) => 20_000_000.0,
- crate::ranking::SignalEnum::from(crate::ranking::core::HostCentrality) => 0.0,
+ crate::ranking::SignalEnum::from(crate::ranking::signals::Bm25Title) => 20_000_000.0,
+ crate::ranking::SignalEnum::from(crate::ranking::signals::HostCentrality) => 0.0,
}.into(),
..Default::default()
@@ -572,7 +577,7 @@ mod tests {
query: "example".to_string(),
signal_coefficients: crate::enum_map! {
- crate::ranking::SignalEnum::from(crate::ranking::core::HostCentrality) => 2_000_000.0,
+ crate::ranking::SignalEnum::from(crate::ranking::signals::HostCentrality) => 2_000_000.0,
}.into(),
..Default::default()
@@ -641,7 +646,7 @@ mod tests {
.search(&SearchQuery {
query: "test".to_string(),
signal_coefficients: crate::enum_map! {
- crate::ranking::SignalEnum::from(crate::ranking::core::FetchTimeMs) => 100_000.0,
+ crate::ranking::SignalEnum::from(crate::ranking::signals::FetchTimeMs) => 100_000.0,
}.into(),
..Default::default()
})
@@ -735,8 +740,8 @@ mod tests {
query: "test".to_string(),
signal_coefficients: crate::enum_map! {
- crate::ranking::SignalEnum::from(crate::ranking::core::UrlSlashes) => 100_000.0,
- crate::ranking::SignalEnum::from(crate::ranking::core::UrlDigits) => 100_000.0,
+ crate::ranking::SignalEnum::from(crate::ranking::signals::UrlSlashes) => 100_000.0,
+ crate::ranking::SignalEnum::from(crate::ranking::signals::UrlDigits) => 100_000.0,
}
.into(),
@@ -842,7 +847,7 @@ mod tests {
query: "best chocolate cake".to_string(),
signal_coefficients: crate::enum_map! {
- crate::ranking::SignalEnum::from(crate::ranking::core::TitleEmbeddingSimilarity) => 100_000.0,
+ crate::ranking::SignalEnum::from(crate::ranking::signals::TitleEmbeddingSimilarity) => 100_000.0,
}.into(),
..Default::default()
@@ -930,7 +935,7 @@ mod tests {
query: "best chocolate cake".to_string(),
signal_coefficients: crate::enum_map! {
- crate::ranking::SignalEnum::from(crate::ranking::core::KeywordEmbeddingSimilarity) => 100_000.0,
+ crate::ranking::SignalEnum::from(crate::ranking::signals::KeywordEmbeddingSimilarity) => 100_000.0,
}.into(),
..Default::default()
@@ -986,7 +991,10 @@ mod tests {
.ranking_signals
.as_ref()
.unwrap()
- .get(&crate::ranking::SignalEnum::from(crate::ranking::core::TitleCoverage).into())
+ .get(
+ &crate::ranking::SignalEnum::from(crate::ranking::signals::TitleCoverage)
+ .into()
+ )
.unwrap()
.value,
1.0,
@@ -1006,7 +1014,10 @@ mod tests {
.ranking_signals
.as_ref()
.unwrap()
- .get(&crate::ranking::SignalEnum::from(crate::ranking::core::TitleCoverage).into())
+ .get(
+ &crate::ranking::SignalEnum::from(crate::ranking::signals::TitleCoverage)
+ .into()
+ )
.unwrap()
.value,
0.5,
@@ -1061,7 +1072,7 @@ mod tests {
.as_ref()
.unwrap()
.get(
- &crate::ranking::SignalEnum::from(crate::ranking::core::CleanBodyCoverage)
+ &crate::ranking::SignalEnum::from(crate::ranking::signals::CleanBodyCoverage)
.into()
)
.unwrap()
@@ -1084,7 +1095,7 @@ mod tests {
.as_ref()
.unwrap()
.get(
- &crate::ranking::SignalEnum::from(crate::ranking::core::CleanBodyCoverage)
+ &crate::ranking::SignalEnum::from(crate::ranking::signals::CleanBodyCoverage)
.into()
)
.unwrap()
diff --git a/crates/core/src/ranking/models/lambdamart.rs b/crates/core/src/ranking/models/lambdamart.rs
index 5d1ee789..0ec3b472 100644
--- a/crates/core/src/ranking/models/lambdamart.rs
+++ b/crates/core/src/ranking/models/lambdamart.rs
@@ -16,11 +16,9 @@
use std::{path::Path, str::FromStr};
-use core::SignalEnumDiscriminants;
-
use crate::{
enum_map::EnumMap,
- ranking::{core, SignalCalculation, SignalEnum},
+ ranking::signals::{SignalCalculation, SignalEnum, SignalEnumDiscriminants},
};
type Result<T> = std::result::Result<T, Error>;
@@ -35,9 +33,6 @@ pub enum Error {
#[error("couldn't find end of trees")]
NoEndOfTrees,
- #[error("Signal error: {0}")]
- Signal(#[from] core::Error),
-
#[error("ParseInt error: {0}")]
ParseInt(#[from] std::num::ParseIntError),
@@ -46,6 +41,9 @@ pub enum Error {
#[error("IO error: {0}")]
Io(#[from] std::io::Error),
+
+ #[error("Unknown signal: {0}")]
+ UnknownSignal(String),
}
#[derive(Clone, Debug)]
@@ -225,7 +223,10 @@ impl Header {
if let Some((key, value)) = lin.split_once('=') {
if key == "feature_names" {
for name in value.split(' ') {
- features.push(SignalEnum::from(SignalEnumDiscriminants::from_str(name)?));
+ features.push(SignalEnum::from(
+ SignalEnumDiscriminants::from_str(name)
+ .map_err(|_| Error::UnknownSignal(name.to_string()))?,
+ ));
}
}
}
@@ -320,41 +321,65 @@ mod tests {
assert!(!model.trees.is_empty());
let mut features = EnumMap::new();
- features.insert(ranking::core::Bm25BacklinkText.into(), 85.7750244140625);
- features.insert(ranking::core::Bm25CleanBody.into(), 67.41311645507812);
- features.insert(ranking::core::Bm25CleanBodyBigrams.into(), 0.0);
- features.insert(ranking::core::Bm25CleanBodyTrigrams.into(), 0.0);
- features.insert(ranking::core::IdfSumDomain.into(), 43.332096099853516);
- features.insert(ranking::core::IdfSumDomainIfHomepage.into(), 0.0);
- features.insert(ranking::core::IdfSumDomainIfHomepageNoTokenizer.into(), 0.0);
features.insert(
- ranking::core::IdfSumDomainNameIfHomepageNoTokenizer.into(),
+ ranking::signals::core::Bm25BacklinkText.into(),
+ 85.7750244140625,
+ );
+ features.insert(
+ ranking::signals::core::Bm25CleanBody.into(),
+ 67.41311645507812,
+ );
+ features.insert(ranking::signals::core::Bm25CleanBodyBigrams.into(), 0.0);
+ features.insert(ranking::signals::core::Bm25CleanBodyTrigrams.into(), 0.0);
+ features.insert(
+ ranking::signals::core::IdfSumDomain.into(),
+ 43.332096099853516,
+ );
+ features.insert(ranking::signals::core::IdfSumDomainIfHomepage.into(), 0.0);
+ features.insert(
+ ranking::signals::core::IdfSumDomainIfHomepageNoTokenizer.into(),
0.0,
);
- features.insert(ranking::core::IdfSumDomainNameNoTokenizer.into(), 0.0);
- features.insert(ranking::core::IdfSumDomainNoTokenizer.into(), 0.0);
- features.insert(ranking::core::IdfSumSite.into(), 61.47410202026367);
- features.insert(ranking::core::IdfSumSiteNoTokenizer.into(), 0.0);
features.insert(
- ranking::core::Bm25StemmedCleanBody.into(),
+ ranking::signals::core::IdfSumDomainNameIfHomepageNoTokenizer.into(),
+ 0.0,
+ );
+ features.insert(
+ ranking::signals::core::IdfSumDomainNameNoTokenizer.into(),
+ 0.0,
+ );
+ features.insert(ranking::signals::core::IdfSumDomainNoTokenizer.into(), 0.0);
+ features.insert(ranking::signals::core::IdfSumSite.into(), 61.47410202026367);
+ features.insert(ranking::signals::core::IdfSumSiteNoTokenizer.into(), 0.0);
+ features.insert(
+ ranking::signals::core::Bm25StemmedCleanBody.into(),
65.94627380371094,
);
- features.insert(ranking::core::Bm25StemmedTitle.into(), 0.0);
- features.insert(ranking::core::Bm25Title.into(), 59.817813873291016);
- features.insert(ranking::core::Bm25TitleBigrams.into(), 0.0);
- features.insert(ranking::core::IdfSumTitleIfHomepage.into(), 0.0);
- features.insert(ranking::core::Bm25TitleTrigrams.into(), 0.0);
- features.insert(ranking::core::IdfSumUrl.into(), 57.07925033569336);
- features.insert(ranking::core::FetchTimeMs.into(), 0.023255813953488372);
- features.insert(ranking::core::HostCentrality.into(), 0.017958538);
- features.insert(ranking::core::InboundSimilarity.into(), 0.0);
- features.insert(ranking::core::IsHomepage.into(), 0.0);
- features.insert(ranking::core::PageCentrality.into(), 0.008253236);
- features.insert(ranking::core::Region.into(), 0.16622349570454012);
- features.insert(ranking::core::TrackerScore.into(), 0.07692307692307693);
- features.insert(ranking::core::UpdateTimestamp.into(), 0.0);
- features.insert(ranking::core::UrlDigits.into(), 0.25);
- features.insert(ranking::core::UrlSlashes.into(), 0.3333333333333333);
+ features.insert(ranking::signals::core::Bm25StemmedTitle.into(), 0.0);
+ features.insert(ranking::signals::core::Bm25Title.into(), 59.817813873291016);
+ features.insert(ranking::signals::core::Bm25TitleBigrams.into(), 0.0);
+ features.insert(ranking::signals::core::IdfSumTitleIfHomepage.into(), 0.0);
+ features.insert(ranking::signals::core::Bm25TitleTrigrams.into(), 0.0);
+ features.insert(ranking::signals::core::IdfSumUrl.into(), 57.07925033569336);
+ features.insert(
+ ranking::signals::core::FetchTimeMs.into(),
+ 0.023255813953488372,
+ );
+ features.insert(ranking::signals::HostCentrality.into(), 0.017958538);
+ features.insert(ranking::signals::InboundSimilarity.into(), 0.0);
+ features.insert(ranking::signals::IsHomepage.into(), 0.0);
+ features.insert(ranking::signals::PageCentrality.into(), 0.008253236);
+ features.insert(ranking::signals::core::Region.into(), 0.16622349570454012);
+ features.insert(
+ ranking::signals::core::TrackerScore.into(),
+ 0.07692307692307693,
+ );
+ features.insert(ranking::signals::core::UpdateTimestamp.into(), 0.0);
+ features.insert(ranking::signals::core::UrlDigits.into(), 0.25);
+ features.insert(
+ ranking::signals::core::UrlSlashes.into(),
+ 0.3333333333333333,
+ );
assert_eq!((model.predict(&features) * 1000.0) as u64, 1050);
}
diff --git a/crates/core/src/ranking/optics.rs b/crates/core/src/ranking/optics.rs
index 56e3f812..148764f0 100644
--- a/crates/core/src/ranking/optics.rs
+++ b/crates/core/src/ranking/optics.rs
@@ -185,7 +185,7 @@ mod tests {
}),
signal_coefficients: crate::enum_map! {
- crate::ranking::SignalEnum::from(crate::ranking::core::InboundSimilarity) => 100_000_000.0,
+ crate::ranking::SignalEnum::from(crate::ranking::signals::InboundSimilarity) => 100_000_000.0,
}.into(),
..Default::default()
@@ -209,7 +209,7 @@ mod tests {
}),
signal_coefficients: crate::enum_map! {
- crate::ranking::SignalEnum::from(crate::ranking::core::InboundSimilarity) => 100_000_000.0,
+ crate::ranking::SignalEnum::from(crate::ranking::signals::InboundSimilarity) => 100_000_000.0,
}.into(),
return_ranking_signals: true,
@@ -233,7 +233,7 @@ mod tests {
}),
signal_coefficients: crate::enum_map! {
- crate::ranking::SignalEnum::from(crate::ranking::core::InboundSimilarity) => 100_000_000.0,
+ crate::ranking::SignalEnum::from(crate::ranking::signals::InboundSimilarity) => 100_000_000.0,
}.into(),
return_ranking_signals: true,
diff --git a/crates/core/src/ranking/pipeline/mod.rs b/crates/core/src/ranking/pipeline/mod.rs
index 4de438fb..d95c73d8 100644
--- a/crates/core/src/ranking/pipeline/mod.rs
+++ b/crates/core/src/ranking/pipeline/mod.rs
@@ -207,7 +207,7 @@ mod tests {
value: i as f64,
score,
};
- signals.insert(ranking::core::HostCentrality.into(), calc);
+ signals.insert(ranking::signals::HostCentrality.into(), calc);
LocalRecallRankingWebpage::new_testing(pointer, signals, calc.score)
})
.map(|local| {
diff --git a/crates/core/src/ranking/pipeline/modifiers/inbound_similarity.rs b/crates/core/src/ranking/pipeline/modifiers/inbound_similarity.rs
index 09874545..111d6298 100644
--- a/crates/core/src/ranking/pipeline/modifiers/inbound_similarity.rs
+++ b/crates/core/src/ranking/pipeline/modifiers/inbound_similarity.rs
@@ -32,7 +32,7 @@ impl Modifier for InboundSimilarity {
webpage
.as_ranking()
.signals()
- .get(ranking::InboundSimilarity.into())
+ .get(ranking::signals::InboundSimilarity.into())
.map(|calc| calc.value)
.unwrap_or(0.0)
+ INBOUND_SIMILARITY_SMOOTHING
diff --git a/crates/core/src/ranking/pipeline/scorers/embedding.rs b/crates/core/src/ranking/pipeline/scorers/embedding.rs
index bba0fb5c..a0639cfa 100644
--- a/crates/core/src/ranking/pipeline/scorers/embedding.rs
+++ b/crates/core/src/ranking/pipeline/scorers/embedding.rs
@@ -148,7 +148,7 @@ pub trait EmbeddingSignal<W>: Send + Sync {
impl EmbeddingSignal<ScoredWebpagePointer> for TitleEmbeddings {
fn signal() -> SignalEnum {
- ranking::core::TitleEmbeddingSimilarity.into()
+ ranking::signals::TitleEmbeddingSimilarity.into()
}
fn has_embedding(webpage: &ScoredWebpagePointer) -> bool {
@@ -170,7 +170,7 @@ impl EmbeddingSignal<ScoredWebpagePointer> for TitleEmbeddings {
impl EmbeddingSignal<ScoredWebpagePointer> for KeywordEmbeddings {
fn signal() -> SignalEnum {
- ranking::core::KeywordEmbeddingSimilarity.into()
+ ranking::signals::KeywordEmbeddingSimilarity.into()
}
fn has_embedding(webpage: &ScoredWebpagePointer) -> bool {
diff --git a/crates/core/src/ranking/pipeline/scorers/inbound_similarity.rs b/crates/core/src/ranking/pipeline/scorers/inbound_similarity.rs
index e5bdf645..5c948767 100644
--- a/crates/core/src/ranking/pipeline/scorers/inbound_similarity.rs
+++ b/crates/core/src/ranking/pipeline/scorers/inbound_similarity.rs
@@ -17,7 +17,7 @@
use std::sync::Mutex;
use crate::{
- ranking::{self, core, inbound_similarity, pipeline::RankableWebpage},
+ ranking::{self, inbound_similarity, pipeline::RankableWebpage},
searcher::api::ScoredWebpagePointer,
};
@@ -47,8 +47,8 @@ impl FullRankingStage for InboundScorer {
webpage.as_ranking().inbound_edges(),
);
webpage.as_ranking_mut().signals_mut().insert(
- core::InboundSimilarity.into(),
- ranking::SignalCalculation::new_symmetrical(score),
+ ranking::signals::InboundSimilarity.into(),
+ ranking::signals::SignalCalculation::new_symmetrical(score),
);
}
}
diff --git a/crates/core/src/ranking/pipeline/scorers/lambdamart.rs b/crates/core/src/ranking/pipeline/scorers/lambdamart.rs
index 9dfc00c9..f86947e2 100644
--- a/crates/core/src/ranking/pipeline/scorers/lambdamart.rs
+++ b/crates/core/src/ranking/pipeline/scorers/lambdamart.rs
@@ -36,7 +36,7 @@ impl RankingStage for Arc<models::LambdaMART> {
.map(|(signal, calc)| (signal, calc.value))
.collect();
(
- ranking::core::LambdaMart.into(),
+ ranking::signals::LambdaMart.into(),
SignalCalculation::new_symmetrical(self.predict(&signals)),
)
}
@@ -59,7 +59,7 @@ impl RankingStage for PrecisionLambda {
fn compute(&self, webpage: &Self::Webpage) -> (SignalEnum, SignalCalculation) {
(
- ranking::core::LambdaMart.into(),
+ ranking::signals::LambdaMart.into(),
SignalCalculation::new_symmetrical(self.0.predict(webpage.ranking().signals())),
)
}
diff --git a/crates/core/src/ranking/pipeline/scorers/reranker.rs b/crates/core/src/ranking/pipeline/scorers/reranker.rs
index 59aa754b..6caac87d 100644
--- a/crates/core/src/ranking/pipeline/scorers/reranker.rs
+++ b/crates/core/src/ranking/pipeline/scorers/reranker.rs
@@ -49,12 +49,12 @@ impl<M: CrossEncoder> ReRanker<M> {
for ((webpage, snippet), title) in webpage.iter_mut().zip(snippet_scores).zip(title_scores)
{
webpage.ranking_mut().signals_mut().insert(
- ranking::core::CrossEncoderSnippet.into(),
+ ranking::signals::CrossEncoderSnippet.into(),
ranking::SignalCalculation::new_symmetrical(snippet),
);
webpage.ranking_mut().signals_mut().insert(
- ranking::core::CrossEncoderTitle.into(),
+ ranking::signals::CrossEncoderTitle.into(),
ranking::SignalCalculation::new_symmetrical(title),
);
}
diff --git a/crates/core/src/ranking/pipeline/scorers/term_distance.rs b/crates/core/src/ranking/pipeline/scorers/term_distance.rs
index 856e1687..b5ebf151 100644
--- a/crates/core/src/ranking/pipeline/scorers/term_distance.rs
+++ b/crates/core/src/ranking/pipeline/scorers/term_distance.rs
@@ -68,7 +68,7 @@ impl super::RankingStage for TitleDistanceScorer {
let score = score_slop(min_slop);
(
- ranking::core::MinTitleSlop.into(),
+ ranking::signals::MinTitleSlop.into(),
ranking::SignalCalculation {
value: min_slop,
score,
@@ -88,7 +88,7 @@ impl super::RankingStage for BodyDistanceScorer {
let score = score_slop(min_slop);
(
- ranking::core::MinCleanBodySlop.into(),
+ ranking::signals::MinCleanBodySlop.into(),
ranking::SignalCalculation {
value: min_slop,
score,
diff --git a/crates/core/src/ranking/signals/core/mod.rs b/crates/core/src/ranking/signals/core/mod.rs
new file mode 100644
index 00000000..ef6302ec
--- /dev/null
+++ b/crates/core/src/ranking/signals/core/mod.rs
@@ -0,0 +1,22 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2024 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+pub use std::result;
+
+pub mod non_text;
+pub mod text;
+
+pub use non_text::*;
+pub use text::*;
diff --git a/crates/core/src/ranking/core/non_text.rs b/crates/core/src/ranking/signals/core/non_text.rs
similarity index 91%
rename from crates/core/src/ranking/core/non_text.rs
rename to crates/core/src/ranking/signals/core/non_text.rs
index 70cd55e6..69e11e88 100644
--- a/crates/core/src/ranking/core/non_text.rs
+++ b/crates/core/src/ranking/signals/core/non_text.rs
@@ -16,7 +16,7 @@
use tantivy::DocId;
-use super::{Signal, SignalCalculation, SignalComputer};
+use crate::ranking::{CoreSignal, SignalCalculation, SignalComputer};
use crate::{
schema::{self, Field},
webpage::Webpage,
@@ -106,7 +106,7 @@ fn score_region(webpage_region: crate::webpage::Region, computer: &SignalCompute
bincode::Decode,
)]
pub struct HostCentrality;
-impl Signal for HostCentrality {
+impl CoreSignal for HostCentrality {
fn default_coefficient(&self) -> f64 {
2.0
}
@@ -146,7 +146,7 @@ impl Signal for HostCentrality {
bincode::Decode,
)]
pub struct HostCentralityRank;
-impl Signal for HostCentralityRank {
+impl CoreSignal for HostCentralityRank {
fn default_coefficient(&self) -> f64 {
0.02
}
@@ -192,7 +192,7 @@ impl Signal for HostCentralityRank {
bincode::Decode,
)]
pub struct PageCentrality;
-impl Signal for PageCentrality {
+impl CoreSignal for PageCentrality {
fn default_coefficient(&self) -> f64 {
2.0
}
@@ -232,7 +232,7 @@ impl Signal for PageCentrality {
bincode::Decode,
)]
pub struct PageCentralityRank;
-impl Signal for PageCentralityRank {
+impl CoreSignal for PageCentralityRank {
fn default_coefficient(&self) -> f64 {
0.02
}
@@ -278,7 +278,7 @@ impl Signal for PageCentralityRank {
bincode::Decode,
)]
pub struct IsHomepage;
-impl Signal for IsHomepage {
+impl CoreSignal for IsHomepage {
fn default_coefficient(&self) -> f64 {
0.01
}
@@ -323,7 +323,7 @@ impl Signal for IsHomepage {
bincode::Decode,
)]
pub struct FetchTimeMs;
-impl Signal for FetchTimeMs {
+impl CoreSignal for FetchTimeMs {
fn default_coefficient(&self) -> f64 {
0.001
}
@@ -387,7 +387,7 @@ impl Signal for FetchTimeMs {
bincode::Decode,
)]
pub struct UpdateTimestamp;
-impl Signal for UpdateTimestamp {
+impl CoreSignal for UpdateTimestamp {
fn default_coefficient(&self) -> f64 {
0.001
}
@@ -448,7 +448,7 @@ impl Signal for UpdateTimestamp {
bincode::Decode,
)]
pub struct TrackerScore;
-impl Signal for TrackerScore {
+impl CoreSignal for TrackerScore {
fn default_coefficient(&self) -> f64 {
0.1
}
@@ -499,7 +499,7 @@ impl Signal for TrackerScore {
bincode::Decode,
)]
pub struct Region;
-impl Signal for Region {
+impl CoreSignal for Region {
fn default_coefficient(&self) -> f64 {
0.15
}
@@ -541,88 +541,6 @@ impl Signal for Region {
}
}
-#[derive(
- Debug,
- Clone,
- Copy,
- PartialEq,
- Eq,
- Hash,
- serde::Serialize,
- serde::Deserialize,
- bincode::Encode,
- bincode::Decode,
-)]
-pub struct QueryCentrality;
-impl Signal for QueryCentrality {
- fn default_coefficient(&self) -> f64 {
- 0.0
- }
-
- fn as_field(&self) -> Option<Field> {
- None
- }
-
- fn compute(&self, _doc: DocId, _signal_computer: &SignalComputer) -> Option<SignalCalculation> {
- // unimplemented!()
- None
- }
-}
-
-#[derive(
- Debug,
- Clone,
- Copy,
- PartialEq,
- Eq,
- Hash,
- serde::Serialize,
- serde::Deserialize,
- bincode::Encode,
- bincode::Decode,
-)]
-pub struct InboundSimilarity;
-impl Signal for InboundSimilarity {
- fn default_coefficient(&self) -> f64 {
- 0.25
- }
-
- fn as_field(&self) -> Option<Field> {
- None
- }
-
- fn compute(&self, _doc: DocId, _signal_computer: &SignalComputer) -> Option<SignalCalculation> {
- None // computed in later ranking stage
- }
-}
-
-#[derive(
- Debug,
- Clone,
- Copy,
- PartialEq,
- Eq,
- Hash,
- serde::Serialize,
- serde::Deserialize,
- bincode::Encode,
- bincode::Decode,
-)]
-pub struct LambdaMart;
-impl Signal for LambdaMart {
- fn default_coefficient(&self) -> f64 {
- 10.0
- }
-
- fn as_field(&self) -> Option<Field> {
- None
- }
-
- fn compute(&self, _: DocId, _: &SignalComputer) -> Option<SignalCalculation> {
- None // computed in later ranking stage
- }
-}
-
#[derive(
Debug,
Clone,
@@ -636,7 +554,7 @@ impl Signal for LambdaMart {
bincode::Decode,
)]
pub struct UrlDigits;
-impl Signal for UrlDigits {
+impl CoreSignal for UrlDigits {
fn default_coefficient(&self) -> f64 {
0.01
}
@@ -699,7 +617,7 @@ impl Signal for UrlDigits {
bincode::Decode,
)]
pub struct UrlSlashes;
-impl Signal for UrlSlashes {
+impl CoreSignal for UrlSlashes {
fn default_coefficient(&self) -> f64 {
0.1
}
@@ -753,7 +671,7 @@ impl Signal for UrlSlashes {
bincode::Decode,
)]
pub struct LinkDensity;
-impl Signal for LinkDensity {
+impl CoreSignal for LinkDensity {
fn default_coefficient(&self) -> f64 {
0.0
}
@@ -801,7 +719,7 @@ impl Signal for LinkDensity {
bincode::Decode,
)]
pub struct HasAds;
-impl Signal for HasAds {
+impl CoreSignal for HasAds {
fn default_coefficient(&self) -> f64 {
0.01
}
diff --git a/crates/core/src/ranking/core/text.rs b/crates/core/src/ranking/signals/core/text.rs
similarity index 82%
rename from crates/core/src/ranking/core/text.rs
rename to crates/core/src/ranking/signals/core/text.rs
index da405f5d..a8785655 100644
--- a/crates/core/src/ranking/core/text.rs
+++ b/crates/core/src/ranking/signals/core/text.rs
@@ -18,7 +18,7 @@ use crate::schema::{self, Field};
use tantivy::DocId;
-use super::{Signal, SignalCalculation, SignalComputer};
+use crate::ranking::{CoreSignal, SignalCalculation, SignalComputer};
#[derive(
Debug,
@@ -33,7 +33,7 @@ use super::{Signal, SignalCalculation, SignalComputer};
bincode::Decode,
)]
pub struct Bm25F;
-impl Signal for Bm25F {
+impl CoreSignal for Bm25F {
fn default_coefficient(&self) -> f64 {
0.1
}
@@ -68,7 +68,7 @@ impl Signal for Bm25F {
bincode::Decode,
)]
pub struct Bm25Title;
-impl Signal for Bm25Title {
+impl CoreSignal for Bm25Title {
fn default_coefficient(&self) -> f64 {
0.0063
}
@@ -105,7 +105,7 @@ impl Signal for Bm25Title {
bincode::Decode,
)]
pub struct TitleCoverage;
-impl Signal for TitleCoverage {
+impl CoreSignal for TitleCoverage {
fn default_coefficient(&self) -> f64 {
0.01
}
@@ -137,7 +137,7 @@ impl Signal for TitleCoverage {
bincode::Decode,
)]
pub struct Bm25TitleBigrams;
-impl Signal for Bm25TitleBigrams {
+impl CoreSignal for Bm25TitleBigrams {
fn default_coefficient(&self) -> f64 {
0.005
}
@@ -174,7 +174,7 @@ impl Signal for Bm25TitleBigrams {
bincode::Decode,
)]
pub struct Bm25TitleTrigrams;
-impl Signal for Bm25TitleTrigrams {
+impl CoreSignal for Bm25TitleTrigrams {
fn default_coefficient(&self) -> f64 {
0.005
}
@@ -211,7 +211,7 @@ impl Signal for Bm25TitleTrigrams {
bincode::Decode,
)]
pub struct Bm25CleanBody;
-impl Signal for Bm25CleanBody {
+impl CoreSignal for Bm25CleanBody {
fn default_coefficient(&self) -> f64 {
0.005
}
@@ -248,7 +248,7 @@ impl Signal for Bm25CleanBody {
bincode::Decode,
)]
pub struct CleanBodyCoverage;
-impl Signal for CleanBodyCoverage {
+impl CoreSignal for CleanBodyCoverage {
fn default_coefficient(&self) -> f64 {
0.01
}
@@ -281,7 +281,7 @@ impl Signal for CleanBodyCoverage {
bincode::Decode,
)]
pub struct Bm25CleanBodyBigrams;
-impl Signal for Bm25CleanBodyBigrams {
+impl CoreSignal for Bm25CleanBodyBigrams {
fn default_coefficient(&self) -> f64 {
0.005
}
@@ -318,7 +318,7 @@ impl Signal for Bm25CleanBodyBigrams {
bincode::Decode,
)]
pub struct Bm25CleanBodyTrigrams;
-impl Signal for Bm25CleanBodyTrigrams {
+impl CoreSignal for Bm25CleanBodyTrigrams {
fn default_coefficient(&self) -> f64 {
0.005
}
@@ -355,7 +355,7 @@ impl Signal for Bm25CleanBodyTrigrams {
bincode::Decode,
)]
pub struct Bm25StemmedTitle;
-impl Signal for Bm25StemmedTitle {
+impl CoreSignal for Bm25StemmedTitle {
fn default_coefficient(&self) -> f64 {
0.003
}
@@ -388,7 +388,7 @@ impl Signal for Bm25StemmedTitle {
bincode::Decode,
)]
pub struct Bm25StemmedCleanBody;
-impl Signal for Bm25StemmedCleanBody {
+impl CoreSignal for Bm25StemmedCleanBody {
fn default_coefficient(&self) -> f64 {
0.001
}
@@ -421,7 +421,7 @@ impl Signal for Bm25StemmedCleanBody {
bincode::Decode,
)]
pub struct Bm25AllBody;
-impl Signal for Bm25AllBody {
+impl CoreSignal for Bm25AllBody {
fn default_coefficient(&self) -> f64 {
0.0
}
@@ -454,7 +454,7 @@ impl Signal for Bm25AllBody {
bincode::Decode,
)]
pub struct Bm25Keywords;
-impl Signal for Bm25Keywords {
+impl CoreSignal for Bm25Keywords {
fn default_coefficient(&self) -> f64 {
0.001
}
@@ -487,7 +487,7 @@ impl Signal for Bm25Keywords {
bincode::Decode,
)]
pub struct Bm25BacklinkText;
-impl Signal for Bm25BacklinkText {
+impl CoreSignal for Bm25BacklinkText {
fn default_coefficient(&self) -> f64 {
0.003
}
@@ -520,7 +520,7 @@ impl Signal for Bm25BacklinkText {
bincode::Decode,
)]
pub struct IdfSumUrl;
-impl Signal for IdfSumUrl {
+impl CoreSignal for IdfSumUrl {
fn default_coefficient(&self) -> f64 {
0.0006
}
@@ -553,7 +553,7 @@ impl Signal for IdfSumUrl {
bincode::Decode,
)]
pub struct IdfSumSite;
-impl Signal for IdfSumSite {
+impl CoreSignal for IdfSumSite {
fn default_coefficient(&self) -> f64 {
0.00015
}
@@ -586,7 +586,7 @@ impl Signal for IdfSumSite {
bincode::Decode,
)]
pub struct IdfSumDomain;
-impl Signal for IdfSumDomain {
+impl CoreSignal for IdfSumDomain {
fn default_coefficient(&self) -> f64 {
0.0003
}
@@ -619,7 +619,7 @@ impl Signal for IdfSumDomain {
bincode::Decode,
)]
pub struct IdfSumSiteNoTokenizer;
-impl Signal for IdfSumSiteNoTokenizer {
+impl CoreSignal for IdfSumSiteNoTokenizer {
fn default_coefficient(&self) -> f64 {
0.00015
}
@@ -652,7 +652,7 @@ impl Signal for IdfSumSiteNoTokenizer {
bincode::Decode,
)]
pub struct IdfSumDomainNoTokenizer;
-impl Signal for IdfSumDomainNoTokenizer {
+impl CoreSignal for IdfSumDomainNoTokenizer {
fn default_coefficient(&self) -> f64 {
0.0036
}
@@ -685,7 +685,7 @@ impl Signal for IdfSumDomainNoTokenizer {
bincode::Decode,
)]
pub struct IdfSumDomainNameNoTokenizer;
-impl Signal for IdfSumDomainNameNoTokenizer {
+impl CoreSignal for IdfSumDomainNameNoTokenizer {
fn default_coefficient(&self) -> f64 {
0.0002
}
@@ -720,7 +720,7 @@ impl Signal for IdfSumDomainNameNoTokenizer {
bincode::Decode,
)]
pub struct IdfSumDomainIfHomepage;
-impl Signal for IdfSumDomainIfHomepage {
+impl CoreSignal for IdfSumDomainIfHomepage {
fn default_coefficient(&self) -> f64 {
0.0004
}
@@ -753,7 +753,7 @@ impl Signal for IdfSumDomainIfHomepage {
bincode::Decode,
)]
pub struct IdfSumDomainNameIfHomepageNoTokenizer;
-impl Signal for IdfSumDomainNameIfHomepageNoTokenizer {
+impl CoreSignal for IdfSumDomainNameIfHomepageNoTokenizer {
fn default_coefficient(&self) -> f64 {
0.0036
}
@@ -788,7 +788,7 @@ impl Signal for IdfSumDomainNameIfHomepageNoTokenizer {
bincode::Decode,
)]
pub struct IdfSumDomainIfHomepageNoTokenizer;
-impl Signal for IdfSumDomainIfHomepageNoTokenizer {
+impl CoreSignal for IdfSumDomainIfHomepageNoTokenizer {
fn default_coefficient(&self) -> f64 {
0.0036
}
@@ -823,7 +823,7 @@ impl Signal for IdfSumDomainIfHomepageNoTokenizer {
bincode::Decode,
)]
pub struct IdfSumTitleIfHomepage;
-impl Signal for IdfSumTitleIfHomepage {
+impl CoreSignal for IdfSumTitleIfHomepage {
fn default_coefficient(&self) -> f64 {
0.001
}
@@ -842,169 +842,3 @@ impl Signal for IdfSumTitleIfHomepage {
.map(SignalCalculation::new_symmetrical)
}
}
-
-#[derive(
- Debug,
- Clone,
- Copy,
- PartialEq,
- Eq,
- Hash,
- serde::Serialize,
- serde::Deserialize,
- bincode::Encode,
- bincode::Decode,
-)]
-pub struct CrossEncoderSnippet;
-impl Signal for CrossEncoderSnippet {
- fn default_coefficient(&self) -> f64 {
- 0.17
- }
-
- fn as_field(&self) -> Option<Field> {
- None
- }
-
- fn compute(&self, _: DocId, _: &SignalComputer) -> Option<SignalCalculation> {
- None // computed in later ranking stage
- }
-}
-
-#[derive(
- Debug,
- Clone,
- Copy,
- PartialEq,
- Eq,
- Hash,
- serde::Serialize,
- serde::Deserialize,
- bincode::Encode,
- bincode::Decode,
-)]
-pub struct CrossEncoderTitle;
-impl Signal for CrossEncoderTitle {
- fn default_coefficient(&self) -> f64 {
- 0.17
- }
-
- fn as_field(&self) -> Option<Field> {
- None
- }
-
- fn compute(&self, _: DocId, _: &SignalComputer) -> Option<SignalCalculation> {
- None // computed in later ranking stage
- }
-}
-
-#[derive(
- Debug,
- Clone,
- Copy,
- PartialEq,
- Eq,
- Hash,
- serde::Serialize,
- serde::Deserialize,
- bincode::Encode,
- bincode::Decode,
-)]
-pub struct TitleEmbeddingSimilarity;
-impl Signal for TitleEmbeddingSimilarity {
- fn default_coefficient(&self) -> f64 {
- 0.01
- }
-
- fn as_field(&self) -> Option<Field> {
- Some(Field::Numerical(
- schema::numerical_field::TitleEmbeddings.into(),
- ))
- }
-
- fn compute(&self, _: DocId, _: &SignalComputer) -> Option<SignalCalculation> {
- None // computed in later ranking stage
- }
-}
-
-#[derive(
- Debug,
- Clone,
- Copy,
- PartialEq,
- Eq,
- Hash,
- serde::Serialize,
- serde::Deserialize,
- bincode::Encode,
- bincode::Decode,
-)]
-pub struct KeywordEmbeddingSimilarity;
-impl Signal for KeywordEmbeddingSimilarity {
- fn default_coefficient(&self) -> f64 {
- 0.01
- }
-
- fn as_field(&self) -> Option<Field> {
- Some(Field::Numerical(
- schema::numerical_field::KeywordEmbeddings.into(),
- ))
- }
-
- fn compute(&self, _: DocId, _: &SignalComputer) -> Option<SignalCalculation> {
- None // computed in later ranking stage
- }
-}
-
-#[derive(
- Debug,
- Clone,
- Copy,
- PartialEq,
- Eq,
- Hash,
- serde::Serialize,
- serde::Deserialize,
- bincode::Encode,
- bincode::Decode,
-)]
-pub struct MinTitleSlop;
-impl Signal for MinTitleSlop {
- fn default_coefficient(&self) -> f64 {
- 0.1
- }
-
- fn as_field(&self) -> Option<Field> {
- Some(Field::Text(schema::text_field::Title.into()))
- }
-
- fn compute(&self, _: DocId, _: &SignalComputer) -> Option<SignalCalculation> {
- None // computed in later ranking stage
- }
-}
-
-#[derive(
- Debug,
- Clone,
- Copy,
- PartialEq,
- Eq,
- Hash,
- serde::Serialize,
- serde::Deserialize,
- bincode::Encode,
- bincode::Decode,
-)]
-pub struct MinCleanBodySlop;
-impl Signal for MinCleanBodySlop {
- fn default_coefficient(&self) -> f64 {
- 0.1
- }
-
- fn as_field(&self) -> Option<Field> {
- Some(Field::Text(schema::text_field::CleanBody.into()))
- }
-
- fn compute(&self, _: DocId, _: &SignalComputer) -> Option<SignalCalculation> {
- None // computed in later ranking stage
- }
-}
diff --git a/crates/core/src/ranking/signals/mod.rs b/crates/core/src/ranking/signals/mod.rs
new file mode 100644
index 00000000..8667b041
--- /dev/null
+++ b/crates/core/src/ranking/signals/mod.rs
@@ -0,0 +1,472 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2024 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>
+
+pub mod core;
+pub mod non_core;
+
+use std::str::FromStr;
+
+use crate::enum_dispatch_from_discriminant;
+use crate::enum_map::{EnumMap, GetEnumMapKey, InsertEnumMapKey};
+
+use crate::schema::Field;
+use crate::{
+ schema::{NumericalFieldEnum, TextFieldEnum},
+ webpage::Webpage,
+};
+use enum_dispatch::enum_dispatch;
+
+use strum::{EnumDiscriminants, VariantArray};
+use utoipa::ToSchema;
+
+pub use self::core::{non_text::*, text::*};
+pub use self::non_core::{non_text::*, text::*};
+
+use crate::ranking::computer::SignalComputer;
+use tantivy::DocId;
+
+#[enum_dispatch]
+pub trait CoreSignal: Clone + Copy + std::fmt::Debug + PartialEq + Eq + std::hash::Hash {
+ fn default_coefficient(&self) -> f64;
+ fn as_field(&self) -> Option<Field>;
+ fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<SignalCalculation>;
+
+ fn precompute(
+ self,
+ _webpage: &Webpage,
+ _signal_computer: &SignalComputer,
+ ) -> Option<SignalCalculation> {
+ None
+ }
+
+ fn as_textfield(&self) -> Option<TextFieldEnum> {
+ self.as_field().and_then(|field| field.as_text())
+ }
+
+ fn has_sibling_ngrams(&self) -> bool {
+ false
+ }
+
+ fn as_numericalfield(&self) -> Option<NumericalFieldEnum> {
+ self.as_field().and_then(|field| field.as_numerical())
+ }
+}
+
+#[enum_dispatch]
+pub trait Signal: Clone + Copy + std::fmt::Debug + PartialEq + Eq + std::hash::Hash {
+ fn default_coefficient(&self) -> f64;
+}
+
+impl<T> Signal for T
+where
+ T: CoreSignal,
+{
+ fn default_coefficient(&self) -> f64 {
+ CoreSignal::default_coefficient(self)
+ }
+}
+
+#[enum_dispatch(Signal)]
+#[derive(
+ Debug,
+ serde::Serialize,
+ serde::Deserialize,
+ bincode::Encode,
+ bincode::Decode,
+ Clone,
+ Copy,
+ PartialEq,
+ Eq,
+ Hash,
+ EnumDiscriminants,
+)]
+#[strum_discriminants(derive(
+ VariantArray,
+ serde::Serialize,
+ serde::Deserialize,
+ bincode::Encode,
+ bincode::Decode,
+ PartialOrd,
+ Ord,
+ Hash,
+ utoipa::ToSchema,
+))]
+#[strum_discriminants(serde(rename_all = "snake_case"))]
+pub enum SignalEnum {
+ Bm25F,
+ Bm25Title,
+ TitleCoverage,
+ Bm25TitleBigrams,
+ Bm25TitleTrigrams,
+ Bm25CleanBody,
+ CleanBodyCoverage,
+ Bm25CleanBodyBigrams,
+ Bm25CleanBodyTrigrams,
+ Bm25StemmedTitle,
+ Bm25StemmedCleanBody,
+ Bm25AllBody,
+ Bm25Keywords,
+ Bm25BacklinkText,
+ IdfSumUrl,
+ IdfSumSite,
+ IdfSumDomain,
+ IdfSumSiteNoTokenizer,
+ IdfSumDomainNoTokenizer,
+ IdfSumDomainNameNoTokenizer,
+ IdfSumDomainIfHomepage,
+ IdfSumDomainNameIfHomepageNoTokenizer,
+ IdfSumDomainIfHomepageNoTokenizer,
+ IdfSumTitleIfHomepage,
+ CrossEncoderSnippet,
+ CrossEncoderTitle,
+ HostCentrality,
+ HostCentralityRank,
+ PageCentrality,
+ PageCentralityRank,
+ IsHomepage,
+ FetchTimeMs,
+ UpdateTimestamp,
+ TrackerScore,
+ Region,
+ QueryCentrality,
+ InboundSimilarity,
+ LambdaMart,
+ UrlDigits,
+ UrlSlashes,
+ LinkDensity,
+ TitleEmbeddingSimilarity,
+ KeywordEmbeddingSimilarity,
+ HasAds,
+ MinTitleSlop,
+ MinCleanBodySlop,
+}
+
+#[enum_dispatch(CoreSignal)]
+#[derive(
+ Debug,
+ serde::Serialize,
+ serde::Deserialize,
+ bincode::Encode,
+ bincode::Decode,
+ Clone,
+ Copy,
+ PartialEq,
+ Eq,
+ Hash,
+ EnumDiscriminants,
+)]
+#[strum_discriminants(derive(
+ VariantArray,
+ serde::Serialize,
+ serde::Deserialize,
+ bincode::Encode,
+ bincode::Decode,
+ PartialOrd,
+ Ord,
+ Hash,
+))]
+#[strum_discriminants(serde(rename_all = "snake_case"))]
+pub enum CoreSignalEnum {
+ Bm25F,
+ Bm25Title,
+ TitleCoverage,
+ Bm25TitleBigrams,
+ Bm25TitleTrigrams,
+ Bm25CleanBody,
+ CleanBodyCoverage,
+ Bm25CleanBodyBigrams,
+ Bm25CleanBodyTrigrams,
+ Bm25StemmedTitle,
+ Bm25StemmedCleanBody,
+ Bm25AllBody,
+ Bm25Keywords,
+ Bm25BacklinkText,
+ IdfSumUrl,
+ IdfSumSite,
+ IdfSumDomain,
+ IdfSumSiteNoTokenizer,
+ IdfSumDomainNoTokenizer,
+ IdfSumDomainNameNoTokenizer,
+ IdfSumDomainIfHomepage,
+ IdfSumDomainNameIfHomepageNoTokenizer,
+ IdfSumDomainIfHomepageNoTokenizer,
+ IdfSumTitleIfHomepage,
+ HostCentrality,
+ HostCentralityRank,
+ PageCentrality,
+ PageCentralityRank,
+ IsHomepage,
+ FetchTimeMs,
+ UpdateTimestamp,
+ TrackerScore,
+ Region,
+ UrlDigits,
+ UrlSlashes,
+ LinkDensity,
+ HasAds,
+}
+
+// Note to future self: Tried to get the num definitions
+// into the macro as well, but some of the procedural macros started failing (bincode and enum_dispatch).
+// This is probably due to macro hygiene
+macro_rules! register_signals {
+ (core=[$($core_signal:ident),*$(,)?], rest=[$($signal:ident),*$(,)?]) => {
+ enum_dispatch_from_discriminant!(SignalEnumDiscriminants => SignalEnum,
+ [
+ $($core_signal,)*
+ $($signal,)*
+ ]);
+
+
+ impl From<CoreSignalEnum> for SignalEnum {
+ fn from(core: CoreSignalEnum) -> Self {
+ match core {
+ $(
+ CoreSignalEnum::$core_signal(signal) => SignalEnum::$core_signal(signal),
+ )*
+ }
+ }
+ }
+
+ enum_dispatch_from_discriminant!(CoreSignalEnumDiscriminants => CoreSignalEnum,
+ [
+
+ $($core_signal,)*
+ ]);
+ }
+}
+
+register_signals! {
+ core=[
+ Bm25F,
+ Bm25Title,
+ TitleCoverage,
+ Bm25TitleBigrams,
+ Bm25TitleTrigrams,
+ Bm25CleanBody,
+ CleanBodyCoverage,
+ Bm25CleanBodyBigrams,
+ Bm25CleanBodyTrigrams,
+ Bm25StemmedTitle,
+ Bm25StemmedCleanBody,
+ Bm25AllBody,
+ Bm25Keywords,
+ Bm25BacklinkText,
+ IdfSumUrl,
+ IdfSumSite,
+ IdfSumDomain,
+ IdfSumSiteNoTokenizer,
+ IdfSumDomainNoTokenizer,
+ IdfSumDomainNameNoTokenizer,
+ IdfSumDomainIfHomepage,
+ IdfSumDomainNameIfHomepageNoTokenizer,
+ IdfSumDomainIfHomepageNoTokenizer,
+ IdfSumTitleIfHomepage,
+ HostCentrality,
+ HostCentralityRank,
+ PageCentrality,
+ PageCentralityRank,
+ IsHomepage,
+ FetchTimeMs,
+ UpdateTimestamp,
+ TrackerScore,
+ Region,
+ UrlDigits,
+ UrlSlashes,
+ LinkDensity,
+ HasAds,
+ ],
+ rest=[
+ QueryCentrality,
+ InboundSimilarity,
+ LambdaMart,
+ MinTitleSlop,
+ MinCleanBodySlop,
+ CrossEncoderSnippet,
+ CrossEncoderTitle,
+ TitleEmbeddingSimilarity,
+ KeywordEmbeddingSimilarity,
+ ]
+}
+
+impl SignalEnum {
+ pub fn all() -> impl Iterator<Item = SignalEnum> {
+ SignalEnumDiscriminants::VARIANTS
+ .iter()
+ .copied()
+ .map(|v| v.into())
+ }
+
+ pub fn get(field_id: usize) -> Option<SignalEnum> {
+ SignalEnumDiscriminants::VARIANTS
+ .get(field_id)
+ .copied()
+ .map(SignalEnum::from)
+ }
+}
+
+impl InsertEnumMapKey for SignalEnum {
+ fn into_usize(self) -> usize {
+ SignalEnumDiscriminants::from(self) as usize
+ }
+}
+
+impl GetEnumMapKey for SignalEnum {
+ fn from_usize(value: usize) -> Option<Self> {
+ Self::get(value)
+ }
+}
+
+impl CoreSignalEnum {
+ pub fn num_variants() -> usize {
+ CoreSignalEnumDiscriminants::VARIANTS.len()
+ }
+
+ pub fn all() -> impl Iterator<Item = CoreSignalEnum> {
+ CoreSignalEnumDiscriminants::VARIANTS
+ .iter()
+ .copied()
+ .map(|v| v.into())
+ }
+
+ pub fn get(field_id: usize) -> Option<CoreSignalEnum> {
+ CoreSignalEnumDiscriminants::VARIANTS
+ .get(field_id)
+ .copied()
+ .map(CoreSignalEnum::from)
+ }
+}
+
+impl InsertEnumMapKey for CoreSignalEnum {
+ fn into_usize(self) -> usize {
+ CoreSignalEnumDiscriminants::from(self) as usize
+ }
+}
+
+impl GetEnumMapKey for CoreSignalEnum {
+ fn from_usize(value: usize) -> Option<Self> {
+ Self::get(value)
+ }
+}
+
+impl FromStr for SignalEnumDiscriminants {
+ type Err = anyhow::Error;
+
+ fn from_str(name: &str) -> std::result::Result<Self, Self::Err> {
+ let s = "\"".to_string() + name + "\"";
+ let signal = serde_json::from_str(&s)?;
+ Ok(signal)
+ }
+}
+
+#[derive(Debug, Clone, Copy, bincode::Encode, bincode::Decode)]
+pub struct SignalCalculation {
+ pub value: f64,
+ pub score: f64,
+}
+
+impl SignalCalculation {
+ pub fn new_symmetrical(val: f64) -> Self {
+ Self {
+ value: val,
+ score: val,
+ }
+ }
+}
+
+#[derive(Debug, Clone, Copy)]
+pub struct ComputedSignal {
+ pub signal: SignalEnum,
+ pub calc: SignalCalculation,
+}
+
+#[derive(
+ Debug,
+ Clone,
+ Copy,
+ serde::Serialize,
+ serde::Deserialize,
+ bincode::Encode,
+ bincode::Decode,
+ ToSchema,
+)]
+#[serde(rename_all = "camelCase")]
+pub struct SignalScore {
+ pub coefficient: f64,
+ pub value: f64,
+}
+
+#[derive(Debug, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, Clone)]
+pub struct SignalCoefficients {
+ map: EnumMap<SignalEnum, f64>,
+}
+
+impl Default for SignalCoefficients {
+ fn default() -> Self {
+ Self::new(SignalEnum::all().map(|signal| (signal, signal.default_coefficient())))
+ }
+}
+
+impl From<EnumMap<SignalEnum, f64>> for SignalCoefficients {
+ fn from(map: EnumMap<SignalEnum, f64>) -> Self {
+ Self { map }
+ }
+}
+
+impl SignalCoefficients {
+ pub fn get(&self, signal: &SignalEnum) -> f64 {
+ self.map
+ .get(*signal)
+ .copied()
+ .unwrap_or(signal.default_coefficient())
+ }
+
+ pub fn new(coefficients: impl Iterator<Item = (SignalEnum, f64)>) -> Self {
+ let mut map = EnumMap::default();
+
+ for (signal, coefficient) in coefficients {
+ map.insert(signal, coefficient);
+ }
+
+ Self { map }
+ }
+
+ pub fn merge_add(&mut self, coeffs: SignalCoefficients) {
+ for signal in SignalEnum::all() {
+ if let Some(coeff) = coeffs.map.get(signal).copied() {
+ match self.map.get_mut(signal) {
+ Some(existing_coeff) => *existing_coeff += coeff,
+ None => {
+ self.map.insert(signal, coeff);
+ }
+ }
+ }
+ }
+ }
+
+ pub fn merge_overwrite(&mut self, coeffs: SignalCoefficients) {
+ for signal in SignalEnum::all() {
+ if let Some(coeff) = coeffs.map.get(signal).copied() {
+ match self.map.get_mut(signal) {
+ Some(existing_coeff) => *existing_coeff = coeff,
+ None => {
+ self.map.insert(signal, coeff);
+ }
+ }
+ }
+ }
+ }
+}
diff --git a/crates/core/src/ranking/signals/non_core/mod.rs b/crates/core/src/ranking/signals/non_core/mod.rs
new file mode 100644
index 00000000..69c83d27
--- /dev/null
+++ b/crates/core/src/ranking/signals/non_core/mod.rs
@@ -0,0 +1,21 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2024 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>
+
+pub mod non_text;
+pub mod text;
+
+pub use self::non_text::*;
+pub use self::text::*;
diff --git a/crates/core/src/ranking/core/query.rs b/crates/core/src/ranking/signals/non_core/non_text.rs
similarity index 59%
rename from crates/core/src/ranking/core/query.rs
rename to crates/core/src/ranking/signals/non_core/non_text.rs
index ca1b5f23..26ee9854 100644
--- a/crates/core/src/ranking/core/query.rs
+++ b/crates/core/src/ranking/signals/non_core/non_text.rs
@@ -1,7 +1,3 @@
-use tantivy::DocId;
-
-use crate::schema::Field;
-
// Stract is an open source web search engine.
// Copyright (C) 2024 Stract ApS
//
@@ -16,9 +12,9 @@ use crate::schema::Field;
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
-// along with this program. If not, see <https://www.gnu.org/licenses/>.
+// along with this program. If not, see <https://www.gnu.org/licenses/>
-use super::{Signal, SignalCalculation, SignalComputer};
+use crate::ranking::Signal;
#[derive(
Debug,
@@ -32,20 +28,47 @@ use super::{Signal, SignalCalculation, SignalComputer};
bincode::Encode,
bincode::Decode,
)]
-pub struct NumQueryTerms;
-impl Signal for NumQueryTerms {
+pub struct QueryCentrality;
+impl Signal for QueryCentrality {
fn default_coefficient(&self) -> f64 {
0.0
}
+}
- fn as_field(&self) -> Option<Field> {
- None
+#[derive(
+ Debug,
+ Clone,
+ Copy,
+ PartialEq,
+ Eq,
+ Hash,
+ serde::Serialize,
+ serde::Deserialize,
+ bincode::Encode,
+ bincode::Decode,
+)]
+pub struct InboundSimilarity;
+impl Signal for InboundSimilarity {
+ fn default_coefficient(&self) -> f64 {
+ 0.25
}
+}
- fn compute(&self, _: DocId, signal_computer: &SignalComputer) -> Option<SignalCalculation> {
- signal_computer
- .query_data()
- .map(|d| d.simple_terms().len() as f64)
- .map(SignalCalculation::new_symmetrical)
+#[derive(
+ Debug,
+ Clone,
+ Copy,
+ PartialEq,
+ Eq,
+ Hash,
+ serde::Serialize,
+ serde::Deserialize,
+ bincode::Encode,
+ bincode::Decode,
+)]
+pub struct LambdaMart;
+impl Signal for LambdaMart {
+ fn default_coefficient(&self) -> f64 {
+ 10.0
}
}
diff --git a/crates/core/src/ranking/signals/non_core/text.rs b/crates/core/src/ranking/signals/non_core/text.rs
new file mode 100644
index 00000000..e03d6f92
--- /dev/null
+++ b/crates/core/src/ranking/signals/non_core/text.rs
@@ -0,0 +1,131 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2024 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>
+
+use crate::ranking::Signal;
+
+#[derive(
+ Debug,
+ Clone,
+ Copy,
+ PartialEq,
+ Eq,
+ Hash,
+ serde::Serialize,
+ serde::Deserialize,
+ bincode::Encode,
+ bincode::Decode,
+)]
+pub struct MinTitleSlop;
+impl Signal for MinTitleSlop {
+ fn default_coefficient(&self) -> f64 {
+ 0.1
+ }
+}
+
+#[derive(
+ Debug,
+ Clone,
+ Copy,
+ PartialEq,
+ Eq,
+ Hash,
+ serde::Serialize,
+ serde::Deserialize,
+ bincode::Encode,
+ bincode::Decode,
+)]
+pub struct MinCleanBodySlop;
+impl Signal for MinCleanBodySlop {
+ fn default_coefficient(&self) -> f64 {
+ 0.1
+ }
+}
+
+#[derive(
+ Debug,
+ Clone,
+ Copy,
+ PartialEq,
+ Eq,
+ Hash,
+ serde::Serialize,
+ serde::Deserialize,
+ bincode::Encode,
+ bincode::Decode,
+)]
+pub struct CrossEncoderSnippet;
+impl Signal for CrossEncoderSnippet {
+ fn default_coefficient(&self) -> f64 {
+ 0.17
+ }
+}
+
+#[derive(
+ Debug,
+ Clone,
+ Copy,
+ PartialEq,
+ Eq,
+ Hash,
+ serde::Serialize,
+ serde::Deserialize,
+ bincode::Encode,
+ bincode::Decode,
+)]
+pub struct CrossEncoderTitle;
+impl Signal for CrossEncoderTitle {
+ fn default_coefficient(&self) -> f64 {
+ 0.17
+ }
+}
+
+#[derive(
+ Debug,
+ Clone,
+ Copy,
+ PartialEq,
+ Eq,
+ Hash,
+ serde::Serialize,
+ serde::Deserialize,
+ bincode::Encode,
+ bincode::Decode,
+)]
+pub struct TitleEmbeddingSimilarity;
+impl Signal for TitleEmbeddingSimilarity {
+ fn default_coefficient(&self) -> f64 {
+ 0.01
+ }
+}
+
+#[derive(
+ Debug,
+ Clone,
+ Copy,
+ PartialEq,
+ Eq,
+ Hash,
+ serde::Serialize,
+ serde::Deserialize,
+ bincode::Encode,
+ bincode::Decode,
+)]
+pub struct KeywordEmbeddingSimilarity;
+impl Signal for KeywordEmbeddingSimilarity {
+ fn default_coefficient(&self) -> f64 {
+ 0.01
+ }
+}
diff --git a/frontend/src/lib/api/index.ts b/frontend/src/lib/api/index.ts
index a11a8944..b5a9103e 100644
--- a/frontend/src/lib/api/index.ts
+++ b/frontend/src/lib/api/index.ts
@@ -414,7 +414,6 @@ export type SignalEnumDiscriminants =
| 'title_embedding_similarity'
| 'keyword_embedding_similarity'
| 'has_ads'
- | 'num_query_terms'
| 'min_title_slop'
| 'min_clean_body_slop';
export const SIGNAL_ENUM_DISCRIMINANTS = [
@@ -462,7 +461,6 @@ export const SIGNAL_ENUM_DISCRIMINANTS = [
'title_embedding_similarity',
'keyword_embedding_similarity',
'has_ads',
- 'num_query_terms',
'min_title_slop',
'min_clean_body_slop',
] satisfies SignalEnumDiscriminants[];
diff --git a/frontend/src/routes/search/Modal.svelte b/frontend/src/routes/search/Modal.svelte
index a85ac60e..8528cd05 100644
--- a/frontend/src/routes/search/Modal.svelte
+++ b/frontend/src/routes/search/Modal.svelte
@@ -83,7 +83,6 @@
.with('query_centrality', () => 'queryCentrality' as const)
.with('inbound_similarity', () => 'inboundSimilarity' as const)
.with('lambda_mart', () => undefined)
- .with('num_query_terms', () => undefined)
.with('url_digits', () => 'urlSymbols' as const)
.with('url_slashes', () => 'urlSymbols' as const)
.with('link_density', () => 'linkDensity' as const)