web search done right
split 'Signal' trait into 'CoreSignal' and 'Signal' to distinguish between the signals that are calculated initially and later during ranking
Mikkel Denker9cba6c13fdbd
diff --git a/crates/core/src/api/docs.rs b/crates/core/src/api/docs.rsindex 514e52fc..78ad892a 100644--- a/crates/core/src/api/docs.rs+++ b/crates/core/src/api/docs.rs@@ -77,8 +77,8 @@ use utoipa_swagger_ui::SwaggerUi;crate::widgets::thesaurus::PartOfSpeech,crate::widgets::thesaurus::PartOfSpeechMeaning,- crate::ranking::core::SignalEnumDiscriminants,- crate::ranking::core::SignalScore,+ crate::ranking::SignalEnumDiscriminants,+ crate::ranking::SignalScore,crate::bangs::BangHit,crate::bangs::Bang,diff --git a/crates/core/src/index.rs b/crates/core/src/index.rsindex d39056ba..9a2fc4d2 100644--- a/crates/core/src/index.rs+++ b/crates/core/src/index.rs@@ -246,7 +246,7 @@ mod tests {.ranking_signals.as_ref().unwrap()- .get(&crate::ranking::SignalEnum::from(ranking::core::Bm25Title).into())+ .get(&crate::ranking::SignalEnum::from(ranking::signals::Bm25Title).into()).unwrap()).all(|&v| v.value > 0.0));}diff --git a/crates/core/src/query/optic.rs b/crates/core/src/query/optic.rsindex 0b9be805..3b607dd6 100644--- a/crates/core/src/query/optic.rs+++ b/crates/core/src/query/optic.rs@@ -352,7 +352,7 @@ mod tests {.search(&SearchQuery {query: "website".to_string(),signal_coefficients: enum_map! {- crate::ranking::SignalEnum::from(crate::ranking::core::Bm25Title) => 1_000_000.0+ crate::ranking::SignalEnum::from(crate::ranking::signals::core::Bm25Title) => 1_000_000.0}.into(),@@ -760,7 +760,7 @@ mod tests {.search(&SearchQuery {query: "website".to_string(),signal_coefficients: crate::enum_map! {- crate::ranking::SignalEnum::from(crate::ranking::core::InboundSimilarity) => 100_000.0+ crate::ranking::SignalEnum::from(crate::ranking::signals::InboundSimilarity) => 100_000.0}.into(),optic: Some(Optic {diff --git a/crates/core/src/ranking/core/computer/mod.rs b/crates/core/src/ranking/computer/mod.rssimilarity index 97%rename from crates/core/src/ranking/core/computer/mod.rsrename to crates/core/src/ranking/computer/mod.rsindex ebaed70c..7f2ef108 100644--- a/crates/core/src/ranking/core/computer/mod.rs+++ b/crates/core/src/ranking/computer/mod.rs@@ -40,7 +40,7 @@ use crate::webpage::region::RegionCount;use crate::ranking::bm25::MultiBm25Weight;use crate::ranking::models::linear::LinearRegression;-use super::{ComputedSignal, Signal, SignalCoefficients, SignalEnum};+use super::{ComputedSignal, CoreSignal, CoreSignalEnum, Signal, SignalCoefficients, SignalEnum};mod order;pub use order::SignalComputeOrder;@@ -288,7 +288,7 @@ impl SignalComputer {if let Some(query) = &self.query_data {if !query.simple_terms.is_empty() {- for signal in SignalEnum::all() {+ for signal in CoreSignalEnum::all() {if let Some((text_field, tv_field)) = signal.as_textfield().and_then(|f| (f.tantivy_field(schema).map(|tv_field| (f, tv_field))))@@ -350,7 +350,7 @@ impl SignalComputer {bm25,bm25f,fieldnorm_reader,- signal_coefficient: self.coefficient(&signal),+ signal_coefficient: self.coefficient(&signal.into()),num_query_terms: terms.len(),},);@@ -470,11 +470,12 @@ impl SignalComputer {}pub fn precompute_score(&self, webpage: &Webpage) -> f64 {- SignalEnum::all()+ CoreSignalEnum::all().filter_map(|signal| {- signal- .precompute(webpage, self)- .map(|calc| ComputedSignal { signal, calc })+ signal.precompute(webpage, self).map(|calc| ComputedSignal {+ signal: signal.into(),+ calc,+ })}).map(|computed| self.coefficient(&computed.signal) * computed.calc.score).sum()diff --git a/crates/core/src/ranking/core/computer/order.rs b/crates/core/src/ranking/computer/order.rssimilarity index 85%rename from crates/core/src/ranking/core/computer/order.rsrename to crates/core/src/ranking/computer/order.rsindex 53c93578..61b33821 100644--- a/crates/core/src/ranking/core/computer/order.rs+++ b/crates/core/src/ranking/computer/order.rs@@ -18,7 +18,7 @@ use tantivy::DocId;use crate::{enum_map::EnumMap,- ranking::{ComputedSignal, Signal, SignalCalculation, SignalEnum},+ ranking::{ComputedSignal, CoreSignal, CoreSignalEnum, SignalCalculation},schema::{text_field::TextField, TextFieldEnum},};@@ -27,7 +27,7 @@ use super::SignalComputer;#[derive(Clone)]pub struct SignalComputeOrder {text_signals: EnumMap<TextFieldEnum, NGramComputeOrder>,- other_signals: Vec<SignalEnum>,+ other_signals: Vec<CoreSignalEnum>,}impl SignalComputeOrder {@@ -35,7 +35,7 @@ impl SignalComputeOrder {let mut text_signals = EnumMap::new();let mut other_signals = Vec::new();- for signal in SignalEnum::all() {+ for signal in CoreSignalEnum::all() {if let Some(text_field) = signal.as_textfield() {if signal.has_sibling_ngrams() {let mono = text_field.monogram_field();@@ -68,15 +68,15 @@ impl SignalComputeOrder {self.text_signals.values().flat_map(move |ngram| ngram.compute(doc, signal_computer))- .chain(self.other_signals.iter().map(move |signal| {+ .chain(self.other_signals.iter().map(move |&signal| {signal.compute(doc, signal_computer).map(|calc| ComputedSignal {- signal: *signal,+ signal: signal.into(),calc,}).unwrap_or_else(|| ComputedSignal {- signal: *signal,+ signal: signal.into(),calc: SignalCalculation {score: 0.0,value: 0.0,@@ -86,6 +86,12 @@ impl SignalComputeOrder {}}+impl Default for SignalComputeOrder {+ fn default() -> Self {+ Self::new()+ }+}+/// If an ngram of size n matches the query for a given document in a given field,/// the score of all ngrams where n' < n is dampened by NGRAM_DAMPENING.///@@ -96,11 +102,11 @@ const NGRAM_DAMPENING: f64 = 0.4;#[derive(Debug, Default, Clone)]pub struct NGramComputeOrder {/// ordered by descending ngram size. e.g. [title_bm25_trigram, title_bm25_bigram, title_bm25]- signals: Vec<(usize, SignalEnum)>,+ signals: Vec<(usize, CoreSignalEnum)>,}impl NGramComputeOrder {- fn push(&mut self, signal: SignalEnum, ngram: usize) {+ fn push(&mut self, signal: CoreSignalEnum, ngram: usize) {self.signals.push((ngram, signal));self.signals.sort_unstable_by(|(a, _), (b, _)| b.cmp(a));}@@ -112,11 +118,11 @@ impl NGramComputeOrder {) -> impl Iterator<Item = ComputedSignal> + 'a {let mut hits = 0;- self.signals.iter().map(|(_, s)| s).map(move |signal| {+ self.signals.iter().map(|(_, s)| s).map(move |&signal| {signal.compute(doc, signal_computer).map(|calc| ComputedSignal {- signal: *signal,+ signal: signal.into(),calc,}).map(|mut c| {@@ -129,7 +135,7 @@ impl NGramComputeOrder {c}).unwrap_or_else(|| ComputedSignal {- signal: *signal,+ signal: signal.into(),calc: SignalCalculation {value: 0.0,score: 0.0,diff --git a/crates/core/src/ranking/core/mod.rs b/crates/core/src/ranking/core/mod.rsdeleted file mode 100644index 9d818502..00000000--- a/crates/core/src/ranking/core/mod.rs+++ /dev/null@@ -1,148 +0,0 @@-// Stract is an open source web search engine.-// Copyright (C) 2024 Stract ApS-//-// This program is free software: you can redistribute it and/or modify-// it under the terms of the GNU Affero General Public License as-// published by the Free Software Foundation, either version 3 of the-// License, or (at your option) any later version.-//-// This program is distributed in the hope that it will be useful,-// but WITHOUT ANY WARRANTY; without even the implied warranty of-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the-// GNU Affero General Public License for more details.-//-// You should have received a copy of the GNU Affero General Public License-// along with this program. If not, see <https://www.gnu.org/licenses/>.--use crate::enum_map::EnumMap;--use std::str::FromStr;--use thiserror::Error;-use utoipa::ToSchema;--mod computer;-mod non_text;-mod prelude;-mod query;-mod text;--pub use computer::SignalComputer;-pub use non_text::*;-pub use prelude::*;-pub use text::*;--#[derive(Debug, Error)]-pub enum Error {- #[error("unknown signal: {0}")]- UnknownSignal(#[from] serde_json::Error),-}--impl FromStr for SignalEnumDiscriminants {- type Err = Error;-- fn from_str(name: &str) -> std::result::Result<Self, Self::Err> {- let s = "\"".to_string() + name + "\"";- let signal = serde_json::from_str(&s)?;- Ok(signal)- }-}--#[derive(Debug, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, Clone)]-pub struct SignalCoefficients {- map: EnumMap<SignalEnum, f64>,-}--impl Default for SignalCoefficients {- fn default() -> Self {- Self::new(SignalEnum::all().map(|signal| (signal, signal.default_coefficient())))- }-}--impl From<EnumMap<SignalEnum, f64>> for SignalCoefficients {- fn from(map: EnumMap<SignalEnum, f64>) -> Self {- Self { map }- }-}--impl SignalCoefficients {- pub fn get(&self, signal: &SignalEnum) -> f64 {- self.map- .get(*signal)- .copied()- .unwrap_or(signal.default_coefficient())- }-- pub fn new(coefficients: impl Iterator<Item = (SignalEnum, f64)>) -> Self {- let mut map = EnumMap::default();-- for (signal, coefficient) in coefficients {- map.insert(signal, coefficient);- }-- Self { map }- }-- pub fn merge_add(&mut self, coeffs: SignalCoefficients) {- for signal in SignalEnum::all() {- if let Some(coeff) = coeffs.map.get(signal).copied() {- match self.map.get_mut(signal) {- Some(existing_coeff) => *existing_coeff += coeff,- None => {- self.map.insert(signal, coeff);- }- }- }- }- }-- pub fn merge_overwrite(&mut self, coeffs: SignalCoefficients) {- for signal in SignalEnum::all() {- if let Some(coeff) = coeffs.map.get(signal).copied() {- match self.map.get_mut(signal) {- Some(existing_coeff) => *existing_coeff = coeff,- None => {- self.map.insert(signal, coeff);- }- }- }- }- }-}--#[derive(Debug, Clone, Copy, bincode::Encode, bincode::Decode)]-pub struct SignalCalculation {- pub value: f64,- pub score: f64,-}--impl SignalCalculation {- pub fn new_symmetrical(val: f64) -> Self {- Self {- value: val,- score: val,- }- }-}--#[derive(Debug, Clone, Copy)]-pub struct ComputedSignal {- pub signal: SignalEnum,- pub calc: SignalCalculation,-}--#[derive(- Debug,- Clone,- Copy,- serde::Serialize,- serde::Deserialize,- bincode::Encode,- bincode::Decode,- ToSchema,-)]-#[serde(rename_all = "camelCase")]-pub struct SignalScore {- pub coefficient: f64,- pub value: f64,-}diff --git a/crates/core/src/ranking/core/prelude.rs b/crates/core/src/ranking/core/prelude.rsdeleted file mode 100644index a5164e64..00000000--- a/crates/core/src/ranking/core/prelude.rs+++ /dev/null@@ -1,224 +0,0 @@-// Stract is an open source web search engine.-// Copyright (C) 2024 Stract ApS-//-// This program is free software: you can redistribute it and/or modify-// it under the terms of the GNU Affero General Public License as-// published by the Free Software Foundation, either version 3 of the-// License, or (at your option) any later version.-//-// This program is distributed in the hope that it will be useful,-// but WITHOUT ANY WARRANTY; without even the implied warranty of-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the-// GNU Affero General Public License for more details.-//-// You should have received a copy of the GNU Affero General Public License-// along with this program. If not, see <https://www.gnu.org/licenses/>--use crate::enum_dispatch_from_discriminant;-use crate::enum_map::{GetEnumMapKey, InsertEnumMapKey};--use crate::schema::Field;-use crate::{- schema::{NumericalFieldEnum, TextFieldEnum},- webpage::Webpage,-};-use enum_dispatch::enum_dispatch;--use strum::{EnumDiscriminants, VariantArray};--use super::query::*;-use super::text::*;-use super::SignalComputer;-use super::{non_text::*, SignalCalculation};-use tantivy::DocId;--#[enum_dispatch]-pub trait Signal:- Clone + Copy + std::fmt::Debug + PartialEq + Eq + std::hash::Hash + Into<SignalEnum>-{- fn default_coefficient(&self) -> f64;- fn as_field(&self) -> Option<Field>;- fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<SignalCalculation>;-- fn precompute(- self,- _webpage: &Webpage,- _signal_computer: &SignalComputer,- ) -> Option<SignalCalculation> {- None- }-- fn as_textfield(&self) -> Option<TextFieldEnum> {- self.as_field().and_then(|field| field.as_text())- }-- fn has_sibling_ngrams(&self) -> bool {- false- }-- fn as_numericalfield(&self) -> Option<NumericalFieldEnum> {- self.as_field().and_then(|field| field.as_numerical())- }-}--#[enum_dispatch(Signal)]-#[derive(- Debug,- serde::Serialize,- serde::Deserialize,- bincode::Encode,- bincode::Decode,- Clone,- Copy,- PartialEq,- Eq,- Hash,- EnumDiscriminants,-)]-#[strum_discriminants(derive(- VariantArray,- serde::Serialize,- serde::Deserialize,- bincode::Encode,- bincode::Decode,- PartialOrd,- Ord,- Hash,- utoipa::ToSchema,-))]-#[strum_discriminants(serde(rename_all = "snake_case"))]-pub enum SignalEnum {- Bm25F,- Bm25Title,- TitleCoverage,- Bm25TitleBigrams,- Bm25TitleTrigrams,- Bm25CleanBody,- CleanBodyCoverage,- Bm25CleanBodyBigrams,- Bm25CleanBodyTrigrams,- Bm25StemmedTitle,- Bm25StemmedCleanBody,- Bm25AllBody,- Bm25Keywords,- Bm25BacklinkText,- IdfSumUrl,- IdfSumSite,- IdfSumDomain,- IdfSumSiteNoTokenizer,- IdfSumDomainNoTokenizer,- IdfSumDomainNameNoTokenizer,- IdfSumDomainIfHomepage,- IdfSumDomainNameIfHomepageNoTokenizer,- IdfSumDomainIfHomepageNoTokenizer,- IdfSumTitleIfHomepage,- CrossEncoderSnippet,- CrossEncoderTitle,- HostCentrality,- HostCentralityRank,- PageCentrality,- PageCentralityRank,- IsHomepage,- FetchTimeMs,- UpdateTimestamp,- TrackerScore,- Region,- QueryCentrality,- InboundSimilarity,- LambdaMart,- UrlDigits,- UrlSlashes,- LinkDensity,- TitleEmbeddingSimilarity,- KeywordEmbeddingSimilarity,- HasAds,- NumQueryTerms,- MinTitleSlop,- MinCleanBodySlop,-}--enum_dispatch_from_discriminant!(SignalEnumDiscriminants => SignalEnum,-[- Bm25F,- Bm25Title,- TitleCoverage,- Bm25TitleBigrams,- Bm25TitleTrigrams,- Bm25CleanBody,- CleanBodyCoverage,- Bm25CleanBodyBigrams,- Bm25CleanBodyTrigrams,- Bm25StemmedTitle,- Bm25StemmedCleanBody,- Bm25AllBody,- Bm25Keywords,- Bm25BacklinkText,- IdfSumUrl,- IdfSumSite,- IdfSumDomain,- IdfSumSiteNoTokenizer,- IdfSumDomainNoTokenizer,- IdfSumDomainNameNoTokenizer,- IdfSumDomainIfHomepage,- IdfSumDomainNameIfHomepageNoTokenizer,- IdfSumDomainIfHomepageNoTokenizer,- IdfSumTitleIfHomepage,- CrossEncoderSnippet,- CrossEncoderTitle,- HostCentrality,- HostCentralityRank,- PageCentrality,- PageCentralityRank,- IsHomepage,- FetchTimeMs,- UpdateTimestamp,- TrackerScore,- Region,- QueryCentrality,- InboundSimilarity,- LambdaMart,- UrlDigits,- UrlSlashes,- LinkDensity,- TitleEmbeddingSimilarity,- KeywordEmbeddingSimilarity,- HasAds,- NumQueryTerms,- MinTitleSlop,- MinCleanBodySlop,-]);--impl SignalEnum {- pub fn num_variants() -> usize {- SignalEnumDiscriminants::VARIANTS.len()- }-- pub fn all() -> impl Iterator<Item = SignalEnum> {- SignalEnumDiscriminants::VARIANTS- .iter()- .copied()- .map(|v| v.into())- }-- pub fn get(field_id: usize) -> Option<SignalEnum> {- SignalEnumDiscriminants::VARIANTS- .get(field_id)- .copied()- .map(SignalEnum::from)- }-}--impl InsertEnumMapKey for SignalEnum {- fn into_usize(self) -> usize {- SignalEnumDiscriminants::from(self) as usize- }-}--impl GetEnumMapKey for SignalEnum {- fn from_usize(value: usize) -> Option<Self> {- SignalEnumDiscriminants::VARIANTS- .get(value)- .copied()- .map(SignalEnum::from)- }-}diff --git a/crates/core/src/ranking/mod.rs b/crates/core/src/ranking/mod.rsindex a67c8c77..3c58526c 100644--- a/crates/core/src/ranking/mod.rs+++ b/crates/core/src/ranking/mod.rs@@ -17,15 +17,22 @@pub mod bitvec_similarity;pub mod bm25;pub mod bm25f;-pub mod core;+pub mod computer;pub mod inbound_similarity;pub mod initial;pub mod models;pub mod optics;pub mod pipeline;+pub mod signals;+pub use computer::SignalComputer;use initial::InitialScoreTweaker;+pub use signals::{+ ComputedSignal, CoreSignal, CoreSignalEnum, Signal, SignalCalculation, SignalCoefficients,+ SignalEnum, SignalEnumDiscriminants, SignalScore,+};+use crate::{collector::{MainCollector, MaxDocsConsidered, TopDocs},config::CollectorConfig,@@ -34,8 +41,6 @@ use crate::{searcher::NUM_RESULTS_PER_PAGE,};-pub use self::core::*;-#[derive(Clone)]pub struct LocalRanker {max_docs: Option<MaxDocsConsidered>,@@ -331,7 +336,7 @@ mod tests {query: "title".to_string(),return_ranking_signals: true,signal_coefficients: crate::enum_map! {- crate::ranking::SignalEnum::from(crate::ranking::core::UpdateTimestamp) => 100_000.0,+ crate::ranking::SignalEnum::from(crate::ranking::signals::UpdateTimestamp) => 100_000.0,}.into(),..Default::default()})@@ -556,8 +561,8 @@ mod tests {query: "example".to_string(),signal_coefficients: crate::enum_map! {- crate::ranking::SignalEnum::from(crate::ranking::core::Bm25Title) => 20_000_000.0,- crate::ranking::SignalEnum::from(crate::ranking::core::HostCentrality) => 0.0,+ crate::ranking::SignalEnum::from(crate::ranking::signals::Bm25Title) => 20_000_000.0,+ crate::ranking::SignalEnum::from(crate::ranking::signals::HostCentrality) => 0.0,}.into(),..Default::default()@@ -572,7 +577,7 @@ mod tests {query: "example".to_string(),signal_coefficients: crate::enum_map! {- crate::ranking::SignalEnum::from(crate::ranking::core::HostCentrality) => 2_000_000.0,+ crate::ranking::SignalEnum::from(crate::ranking::signals::HostCentrality) => 2_000_000.0,}.into(),..Default::default()@@ -641,7 +646,7 @@ mod tests {.search(&SearchQuery {query: "test".to_string(),signal_coefficients: crate::enum_map! {- crate::ranking::SignalEnum::from(crate::ranking::core::FetchTimeMs) => 100_000.0,+ crate::ranking::SignalEnum::from(crate::ranking::signals::FetchTimeMs) => 100_000.0,}.into(),..Default::default()})@@ -735,8 +740,8 @@ mod tests {query: "test".to_string(),signal_coefficients: crate::enum_map! {- crate::ranking::SignalEnum::from(crate::ranking::core::UrlSlashes) => 100_000.0,- crate::ranking::SignalEnum::from(crate::ranking::core::UrlDigits) => 100_000.0,+ crate::ranking::SignalEnum::from(crate::ranking::signals::UrlSlashes) => 100_000.0,+ crate::ranking::SignalEnum::from(crate::ranking::signals::UrlDigits) => 100_000.0,}.into(),@@ -842,7 +847,7 @@ mod tests {query: "best chocolate cake".to_string(),signal_coefficients: crate::enum_map! {- crate::ranking::SignalEnum::from(crate::ranking::core::TitleEmbeddingSimilarity) => 100_000.0,+ crate::ranking::SignalEnum::from(crate::ranking::signals::TitleEmbeddingSimilarity) => 100_000.0,}.into(),..Default::default()@@ -930,7 +935,7 @@ mod tests {query: "best chocolate cake".to_string(),signal_coefficients: crate::enum_map! {- crate::ranking::SignalEnum::from(crate::ranking::core::KeywordEmbeddingSimilarity) => 100_000.0,+ crate::ranking::SignalEnum::from(crate::ranking::signals::KeywordEmbeddingSimilarity) => 100_000.0,}.into(),..Default::default()@@ -986,7 +991,10 @@ mod tests {.ranking_signals.as_ref().unwrap()- .get(&crate::ranking::SignalEnum::from(crate::ranking::core::TitleCoverage).into())+ .get(+ &crate::ranking::SignalEnum::from(crate::ranking::signals::TitleCoverage)+ .into()+ ).unwrap().value,1.0,@@ -1006,7 +1014,10 @@ mod tests {.ranking_signals.as_ref().unwrap()- .get(&crate::ranking::SignalEnum::from(crate::ranking::core::TitleCoverage).into())+ .get(+ &crate::ranking::SignalEnum::from(crate::ranking::signals::TitleCoverage)+ .into()+ ).unwrap().value,0.5,@@ -1061,7 +1072,7 @@ mod tests {.as_ref().unwrap().get(- &crate::ranking::SignalEnum::from(crate::ranking::core::CleanBodyCoverage)+ &crate::ranking::SignalEnum::from(crate::ranking::signals::CleanBodyCoverage).into()).unwrap()@@ -1084,7 +1095,7 @@ mod tests {.as_ref().unwrap().get(- &crate::ranking::SignalEnum::from(crate::ranking::core::CleanBodyCoverage)+ &crate::ranking::SignalEnum::from(crate::ranking::signals::CleanBodyCoverage).into()).unwrap()diff --git a/crates/core/src/ranking/models/lambdamart.rs b/crates/core/src/ranking/models/lambdamart.rsindex 5d1ee789..0ec3b472 100644--- a/crates/core/src/ranking/models/lambdamart.rs+++ b/crates/core/src/ranking/models/lambdamart.rs@@ -16,11 +16,9 @@use std::{path::Path, str::FromStr};-use core::SignalEnumDiscriminants;-use crate::{enum_map::EnumMap,- ranking::{core, SignalCalculation, SignalEnum},+ ranking::signals::{SignalCalculation, SignalEnum, SignalEnumDiscriminants},};type Result<T> = std::result::Result<T, Error>;@@ -35,9 +33,6 @@ pub enum Error {#[error("couldn't find end of trees")]NoEndOfTrees,- #[error("Signal error: {0}")]- Signal(#[from] core::Error),-#[error("ParseInt error: {0}")]ParseInt(#[from] std::num::ParseIntError),@@ -46,6 +41,9 @@ pub enum Error {#[error("IO error: {0}")]Io(#[from] std::io::Error),++ #[error("Unknown signal: {0}")]+ UnknownSignal(String),}#[derive(Clone, Debug)]@@ -225,7 +223,10 @@ impl Header {if let Some((key, value)) = lin.split_once('=') {if key == "feature_names" {for name in value.split(' ') {- features.push(SignalEnum::from(SignalEnumDiscriminants::from_str(name)?));+ features.push(SignalEnum::from(+ SignalEnumDiscriminants::from_str(name)+ .map_err(|_| Error::UnknownSignal(name.to_string()))?,+ ));}}}@@ -320,41 +321,65 @@ mod tests {assert!(!model.trees.is_empty());let mut features = EnumMap::new();- features.insert(ranking::core::Bm25BacklinkText.into(), 85.7750244140625);- features.insert(ranking::core::Bm25CleanBody.into(), 67.41311645507812);- features.insert(ranking::core::Bm25CleanBodyBigrams.into(), 0.0);- features.insert(ranking::core::Bm25CleanBodyTrigrams.into(), 0.0);- features.insert(ranking::core::IdfSumDomain.into(), 43.332096099853516);- features.insert(ranking::core::IdfSumDomainIfHomepage.into(), 0.0);- features.insert(ranking::core::IdfSumDomainIfHomepageNoTokenizer.into(), 0.0);features.insert(- ranking::core::IdfSumDomainNameIfHomepageNoTokenizer.into(),+ ranking::signals::core::Bm25BacklinkText.into(),+ 85.7750244140625,+ );+ features.insert(+ ranking::signals::core::Bm25CleanBody.into(),+ 67.41311645507812,+ );+ features.insert(ranking::signals::core::Bm25CleanBodyBigrams.into(), 0.0);+ features.insert(ranking::signals::core::Bm25CleanBodyTrigrams.into(), 0.0);+ features.insert(+ ranking::signals::core::IdfSumDomain.into(),+ 43.332096099853516,+ );+ features.insert(ranking::signals::core::IdfSumDomainIfHomepage.into(), 0.0);+ features.insert(+ ranking::signals::core::IdfSumDomainIfHomepageNoTokenizer.into(),0.0,);- features.insert(ranking::core::IdfSumDomainNameNoTokenizer.into(), 0.0);- features.insert(ranking::core::IdfSumDomainNoTokenizer.into(), 0.0);- features.insert(ranking::core::IdfSumSite.into(), 61.47410202026367);- features.insert(ranking::core::IdfSumSiteNoTokenizer.into(), 0.0);features.insert(- ranking::core::Bm25StemmedCleanBody.into(),+ ranking::signals::core::IdfSumDomainNameIfHomepageNoTokenizer.into(),+ 0.0,+ );+ features.insert(+ ranking::signals::core::IdfSumDomainNameNoTokenizer.into(),+ 0.0,+ );+ features.insert(ranking::signals::core::IdfSumDomainNoTokenizer.into(), 0.0);+ features.insert(ranking::signals::core::IdfSumSite.into(), 61.47410202026367);+ features.insert(ranking::signals::core::IdfSumSiteNoTokenizer.into(), 0.0);+ features.insert(+ ranking::signals::core::Bm25StemmedCleanBody.into(),65.94627380371094,);- features.insert(ranking::core::Bm25StemmedTitle.into(), 0.0);- features.insert(ranking::core::Bm25Title.into(), 59.817813873291016);- features.insert(ranking::core::Bm25TitleBigrams.into(), 0.0);- features.insert(ranking::core::IdfSumTitleIfHomepage.into(), 0.0);- features.insert(ranking::core::Bm25TitleTrigrams.into(), 0.0);- features.insert(ranking::core::IdfSumUrl.into(), 57.07925033569336);- features.insert(ranking::core::FetchTimeMs.into(), 0.023255813953488372);- features.insert(ranking::core::HostCentrality.into(), 0.017958538);- features.insert(ranking::core::InboundSimilarity.into(), 0.0);- features.insert(ranking::core::IsHomepage.into(), 0.0);- features.insert(ranking::core::PageCentrality.into(), 0.008253236);- features.insert(ranking::core::Region.into(), 0.16622349570454012);- features.insert(ranking::core::TrackerScore.into(), 0.07692307692307693);- features.insert(ranking::core::UpdateTimestamp.into(), 0.0);- features.insert(ranking::core::UrlDigits.into(), 0.25);- features.insert(ranking::core::UrlSlashes.into(), 0.3333333333333333);+ features.insert(ranking::signals::core::Bm25StemmedTitle.into(), 0.0);+ features.insert(ranking::signals::core::Bm25Title.into(), 59.817813873291016);+ features.insert(ranking::signals::core::Bm25TitleBigrams.into(), 0.0);+ features.insert(ranking::signals::core::IdfSumTitleIfHomepage.into(), 0.0);+ features.insert(ranking::signals::core::Bm25TitleTrigrams.into(), 0.0);+ features.insert(ranking::signals::core::IdfSumUrl.into(), 57.07925033569336);+ features.insert(+ ranking::signals::core::FetchTimeMs.into(),+ 0.023255813953488372,+ );+ features.insert(ranking::signals::HostCentrality.into(), 0.017958538);+ features.insert(ranking::signals::InboundSimilarity.into(), 0.0);+ features.insert(ranking::signals::IsHomepage.into(), 0.0);+ features.insert(ranking::signals::PageCentrality.into(), 0.008253236);+ features.insert(ranking::signals::core::Region.into(), 0.16622349570454012);+ features.insert(+ ranking::signals::core::TrackerScore.into(),+ 0.07692307692307693,+ );+ features.insert(ranking::signals::core::UpdateTimestamp.into(), 0.0);+ features.insert(ranking::signals::core::UrlDigits.into(), 0.25);+ features.insert(+ ranking::signals::core::UrlSlashes.into(),+ 0.3333333333333333,+ );assert_eq!((model.predict(&features) * 1000.0) as u64, 1050);}diff --git a/crates/core/src/ranking/optics.rs b/crates/core/src/ranking/optics.rsindex 56e3f812..148764f0 100644--- a/crates/core/src/ranking/optics.rs+++ b/crates/core/src/ranking/optics.rs@@ -185,7 +185,7 @@ mod tests {}),signal_coefficients: crate::enum_map! {- crate::ranking::SignalEnum::from(crate::ranking::core::InboundSimilarity) => 100_000_000.0,+ crate::ranking::SignalEnum::from(crate::ranking::signals::InboundSimilarity) => 100_000_000.0,}.into(),..Default::default()@@ -209,7 +209,7 @@ mod tests {}),signal_coefficients: crate::enum_map! {- crate::ranking::SignalEnum::from(crate::ranking::core::InboundSimilarity) => 100_000_000.0,+ crate::ranking::SignalEnum::from(crate::ranking::signals::InboundSimilarity) => 100_000_000.0,}.into(),return_ranking_signals: true,@@ -233,7 +233,7 @@ mod tests {}),signal_coefficients: crate::enum_map! {- crate::ranking::SignalEnum::from(crate::ranking::core::InboundSimilarity) => 100_000_000.0,+ crate::ranking::SignalEnum::from(crate::ranking::signals::InboundSimilarity) => 100_000_000.0,}.into(),return_ranking_signals: true,diff --git a/crates/core/src/ranking/pipeline/mod.rs b/crates/core/src/ranking/pipeline/mod.rsindex 4de438fb..d95c73d8 100644--- a/crates/core/src/ranking/pipeline/mod.rs+++ b/crates/core/src/ranking/pipeline/mod.rs@@ -207,7 +207,7 @@ mod tests {value: i as f64,score,};- signals.insert(ranking::core::HostCentrality.into(), calc);+ signals.insert(ranking::signals::HostCentrality.into(), calc);LocalRecallRankingWebpage::new_testing(pointer, signals, calc.score)}).map(|local| {diff --git a/crates/core/src/ranking/pipeline/modifiers/inbound_similarity.rs b/crates/core/src/ranking/pipeline/modifiers/inbound_similarity.rsindex 09874545..111d6298 100644--- a/crates/core/src/ranking/pipeline/modifiers/inbound_similarity.rs+++ b/crates/core/src/ranking/pipeline/modifiers/inbound_similarity.rs@@ -32,7 +32,7 @@ impl Modifier for InboundSimilarity {webpage.as_ranking().signals()- .get(ranking::InboundSimilarity.into())+ .get(ranking::signals::InboundSimilarity.into()).map(|calc| calc.value).unwrap_or(0.0)+ INBOUND_SIMILARITY_SMOOTHINGdiff --git a/crates/core/src/ranking/pipeline/scorers/embedding.rs b/crates/core/src/ranking/pipeline/scorers/embedding.rsindex bba0fb5c..a0639cfa 100644--- a/crates/core/src/ranking/pipeline/scorers/embedding.rs+++ b/crates/core/src/ranking/pipeline/scorers/embedding.rs@@ -148,7 +148,7 @@ pub trait EmbeddingSignal<W>: Send + Sync {impl EmbeddingSignal<ScoredWebpagePointer> for TitleEmbeddings {fn signal() -> SignalEnum {- ranking::core::TitleEmbeddingSimilarity.into()+ ranking::signals::TitleEmbeddingSimilarity.into()}fn has_embedding(webpage: &ScoredWebpagePointer) -> bool {@@ -170,7 +170,7 @@ impl EmbeddingSignal<ScoredWebpagePointer> for TitleEmbeddings {impl EmbeddingSignal<ScoredWebpagePointer> for KeywordEmbeddings {fn signal() -> SignalEnum {- ranking::core::KeywordEmbeddingSimilarity.into()+ ranking::signals::KeywordEmbeddingSimilarity.into()}fn has_embedding(webpage: &ScoredWebpagePointer) -> bool {diff --git a/crates/core/src/ranking/pipeline/scorers/inbound_similarity.rs b/crates/core/src/ranking/pipeline/scorers/inbound_similarity.rsindex e5bdf645..5c948767 100644--- a/crates/core/src/ranking/pipeline/scorers/inbound_similarity.rs+++ b/crates/core/src/ranking/pipeline/scorers/inbound_similarity.rs@@ -17,7 +17,7 @@use std::sync::Mutex;use crate::{- ranking::{self, core, inbound_similarity, pipeline::RankableWebpage},+ ranking::{self, inbound_similarity, pipeline::RankableWebpage},searcher::api::ScoredWebpagePointer,};@@ -47,8 +47,8 @@ impl FullRankingStage for InboundScorer {webpage.as_ranking().inbound_edges(),);webpage.as_ranking_mut().signals_mut().insert(- core::InboundSimilarity.into(),- ranking::SignalCalculation::new_symmetrical(score),+ ranking::signals::InboundSimilarity.into(),+ ranking::signals::SignalCalculation::new_symmetrical(score),);}}diff --git a/crates/core/src/ranking/pipeline/scorers/lambdamart.rs b/crates/core/src/ranking/pipeline/scorers/lambdamart.rsindex 9dfc00c9..f86947e2 100644--- a/crates/core/src/ranking/pipeline/scorers/lambdamart.rs+++ b/crates/core/src/ranking/pipeline/scorers/lambdamart.rs@@ -36,7 +36,7 @@ impl RankingStage for Arc<models::LambdaMART> {.map(|(signal, calc)| (signal, calc.value)).collect();(- ranking::core::LambdaMart.into(),+ ranking::signals::LambdaMart.into(),SignalCalculation::new_symmetrical(self.predict(&signals)),)}@@ -59,7 +59,7 @@ impl RankingStage for PrecisionLambda {fn compute(&self, webpage: &Self::Webpage) -> (SignalEnum, SignalCalculation) {(- ranking::core::LambdaMart.into(),+ ranking::signals::LambdaMart.into(),SignalCalculation::new_symmetrical(self.0.predict(webpage.ranking().signals())),)}diff --git a/crates/core/src/ranking/pipeline/scorers/reranker.rs b/crates/core/src/ranking/pipeline/scorers/reranker.rsindex 59aa754b..6caac87d 100644--- a/crates/core/src/ranking/pipeline/scorers/reranker.rs+++ b/crates/core/src/ranking/pipeline/scorers/reranker.rs@@ -49,12 +49,12 @@ impl<M: CrossEncoder> ReRanker<M> {for ((webpage, snippet), title) in webpage.iter_mut().zip(snippet_scores).zip(title_scores){webpage.ranking_mut().signals_mut().insert(- ranking::core::CrossEncoderSnippet.into(),+ ranking::signals::CrossEncoderSnippet.into(),ranking::SignalCalculation::new_symmetrical(snippet),);webpage.ranking_mut().signals_mut().insert(- ranking::core::CrossEncoderTitle.into(),+ ranking::signals::CrossEncoderTitle.into(),ranking::SignalCalculation::new_symmetrical(title),);}diff --git a/crates/core/src/ranking/pipeline/scorers/term_distance.rs b/crates/core/src/ranking/pipeline/scorers/term_distance.rsindex 856e1687..b5ebf151 100644--- a/crates/core/src/ranking/pipeline/scorers/term_distance.rs+++ b/crates/core/src/ranking/pipeline/scorers/term_distance.rs@@ -68,7 +68,7 @@ impl super::RankingStage for TitleDistanceScorer {let score = score_slop(min_slop);(- ranking::core::MinTitleSlop.into(),+ ranking::signals::MinTitleSlop.into(),ranking::SignalCalculation {value: min_slop,score,@@ -88,7 +88,7 @@ impl super::RankingStage for BodyDistanceScorer {let score = score_slop(min_slop);(- ranking::core::MinCleanBodySlop.into(),+ ranking::signals::MinCleanBodySlop.into(),ranking::SignalCalculation {value: min_slop,score,diff --git a/crates/core/src/ranking/signals/core/mod.rs b/crates/core/src/ranking/signals/core/mod.rsnew file mode 100644index 00000000..ef6302ec--- /dev/null+++ b/crates/core/src/ranking/signals/core/mod.rs@@ -0,0 +1,22 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.+pub use std::result;++pub mod non_text;+pub mod text;++pub use non_text::*;+pub use text::*;diff --git a/crates/core/src/ranking/core/non_text.rs b/crates/core/src/ranking/signals/core/non_text.rssimilarity index 91%rename from crates/core/src/ranking/core/non_text.rsrename to crates/core/src/ranking/signals/core/non_text.rsindex 70cd55e6..69e11e88 100644--- a/crates/core/src/ranking/core/non_text.rs+++ b/crates/core/src/ranking/signals/core/non_text.rs@@ -16,7 +16,7 @@use tantivy::DocId;-use super::{Signal, SignalCalculation, SignalComputer};+use crate::ranking::{CoreSignal, SignalCalculation, SignalComputer};use crate::{schema::{self, Field},webpage::Webpage,@@ -106,7 +106,7 @@ fn score_region(webpage_region: crate::webpage::Region, computer: &SignalComputebincode::Decode,)]pub struct HostCentrality;-impl Signal for HostCentrality {+impl CoreSignal for HostCentrality {fn default_coefficient(&self) -> f64 {2.0}@@ -146,7 +146,7 @@ impl Signal for HostCentrality {bincode::Decode,)]pub struct HostCentralityRank;-impl Signal for HostCentralityRank {+impl CoreSignal for HostCentralityRank {fn default_coefficient(&self) -> f64 {0.02}@@ -192,7 +192,7 @@ impl Signal for HostCentralityRank {bincode::Decode,)]pub struct PageCentrality;-impl Signal for PageCentrality {+impl CoreSignal for PageCentrality {fn default_coefficient(&self) -> f64 {2.0}@@ -232,7 +232,7 @@ impl Signal for PageCentrality {bincode::Decode,)]pub struct PageCentralityRank;-impl Signal for PageCentralityRank {+impl CoreSignal for PageCentralityRank {fn default_coefficient(&self) -> f64 {0.02}@@ -278,7 +278,7 @@ impl Signal for PageCentralityRank {bincode::Decode,)]pub struct IsHomepage;-impl Signal for IsHomepage {+impl CoreSignal for IsHomepage {fn default_coefficient(&self) -> f64 {0.01}@@ -323,7 +323,7 @@ impl Signal for IsHomepage {bincode::Decode,)]pub struct FetchTimeMs;-impl Signal for FetchTimeMs {+impl CoreSignal for FetchTimeMs {fn default_coefficient(&self) -> f64 {0.001}@@ -387,7 +387,7 @@ impl Signal for FetchTimeMs {bincode::Decode,)]pub struct UpdateTimestamp;-impl Signal for UpdateTimestamp {+impl CoreSignal for UpdateTimestamp {fn default_coefficient(&self) -> f64 {0.001}@@ -448,7 +448,7 @@ impl Signal for UpdateTimestamp {bincode::Decode,)]pub struct TrackerScore;-impl Signal for TrackerScore {+impl CoreSignal for TrackerScore {fn default_coefficient(&self) -> f64 {0.1}@@ -499,7 +499,7 @@ impl Signal for TrackerScore {bincode::Decode,)]pub struct Region;-impl Signal for Region {+impl CoreSignal for Region {fn default_coefficient(&self) -> f64 {0.15}@@ -541,88 +541,6 @@ impl Signal for Region {}}-#[derive(- Debug,- Clone,- Copy,- PartialEq,- Eq,- Hash,- serde::Serialize,- serde::Deserialize,- bincode::Encode,- bincode::Decode,-)]-pub struct QueryCentrality;-impl Signal for QueryCentrality {- fn default_coefficient(&self) -> f64 {- 0.0- }-- fn as_field(&self) -> Option<Field> {- None- }-- fn compute(&self, _doc: DocId, _signal_computer: &SignalComputer) -> Option<SignalCalculation> {- // unimplemented!()- None- }-}--#[derive(- Debug,- Clone,- Copy,- PartialEq,- Eq,- Hash,- serde::Serialize,- serde::Deserialize,- bincode::Encode,- bincode::Decode,-)]-pub struct InboundSimilarity;-impl Signal for InboundSimilarity {- fn default_coefficient(&self) -> f64 {- 0.25- }-- fn as_field(&self) -> Option<Field> {- None- }-- fn compute(&self, _doc: DocId, _signal_computer: &SignalComputer) -> Option<SignalCalculation> {- None // computed in later ranking stage- }-}--#[derive(- Debug,- Clone,- Copy,- PartialEq,- Eq,- Hash,- serde::Serialize,- serde::Deserialize,- bincode::Encode,- bincode::Decode,-)]-pub struct LambdaMart;-impl Signal for LambdaMart {- fn default_coefficient(&self) -> f64 {- 10.0- }-- fn as_field(&self) -> Option<Field> {- None- }-- fn compute(&self, _: DocId, _: &SignalComputer) -> Option<SignalCalculation> {- None // computed in later ranking stage- }-}-#[derive(Debug,Clone,@@ -636,7 +554,7 @@ impl Signal for LambdaMart {bincode::Decode,)]pub struct UrlDigits;-impl Signal for UrlDigits {+impl CoreSignal for UrlDigits {fn default_coefficient(&self) -> f64 {0.01}@@ -699,7 +617,7 @@ impl Signal for UrlDigits {bincode::Decode,)]pub struct UrlSlashes;-impl Signal for UrlSlashes {+impl CoreSignal for UrlSlashes {fn default_coefficient(&self) -> f64 {0.1}@@ -753,7 +671,7 @@ impl Signal for UrlSlashes {bincode::Decode,)]pub struct LinkDensity;-impl Signal for LinkDensity {+impl CoreSignal for LinkDensity {fn default_coefficient(&self) -> f64 {0.0}@@ -801,7 +719,7 @@ impl Signal for LinkDensity {bincode::Decode,)]pub struct HasAds;-impl Signal for HasAds {+impl CoreSignal for HasAds {fn default_coefficient(&self) -> f64 {0.01}diff --git a/crates/core/src/ranking/core/text.rs b/crates/core/src/ranking/signals/core/text.rssimilarity index 82%rename from crates/core/src/ranking/core/text.rsrename to crates/core/src/ranking/signals/core/text.rsindex da405f5d..a8785655 100644--- a/crates/core/src/ranking/core/text.rs+++ b/crates/core/src/ranking/signals/core/text.rs@@ -18,7 +18,7 @@ use crate::schema::{self, Field};use tantivy::DocId;-use super::{Signal, SignalCalculation, SignalComputer};+use crate::ranking::{CoreSignal, SignalCalculation, SignalComputer};#[derive(Debug,@@ -33,7 +33,7 @@ use super::{Signal, SignalCalculation, SignalComputer};bincode::Decode,)]pub struct Bm25F;-impl Signal for Bm25F {+impl CoreSignal for Bm25F {fn default_coefficient(&self) -> f64 {0.1}@@ -68,7 +68,7 @@ impl Signal for Bm25F {bincode::Decode,)]pub struct Bm25Title;-impl Signal for Bm25Title {+impl CoreSignal for Bm25Title {fn default_coefficient(&self) -> f64 {0.0063}@@ -105,7 +105,7 @@ impl Signal for Bm25Title {bincode::Decode,)]pub struct TitleCoverage;-impl Signal for TitleCoverage {+impl CoreSignal for TitleCoverage {fn default_coefficient(&self) -> f64 {0.01}@@ -137,7 +137,7 @@ impl Signal for TitleCoverage {bincode::Decode,)]pub struct Bm25TitleBigrams;-impl Signal for Bm25TitleBigrams {+impl CoreSignal for Bm25TitleBigrams {fn default_coefficient(&self) -> f64 {0.005}@@ -174,7 +174,7 @@ impl Signal for Bm25TitleBigrams {bincode::Decode,)]pub struct Bm25TitleTrigrams;-impl Signal for Bm25TitleTrigrams {+impl CoreSignal for Bm25TitleTrigrams {fn default_coefficient(&self) -> f64 {0.005}@@ -211,7 +211,7 @@ impl Signal for Bm25TitleTrigrams {bincode::Decode,)]pub struct Bm25CleanBody;-impl Signal for Bm25CleanBody {+impl CoreSignal for Bm25CleanBody {fn default_coefficient(&self) -> f64 {0.005}@@ -248,7 +248,7 @@ impl Signal for Bm25CleanBody {bincode::Decode,)]pub struct CleanBodyCoverage;-impl Signal for CleanBodyCoverage {+impl CoreSignal for CleanBodyCoverage {fn default_coefficient(&self) -> f64 {0.01}@@ -281,7 +281,7 @@ impl Signal for CleanBodyCoverage {bincode::Decode,)]pub struct Bm25CleanBodyBigrams;-impl Signal for Bm25CleanBodyBigrams {+impl CoreSignal for Bm25CleanBodyBigrams {fn default_coefficient(&self) -> f64 {0.005}@@ -318,7 +318,7 @@ impl Signal for Bm25CleanBodyBigrams {bincode::Decode,)]pub struct Bm25CleanBodyTrigrams;-impl Signal for Bm25CleanBodyTrigrams {+impl CoreSignal for Bm25CleanBodyTrigrams {fn default_coefficient(&self) -> f64 {0.005}@@ -355,7 +355,7 @@ impl Signal for Bm25CleanBodyTrigrams {bincode::Decode,)]pub struct Bm25StemmedTitle;-impl Signal for Bm25StemmedTitle {+impl CoreSignal for Bm25StemmedTitle {fn default_coefficient(&self) -> f64 {0.003}@@ -388,7 +388,7 @@ impl Signal for Bm25StemmedTitle {bincode::Decode,)]pub struct Bm25StemmedCleanBody;-impl Signal for Bm25StemmedCleanBody {+impl CoreSignal for Bm25StemmedCleanBody {fn default_coefficient(&self) -> f64 {0.001}@@ -421,7 +421,7 @@ impl Signal for Bm25StemmedCleanBody {bincode::Decode,)]pub struct Bm25AllBody;-impl Signal for Bm25AllBody {+impl CoreSignal for Bm25AllBody {fn default_coefficient(&self) -> f64 {0.0}@@ -454,7 +454,7 @@ impl Signal for Bm25AllBody {bincode::Decode,)]pub struct Bm25Keywords;-impl Signal for Bm25Keywords {+impl CoreSignal for Bm25Keywords {fn default_coefficient(&self) -> f64 {0.001}@@ -487,7 +487,7 @@ impl Signal for Bm25Keywords {bincode::Decode,)]pub struct Bm25BacklinkText;-impl Signal for Bm25BacklinkText {+impl CoreSignal for Bm25BacklinkText {fn default_coefficient(&self) -> f64 {0.003}@@ -520,7 +520,7 @@ impl Signal for Bm25BacklinkText {bincode::Decode,)]pub struct IdfSumUrl;-impl Signal for IdfSumUrl {+impl CoreSignal for IdfSumUrl {fn default_coefficient(&self) -> f64 {0.0006}@@ -553,7 +553,7 @@ impl Signal for IdfSumUrl {bincode::Decode,)]pub struct IdfSumSite;-impl Signal for IdfSumSite {+impl CoreSignal for IdfSumSite {fn default_coefficient(&self) -> f64 {0.00015}@@ -586,7 +586,7 @@ impl Signal for IdfSumSite {bincode::Decode,)]pub struct IdfSumDomain;-impl Signal for IdfSumDomain {+impl CoreSignal for IdfSumDomain {fn default_coefficient(&self) -> f64 {0.0003}@@ -619,7 +619,7 @@ impl Signal for IdfSumDomain {bincode::Decode,)]pub struct IdfSumSiteNoTokenizer;-impl Signal for IdfSumSiteNoTokenizer {+impl CoreSignal for IdfSumSiteNoTokenizer {fn default_coefficient(&self) -> f64 {0.00015}@@ -652,7 +652,7 @@ impl Signal for IdfSumSiteNoTokenizer {bincode::Decode,)]pub struct IdfSumDomainNoTokenizer;-impl Signal for IdfSumDomainNoTokenizer {+impl CoreSignal for IdfSumDomainNoTokenizer {fn default_coefficient(&self) -> f64 {0.0036}@@ -685,7 +685,7 @@ impl Signal for IdfSumDomainNoTokenizer {bincode::Decode,)]pub struct IdfSumDomainNameNoTokenizer;-impl Signal for IdfSumDomainNameNoTokenizer {+impl CoreSignal for IdfSumDomainNameNoTokenizer {fn default_coefficient(&self) -> f64 {0.0002}@@ -720,7 +720,7 @@ impl Signal for IdfSumDomainNameNoTokenizer {bincode::Decode,)]pub struct IdfSumDomainIfHomepage;-impl Signal for IdfSumDomainIfHomepage {+impl CoreSignal for IdfSumDomainIfHomepage {fn default_coefficient(&self) -> f64 {0.0004}@@ -753,7 +753,7 @@ impl Signal for IdfSumDomainIfHomepage {bincode::Decode,)]pub struct IdfSumDomainNameIfHomepageNoTokenizer;-impl Signal for IdfSumDomainNameIfHomepageNoTokenizer {+impl CoreSignal for IdfSumDomainNameIfHomepageNoTokenizer {fn default_coefficient(&self) -> f64 {0.0036}@@ -788,7 +788,7 @@ impl Signal for IdfSumDomainNameIfHomepageNoTokenizer {bincode::Decode,)]pub struct IdfSumDomainIfHomepageNoTokenizer;-impl Signal for IdfSumDomainIfHomepageNoTokenizer {+impl CoreSignal for IdfSumDomainIfHomepageNoTokenizer {fn default_coefficient(&self) -> f64 {0.0036}@@ -823,7 +823,7 @@ impl Signal for IdfSumDomainIfHomepageNoTokenizer {bincode::Decode,)]pub struct IdfSumTitleIfHomepage;-impl Signal for IdfSumTitleIfHomepage {+impl CoreSignal for IdfSumTitleIfHomepage {fn default_coefficient(&self) -> f64 {0.001}@@ -842,169 +842,3 @@ impl Signal for IdfSumTitleIfHomepage {.map(SignalCalculation::new_symmetrical)}}--#[derive(- Debug,- Clone,- Copy,- PartialEq,- Eq,- Hash,- serde::Serialize,- serde::Deserialize,- bincode::Encode,- bincode::Decode,-)]-pub struct CrossEncoderSnippet;-impl Signal for CrossEncoderSnippet {- fn default_coefficient(&self) -> f64 {- 0.17- }-- fn as_field(&self) -> Option<Field> {- None- }-- fn compute(&self, _: DocId, _: &SignalComputer) -> Option<SignalCalculation> {- None // computed in later ranking stage- }-}--#[derive(- Debug,- Clone,- Copy,- PartialEq,- Eq,- Hash,- serde::Serialize,- serde::Deserialize,- bincode::Encode,- bincode::Decode,-)]-pub struct CrossEncoderTitle;-impl Signal for CrossEncoderTitle {- fn default_coefficient(&self) -> f64 {- 0.17- }-- fn as_field(&self) -> Option<Field> {- None- }-- fn compute(&self, _: DocId, _: &SignalComputer) -> Option<SignalCalculation> {- None // computed in later ranking stage- }-}--#[derive(- Debug,- Clone,- Copy,- PartialEq,- Eq,- Hash,- serde::Serialize,- serde::Deserialize,- bincode::Encode,- bincode::Decode,-)]-pub struct TitleEmbeddingSimilarity;-impl Signal for TitleEmbeddingSimilarity {- fn default_coefficient(&self) -> f64 {- 0.01- }-- fn as_field(&self) -> Option<Field> {- Some(Field::Numerical(- schema::numerical_field::TitleEmbeddings.into(),- ))- }-- fn compute(&self, _: DocId, _: &SignalComputer) -> Option<SignalCalculation> {- None // computed in later ranking stage- }-}--#[derive(- Debug,- Clone,- Copy,- PartialEq,- Eq,- Hash,- serde::Serialize,- serde::Deserialize,- bincode::Encode,- bincode::Decode,-)]-pub struct KeywordEmbeddingSimilarity;-impl Signal for KeywordEmbeddingSimilarity {- fn default_coefficient(&self) -> f64 {- 0.01- }-- fn as_field(&self) -> Option<Field> {- Some(Field::Numerical(- schema::numerical_field::KeywordEmbeddings.into(),- ))- }-- fn compute(&self, _: DocId, _: &SignalComputer) -> Option<SignalCalculation> {- None // computed in later ranking stage- }-}--#[derive(- Debug,- Clone,- Copy,- PartialEq,- Eq,- Hash,- serde::Serialize,- serde::Deserialize,- bincode::Encode,- bincode::Decode,-)]-pub struct MinTitleSlop;-impl Signal for MinTitleSlop {- fn default_coefficient(&self) -> f64 {- 0.1- }-- fn as_field(&self) -> Option<Field> {- Some(Field::Text(schema::text_field::Title.into()))- }-- fn compute(&self, _: DocId, _: &SignalComputer) -> Option<SignalCalculation> {- None // computed in later ranking stage- }-}--#[derive(- Debug,- Clone,- Copy,- PartialEq,- Eq,- Hash,- serde::Serialize,- serde::Deserialize,- bincode::Encode,- bincode::Decode,-)]-pub struct MinCleanBodySlop;-impl Signal for MinCleanBodySlop {- fn default_coefficient(&self) -> f64 {- 0.1- }-- fn as_field(&self) -> Option<Field> {- Some(Field::Text(schema::text_field::CleanBody.into()))- }-- fn compute(&self, _: DocId, _: &SignalComputer) -> Option<SignalCalculation> {- None // computed in later ranking stage- }-}diff --git a/crates/core/src/ranking/signals/mod.rs b/crates/core/src/ranking/signals/mod.rsnew file mode 100644index 00000000..8667b041--- /dev/null+++ b/crates/core/src/ranking/signals/mod.rs@@ -0,0 +1,472 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>++pub mod core;+pub mod non_core;++use std::str::FromStr;++use crate::enum_dispatch_from_discriminant;+use crate::enum_map::{EnumMap, GetEnumMapKey, InsertEnumMapKey};++use crate::schema::Field;+use crate::{+ schema::{NumericalFieldEnum, TextFieldEnum},+ webpage::Webpage,+};+use enum_dispatch::enum_dispatch;++use strum::{EnumDiscriminants, VariantArray};+use utoipa::ToSchema;++pub use self::core::{non_text::*, text::*};+pub use self::non_core::{non_text::*, text::*};++use crate::ranking::computer::SignalComputer;+use tantivy::DocId;++#[enum_dispatch]+pub trait CoreSignal: Clone + Copy + std::fmt::Debug + PartialEq + Eq + std::hash::Hash {+ fn default_coefficient(&self) -> f64;+ fn as_field(&self) -> Option<Field>;+ fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<SignalCalculation>;++ fn precompute(+ self,+ _webpage: &Webpage,+ _signal_computer: &SignalComputer,+ ) -> Option<SignalCalculation> {+ None+ }++ fn as_textfield(&self) -> Option<TextFieldEnum> {+ self.as_field().and_then(|field| field.as_text())+ }++ fn has_sibling_ngrams(&self) -> bool {+ false+ }++ fn as_numericalfield(&self) -> Option<NumericalFieldEnum> {+ self.as_field().and_then(|field| field.as_numerical())+ }+}++#[enum_dispatch]+pub trait Signal: Clone + Copy + std::fmt::Debug + PartialEq + Eq + std::hash::Hash {+ fn default_coefficient(&self) -> f64;+}++impl<T> Signal for T+where+ T: CoreSignal,+{+ fn default_coefficient(&self) -> f64 {+ CoreSignal::default_coefficient(self)+ }+}++#[enum_dispatch(Signal)]+#[derive(+ Debug,+ serde::Serialize,+ serde::Deserialize,+ bincode::Encode,+ bincode::Decode,+ Clone,+ Copy,+ PartialEq,+ Eq,+ Hash,+ EnumDiscriminants,+)]+#[strum_discriminants(derive(+ VariantArray,+ serde::Serialize,+ serde::Deserialize,+ bincode::Encode,+ bincode::Decode,+ PartialOrd,+ Ord,+ Hash,+ utoipa::ToSchema,+))]+#[strum_discriminants(serde(rename_all = "snake_case"))]+pub enum SignalEnum {+ Bm25F,+ Bm25Title,+ TitleCoverage,+ Bm25TitleBigrams,+ Bm25TitleTrigrams,+ Bm25CleanBody,+ CleanBodyCoverage,+ Bm25CleanBodyBigrams,+ Bm25CleanBodyTrigrams,+ Bm25StemmedTitle,+ Bm25StemmedCleanBody,+ Bm25AllBody,+ Bm25Keywords,+ Bm25BacklinkText,+ IdfSumUrl,+ IdfSumSite,+ IdfSumDomain,+ IdfSumSiteNoTokenizer,+ IdfSumDomainNoTokenizer,+ IdfSumDomainNameNoTokenizer,+ IdfSumDomainIfHomepage,+ IdfSumDomainNameIfHomepageNoTokenizer,+ IdfSumDomainIfHomepageNoTokenizer,+ IdfSumTitleIfHomepage,+ CrossEncoderSnippet,+ CrossEncoderTitle,+ HostCentrality,+ HostCentralityRank,+ PageCentrality,+ PageCentralityRank,+ IsHomepage,+ FetchTimeMs,+ UpdateTimestamp,+ TrackerScore,+ Region,+ QueryCentrality,+ InboundSimilarity,+ LambdaMart,+ UrlDigits,+ UrlSlashes,+ LinkDensity,+ TitleEmbeddingSimilarity,+ KeywordEmbeddingSimilarity,+ HasAds,+ MinTitleSlop,+ MinCleanBodySlop,+}++#[enum_dispatch(CoreSignal)]+#[derive(+ Debug,+ serde::Serialize,+ serde::Deserialize,+ bincode::Encode,+ bincode::Decode,+ Clone,+ Copy,+ PartialEq,+ Eq,+ Hash,+ EnumDiscriminants,+)]+#[strum_discriminants(derive(+ VariantArray,+ serde::Serialize,+ serde::Deserialize,+ bincode::Encode,+ bincode::Decode,+ PartialOrd,+ Ord,+ Hash,+))]+#[strum_discriminants(serde(rename_all = "snake_case"))]+pub enum CoreSignalEnum {+ Bm25F,+ Bm25Title,+ TitleCoverage,+ Bm25TitleBigrams,+ Bm25TitleTrigrams,+ Bm25CleanBody,+ CleanBodyCoverage,+ Bm25CleanBodyBigrams,+ Bm25CleanBodyTrigrams,+ Bm25StemmedTitle,+ Bm25StemmedCleanBody,+ Bm25AllBody,+ Bm25Keywords,+ Bm25BacklinkText,+ IdfSumUrl,+ IdfSumSite,+ IdfSumDomain,+ IdfSumSiteNoTokenizer,+ IdfSumDomainNoTokenizer,+ IdfSumDomainNameNoTokenizer,+ IdfSumDomainIfHomepage,+ IdfSumDomainNameIfHomepageNoTokenizer,+ IdfSumDomainIfHomepageNoTokenizer,+ IdfSumTitleIfHomepage,+ HostCentrality,+ HostCentralityRank,+ PageCentrality,+ PageCentralityRank,+ IsHomepage,+ FetchTimeMs,+ UpdateTimestamp,+ TrackerScore,+ Region,+ UrlDigits,+ UrlSlashes,+ LinkDensity,+ HasAds,+}++// Note to future self: Tried to get the num definitions+// into the macro as well, but some of the procedural macros started failing (bincode and enum_dispatch).+// This is probably due to macro hygiene+macro_rules! register_signals {+ (core=[$($core_signal:ident),*$(,)?], rest=[$($signal:ident),*$(,)?]) => {+ enum_dispatch_from_discriminant!(SignalEnumDiscriminants => SignalEnum,+ [+ $($core_signal,)*+ $($signal,)*+ ]);+++ impl From<CoreSignalEnum> for SignalEnum {+ fn from(core: CoreSignalEnum) -> Self {+ match core {+ $(+ CoreSignalEnum::$core_signal(signal) => SignalEnum::$core_signal(signal),+ )*+ }+ }+ }++ enum_dispatch_from_discriminant!(CoreSignalEnumDiscriminants => CoreSignalEnum,+ [++ $($core_signal,)*+ ]);+ }+}++register_signals! {+ core=[+ Bm25F,+ Bm25Title,+ TitleCoverage,+ Bm25TitleBigrams,+ Bm25TitleTrigrams,+ Bm25CleanBody,+ CleanBodyCoverage,+ Bm25CleanBodyBigrams,+ Bm25CleanBodyTrigrams,+ Bm25StemmedTitle,+ Bm25StemmedCleanBody,+ Bm25AllBody,+ Bm25Keywords,+ Bm25BacklinkText,+ IdfSumUrl,+ IdfSumSite,+ IdfSumDomain,+ IdfSumSiteNoTokenizer,+ IdfSumDomainNoTokenizer,+ IdfSumDomainNameNoTokenizer,+ IdfSumDomainIfHomepage,+ IdfSumDomainNameIfHomepageNoTokenizer,+ IdfSumDomainIfHomepageNoTokenizer,+ IdfSumTitleIfHomepage,+ HostCentrality,+ HostCentralityRank,+ PageCentrality,+ PageCentralityRank,+ IsHomepage,+ FetchTimeMs,+ UpdateTimestamp,+ TrackerScore,+ Region,+ UrlDigits,+ UrlSlashes,+ LinkDensity,+ HasAds,+ ],+ rest=[+ QueryCentrality,+ InboundSimilarity,+ LambdaMart,+ MinTitleSlop,+ MinCleanBodySlop,+ CrossEncoderSnippet,+ CrossEncoderTitle,+ TitleEmbeddingSimilarity,+ KeywordEmbeddingSimilarity,+ ]+}++impl SignalEnum {+ pub fn all() -> impl Iterator<Item = SignalEnum> {+ SignalEnumDiscriminants::VARIANTS+ .iter()+ .copied()+ .map(|v| v.into())+ }++ pub fn get(field_id: usize) -> Option<SignalEnum> {+ SignalEnumDiscriminants::VARIANTS+ .get(field_id)+ .copied()+ .map(SignalEnum::from)+ }+}++impl InsertEnumMapKey for SignalEnum {+ fn into_usize(self) -> usize {+ SignalEnumDiscriminants::from(self) as usize+ }+}++impl GetEnumMapKey for SignalEnum {+ fn from_usize(value: usize) -> Option<Self> {+ Self::get(value)+ }+}++impl CoreSignalEnum {+ pub fn num_variants() -> usize {+ CoreSignalEnumDiscriminants::VARIANTS.len()+ }++ pub fn all() -> impl Iterator<Item = CoreSignalEnum> {+ CoreSignalEnumDiscriminants::VARIANTS+ .iter()+ .copied()+ .map(|v| v.into())+ }++ pub fn get(field_id: usize) -> Option<CoreSignalEnum> {+ CoreSignalEnumDiscriminants::VARIANTS+ .get(field_id)+ .copied()+ .map(CoreSignalEnum::from)+ }+}++impl InsertEnumMapKey for CoreSignalEnum {+ fn into_usize(self) -> usize {+ CoreSignalEnumDiscriminants::from(self) as usize+ }+}++impl GetEnumMapKey for CoreSignalEnum {+ fn from_usize(value: usize) -> Option<Self> {+ Self::get(value)+ }+}++impl FromStr for SignalEnumDiscriminants {+ type Err = anyhow::Error;++ fn from_str(name: &str) -> std::result::Result<Self, Self::Err> {+ let s = "\"".to_string() + name + "\"";+ let signal = serde_json::from_str(&s)?;+ Ok(signal)+ }+}++#[derive(Debug, Clone, Copy, bincode::Encode, bincode::Decode)]+pub struct SignalCalculation {+ pub value: f64,+ pub score: f64,+}++impl SignalCalculation {+ pub fn new_symmetrical(val: f64) -> Self {+ Self {+ value: val,+ score: val,+ }+ }+}++#[derive(Debug, Clone, Copy)]+pub struct ComputedSignal {+ pub signal: SignalEnum,+ pub calc: SignalCalculation,+}++#[derive(+ Debug,+ Clone,+ Copy,+ serde::Serialize,+ serde::Deserialize,+ bincode::Encode,+ bincode::Decode,+ ToSchema,+)]+#[serde(rename_all = "camelCase")]+pub struct SignalScore {+ pub coefficient: f64,+ pub value: f64,+}++#[derive(Debug, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, Clone)]+pub struct SignalCoefficients {+ map: EnumMap<SignalEnum, f64>,+}++impl Default for SignalCoefficients {+ fn default() -> Self {+ Self::new(SignalEnum::all().map(|signal| (signal, signal.default_coefficient())))+ }+}++impl From<EnumMap<SignalEnum, f64>> for SignalCoefficients {+ fn from(map: EnumMap<SignalEnum, f64>) -> Self {+ Self { map }+ }+}++impl SignalCoefficients {+ pub fn get(&self, signal: &SignalEnum) -> f64 {+ self.map+ .get(*signal)+ .copied()+ .unwrap_or(signal.default_coefficient())+ }++ pub fn new(coefficients: impl Iterator<Item = (SignalEnum, f64)>) -> Self {+ let mut map = EnumMap::default();++ for (signal, coefficient) in coefficients {+ map.insert(signal, coefficient);+ }++ Self { map }+ }++ pub fn merge_add(&mut self, coeffs: SignalCoefficients) {+ for signal in SignalEnum::all() {+ if let Some(coeff) = coeffs.map.get(signal).copied() {+ match self.map.get_mut(signal) {+ Some(existing_coeff) => *existing_coeff += coeff,+ None => {+ self.map.insert(signal, coeff);+ }+ }+ }+ }+ }++ pub fn merge_overwrite(&mut self, coeffs: SignalCoefficients) {+ for signal in SignalEnum::all() {+ if let Some(coeff) = coeffs.map.get(signal).copied() {+ match self.map.get_mut(signal) {+ Some(existing_coeff) => *existing_coeff = coeff,+ None => {+ self.map.insert(signal, coeff);+ }+ }+ }+ }+ }+}diff --git a/crates/core/src/ranking/signals/non_core/mod.rs b/crates/core/src/ranking/signals/non_core/mod.rsnew file mode 100644index 00000000..69c83d27--- /dev/null+++ b/crates/core/src/ranking/signals/non_core/mod.rs@@ -0,0 +1,21 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>++pub mod non_text;+pub mod text;++pub use self::non_text::*;+pub use self::text::*;diff --git a/crates/core/src/ranking/core/query.rs b/crates/core/src/ranking/signals/non_core/non_text.rssimilarity index 59%rename from crates/core/src/ranking/core/query.rsrename to crates/core/src/ranking/signals/non_core/non_text.rsindex ca1b5f23..26ee9854 100644--- a/crates/core/src/ranking/core/query.rs+++ b/crates/core/src/ranking/signals/non_core/non_text.rs@@ -1,7 +1,3 @@-use tantivy::DocId;--use crate::schema::Field;-// Stract is an open source web search engine.// Copyright (C) 2024 Stract ApS//@@ -16,9 +12,9 @@ use crate::schema::Field;// GNU Affero General Public License for more details.//// You should have received a copy of the GNU Affero General Public License-// along with this program. If not, see <https://www.gnu.org/licenses/>.+// along with this program. If not, see <https://www.gnu.org/licenses/>-use super::{Signal, SignalCalculation, SignalComputer};+use crate::ranking::Signal;#[derive(Debug,@@ -32,20 +28,47 @@ use super::{Signal, SignalCalculation, SignalComputer};bincode::Encode,bincode::Decode,)]-pub struct NumQueryTerms;-impl Signal for NumQueryTerms {+pub struct QueryCentrality;+impl Signal for QueryCentrality {fn default_coefficient(&self) -> f64 {0.0}+}- fn as_field(&self) -> Option<Field> {- None+#[derive(+ Debug,+ Clone,+ Copy,+ PartialEq,+ Eq,+ Hash,+ serde::Serialize,+ serde::Deserialize,+ bincode::Encode,+ bincode::Decode,+)]+pub struct InboundSimilarity;+impl Signal for InboundSimilarity {+ fn default_coefficient(&self) -> f64 {+ 0.25}+}- fn compute(&self, _: DocId, signal_computer: &SignalComputer) -> Option<SignalCalculation> {- signal_computer- .query_data()- .map(|d| d.simple_terms().len() as f64)- .map(SignalCalculation::new_symmetrical)+#[derive(+ Debug,+ Clone,+ Copy,+ PartialEq,+ Eq,+ Hash,+ serde::Serialize,+ serde::Deserialize,+ bincode::Encode,+ bincode::Decode,+)]+pub struct LambdaMart;+impl Signal for LambdaMart {+ fn default_coefficient(&self) -> f64 {+ 10.0}}diff --git a/crates/core/src/ranking/signals/non_core/text.rs b/crates/core/src/ranking/signals/non_core/text.rsnew file mode 100644index 00000000..e03d6f92--- /dev/null+++ b/crates/core/src/ranking/signals/non_core/text.rs@@ -0,0 +1,131 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>++use crate::ranking::Signal;++#[derive(+ Debug,+ Clone,+ Copy,+ PartialEq,+ Eq,+ Hash,+ serde::Serialize,+ serde::Deserialize,+ bincode::Encode,+ bincode::Decode,+)]+pub struct MinTitleSlop;+impl Signal for MinTitleSlop {+ fn default_coefficient(&self) -> f64 {+ 0.1+ }+}++#[derive(+ Debug,+ Clone,+ Copy,+ PartialEq,+ Eq,+ Hash,+ serde::Serialize,+ serde::Deserialize,+ bincode::Encode,+ bincode::Decode,+)]+pub struct MinCleanBodySlop;+impl Signal for MinCleanBodySlop {+ fn default_coefficient(&self) -> f64 {+ 0.1+ }+}++#[derive(+ Debug,+ Clone,+ Copy,+ PartialEq,+ Eq,+ Hash,+ serde::Serialize,+ serde::Deserialize,+ bincode::Encode,+ bincode::Decode,+)]+pub struct CrossEncoderSnippet;+impl Signal for CrossEncoderSnippet {+ fn default_coefficient(&self) -> f64 {+ 0.17+ }+}++#[derive(+ Debug,+ Clone,+ Copy,+ PartialEq,+ Eq,+ Hash,+ serde::Serialize,+ serde::Deserialize,+ bincode::Encode,+ bincode::Decode,+)]+pub struct CrossEncoderTitle;+impl Signal for CrossEncoderTitle {+ fn default_coefficient(&self) -> f64 {+ 0.17+ }+}++#[derive(+ Debug,+ Clone,+ Copy,+ PartialEq,+ Eq,+ Hash,+ serde::Serialize,+ serde::Deserialize,+ bincode::Encode,+ bincode::Decode,+)]+pub struct TitleEmbeddingSimilarity;+impl Signal for TitleEmbeddingSimilarity {+ fn default_coefficient(&self) -> f64 {+ 0.01+ }+}++#[derive(+ Debug,+ Clone,+ Copy,+ PartialEq,+ Eq,+ Hash,+ serde::Serialize,+ serde::Deserialize,+ bincode::Encode,+ bincode::Decode,+)]+pub struct KeywordEmbeddingSimilarity;+impl Signal for KeywordEmbeddingSimilarity {+ fn default_coefficient(&self) -> f64 {+ 0.01+ }+}diff --git a/frontend/src/lib/api/index.ts b/frontend/src/lib/api/index.tsindex a11a8944..b5a9103e 100644--- a/frontend/src/lib/api/index.ts+++ b/frontend/src/lib/api/index.ts@@ -414,7 +414,6 @@ export type SignalEnumDiscriminants =| 'title_embedding_similarity'| 'keyword_embedding_similarity'| 'has_ads'- | 'num_query_terms'| 'min_title_slop'| 'min_clean_body_slop';export const SIGNAL_ENUM_DISCRIMINANTS = [@@ -462,7 +461,6 @@ export const SIGNAL_ENUM_DISCRIMINANTS = ['title_embedding_similarity','keyword_embedding_similarity','has_ads',- 'num_query_terms','min_title_slop','min_clean_body_slop',] satisfies SignalEnumDiscriminants[];diff --git a/frontend/src/routes/search/Modal.svelte b/frontend/src/routes/search/Modal.svelteindex a85ac60e..8528cd05 100644--- a/frontend/src/routes/search/Modal.svelte+++ b/frontend/src/routes/search/Modal.svelte@@ -83,7 +83,6 @@.with('query_centrality', () => 'queryCentrality' as const).with('inbound_similarity', () => 'inboundSimilarity' as const).with('lambda_mart', () => undefined)- .with('num_query_terms', () => undefined).with('url_digits', () => 'urlSymbols' as const).with('url_slashes', () => 'urlSymbols' as const).with('link_density', () => 'linkDensity' as const)