web search done right
refactor signal into trait with enum dispatch
Mikkel Denkerbf23701d165e
diff --git a/crates/core/src/index.rs b/crates/core/src/index.rsindex f14ae105..fc54185b 100644--- a/crates/core/src/index.rs+++ b/crates/core/src/index.rs@@ -153,7 +153,10 @@ impl Index {#[cfg(test)]mod tests {- use crate::searcher::{LocalSearcher, SearchQuery};+ use crate::{+ ranking,+ searcher::{LocalSearcher, SearchQuery},+ };use super::*;@@ -245,7 +248,7 @@ mod tests {.ranking_signals.as_ref().unwrap()- .get(&crate::ranking::Signal::Bm25Title)+ .get(&crate::ranking::SignalEnum::from(ranking::signal::Bm25Title).into()).unwrap()).all(|&v| v.value > 0.0));}diff --git a/crates/core/src/lib.rs b/crates/core/src/lib.rsindex 20dccb0b..78feed64 100644--- a/crates/core/src/lib.rs+++ b/crates/core/src/lib.rs@@ -254,3 +254,18 @@ mod tests {}}}++#[macro_export]+macro_rules! enum_dispatch_from_discriminant {+ ($discenum:ident => $enum:ident, [$($disc:ident),*$(,)?]) => {+ impl From<$discenum> for $enum {+ fn from(value: $discenum) -> Self {+ match value {+ $(+ $discenum::$disc => $disc.into(),+ )*+ }+ }+ }+ };+}diff --git a/crates/core/src/ranking/models/lambdamart.rs b/crates/core/src/ranking/models/lambdamart.rsindex f1962260..aac7b839 100644--- a/crates/core/src/ranking/models/lambdamart.rs+++ b/crates/core/src/ranking/models/lambdamart.rs@@ -16,9 +16,11 @@use std::{path::Path, str::FromStr};+use signal::SignalEnumDiscriminants;+use crate::{enum_map::EnumMap,- ranking::{signal, Signal},+ ranking::{signal, SignalEnum},};type Result<T> = std::result::Result<T, Error>;@@ -65,14 +67,14 @@ impl AsValue for f64 {#[derive(Debug)]struct Node {threshold: f64,- feature: Option<Signal>,+ feature: Option<SignalEnum>,leaf_value: f64,left: Option<NodeOrLeaf>,right: Option<NodeOrLeaf>,}impl Node {- fn next<V: AsValue>(&self, features: &EnumMap<Signal, V>) -> Option<&NodeOrLeaf> {+ fn next<V: AsValue>(&self, features: &EnumMap<SignalEnum, V>) -> Option<&NodeOrLeaf> {self.feature.and_then(|feature| {let value = features.get(feature).map(|v| v.as_value()).unwrap_or(0.0);if value <= self.threshold {@@ -192,7 +194,7 @@ impl Tree {Ok(Self { nodes })}- fn predict<V: AsValue>(&self, features: &EnumMap<Signal, V>) -> Result<f64> {+ fn predict<V: AsValue>(&self, features: &EnumMap<SignalEnum, V>) -> Result<f64> {let mut node = &self.nodes[0];while let Some(next) = node.next(features) {node = match next {@@ -206,7 +208,7 @@ impl Tree {}struct Header {- features: Vec<Signal>,+ features: Vec<SignalEnum>,}impl Header {@@ -217,7 +219,7 @@ impl Header {if let Some((key, value)) = lin.split_once('=') {if key == "feature_names" {for name in value.split(' ') {- features.push(Signal::from_str(name)?);+ features.push(SignalEnum::from(SignalEnumDiscriminants::from_str(name)?));}}}@@ -290,7 +292,7 @@ impl LambdaMART {Self::parse(&s)}- pub fn predict<V: AsValue>(&self, features: &EnumMap<Signal, V>) -> f64 {+ pub fn predict<V: AsValue>(&self, features: &EnumMap<SignalEnum, V>) -> f64 {self.trees.iter().map(|t| t.predict(features).unwrap())@@ -301,6 +303,8 @@ impl LambdaMART {#[cfg(test)]mod tests {+ use crate::ranking;+use super::*;#[test]@@ -310,35 +314,44 @@ mod tests {assert!(!model.trees.is_empty());let mut features = EnumMap::new();- features.insert(Signal::Bm25BacklinkText, 85.7750244140625);- features.insert(Signal::Bm25CleanBody, 67.41311645507812);- features.insert(Signal::Bm25CleanBodyBigrams, 0.0);- features.insert(Signal::Bm25CleanBodyTrigrams, 0.0);- features.insert(Signal::IdfSumDomain, 43.332096099853516);- features.insert(Signal::IdfSumDomainIfHomepage, 0.0);- features.insert(Signal::IdfSumDomainIfHomepageNoTokenizer, 0.0);- features.insert(Signal::IdfSumDomainNameIfHomepageNoTokenizer, 0.0);- features.insert(Signal::IdfSumDomainNameNoTokenizer, 0.0);- features.insert(Signal::IdfSumDomainNoTokenizer, 0.0);- features.insert(Signal::IdfSumSite, 61.47410202026367);- features.insert(Signal::IdfSumSiteNoTokenizer, 0.0);- features.insert(Signal::Bm25StemmedCleanBody, 65.94627380371094);- features.insert(Signal::Bm25StemmedTitle, 0.0);- features.insert(Signal::Bm25Title, 59.817813873291016);- features.insert(Signal::Bm25TitleBigrams, 0.0);- features.insert(Signal::IdfSumTitleIfHomepage, 0.0);- features.insert(Signal::Bm25TitleTrigrams, 0.0);- features.insert(Signal::IdfSumUrl, 57.07925033569336);- features.insert(Signal::FetchTimeMs, 0.023255813953488372);- features.insert(Signal::HostCentrality, 0.017958538);- features.insert(Signal::InboundSimilarity, 0.0);- features.insert(Signal::IsHomepage, 0.0);- features.insert(Signal::PageCentrality, 0.008253236);- features.insert(Signal::Region, 0.16622349570454012);- features.insert(Signal::TrackerScore, 0.07692307692307693);- features.insert(Signal::UpdateTimestamp, 0.0);- features.insert(Signal::UrlDigits, 0.25);- features.insert(Signal::UrlSlashes, 0.3333333333333333);+ features.insert(ranking::signal::Bm25BacklinkText.into(), 85.7750244140625);+ features.insert(ranking::signal::Bm25CleanBody.into(), 67.41311645507812);+ features.insert(ranking::signal::Bm25CleanBodyBigrams.into(), 0.0);+ features.insert(ranking::signal::Bm25CleanBodyTrigrams.into(), 0.0);+ features.insert(ranking::signal::IdfSumDomain.into(), 43.332096099853516);+ features.insert(ranking::signal::IdfSumDomainIfHomepage.into(), 0.0);+ features.insert(+ ranking::signal::IdfSumDomainIfHomepageNoTokenizer.into(),+ 0.0,+ );+ features.insert(+ ranking::signal::IdfSumDomainNameIfHomepageNoTokenizer.into(),+ 0.0,+ );+ features.insert(ranking::signal::IdfSumDomainNameNoTokenizer.into(), 0.0);+ features.insert(ranking::signal::IdfSumDomainNoTokenizer.into(), 0.0);+ features.insert(ranking::signal::IdfSumSite.into(), 61.47410202026367);+ features.insert(ranking::signal::IdfSumSiteNoTokenizer.into(), 0.0);+ features.insert(+ ranking::signal::Bm25StemmedCleanBody.into(),+ 65.94627380371094,+ );+ features.insert(ranking::signal::Bm25StemmedTitle.into(), 0.0);+ features.insert(ranking::signal::Bm25Title.into(), 59.817813873291016);+ features.insert(ranking::signal::Bm25TitleBigrams.into(), 0.0);+ features.insert(ranking::signal::IdfSumTitleIfHomepage.into(), 0.0);+ features.insert(ranking::signal::Bm25TitleTrigrams.into(), 0.0);+ features.insert(ranking::signal::IdfSumUrl.into(), 57.07925033569336);+ features.insert(ranking::signal::FetchTimeMs.into(), 0.023255813953488372);+ features.insert(ranking::signal::HostCentrality.into(), 0.017958538);+ features.insert(ranking::signal::InboundSimilarity.into(), 0.0);+ features.insert(ranking::signal::IsHomepage.into(), 0.0);+ features.insert(ranking::signal::PageCentrality.into(), 0.008253236);+ features.insert(ranking::signal::Region.into(), 0.16622349570454012);+ features.insert(ranking::signal::TrackerScore.into(), 0.07692307692307693);+ features.insert(ranking::signal::UpdateTimestamp.into(), 0.0);+ features.insert(ranking::signal::UrlDigits.into(), 0.25);+ features.insert(ranking::signal::UrlSlashes.into(), 0.3333333333333333);assert_eq!((model.predict(&features) * 1000.0) as u64, 1050);}diff --git a/crates/core/src/ranking/models/linear.rs b/crates/core/src/ranking/models/linear.rsindex 81ed458e..c2abad17 100644--- a/crates/core/src/ranking/models/linear.rs+++ b/crates/core/src/ranking/models/linear.rs@@ -19,12 +19,12 @@ use std::io::BufReader;use std::{collections::HashMap, path::Path};use crate::enum_map::EnumMap;-use crate::ranking::Signal;+use crate::ranking::SignalEnum;use crate::Result;#[derive(serde::Serialize, serde::Deserialize, Debug)]struct SerialziedLinearRegression {- weights: HashMap<Signal, f64>,+ weights: HashMap<SignalEnum, f64>,}impl From<SerialziedLinearRegression> for LinearRegression {@@ -41,7 +41,7 @@ impl From<SerialziedLinearRegression> for LinearRegression {#[derive(serde::Serialize, serde::Deserialize, Debug)]pub struct LinearRegression {- pub weights: EnumMap<Signal, f64>,+ pub weights: EnumMap<SignalEnum, f64>,}impl LinearRegression {diff --git a/crates/core/src/ranking/pipeline/mod.rs b/crates/core/src/ranking/pipeline/mod.rsindex 2a2b5ff8..c3d11e14 100644--- a/crates/core/src/ranking/pipeline/mod.rs+++ b/crates/core/src/ranking/pipeline/mod.rs@@ -187,7 +187,7 @@ mod tests {enum_map::EnumMap,inverted_index::{DocAddress, WebpagePointer},prehashed::Prehashed,- ranking::{initial::Score, Signal},+ ranking::{self, initial::Score},};use super::*;@@ -197,7 +197,7 @@ mod tests {.map(|i| -> RecallRankingWebpage {let mut signals = EnumMap::new();signals.insert(- Signal::HostCentrality,+ ranking::signal::HostCentrality.into(),SignalScore {coefficient: 1.0,value: 1.0 / i as f64,diff --git a/crates/core/src/ranking/pipeline/scorers/embedding.rs b/crates/core/src/ranking/pipeline/scorers/embedding.rsindex 966b9442..fc819c56 100644--- a/crates/core/src/ranking/pipeline/scorers/embedding.rs+++ b/crates/core/src/ranking/pipeline/scorers/embedding.rs@@ -21,8 +21,9 @@ use candle_core::Tensor;use crate::{models::dual_encoder::DualEncoder,ranking::{+ self,pipeline::{stages::StoredEmbeddings, RankableWebpage, RecallRankingWebpage},- Signal, SignalCoefficient, SignalScore,+ SignalCoefficient, SignalEnum, SignalScore,},searcher::{api::ScoredWebpagePointer, SearchQuery},Result,@@ -100,7 +101,7 @@ impl<W, E: EmbeddingSignal<W>> EmbeddingScorer<W, E> {})}- fn query_emb_and_coefficient(&self, coeff_signal: Signal) -> Option<(Embedding, f64)> {+ fn query_emb_and_coefficient(&self, coeff_signal: SignalEnum) -> Option<(Embedding, f64)> {self.query_emb().and_then(|query_emb| {self.signal_coefficients.as_ref()@@ -167,15 +168,15 @@ pub struct TitleEmbeddings;pub struct KeywordEmbeddings;pub trait EmbeddingSignal<W>: Send + Sync {- fn signal() -> Signal;+ fn signal() -> SignalEnum;fn has_embedding(webpage: &W) -> bool;fn embedding(webpage: &W, hidden_size: usize) -> Option<Embedding>;fn insert_signal(webpage: &mut W, score: f64, coefficient: f64);}impl EmbeddingSignal<ScoredWebpagePointer> for TitleEmbeddings {- fn signal() -> Signal {- Signal::TitleEmbeddingSimilarity+ fn signal() -> SignalEnum {+ ranking::signal::TitleEmbeddingSimilarity.into()}fn has_embedding(webpage: &ScoredWebpagePointer) -> bool {@@ -199,8 +200,8 @@ impl EmbeddingSignal<ScoredWebpagePointer> for TitleEmbeddings {}impl EmbeddingSignal<RecallRankingWebpage> for TitleEmbeddings {- fn signal() -> Signal {- Signal::TitleEmbeddingSimilarity+ fn signal() -> SignalEnum {+ ranking::signal::TitleEmbeddingSimilarity.into()}fn has_embedding(webpage: &RecallRankingWebpage) -> bool {@@ -225,8 +226,8 @@ impl EmbeddingSignal<RecallRankingWebpage> for TitleEmbeddings {}impl EmbeddingSignal<ScoredWebpagePointer> for KeywordEmbeddings {- fn signal() -> Signal {- Signal::KeywordEmbeddingSimilarity+ fn signal() -> SignalEnum {+ ranking::signal::KeywordEmbeddingSimilarity.into()}fn has_embedding(webpage: &ScoredWebpagePointer) -> bool {@@ -250,8 +251,8 @@ impl EmbeddingSignal<ScoredWebpagePointer> for KeywordEmbeddings {}impl EmbeddingSignal<RecallRankingWebpage> for KeywordEmbeddings {- fn signal() -> Signal {- Signal::KeywordEmbeddingSimilarity+ fn signal() -> SignalEnum {+ ranking::signal::KeywordEmbeddingSimilarity.into()}fn has_embedding(webpage: &RecallRankingWebpage) -> bool {diff --git a/crates/core/src/ranking/pipeline/scorers/mod.rs b/crates/core/src/ranking/pipeline/scorers/mod.rsindex 1c0ffc53..98032546 100644--- a/crates/core/src/ranking/pipeline/scorers/mod.rs+++ b/crates/core/src/ranking/pipeline/scorers/mod.rs@@ -25,7 +25,7 @@ pub use reranker::ReRanker;use crate::{enum_map::EnumMap,- ranking::{models::lambdamart::LambdaMART, Signal, SignalCoefficient, SignalScore},+ ranking::{self, models::lambdamart::LambdaMART, SignalCoefficient, SignalEnum, SignalScore},searcher::SearchQuery,};@@ -45,11 +45,11 @@ impl<T: RankableWebpage> Scorer<T> for IdentityScorer {fn calculate_score(model: &Option<Arc<LambdaMART>>,coefficients: SignalCoefficient,- signals: &EnumMap<Signal, SignalScore>,+ signals: &EnumMap<SignalEnum, SignalScore>,) -> f64 {let lambda_score = match model {Some(model) => {- let coeff = coefficients.get(&Signal::LambdaMART);+ let coeff = coefficients.get(&ranking::signal::LambdaMart.into());if coeff == 0.0 {signals.values()diff --git a/crates/core/src/ranking/pipeline/scorers/reranker.rs b/crates/core/src/ranking/pipeline/scorers/reranker.rsindex b450201d..6ef1a56c 100644--- a/crates/core/src/ranking/pipeline/scorers/reranker.rs+++ b/crates/core/src/ranking/pipeline/scorers/reranker.rs@@ -18,8 +18,9 @@ use std::sync::Arc;use crate::{ranking::{+ self,models::{cross_encoder::CrossEncoder, lambdamart::LambdaMART},- Signal, SignalCoefficient, SignalScore,+ Signal, SignalCoefficient, SignalEnum, SignalScore,},searcher::SearchQuery,};@@ -48,15 +49,19 @@ impl<M: CrossEncoder> ReRanker<M> {fn crossencoder_snippet_coeff(&self) -> f64 {self.signal_coefficients.as_ref()- .map(|coeff| coeff.get(&Signal::CrossEncoderSnippet))- .unwrap_or_else(|| Signal::CrossEncoderSnippet.default_coefficient())+ .map(|coeff| coeff.get(&ranking::signal::CrossEncoderSnippet.into()))+ .unwrap_or_else(|| {+ SignalEnum::from(ranking::signal::CrossEncoderSnippet).default_coefficient()+ })}fn crossencoder_title_coeff(&self) -> f64 {self.signal_coefficients.as_ref()- .map(|coeff| coeff.get(&Signal::CrossEncoderTitle))- .unwrap_or_else(|| Signal::CrossEncoderTitle.default_coefficient())+ .map(|coeff| coeff.get(&ranking::signal::CrossEncoderTitle.into()))+ .unwrap_or_else(|| {+ SignalEnum::from(ranking::signal::CrossEncoderTitle).default_coefficient()+ })}fn crossencoder_score_webpages(&self, webpage: &mut [PrecisionRankingWebpage]) {@@ -75,7 +80,7 @@ impl<M: CrossEncoder> ReRanker<M> {for ((webpage, snippet), title) in webpage.iter_mut().zip(snippet_scores).zip(title_scores){webpage.ranking.signals.insert(- Signal::CrossEncoderSnippet,+ ranking::signal::CrossEncoderSnippet.into(),SignalScore {coefficient: self.crossencoder_snippet_coeff(),value: snippet,@@ -83,7 +88,7 @@ impl<M: CrossEncoder> ReRanker<M> {);webpage.ranking.signals.insert(- Signal::CrossEncoderTitle,+ ranking::signal::CrossEncoderTitle.into(),SignalScore {coefficient: self.crossencoder_title_coeff(),value: title,diff --git a/crates/core/src/ranking/pipeline/stages/recall.rs b/crates/core/src/ranking/pipeline/stages/recall.rsindex 4267b779..3bd3daf6 100644--- a/crates/core/src/ranking/pipeline/stages/recall.rs+++ b/crates/core/src/ranking/pipeline/stages/recall.rs@@ -28,7 +28,7 @@ use crate::{ranking::{models::lambdamart::LambdaMART,pipeline::{RankableWebpage, RankingPipeline, RankingStage, Recall, Scorer},- Signal, SignalAggregator, SignalScore,+ SignalAggregator, SignalEnum, SignalScore,},schema::fast_field,searcher::SearchQuery,@@ -46,7 +46,7 @@ impl StoredEmbeddings {#[derive(Clone, Debug, Serialize, Deserialize)]pub struct RecallRankingWebpage {pub pointer: WebpagePointer,- pub signals: EnumMap<Signal, SignalScore>,+ pub signals: EnumMap<SignalEnum, SignalScore>,pub optic_boost: Option<f64>,pub title_embedding: Option<StoredEmbeddings>,pub keyword_embedding: Option<StoredEmbeddings>,diff --git a/crates/core/src/ranking/signal.rs b/crates/core/src/ranking/signal.rsindex 61ed32ef..64b1de68 100644--- a/crates/core/src/ranking/signal.rs+++ b/crates/core/src/ranking/signal.rs@@ -15,11 +15,12 @@// along with this program. If not, see <https://www.gnu.org/licenses/>.use crate::enum_map::InsertEnumMapKey;+use crate::fastfield_reader::FieldReader;use crate::query::optic::AsSearchableRule;use crate::query::Query;use crate::schema::text_field::TextField;-use crate::schema::{fast_field, text_field};-use crate::Result;+use crate::schema::{self, Field};+use crate::{enum_dispatch_from_discriminant, Result};use crate::{enum_map::EnumMap,fastfield_reader,@@ -27,6 +28,7 @@ use crate::{webgraph::NodeID,webpage::Webpage,};+use enum_dispatch::enum_dispatch;use itertools::Itertools;use optics::ast::RankingTarget;use optics::Optic;@@ -34,7 +36,7 @@ use serde::{Deserialize, Serialize};use std::cell::RefCell;use std::str::FromStr;use std::sync::Arc;-use strum::VariantArray;+use strum::{EnumDiscriminants, VariantArray};use tantivy::fieldnorm::FieldNormReader;use tantivy::postings::SegmentPostings;use tantivy::query::{Query as _, Scorer};@@ -45,110 +47,1235 @@ use utoipa::ToSchema;use tantivy::DocSet;use tantivy::{DocId, Postings};-use crate::{- schema::FLOAT_SCALING,- webpage::region::{Region, RegionCount},-};+use crate::{schema::FLOAT_SCALING, webpage::region::RegionCount};++use super::bm25::MultiBm25Weight;+use super::models::linear::LinearRegression;+use super::{inbound_similarity, query_centrality};++#[derive(Debug, Error)]+pub enum Error {+ #[error("unknown signal: {0}")]+ UnknownSignal(#[from] serde_json::Error),+}++#[enum_dispatch]+pub trait Signal:+ Clone + Copy + std::fmt::Debug + PartialEq + Eq + std::hash::Hash + Into<SignalEnum>+{+ fn default_coefficient(&self) -> f64;+ fn as_field(&self) -> Option<Field>;+ fn compute(&self, doc: DocId, signal_aggregator: &SignalAggregator) -> Option<f64>;++ fn precompute(self, _webpage: &Webpage, _signal_aggregator: &SignalAggregator) -> Option<f64> {+ None+ }++ fn as_textfield(&self) -> Option<TextFieldEnum> {+ self.as_field().and_then(|field| field.as_text())+ }++ fn as_fastfield(&self) -> Option<FastFieldEnum> {+ self.as_field().and_then(|field| field.as_fast())+ }+}++#[enum_dispatch(Signal)]+#[derive(+ Debug, serde::Serialize, serde::Deserialize, Clone, Copy, PartialEq, Eq, Hash, EnumDiscriminants,+)]+#[strum_discriminants(derive(VariantArray, serde::Serialize, serde::Deserialize, Hash))]+#[strum_discriminants(serde(rename_all = "snake_case"))]+pub enum SignalEnum {+ Bm25Title,+ Bm25TitleBigrams,+ Bm25TitleTrigrams,+ Bm25CleanBody,+ Bm25CleanBodyBigrams,+ Bm25CleanBodyTrigrams,+ Bm25StemmedTitle,+ Bm25StemmedCleanBody,+ Bm25AllBody,+ Bm25Keywords,+ Bm25BacklinkText,+ IdfSumUrl,+ IdfSumSite,+ IdfSumDomain,+ IdfSumSiteNoTokenizer,+ IdfSumDomainNoTokenizer,+ IdfSumDomainNameNoTokenizer,+ IdfSumDomainIfHomepage,+ IdfSumDomainNameIfHomepageNoTokenizer,+ IdfSumDomainIfHomepageNoTokenizer,+ IdfSumTitleIfHomepage,+ CrossEncoderSnippet,+ CrossEncoderTitle,+ HostCentrality,+ HostCentralityRank,+ PageCentrality,+ PageCentralityRank,+ IsHomepage,+ FetchTimeMs,+ UpdateTimestamp,+ TrackerScore,+ Region,+ QueryCentrality,+ InboundSimilarity,+ LambdaMart,+ UrlDigits,+ UrlSlashes,+ LinkDensity,+ TitleEmbeddingSimilarity,+ KeywordEmbeddingSimilarity,+}++enum_dispatch_from_discriminant!(SignalEnumDiscriminants => SignalEnum,+[+ Bm25Title,+ Bm25TitleBigrams,+ Bm25TitleTrigrams,+ Bm25CleanBody,+ Bm25CleanBodyBigrams,+ Bm25CleanBodyTrigrams,+ Bm25StemmedTitle,+ Bm25StemmedCleanBody,+ Bm25AllBody,+ Bm25Keywords,+ Bm25BacklinkText,+ IdfSumUrl,+ IdfSumSite,+ IdfSumDomain,+ IdfSumSiteNoTokenizer,+ IdfSumDomainNoTokenizer,+ IdfSumDomainNameNoTokenizer,+ IdfSumDomainIfHomepage,+ IdfSumDomainNameIfHomepageNoTokenizer,+ IdfSumDomainIfHomepageNoTokenizer,+ IdfSumTitleIfHomepage,+ CrossEncoderSnippet,+ CrossEncoderTitle,+ HostCentrality,+ HostCentralityRank,+ PageCentrality,+ PageCentralityRank,+ IsHomepage,+ FetchTimeMs,+ UpdateTimestamp,+ TrackerScore,+ Region,+ QueryCentrality,+ InboundSimilarity,+ LambdaMart,+ UrlDigits,+ UrlSlashes,+ LinkDensity,+ TitleEmbeddingSimilarity,+ KeywordEmbeddingSimilarity,+]);++#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]+pub struct Bm25Title;+impl Signal for Bm25Title {+ fn default_coefficient(&self) -> f64 {+ 0.0063+ }++ fn as_field(&self) -> Option<Field> {+ Some(Field::Text(schema::text_field::Title.into()))+ }++ fn compute(&self, doc: DocId, signal_aggregator: &SignalAggregator) -> Option<f64> {+ let mut seg_reader = signal_aggregator+ .segment_reader+ .as_ref()+ .unwrap()+ .borrow_mut();++ seg_reader+ .text_fields+ .get_mut(self.as_textfield().unwrap())+ .map(|field| bm25(field, doc))+ }+}++#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]+pub struct Bm25TitleBigrams;+impl Signal for Bm25TitleBigrams {+ fn default_coefficient(&self) -> f64 {+ 0.01+ }++ fn as_field(&self) -> Option<Field> {+ Some(Field::Text(schema::text_field::TitleBigrams.into()))+ }++ fn compute(&self, doc: DocId, signal_aggregator: &SignalAggregator) -> Option<f64> {+ let mut seg_reader = signal_aggregator+ .segment_reader+ .as_ref()+ .unwrap()+ .borrow_mut();++ seg_reader+ .text_fields+ .get_mut(self.as_textfield().unwrap())+ .map(|field| bm25(field, doc))+ }+}++#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]+pub struct Bm25TitleTrigrams;+impl Signal for Bm25TitleTrigrams {+ fn default_coefficient(&self) -> f64 {+ 0.01+ }++ fn as_field(&self) -> Option<Field> {+ Some(Field::Text(schema::text_field::TitleTrigrams.into()))+ }++ fn compute(&self, doc: DocId, signal_aggregator: &SignalAggregator) -> Option<f64> {+ let mut seg_reader = signal_aggregator+ .segment_reader+ .as_ref()+ .unwrap()+ .borrow_mut();++ seg_reader+ .text_fields+ .get_mut(self.as_textfield().unwrap())+ .map(|field| bm25(field, doc))+ }+}++#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]+pub struct Bm25CleanBody;+impl Signal for Bm25CleanBody {+ fn default_coefficient(&self) -> f64 {+ 0.0063+ }++ fn as_field(&self) -> Option<Field> {+ Some(Field::Text(schema::text_field::CleanBody.into()))+ }++ fn compute(&self, doc: DocId, signal_aggregator: &SignalAggregator) -> Option<f64> {+ let mut seg_reader = signal_aggregator+ .segment_reader+ .as_ref()+ .unwrap()+ .borrow_mut();++ seg_reader+ .text_fields+ .get_mut(self.as_textfield().unwrap())+ .map(|field| bm25(field, doc))+ }+}++#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]+pub struct Bm25CleanBodyBigrams;+impl Signal for Bm25CleanBodyBigrams {+ fn default_coefficient(&self) -> f64 {+ 0.005+ }++ fn as_field(&self) -> Option<Field> {+ Some(Field::Text(schema::text_field::CleanBodyBigrams.into()))+ }++ fn compute(&self, doc: DocId, signal_aggregator: &SignalAggregator) -> Option<f64> {+ let mut seg_reader = signal_aggregator+ .segment_reader+ .as_ref()+ .unwrap()+ .borrow_mut();++ seg_reader+ .text_fields+ .get_mut(self.as_textfield().unwrap())+ .map(|field| bm25(field, doc))+ }+}++#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]+pub struct Bm25CleanBodyTrigrams;+impl Signal for Bm25CleanBodyTrigrams {+ fn default_coefficient(&self) -> f64 {+ 0.005+ }++ fn as_field(&self) -> Option<Field> {+ Some(Field::Text(schema::text_field::CleanBodyTrigrams.into()))+ }++ fn compute(&self, doc: DocId, signal_aggregator: &SignalAggregator) -> Option<f64> {+ let mut seg_reader = signal_aggregator+ .segment_reader+ .as_ref()+ .unwrap()+ .borrow_mut();++ seg_reader+ .text_fields+ .get_mut(self.as_textfield().unwrap())+ .map(|field| bm25(field, doc))+ }+}++#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]+pub struct Bm25StemmedTitle;+impl Signal for Bm25StemmedTitle {+ fn default_coefficient(&self) -> f64 {+ 0.003+ }++ fn as_field(&self) -> Option<Field> {+ Some(Field::Text(schema::text_field::StemmedTitle.into()))+ }++ fn compute(&self, doc: DocId, signal_aggregator: &SignalAggregator) -> Option<f64> {+ let mut seg_reader = signal_aggregator+ .segment_reader+ .as_ref()+ .unwrap()+ .borrow_mut();++ seg_reader+ .text_fields+ .get_mut(self.as_textfield().unwrap())+ .map(|field| bm25(field, doc))+ }+}++#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]+pub struct Bm25StemmedCleanBody;+impl Signal for Bm25StemmedCleanBody {+ fn default_coefficient(&self) -> f64 {+ 0.001+ }++ fn as_field(&self) -> Option<Field> {+ Some(Field::Text(schema::text_field::StemmedCleanBody.into()))+ }++ fn compute(&self, doc: DocId, signal_aggregator: &SignalAggregator) -> Option<f64> {+ let mut seg_reader = signal_aggregator+ .segment_reader+ .as_ref()+ .unwrap()+ .borrow_mut();++ seg_reader+ .text_fields+ .get_mut(self.as_textfield().unwrap())+ .map(|field| bm25(field, doc))+ }+}++#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]+pub struct Bm25AllBody;+impl Signal for Bm25AllBody {+ fn default_coefficient(&self) -> f64 {+ 0.0+ }++ fn as_field(&self) -> Option<Field> {+ Some(Field::Text(schema::text_field::AllBody.into()))+ }++ fn compute(&self, doc: DocId, signal_aggregator: &SignalAggregator) -> Option<f64> {+ let mut seg_reader = signal_aggregator+ .segment_reader+ .as_ref()+ .unwrap()+ .borrow_mut();++ seg_reader+ .text_fields+ .get_mut(self.as_textfield().unwrap())+ .map(|field| bm25(field, doc))+ }+}++#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]+pub struct Bm25Keywords;+impl Signal for Bm25Keywords {+ fn default_coefficient(&self) -> f64 {+ 0.001+ }++ fn as_field(&self) -> Option<Field> {+ Some(Field::Text(schema::text_field::Keywords.into()))+ }++ fn compute(&self, doc: DocId, signal_aggregator: &SignalAggregator) -> Option<f64> {+ let mut seg_reader = signal_aggregator+ .segment_reader+ .as_ref()+ .unwrap()+ .borrow_mut();++ seg_reader+ .text_fields+ .get_mut(self.as_textfield().unwrap())+ .map(|field| bm25(field, doc))+ }+}++#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]+pub struct Bm25BacklinkText;+impl Signal for Bm25BacklinkText {+ fn default_coefficient(&self) -> f64 {+ 0.003+ }++ fn as_field(&self) -> Option<Field> {+ Some(Field::Text(schema::text_field::BacklinkText.into()))+ }++ fn compute(&self, doc: DocId, signal_aggregator: &SignalAggregator) -> Option<f64> {+ let mut seg_reader = signal_aggregator+ .segment_reader+ .as_ref()+ .unwrap()+ .borrow_mut();++ seg_reader+ .text_fields+ .get_mut(self.as_textfield().unwrap())+ .map(|field| bm25(field, doc))+ }+}++#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]+pub struct IdfSumUrl;+impl Signal for IdfSumUrl {+ fn default_coefficient(&self) -> f64 {+ 0.0003+ }++ fn as_field(&self) -> Option<Field> {+ Some(Field::Text(schema::text_field::Url.into()))+ }++ fn compute(&self, doc: DocId, signal_aggregator: &SignalAggregator) -> Option<f64> {+ let mut seg_reader = signal_aggregator+ .segment_reader+ .as_ref()+ .unwrap()+ .borrow_mut();++ seg_reader+ .text_fields+ .get_mut(self.as_textfield().unwrap())+ .map(|field| idf_sum(field, doc))+ }+}++#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]+pub struct IdfSumSite;+impl Signal for IdfSumSite {+ fn default_coefficient(&self) -> f64 {+ 0.00015+ }++ fn as_field(&self) -> Option<Field> {+ Some(Field::Text(schema::text_field::SiteWithout.into()))+ }++ fn compute(&self, doc: DocId, signal_aggregator: &SignalAggregator) -> Option<f64> {+ let mut seg_reader = signal_aggregator+ .segment_reader+ .as_ref()+ .unwrap()+ .borrow_mut();++ seg_reader+ .text_fields+ .get_mut(self.as_textfield().unwrap())+ .map(|field| idf_sum(field, doc))+ }+}++#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]+pub struct IdfSumDomain;+impl Signal for IdfSumDomain {+ fn default_coefficient(&self) -> f64 {+ 0.0003+ }++ fn as_field(&self) -> Option<Field> {+ Some(Field::Text(schema::text_field::Domain.into()))+ }++ fn compute(&self, doc: DocId, signal_aggregator: &SignalAggregator) -> Option<f64> {+ let mut seg_reader = signal_aggregator+ .segment_reader+ .as_ref()+ .unwrap()+ .borrow_mut();++ seg_reader+ .text_fields+ .get_mut(self.as_textfield().unwrap())+ .map(|field| idf_sum(field, doc))+ }+}++#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]+pub struct IdfSumSiteNoTokenizer;+impl Signal for IdfSumSiteNoTokenizer {+ fn default_coefficient(&self) -> f64 {+ 0.00015+ }++ fn as_field(&self) -> Option<Field> {+ Some(Field::Text(schema::text_field::SiteNoTokenizer.into()))+ }++ fn compute(&self, doc: DocId, signal_aggregator: &SignalAggregator) -> Option<f64> {+ let mut seg_reader = signal_aggregator+ .segment_reader+ .as_ref()+ .unwrap()+ .borrow_mut();++ seg_reader+ .text_fields+ .get_mut(self.as_textfield().unwrap())+ .map(|field| idf_sum(field, doc))+ }+}++#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]+pub struct IdfSumDomainNoTokenizer;+impl Signal for IdfSumDomainNoTokenizer {+ fn default_coefficient(&self) -> f64 {+ 0.0002+ }++ fn as_field(&self) -> Option<Field> {+ Some(Field::Text(schema::text_field::DomainNoTokenizer.into()))+ }++ fn compute(&self, doc: DocId, signal_aggregator: &SignalAggregator) -> Option<f64> {+ let mut seg_reader = signal_aggregator+ .segment_reader+ .as_ref()+ .unwrap()+ .borrow_mut();++ seg_reader+ .text_fields+ .get_mut(self.as_textfield().unwrap())+ .map(|field| idf_sum(field, doc))+ }+}++#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]+pub struct IdfSumDomainNameNoTokenizer;+impl Signal for IdfSumDomainNameNoTokenizer {+ fn default_coefficient(&self) -> f64 {+ 0.0002+ }++ fn as_field(&self) -> Option<Field> {+ Some(Field::Text(+ schema::text_field::DomainNameNoTokenizer.into(),+ ))+ }++ fn compute(&self, doc: DocId, signal_aggregator: &SignalAggregator) -> Option<f64> {+ let mut seg_reader = signal_aggregator+ .segment_reader+ .as_ref()+ .unwrap()+ .borrow_mut();++ seg_reader+ .text_fields+ .get_mut(self.as_textfield().unwrap())+ .map(|field| idf_sum(field, doc))+ }+}++#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]+pub struct IdfSumDomainIfHomepage;+impl Signal for IdfSumDomainIfHomepage {+ fn default_coefficient(&self) -> f64 {+ 0.0004+ }++ fn as_field(&self) -> Option<Field> {+ Some(Field::Text(schema::text_field::DomainIfHomepage.into()))+ }++ fn compute(&self, doc: DocId, signal_aggregator: &SignalAggregator) -> Option<f64> {+ let mut seg_reader = signal_aggregator+ .segment_reader+ .as_ref()+ .unwrap()+ .borrow_mut();++ seg_reader+ .text_fields+ .get_mut(self.as_textfield().unwrap())+ .map(|field| idf_sum(field, doc))+ }+}++#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]+pub struct IdfSumDomainNameIfHomepageNoTokenizer;+impl Signal for IdfSumDomainNameIfHomepageNoTokenizer {+ fn default_coefficient(&self) -> f64 {+ 0.0036+ }++ fn as_field(&self) -> Option<Field> {+ Some(Field::Text(+ schema::text_field::DomainNameIfHomepageNoTokenizer.into(),+ ))+ }++ fn compute(&self, doc: DocId, signal_aggregator: &SignalAggregator) -> Option<f64> {+ let mut seg_reader = signal_aggregator+ .segment_reader+ .as_ref()+ .unwrap()+ .borrow_mut();++ seg_reader+ .text_fields+ .get_mut(self.as_textfield().unwrap())+ .map(|field| idf_sum(field, doc))+ }+}++#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]+pub struct IdfSumDomainIfHomepageNoTokenizer;+impl Signal for IdfSumDomainIfHomepageNoTokenizer {+ fn default_coefficient(&self) -> f64 {+ 0.0036+ }++ fn as_field(&self) -> Option<Field> {+ Some(Field::Text(+ schema::text_field::DomainIfHomepageNoTokenizer.into(),+ ))+ }++ fn compute(&self, doc: DocId, signal_aggregator: &SignalAggregator) -> Option<f64> {+ let mut seg_reader = signal_aggregator+ .segment_reader+ .as_ref()+ .unwrap()+ .borrow_mut();++ seg_reader+ .text_fields+ .get_mut(self.as_textfield().unwrap())+ .map(|field| idf_sum(field, doc))+ }+}++#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]+pub struct IdfSumTitleIfHomepage;+impl Signal for IdfSumTitleIfHomepage {+ fn default_coefficient(&self) -> f64 {+ 0.00022+ }++ fn as_field(&self) -> Option<Field> {+ Some(Field::Text(schema::text_field::TitleIfHomepage.into()))+ }++ fn compute(&self, doc: DocId, signal_aggregator: &SignalAggregator) -> Option<f64> {+ let mut seg_reader = signal_aggregator+ .segment_reader+ .as_ref()+ .unwrap()+ .borrow_mut();++ seg_reader+ .text_fields+ .get_mut(self.as_textfield().unwrap())+ .map(|field| idf_sum(field, doc))+ }+}++#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]+pub struct CrossEncoderSnippet;+impl Signal for CrossEncoderSnippet {+ fn default_coefficient(&self) -> f64 {+ 0.17+ }++ fn as_field(&self) -> Option<Field> {+ None+ }++ fn compute(&self, _: DocId, _: &SignalAggregator) -> Option<f64> {+ None // computed in later ranking stage+ }+}++#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]+pub struct CrossEncoderTitle;+impl Signal for CrossEncoderTitle {+ fn default_coefficient(&self) -> f64 {+ 0.17+ }++ fn as_field(&self) -> Option<Field> {+ None+ }++ fn compute(&self, _: DocId, _: &SignalAggregator) -> Option<f64> {+ None // computed in later ranking stage+ }+}++#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]+pub struct HostCentrality;+impl Signal for HostCentrality {+ fn default_coefficient(&self) -> f64 {+ 0.5+ }++ fn as_field(&self) -> Option<Field> {+ Some(Field::Fast(schema::fast_field::HostCentrality.into()))+ }++ fn precompute(self, webpage: &Webpage, _: &SignalAggregator) -> Option<f64> {+ Some(webpage.host_centrality)+ }++ fn compute(&self, doc: DocId, signal_aggregator: &SignalAggregator) -> Option<f64> {+ let seg_reader = signal_aggregator+ .segment_reader+ .as_ref()+ .unwrap()+ .borrow_mut();+ let fastfield_reader = seg_reader.fastfield_reader.get_field_reader(doc);++ let val = fastfield_reader+ .get(self.as_fastfield().unwrap())+ .and_then(|v| v.as_u64())+ .unwrap();+ Some(val as f64 / FLOAT_SCALING as f64)+ }+}++#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]+pub struct HostCentralityRank;+impl Signal for HostCentralityRank {+ fn default_coefficient(&self) -> f64 {+ 0.0+ }++ fn as_field(&self) -> Option<Field> {+ Some(Field::Fast(schema::fast_field::HostCentralityRank.into()))+ }++ fn precompute(self, webpage: &Webpage, _: &SignalAggregator) -> Option<f64> {+ Some(score_rank(webpage.host_centrality_rank as f64))+ }++ fn compute(&self, doc: DocId, signal_aggregator: &SignalAggregator) -> Option<f64> {+ let seg_reader = signal_aggregator+ .segment_reader+ .as_ref()+ .unwrap()+ .borrow_mut();+ let fastfield_reader = seg_reader.fastfield_reader.get_field_reader(doc);++ let val = fastfield_reader+ .get(self.as_fastfield().unwrap())+ .and_then(|v| v.as_u64())+ .unwrap();+ Some(score_rank(val as f64))+ }+}++#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]+pub struct PageCentrality;+impl Signal for PageCentrality {+ fn default_coefficient(&self) -> f64 {+ 0.25+ }++ fn as_field(&self) -> Option<Field> {+ Some(Field::Fast(schema::fast_field::PageCentrality.into()))+ }++ fn precompute(self, webpage: &Webpage, _: &SignalAggregator) -> Option<f64> {+ Some(webpage.page_centrality)+ }++ fn compute(&self, doc: DocId, signal_aggregator: &SignalAggregator) -> Option<f64> {+ let seg_reader = signal_aggregator+ .segment_reader+ .as_ref()+ .unwrap()+ .borrow_mut();+ let fastfield_reader = seg_reader.fastfield_reader.get_field_reader(doc);++ let val = fastfield_reader+ .get(self.as_fastfield().unwrap())+ .and_then(|v| v.as_u64())+ .unwrap();+ Some(val as f64 / FLOAT_SCALING as f64)+ }+}++#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]+pub struct PageCentralityRank;+impl Signal for PageCentralityRank {+ fn default_coefficient(&self) -> f64 {+ 0.0+ }++ fn as_field(&self) -> Option<Field> {+ Some(Field::Fast(schema::fast_field::PageCentralityRank.into()))+ }++ fn precompute(self, webpage: &Webpage, _: &SignalAggregator) -> Option<f64> {+ Some(score_rank(webpage.page_centrality_rank as f64))+ }++ fn compute(&self, doc: DocId, signal_aggregator: &SignalAggregator) -> Option<f64> {+ let seg_reader = signal_aggregator+ .segment_reader+ .as_ref()+ .unwrap()+ .borrow_mut();+ let fastfield_reader = seg_reader.fastfield_reader.get_field_reader(doc);++ let val = fastfield_reader+ .get(self.as_fastfield().unwrap())+ .and_then(|v| v.as_u64())+ .unwrap();+ Some(score_rank(val as f64))+ }+}++#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]+pub struct IsHomepage;+impl Signal for IsHomepage {+ fn default_coefficient(&self) -> f64 {+ 0.0005+ }++ fn as_field(&self) -> Option<Field> {+ Some(Field::Fast(schema::fast_field::IsHomepage.into()))+ }++ fn precompute(self, webpage: &Webpage, _: &SignalAggregator) -> Option<f64> {+ Some(webpage.html.is_homepage().into())+ }++ fn compute(&self, doc: DocId, signal_aggregator: &SignalAggregator) -> Option<f64> {+ let seg_reader = signal_aggregator+ .segment_reader+ .as_ref()+ .unwrap()+ .borrow_mut();+ let fastfield_reader = seg_reader.fastfield_reader.get_field_reader(doc);++ let val = fastfield_reader+ .get(self.as_fastfield().unwrap())+ .and_then(|v| v.as_u64())+ .unwrap();+ Some(val as f64)+ }+}++#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]+pub struct FetchTimeMs;+impl Signal for FetchTimeMs {+ fn default_coefficient(&self) -> f64 {+ 0.001+ }++ fn as_field(&self) -> Option<Field> {+ Some(Field::Fast(schema::fast_field::FetchTimeMs.into()))+ }++ fn precompute(self, webpage: &Webpage, signal_aggregator: &SignalAggregator) -> Option<f64> {+ let fetch_time_ms = webpage.fetch_time_ms as usize;+ if fetch_time_ms >= signal_aggregator.fetch_time_ms_cache.len() {+ Some(0.0)+ } else {+ Some(signal_aggregator.fetch_time_ms_cache[fetch_time_ms])+ }+ }++ fn compute(&self, doc: DocId, signal_aggregator: &SignalAggregator) -> Option<f64> {+ let seg_reader = signal_aggregator+ .segment_reader+ .as_ref()+ .unwrap()+ .borrow_mut();+ let fastfield_reader = seg_reader.fastfield_reader.get_field_reader(doc);++ let fetch_time_ms = fastfield_reader+ .get(self.as_fastfield().unwrap())+ .and_then(|v| v.as_u64())+ .unwrap() as usize;++ if fetch_time_ms >= signal_aggregator.fetch_time_ms_cache.len() {+ Some(0.0)+ } else {+ Some(signal_aggregator.fetch_time_ms_cache[fetch_time_ms])+ }+ }+}++#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]+pub struct UpdateTimestamp;+impl Signal for UpdateTimestamp {+ fn default_coefficient(&self) -> f64 {+ 0.001+ }++ fn as_field(&self) -> Option<Field> {+ Some(Field::Fast(schema::fast_field::LastUpdated.into()))+ }++ fn precompute(self, webpage: &Webpage, signal_aggregator: &SignalAggregator) -> Option<f64> {+ let update_timestamp = webpage+ .html+ .updated_time()+ .map(|date| date.timestamp().max(0))+ .unwrap_or(0) as usize;++ Some(score_timestamp(update_timestamp, signal_aggregator))+ }++ fn compute(&self, doc: DocId, signal_aggregator: &SignalAggregator) -> Option<f64> {+ let seg_reader = signal_aggregator+ .segment_reader+ .as_ref()+ .unwrap()+ .borrow_mut();+ let fastfield_reader = seg_reader.fastfield_reader.get_field_reader(doc);++ let val = fastfield_reader+ .get(self.as_fastfield().unwrap())+ .and_then(|v| v.as_u64())+ .unwrap() as usize;++ Some(score_timestamp(val, signal_aggregator))+ }+}-use super::bm25::MultiBm25Weight;-use super::models::linear::LinearRegression;-use super::{inbound_similarity, query_centrality};+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]+pub struct TrackerScore;+impl Signal for TrackerScore {+ fn default_coefficient(&self) -> f64 {+ 0.05+ }-#[derive(Debug, Error)]-pub enum Error {- #[error("unknown signal: {0}")]- UnknownSignal(#[from] serde_json::Error),+ fn as_field(&self) -> Option<Field> {+ Some(Field::Fast(schema::fast_field::TrackerScore.into()))+ }++ fn precompute(self, webpage: &Webpage, _: &SignalAggregator) -> Option<f64> {+ let num_trackers = webpage.html.trackers().len() as f64;+ Some(score_trackers(num_trackers))+ }++ fn compute(&self, doc: DocId, signal_aggregator: &SignalAggregator) -> Option<f64> {+ let seg_reader = signal_aggregator+ .segment_reader+ .as_ref()+ .unwrap()+ .borrow_mut();+ let fastfield_reader = seg_reader.fastfield_reader.get_field_reader(doc);++ let val = fastfield_reader+ .get(self.as_fastfield().unwrap())+ .and_then(|v| v.as_u64())+ .unwrap();+ Some(score_trackers(val as f64))+ }}-#[derive(- Debug, serde::Serialize, serde::Deserialize, Clone, Copy, PartialEq, Eq, Hash, VariantArray,-)]-pub enum Signal {- #[serde(rename = "bm25_title")]- Bm25Title,- #[serde(rename = "bm25_title_bigrams")]- Bm25TitleBigrams,- #[serde(rename = "bm25_title_trigrams")]- Bm25TitleTrigrams,- #[serde(rename = "bm25_clean_body")]- Bm25CleanBody,- #[serde(rename = "bm25_clean_body_bigrams")]- Bm25CleanBodyBigrams,- #[serde(rename = "bm25_clean_body_trigrams")]- Bm25CleanBodyTrigrams,- #[serde(rename = "bm25_stemmed_title")]- Bm25StemmedTitle,- #[serde(rename = "bm25_stemmed_clean_body")]- Bm25StemmedCleanBody,- #[serde(rename = "bm25_all_body")]- Bm25AllBody,- #[serde(rename = "bm25_keywords")]- Bm25Keywords,- #[serde(rename = "bm25_backlink_text")]- Bm25BacklinkText,- #[serde(rename = "idf_sum_url")]- IdfSumUrl,- #[serde(rename = "idf_sum_site")]- IdfSumSite,- #[serde(rename = "idf_sum_domain")]- IdfSumDomain,- #[serde(rename = "idf_sum_site_no_tokenizer")]- IdfSumSiteNoTokenizer,- #[serde(rename = "idf_sum_domain_no_tokenizer")]- IdfSumDomainNoTokenizer,- #[serde(rename = "idf_sum_domain_name_no_tokenizer")]- IdfSumDomainNameNoTokenizer,- #[serde(rename = "idf_sum_domain_if_homepage")]- IdfSumDomainIfHomepage,- #[serde(rename = "idf_sum_domain_name_if_homepage_no_tokenizer")]- IdfSumDomainNameIfHomepageNoTokenizer,- #[serde(rename = "idf_sum_domain_if_homepage_no_tokenizer")]- IdfSumDomainIfHomepageNoTokenizer,- #[serde(rename = "idf_sum_title_if_homepage")]- IdfSumTitleIfHomepage,- #[serde(rename = "cross_encoder_snippet")]- CrossEncoderSnippet,- #[serde(rename = "cross_encoder_title")]- CrossEncoderTitle,- #[serde(rename = "host_centrality")]- HostCentrality,- #[serde(rename = "host_centrality_rank")]- HostCentralityRank,- #[serde(rename = "page_centrality")]- PageCentrality,- #[serde(rename = "page_centrality_rank")]- PageCentralityRank,- #[serde(rename = "is_homepage")]- IsHomepage,- #[serde(rename = "fetch_time_ms")]- FetchTimeMs,- #[serde(rename = "update_timestamp")]- UpdateTimestamp,- #[serde(rename = "tracker_score")]- TrackerScore,- #[serde(rename = "region")]- Region,- #[serde(rename = "query_centrality")]- QueryCentrality,- #[serde(rename = "inbound_similarity")]- InboundSimilarity,- #[serde(rename = "lambda_mart")]- LambdaMART,- #[serde(rename = "url_digits")]- UrlDigits,- #[serde(rename = "url_slashes")]- UrlSlashes,- #[serde(rename = "link_density")]- LinkDensity,- #[serde(rename = "title_embedding_similarity")]- TitleEmbeddingSimilarity,- #[serde(rename = "keyword_embedding_similarity")]- KeywordEmbeddingSimilarity,+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]+pub struct Region;+impl Signal for Region {+ fn default_coefficient(&self) -> f64 {+ 0.15+ }++ fn as_field(&self) -> Option<Field> {+ Some(Field::Fast(schema::fast_field::Region.into()))+ }++ fn precompute(self, webpage: &Webpage, signal_aggregator: &SignalAggregator) -> Option<f64> {+ let region =+ crate::webpage::Region::guess_from(webpage).unwrap_or(crate::webpage::Region::All);+ Some(score_region(region, signal_aggregator))+ }++ fn compute(&self, doc: DocId, signal_aggregator: &SignalAggregator) -> Option<f64> {+ let seg_reader = signal_aggregator+ .segment_reader+ .as_ref()+ .unwrap()+ .borrow_mut();+ let fastfield_reader = seg_reader.fastfield_reader.get_field_reader(doc);++ let val = fastfield_reader+ .get(self.as_fastfield().unwrap())+ .and_then(|v| v.as_u64())+ .unwrap();+ let region = crate::webpage::Region::from_id(val);+ Some(score_region(region, signal_aggregator))+ }+}++#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]+pub struct QueryCentrality;+impl Signal for QueryCentrality {+ fn default_coefficient(&self) -> f64 {+ 0.0+ }++ fn as_field(&self) -> Option<Field> {+ None+ }++ fn compute(&self, doc: DocId, signal_aggregator: &SignalAggregator) -> Option<f64> {+ let seg_reader = signal_aggregator+ .segment_reader+ .as_ref()+ .unwrap()+ .borrow_mut();+ let fastfield_reader = seg_reader.fastfield_reader.get_field_reader(doc);+ let host_id = host_id(&fastfield_reader);++ host_id.and_then(|host_id| signal_aggregator.query_centrality(host_id))+ }+}++#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]+pub struct InboundSimilarity;+impl Signal for InboundSimilarity {+ fn default_coefficient(&self) -> f64 {+ 0.25+ }++ fn as_field(&self) -> Option<Field> {+ None+ }++ fn compute(&self, doc: DocId, signal_aggregator: &SignalAggregator) -> Option<f64> {+ let seg_reader = signal_aggregator+ .segment_reader+ .as_ref()+ .unwrap()+ .borrow_mut();+ let fastfield_reader = seg_reader.fastfield_reader.get_field_reader(doc);+ let host_id = host_id(&fastfield_reader);++ host_id.map(|host_id| signal_aggregator.inbound_similarity(host_id))+ }+}++#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]+pub struct LambdaMart;+impl Signal for LambdaMart {+ fn default_coefficient(&self) -> f64 {+ 10.0+ }++ fn as_field(&self) -> Option<Field> {+ None+ }++ fn compute(&self, _: DocId, _: &SignalAggregator) -> Option<f64> {+ None // computed in later ranking stage+ }+}++#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]+pub struct UrlDigits;+impl Signal for UrlDigits {+ fn default_coefficient(&self) -> f64 {+ 0.01+ }++ fn as_field(&self) -> Option<Field> {+ Some(Field::Fast(+ schema::fast_field::NumPathAndQueryDigits.into(),+ ))+ }++ fn precompute(self, webpage: &Webpage, _: &SignalAggregator) -> Option<f64> {+ let num_digits = (webpage+ .html+ .url()+ .path()+ .chars()+ .filter(|c| c.is_ascii_digit())+ .count()+ + webpage+ .html+ .url()+ .query()+ .unwrap_or_default()+ .chars()+ .filter(|c| c.is_ascii_digit())+ .count()) as f64;++ Some(score_digits(num_digits))+ }++ fn compute(&self, doc: DocId, signal_aggregator: &SignalAggregator) -> Option<f64> {+ let seg_reader = signal_aggregator+ .segment_reader+ .as_ref()+ .unwrap()+ .borrow_mut();+ let fastfield_reader = seg_reader.fastfield_reader.get_field_reader(doc);++ let val = fastfield_reader+ .get(self.as_fastfield().unwrap())+ .and_then(|v| v.as_u64())+ .unwrap();+ Some(score_digits(val as f64))+ }+}++#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]+pub struct UrlSlashes;+impl Signal for UrlSlashes {+ fn default_coefficient(&self) -> f64 {+ 0.01+ }++ fn as_field(&self) -> Option<Field> {+ Some(Field::Fast(+ schema::fast_field::NumPathAndQuerySlashes.into(),+ ))+ }++ fn precompute(self, webpage: &Webpage, _: &SignalAggregator) -> Option<f64> {+ let num_slashes = webpage+ .html+ .url()+ .path()+ .chars()+ .filter(|c| c == &'/')+ .count() as f64;+ Some(score_slashes(num_slashes))+ }++ fn compute(&self, doc: DocId, signal_aggregator: &SignalAggregator) -> Option<f64> {+ let seg_reader = signal_aggregator+ .segment_reader+ .as_ref()+ .unwrap()+ .borrow_mut();+ let fastfield_reader = seg_reader.fastfield_reader.get_field_reader(doc);++ let val = fastfield_reader+ .get(self.as_fastfield().unwrap())+ .and_then(|v| v.as_u64())+ .unwrap();+ Some(score_slashes(val as f64))+ }+}++#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]+pub struct LinkDensity;+impl Signal for LinkDensity {+ fn default_coefficient(&self) -> f64 {+ 0.0+ }++ fn as_field(&self) -> Option<Field> {+ Some(Field::Fast(schema::fast_field::LinkDensity.into()))+ }++ fn precompute(self, webpage: &Webpage, _: &SignalAggregator) -> Option<f64> {+ let link_density = webpage.html.link_density();+ Some(score_link_density(link_density))+ }++ fn compute(&self, doc: DocId, signal_aggregator: &SignalAggregator) -> Option<f64> {+ let seg_reader = signal_aggregator+ .segment_reader+ .as_ref()+ .unwrap()+ .borrow_mut();+ let fastfield_reader = seg_reader.fastfield_reader.get_field_reader(doc);++ let val = fastfield_reader+ .get(self.as_fastfield().unwrap())+ .and_then(|v| v.as_u64())+ .unwrap();+ Some(score_link_density(val as f64 / FLOAT_SCALING as f64))+ }+}++#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]+pub struct TitleEmbeddingSimilarity;+impl Signal for TitleEmbeddingSimilarity {+ fn default_coefficient(&self) -> f64 {+ 0.01+ }++ fn as_field(&self) -> Option<Field> {+ Some(Field::Fast(schema::fast_field::TitleEmbeddings.into()))+ }++ fn compute(&self, _: DocId, _: &SignalAggregator) -> Option<f64> {+ None // computed in later ranking stage+ }+}++#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]+pub struct KeywordEmbeddingSimilarity;+impl Signal for KeywordEmbeddingSimilarity {+ fn default_coefficient(&self) -> f64 {+ 0.01+ }++ fn as_field(&self) -> Option<Field> {+ Some(Field::Fast(schema::fast_field::KeywordEmbeddings.into()))+ }++ fn compute(&self, _: DocId, _: &SignalAggregator) -> Option<f64> {+ None // computed in later ranking stage+ }+}++impl SignalEnum {+ pub fn num_variants() -> usize {+ SignalEnumDiscriminants::VARIANTS.len()+ }++ pub fn all() -> impl Iterator<Item = SignalEnum> {+ SignalEnumDiscriminants::VARIANTS+ .iter()+ .copied()+ .map(|v| v.into())+ }++ pub fn get(field_id: usize) -> Option<SignalEnum> {+ SignalEnumDiscriminants::VARIANTS+ .get(field_id)+ .copied()+ .map(SignalEnum::from)+ }}-impl InsertEnumMapKey for Signal {+impl InsertEnumMapKey for SignalEnum {fn into_usize(self) -> usize {- self as usize+ SignalEnumDiscriminants::from(self) as usize}}@@ -196,7 +1323,7 @@ fn score_link_density(link_density: f64) -> f64 {}}-fn score_region(webpage_region: Region, aggregator: &SignalAggregator) -> f64 {+fn score_region(webpage_region: crate::webpage::Region, aggregator: &SignalAggregator) -> f64 {match aggregator.region_count.as_ref() {Some(region_count) => {let boost = aggregator@@ -204,7 +1331,7 @@ fn score_region(webpage_region: Region, aggregator: &SignalAggregator) -> f64 {.as_ref().and_then(|q| q.selected_region).map_or(0.0, |region| {- if region != Region::All && region == webpage_region {+ if region != crate::webpage::Region::All && region == webpage_region {50.0} else {0.0@@ -254,382 +1381,20 @@ fn idf_sum(field: &mut TextFieldData, doc: DocId) -> f64 {.sum::<f32>() as f64}-impl Signal {- fn is_computable_before_search(&self) -> bool {- self.as_fastfield().is_some()- && !matches!(- self,- Signal::TitleEmbeddingSimilarity | Signal::KeywordEmbeddingSimilarity- )- }-- pub fn default_coefficient(&self) -> f64 {- match self {- Signal::Bm25Title => 0.0063,- Signal::Bm25TitleBigrams => 0.01,- Signal::Bm25TitleTrigrams => 0.01,- Signal::Bm25CleanBody => 0.0063,- Signal::Bm25CleanBodyBigrams => 0.005,- Signal::Bm25CleanBodyTrigrams => 0.005,- Signal::Bm25StemmedTitle => 0.003,- Signal::Bm25StemmedCleanBody => 0.001,- Signal::Bm25AllBody => 0.0,- Signal::IdfSumUrl => 0.0003,- Signal::IdfSumSite => 0.00015,- Signal::IdfSumDomain => 0.0003,- Signal::IdfSumSiteNoTokenizer => 0.00015,- Signal::IdfSumDomainNoTokenizer => 0.0002,- Signal::IdfSumDomainNameNoTokenizer => 0.0002,- Signal::IdfSumDomainIfHomepage => 0.0004,- Signal::IdfSumDomainNameIfHomepageNoTokenizer => 0.0036,- Signal::IdfSumDomainIfHomepageNoTokenizer => 0.0036,- Signal::IdfSumTitleIfHomepage => 0.00022,- Signal::Bm25BacklinkText => 0.003,- Signal::Bm25Keywords => 0.001,- Signal::CrossEncoderSnippet => 0.17,- Signal::CrossEncoderTitle => 0.17,- Signal::HostCentrality => 0.5,- Signal::HostCentralityRank => 0.0,- Signal::PageCentrality => 0.25,- Signal::PageCentralityRank => 0.0,- Signal::QueryCentrality => 0.0,- Signal::IsHomepage => 0.0005,- Signal::FetchTimeMs => 0.001,- Signal::UpdateTimestamp => 0.001,- Signal::TrackerScore => 0.05,- Signal::Region => 0.15,- Signal::InboundSimilarity => 0.25,- Signal::LambdaMART => 10.0,- Signal::UrlSlashes => 0.01,- Signal::UrlDigits => 0.01,- Signal::LinkDensity => 0.0,- Signal::TitleEmbeddingSimilarity => 0.01,- Signal::KeywordEmbeddingSimilarity => 0.01,- }- }-- fn compute(self, signal_aggregator: &SignalAggregator, doc: DocId) -> Option<ComputedSignal> {- let coefficient = signal_aggregator.coefficient(&self);- if coefficient == 0.0 {- return None;- }-- let mut seg_reader = signal_aggregator- .segment_reader- .as_ref()- .unwrap()- .borrow_mut();- let fastfield_reader = seg_reader.fastfield_reader.get_field_reader(doc);-- let node_id = fastfield_reader- .get(fast_field::HostNodeID.into())- .and_then(|n| n.as_u64())- .unwrap();-- let host_id: Option<NodeID> = if node_id == u64::MAX {- None- } else {- Some(node_id.into())- };-- let value: Option<f64> = match self {- Signal::HostCentrality | Signal::PageCentrality => {- let val = fastfield_reader- .get(self.as_fastfield().unwrap())- .and_then(|v| v.as_u64())- .unwrap();- Some(val as f64 / FLOAT_SCALING as f64)- }- Signal::HostCentralityRank | Signal::PageCentralityRank => {- let val = fastfield_reader- .get(self.as_fastfield().unwrap())- .and_then(|v| v.as_u64())- .unwrap();- Some(score_rank(val as f64))- }- Signal::IsHomepage => {- let val = fastfield_reader- .get(self.as_fastfield().unwrap())- .and_then(|v| v.as_u64())- .unwrap();- Some(val as f64)- }- Signal::LinkDensity => {- let val = fastfield_reader- .get(self.as_fastfield().unwrap())- .and_then(|v| v.as_u64())- .unwrap();- Some(score_link_density(val as f64 / FLOAT_SCALING as f64))- }- Signal::FetchTimeMs => {- let fetch_time_ms = fastfield_reader- .get(self.as_fastfield().unwrap())- .and_then(|v| v.as_u64())- .unwrap() as usize;-- if fetch_time_ms >= signal_aggregator.fetch_time_ms_cache.len() {- Some(0.0)- } else {- Some(signal_aggregator.fetch_time_ms_cache[fetch_time_ms])- }- }- Signal::UpdateTimestamp => {- let val = fastfield_reader- .get(self.as_fastfield().unwrap())- .and_then(|v| v.as_u64())- .unwrap() as usize;-- Some(score_timestamp(val, signal_aggregator))- }- Signal::TrackerScore => {- let val = fastfield_reader- .get(self.as_fastfield().unwrap())- .and_then(|v| v.as_u64())- .unwrap();- Some(score_trackers(val as f64))- }- Signal::UrlDigits => {- let val = fastfield_reader- .get(self.as_fastfield().unwrap())- .and_then(|v| v.as_u64())- .unwrap();- Some(score_digits(val as f64))- }- Signal::UrlSlashes => {- let val = fastfield_reader- .get(self.as_fastfield().unwrap())- .and_then(|v| v.as_u64())- .unwrap();- Some(score_slashes(val as f64))- }- Signal::Region => {- let val = fastfield_reader- .get(self.as_fastfield().unwrap())- .and_then(|v| v.as_u64())- .unwrap();- let region = Region::from_id(val);- Some(score_region(region, signal_aggregator))- }- Signal::QueryCentrality => {- host_id.and_then(|host_id| signal_aggregator.query_centrality(host_id))- }- Signal::InboundSimilarity => {- host_id.map(|host_id| signal_aggregator.inbound_similarity(host_id))- }- Signal::Bm25Title- | Signal::Bm25TitleBigrams- | Signal::Bm25TitleTrigrams- | Signal::Bm25CleanBody- | Signal::Bm25CleanBodyBigrams- | Signal::Bm25CleanBodyTrigrams- | Signal::Bm25StemmedTitle- | Signal::Bm25StemmedCleanBody- | Signal::Bm25AllBody- | Signal::Bm25Keywords- | Signal::Bm25BacklinkText => seg_reader- .text_fields- .get_mut(self.as_textfield().unwrap())- .map(|field| bm25(field, doc)),-- Signal::IdfSumUrl- | Signal::IdfSumSite- | Signal::IdfSumDomain- | Signal::IdfSumSiteNoTokenizer- | Signal::IdfSumDomainNoTokenizer- | Signal::IdfSumDomainNameNoTokenizer- | Signal::IdfSumDomainIfHomepage- | Signal::IdfSumDomainNameIfHomepageNoTokenizer- | Signal::IdfSumDomainIfHomepageNoTokenizer- | Signal::IdfSumTitleIfHomepage => seg_reader- .text_fields- .get_mut(self.as_textfield().unwrap())- .map(|field| idf_sum(field, doc)),-- Signal::CrossEncoderSnippet- | Signal::CrossEncoderTitle- | Signal::TitleEmbeddingSimilarity- | Signal::KeywordEmbeddingSimilarity => None, // these are calculated in a later step-- Signal::LambdaMART => None,- };-- value.map(|value| ComputedSignal {- signal: self,- score: SignalScore { coefficient, value },- })- }-- pub fn precompute(- self,- signal_aggregator: &SignalAggregator,- webpage: &Webpage,- ) -> Option<ComputedSignal> {- if !self.is_computable_before_search() {- return None;- }-- let value = match self {- Signal::HostCentrality => Some(webpage.host_centrality),- Signal::PageCentrality => Some(webpage.page_centrality),- Signal::HostCentralityRank => Some(score_rank(webpage.host_centrality_rank as f64)),- Signal::PageCentralityRank => Some(score_rank(webpage.page_centrality_rank as f64)),- Signal::IsHomepage => Some(webpage.html.is_homepage().into()),- Signal::FetchTimeMs => {- let fetch_time_ms = webpage.fetch_time_ms as usize;- if fetch_time_ms >= signal_aggregator.fetch_time_ms_cache.len() {- Some(0.0)- } else {- Some(signal_aggregator.fetch_time_ms_cache[fetch_time_ms])- }- }- Signal::UpdateTimestamp => {- let update_timestamp = webpage- .html- .updated_time()- .map(|date| date.timestamp().max(0))- .unwrap_or(0) as usize;-- Some(score_timestamp(update_timestamp, signal_aggregator))- }- Signal::TrackerScore => {- let num_trackers = webpage.html.trackers().len() as f64;- Some(score_trackers(num_trackers))- }- Signal::Region => {- let region = Region::guess_from(webpage).unwrap_or(Region::All);- Some(score_region(region, signal_aggregator))- }- Signal::UrlDigits => {- let num_digits = (webpage- .html- .url()- .path()- .chars()- .filter(|c| c.is_ascii_digit())- .count()- + webpage- .html- .url()- .query()- .unwrap_or_default()- .chars()- .filter(|c| c.is_ascii_digit())- .count()) as f64;- Some(score_digits(num_digits))- }- Signal::UrlSlashes => {- let num_slashes = webpage- .html- .url()- .path()- .chars()- .filter(|c| c == &'/')- .count() as f64;- Some(score_slashes(num_slashes))- }- Signal::LinkDensity => {- let link_density = webpage.html.link_density();- Some(score_link_density(link_density))- }- Signal::Bm25Title- | Signal::Bm25TitleBigrams- | Signal::Bm25TitleTrigrams- | Signal::Bm25CleanBody- | Signal::Bm25CleanBodyBigrams- | Signal::Bm25CleanBodyTrigrams- | Signal::Bm25StemmedTitle- | Signal::Bm25StemmedCleanBody- | Signal::Bm25AllBody- | Signal::Bm25BacklinkText- | Signal::Bm25Keywords- | Signal::IdfSumUrl- | Signal::IdfSumSite- | Signal::IdfSumDomain- | Signal::IdfSumSiteNoTokenizer- | Signal::IdfSumDomainNoTokenizer- | Signal::IdfSumDomainNameNoTokenizer- | Signal::IdfSumDomainIfHomepage- | Signal::IdfSumDomainNameIfHomepageNoTokenizer- | Signal::IdfSumDomainIfHomepageNoTokenizer- | Signal::IdfSumTitleIfHomepage- | Signal::CrossEncoderSnippet- | Signal::CrossEncoderTitle- | Signal::InboundSimilarity- | Signal::LambdaMART- | Signal::TitleEmbeddingSimilarity- | Signal::KeywordEmbeddingSimilarity- | Signal::QueryCentrality => {- tracing::error!("signal {self:?} cannot be precomputed");- None- }- };-- value.map(|value| ComputedSignal {- signal: self,- score: SignalScore {- coefficient: signal_aggregator.coefficient(&self),- value,- },- })- }-- fn as_fastfield(&self) -> Option<FastFieldEnum> {- match self {- Signal::HostCentrality => Some(fast_field::HostCentrality.into()),- Signal::HostCentralityRank => Some(fast_field::HostCentralityRank.into()),- Signal::PageCentrality => Some(fast_field::PageCentrality.into()),- Signal::PageCentralityRank => Some(fast_field::PageCentralityRank.into()),- Signal::IsHomepage => Some(fast_field::IsHomepage.into()),- Signal::FetchTimeMs => Some(fast_field::FetchTimeMs.into()),- Signal::UpdateTimestamp => Some(fast_field::LastUpdated.into()),- Signal::TrackerScore => Some(fast_field::TrackerScore.into()),- Signal::Region => Some(fast_field::Region.into()),- Signal::UrlSlashes => Some(fast_field::NumPathAndQuerySlashes.into()),- Signal::UrlDigits => Some(fast_field::NumPathAndQueryDigits.into()),- Signal::LinkDensity => Some(fast_field::LinkDensity.into()),- Signal::TitleEmbeddingSimilarity => Some(fast_field::TitleEmbeddings.into()),- Signal::KeywordEmbeddingSimilarity => Some(fast_field::KeywordEmbeddings.into()),- _ => None,- }- }-- fn as_textfield(&self) -> Option<TextFieldEnum> {- match self {- Signal::Bm25Title => Some(text_field::Title.into()),- Signal::Bm25TitleBigrams => Some(text_field::TitleBigrams.into()),- Signal::Bm25TitleTrigrams => Some(text_field::TitleTrigrams.into()),- Signal::Bm25CleanBody => Some(text_field::CleanBody.into()),- Signal::Bm25CleanBodyBigrams => Some(text_field::CleanBodyBigrams.into()),- Signal::Bm25CleanBodyTrigrams => Some(text_field::CleanBodyTrigrams.into()),- Signal::Bm25StemmedTitle => Some(text_field::StemmedTitle.into()),- Signal::Bm25StemmedCleanBody => Some(text_field::StemmedCleanBody.into()),- Signal::Bm25AllBody => Some(text_field::AllBody.into()),- Signal::Bm25BacklinkText => Some(text_field::BacklinkText.into()),- Signal::Bm25Keywords => Some(text_field::Keywords.into()),- Signal::IdfSumUrl => Some(text_field::Url.into()),- Signal::IdfSumSite => Some(text_field::SiteWithout.into()),- Signal::IdfSumDomain => Some(text_field::Domain.into()),- Signal::IdfSumSiteNoTokenizer => Some(text_field::SiteNoTokenizer.into()),- Signal::IdfSumDomainNoTokenizer => Some(text_field::DomainNoTokenizer.into()),- Signal::IdfSumDomainNameNoTokenizer => Some(text_field::DomainNameNoTokenizer.into()),- Signal::IdfSumDomainIfHomepage => Some(text_field::DomainIfHomepage.into()),- Signal::IdfSumDomainNameIfHomepageNoTokenizer => {- Some(text_field::DomainNameIfHomepageNoTokenizer.into())- }- Signal::IdfSumTitleIfHomepage => Some(text_field::TitleIfHomepage.into()),- Signal::IdfSumDomainIfHomepageNoTokenizer => {- Some(text_field::DomainIfHomepageNoTokenizer.into())- }- _ => None,- }- }+fn host_id<'a>(fastfield_reader: &FieldReader<'a>) -> Option<NodeID> {+ let node_id = fastfield_reader+ .get(schema::fast_field::HostNodeID.into())+ .and_then(|n| n.as_u64())+ .unwrap();- pub fn all() -> impl Iterator<Item = Self> {- Self::VARIANTS.iter().copied()+ if node_id == u64::MAX {+ None+ } else {+ Some(node_id.into())}}-impl FromStr for Signal {+impl FromStr for SignalEnumDiscriminants {type Err = Error;fn from_str(name: &str) -> std::result::Result<Self, Self::Err> {@@ -641,18 +1406,18 @@ impl FromStr for Signal {#[derive(Debug, Clone, Default)]pub struct SignalCoefficient {- map: EnumMap<Signal, f64>,+ map: EnumMap<SignalEnum, f64>,}impl SignalCoefficient {- pub fn get(&self, signal: &Signal) -> f64 {+ pub fn get(&self, signal: &SignalEnum) -> f64 {self.map.get(*signal).copied().unwrap_or(signal.default_coefficient())}- pub fn new(coefficients: impl Iterator<Item = (Signal, f64)>) -> Self {+ pub fn new(coefficients: impl Iterator<Item = (SignalEnum, f64)>) -> Self {let mut map = EnumMap::default();for (signal, coefficient) in coefficients {@@ -665,15 +1430,15 @@ impl SignalCoefficient {pub fn from_optic(optic: &Optic) -> Self {SignalCoefficient::new(optic.rankings.iter().filter_map(|coeff| {match &coeff.target {- RankingTarget::Signal(signal) => Signal::from_str(signal)+ RankingTarget::Signal(signal) => SignalEnumDiscriminants::from_str(signal).ok()- .map(|signal| (signal, coeff.value)),+ .map(|signal| (signal.into(), coeff.value)),}}))}pub fn merge_into(&mut self, coeffs: SignalCoefficient) {- for signal in Signal::all() {+ for signal in SignalEnum::all() {if let Some(coeff) = coeffs.map.get(signal).copied() {match self.map.get_mut(signal) {Some(existing_coeff) => *existing_coeff += coeff,@@ -712,7 +1477,7 @@ struct SegmentReader {struct QueryData {simple_terms: Vec<String>,optic_rules: Vec<optics::Rule>,- selected_region: Option<Region>,+ selected_region: Option<crate::webpage::Region>,}pub struct SignalAggregator {@@ -829,7 +1594,7 @@ impl SignalAggregator {if let Some(query) = &self.query_data {if !query.simple_terms.is_empty() {- for signal in Signal::all() {+ for signal in SignalEnum::all() {if let Some(text_field) = signal.as_textfield() {let tv_field = schema.get_field(text_field.name()).unwrap();let simple_query = itertools::intersperse(@@ -1010,13 +1775,23 @@ impl SignalAggregator {}pub fn precompute_score(&self, webpage: &Webpage) -> f64 {- Signal::all()- .filter_map(|signal| signal.precompute(self, webpage))+ SignalEnum::all()+ .filter_map(|signal| {+ signal+ .precompute(webpage, self)+ .map(|value| ComputedSignal {+ signal,+ score: SignalScore {+ coefficient: self.coefficient(&signal),+ value,+ },+ })+ }).map(|computed| computed.score.coefficient * computed.score.value).sum()}- pub fn coefficient(&self, signal: &Signal) -> f64 {+ pub fn coefficient(&self, signal: &SignalEnum) -> f64 {self.query_signal_coefficients.as_ref().map(|coefficients| coefficients.get(signal))@@ -1031,7 +1806,7 @@ impl SignalAggregator {#[derive(Debug, Clone, Copy)]pub struct ComputedSignal {- pub signal: Signal,+ pub signal: SignalEnum,pub score: SignalScore,}@@ -1045,7 +1820,7 @@ pub struct SignalScore {#[derive(Clone)]pub struct SignalOrder {text_signals: EnumMap<TextFieldEnum, NGramSignalOrder>,- other_signals: Vec<Signal>,+ other_signals: Vec<SignalEnum>,}impl SignalOrder {@@ -1060,7 +1835,7 @@ impl SignalOrder {let mut text_signals = EnumMap::new();let mut other_signals = Vec::new();- for signal in Signal::all() {+ for signal in SignalEnum::all() {if signal_aggregator.coefficient(&signal) == 0.0 {continue;}@@ -1097,7 +1872,23 @@ impl SignalOrder {.chain(self.other_signals.iter()- .map(move |signal| signal.compute(signal_aggregator, doc)),+ .filter_map(|signal| {+ let coefficient = signal_aggregator.coefficient(signal);++ if coefficient > 0.0 {+ Some((signal, coefficient))+ } else {+ None+ }+ })+ .map(move |(signal, coefficient)| {+ signal+ .compute(doc, signal_aggregator)+ .map(|value| ComputedSignal {+ signal: *signal,+ score: SignalScore { coefficient, value },+ })+ }),)}}@@ -1112,11 +1903,11 @@ const NGRAM_DAMPENING: f64 = 0.4;#[derive(Debug, Default, Clone)]pub struct NGramSignalOrder {/// ordered by descending ngram size. e.g. [title_bm25_trigram, title_bm25_bigram, title_bm25]- signals: Vec<(usize, Signal)>,+ signals: Vec<(usize, SignalEnum)>,}impl NGramSignalOrder {- fn push(&mut self, signal: Signal, ngram: usize) {+ fn push(&mut self, signal: SignalEnum, ngram: usize) {self.signals.push((ngram, signal));self.signals.sort_unstable_by(|(a, _), (b, _)| b.cmp(a));}@@ -1132,15 +1923,25 @@ impl NGramSignalOrder {.iter().map(|(_, s)| s).filter_map(move |signal| {- signal.compute(signal_aggregator, doc).map(|mut c| {- c.score.coefficient *= NGRAM_DAMPENING.powi(hits);+ signal+ .compute(doc, signal_aggregator)+ .map(|value| {+ let coefficient = signal_aggregator.coefficient(signal);++ ComputedSignal {+ signal: *signal,+ score: SignalScore { coefficient, value },+ }+ })+ .map(|mut c| {+ c.score.coefficient *= NGRAM_DAMPENING.powi(hits);- if c.score.value > 0.0 {- hits += 1;- }+ if c.score.value > 0.0 {+ hits += 1;+ }- c- })+ c+ })})}}diff --git a/crates/core/src/schema/fast_field.rs b/crates/core/src/schema/fast_field.rsindex 44413bce..b372a264 100644--- a/crates/core/src/schema/fast_field.rs+++ b/crates/core/src/schema/fast_field.rs@@ -22,8 +22,9 @@ use tantivy::{};use crate::{+ enum_dispatch_from_discriminant,enum_map::InsertEnumMapKey,- from_discriminant, simhash,+ simhash,webpage::{html::FnCache, Html, Webpage},Result,};@@ -130,7 +131,7 @@ pub enum FastFieldEnum {KeywordEmbeddings,}-from_discriminant!(FastFieldEnumDiscriminants => FastFieldEnum,+enum_dispatch_from_discriminant!(FastFieldEnumDiscriminants => FastFieldEnum,[IsHomepage,HostCentrality,diff --git a/crates/core/src/schema/mod.rs b/crates/core/src/schema/mod.rsindex b5021c18..6409435b 100644--- a/crates/core/src/schema/mod.rs+++ b/crates/core/src/schema/mod.rs@@ -122,18 +122,3 @@ pub enum IndexingOption {DateTime(DateOptions),Bytes(BytesOptions),}--#[macro_export]-macro_rules! from_discriminant {- ($discenum:ident => $enum:ident, [$($disc:ident),*$(,)?]) => {- impl From<$discenum> for $enum {- fn from(value: $discenum) -> Self {- match value {- $(- $discenum::$disc => $disc.into(),- )*- }- }- }- };-}diff --git a/crates/core/src/schema/text_field.rs b/crates/core/src/schema/text_field.rsindex 0bb60d5f..482dd1d1 100644--- a/crates/core/src/schema/text_field.rs+++ b/crates/core/src/schema/text_field.rs@@ -25,8 +25,8 @@ use tantivy::{use whatlang::Lang;use crate::{+ enum_dispatch_from_discriminant,enum_map::InsertEnumMapKey,- from_discriminant,tokenizer::{BigramTokenizer, Identity, JsonField, SiteOperatorUrlTokenizer, Tokenizer, TrigramTokenizer,},@@ -170,7 +170,7 @@ pub enum TextFieldEnum {Keywords,}-from_discriminant!(TextFieldEnumDiscriminants => TextFieldEnum,+enum_dispatch_from_discriminant!(TextFieldEnumDiscriminants => TextFieldEnum,[Title,CleanBody,diff --git a/crates/core/src/search_prettifier/mod.rs b/crates/core/src/search_prettifier/mod.rsindex 1d4e0c45..288e9894 100644--- a/crates/core/src/search_prettifier/mod.rs+++ b/crates/core/src/search_prettifier/mod.rs@@ -27,7 +27,7 @@ use utoipa::ToSchema;use crate::{highlighted::HighlightedFragment,inverted_index::RetrievedWebpage,- ranking::{Signal, SignalScore},+ ranking::{SignalEnumDiscriminants, SignalScore},snippet::TextSnippet,web_spell::{self, CorrectionTerm},webpage::url_ext::UrlExt,@@ -179,7 +179,7 @@ pub struct DisplayedWebpage {pub pretty_url: String,pub snippet: Snippet,pub rich_snippet: Option<RichSnippet>,- pub ranking_signals: Option<HashMap<Signal, SignalScore>>,+ pub ranking_signals: Option<HashMap<SignalEnumDiscriminants, SignalScore>>,pub score: Option<f64>,pub likely_has_ads: bool,pub likely_has_paywall: bool,diff --git a/crates/core/src/searcher/api/mod.rs b/crates/core/src/searcher/api/mod.rsindex c733222f..e6e5f9bf 100644--- a/crates/core/src/searcher/api/mod.rs+++ b/crates/core/src/searcher/api/mod.rs@@ -32,7 +32,7 @@ use crate::inverted_index::RetrievedWebpage;use crate::models::dual_encoder::DualEncoder;use crate::ranking::models::cross_encoder::CrossEncoderModel;use crate::ranking::pipeline::{PrecisionRankingWebpage, RankableWebpage, RecallRankingWebpage};-use crate::ranking::Signal;+use crate::ranking::SignalEnum;use crate::search_prettifier::{DisplayedSidebar, DisplayedWebpage, HighlightedSpellCorrection};use crate::web_spell::SpellChecker;use crate::widgets::{Widget, Widgets};@@ -147,9 +147,9 @@ pub fn add_ranking_signals(websites: &mut [DisplayedWebpage], pointers: &[Scoredfor (website, pointer) in websites.iter_mut().zip(pointers.iter()) {let mut signals = HashMap::new();- for signal in Signal::all() {+ for signal in SignalEnum::all() {if let Some(signal_value) = pointer.as_ranking().signals.get(signal) {- signals.insert(signal, *signal_value);+ signals.insert(signal.into(), *signal_value);}}diff --git a/crates/core/src/searcher/local.rs b/crates/core/src/searcher/local.rsindex df372fb4..899c94bf 100644--- a/crates/core/src/searcher/local.rs+++ b/crates/core/src/searcher/local.rs@@ -29,7 +29,7 @@ use crate::ranking::inbound_similarity::InboundSimilarity;use crate::ranking::models::lambdamart::LambdaMART;use crate::ranking::models::linear::LinearRegression;use crate::ranking::pipeline::{PrecisionRankingWebpage, RankingPipeline, RecallRankingWebpage};-use crate::ranking::{query_centrality, Ranker, Signal, SignalAggregator};+use crate::ranking::{self, query_centrality, Ranker, SignalAggregator, SignalEnum};use crate::search_ctx::Ctx;use crate::search_prettifier::DisplayedWebpage;use crate::webgraph::Node;@@ -186,7 +186,8 @@ wherede_rank_similar: bool,aggregator: SignalAggregator,) -> Result<Ranker> {- let query_centrality_coeff = aggregator.coefficient(&Signal::QueryCentrality);+ let query_centrality_coeff =+ aggregator.coefficient(&ranking::signal::QueryCentrality.into());let mut ranker = Ranker::new(aggregator,@@ -409,9 +410,9 @@ wherefor (webpage, ranking) in webpages.iter_mut().zip(top_websites) {let mut ranking_signals = HashMap::new();- for signal in Signal::all() {+ for signal in SignalEnum::all() {if let Some(score) = ranking.ranking.signals.get(signal) {- ranking_signals.insert(signal, *score);+ ranking_signals.insert(signal.into(), *score);}}diff --git a/crates/core/src/webpage/mod.rs b/crates/core/src/webpage/mod.rsindex ea507f42..63b32ffe 100644--- a/crates/core/src/webpage/mod.rs+++ b/crates/core/src/webpage/mod.rs@@ -35,6 +35,8 @@ pub mod schema_org;pub mod url_ext;pub use self::html::Html;+pub use region::Region;+#[derive(Debug)]pub struct Webpage {pub html: Html,