web search done right
specific likely_has_ads ranking signal in addition to the num_trackers signal
Mikkel Denker8ec40ab9a195
diff --git a/crates/core/src/api/docs.rs b/crates/core/src/api/docs.rsindex cffb5767..514e52fc 100644--- a/crates/core/src/api/docs.rs+++ b/crates/core/src/api/docs.rs@@ -77,8 +77,8 @@ use utoipa_swagger_ui::SwaggerUi;crate::widgets::thesaurus::PartOfSpeech,crate::widgets::thesaurus::PartOfSpeechMeaning,- crate::ranking::signal::SignalEnumDiscriminants,- crate::ranking::signal::SignalScore,+ crate::ranking::core::SignalEnumDiscriminants,+ crate::ranking::core::SignalScore,crate::bangs::BangHit,crate::bangs::Bang,diff --git a/crates/core/src/index.rs b/crates/core/src/index.rsindex e61b771a..d39056ba 100644--- a/crates/core/src/index.rs+++ b/crates/core/src/index.rs@@ -246,7 +246,7 @@ mod tests {.ranking_signals.as_ref().unwrap()- .get(&crate::ranking::SignalEnum::from(ranking::signal::Bm25Title).into())+ .get(&crate::ranking::SignalEnum::from(ranking::core::Bm25Title).into()).unwrap()).all(|&v| v.value > 0.0));}diff --git a/crates/core/src/query/optic.rs b/crates/core/src/query/optic.rsindex 03f1dd39..0b9be805 100644--- a/crates/core/src/query/optic.rs+++ b/crates/core/src/query/optic.rs@@ -351,10 +351,11 @@ mod tests {let res = searcher.search(&SearchQuery {query: "website".to_string(),- signal_coefficients: enum_map!{- crate::ranking::SignalEnum::from(crate::ranking::signal::Bm25Title) => 1_000_000.0-- }.into(),+ signal_coefficients: enum_map! {+ crate::ranking::SignalEnum::from(crate::ranking::core::Bm25Title) => 1_000_000.0++ }+ .into(),..Default::default()}).unwrap()@@ -640,33 +641,33 @@ mod tests {Node::from("https://www.e.com").into_host(),Node::from("https://www.a.com").into_host(),String::new(),- RelFlags::default()+ RelFlags::default(),);writer.insert(Node::from("https://www.a.com").into_host(),Node::from("https://www.e.com").into_host(),String::new(),- RelFlags::default()+ RelFlags::default(),);writer.insert(Node::from("https://www.c.com").into_host(),Node::from("https://www.c.com").into_host(),String::new(),- RelFlags::default()+ RelFlags::default(),);writer.insert(Node::from("https://www.b.com").into_host(),Node::from("https://www.e.com").into_host(),String::new(),- RelFlags::default()+ RelFlags::default(),);writer.insert(Node::from("https://www.e.com").into_host(),Node::from("https://www.b.com").into_host(),String::new(),- RelFlags::default()+ RelFlags::default(),);let graph = writer.finalize();@@ -759,7 +760,7 @@ mod tests {.search(&SearchQuery {query: "website".to_string(),signal_coefficients: crate::enum_map! {- crate::ranking::SignalEnum::from(crate::ranking::signal::InboundSimilarity) => 100_000.0+ crate::ranking::SignalEnum::from(crate::ranking::core::InboundSimilarity) => 100_000.0}.into(),optic: Some(Optic {diff --git a/crates/core/src/ranking/signal/computer/mod.rs b/crates/core/src/ranking/core/computer/mod.rssimilarity index 100%rename from crates/core/src/ranking/signal/computer/mod.rsrename to crates/core/src/ranking/core/computer/mod.rsdiff --git a/crates/core/src/ranking/signal/computer/order.rs b/crates/core/src/ranking/core/computer/order.rssimilarity index 100%rename from crates/core/src/ranking/signal/computer/order.rsrename to crates/core/src/ranking/core/computer/order.rsdiff --git a/crates/core/src/ranking/signal/mod.rs b/crates/core/src/ranking/core/mod.rssimilarity index 100%rename from crates/core/src/ranking/signal/mod.rsrename to crates/core/src/ranking/core/mod.rsdiff --git a/crates/core/src/ranking/signal/non_text.rs b/crates/core/src/ranking/core/non_text.rssimilarity index 93%rename from crates/core/src/ranking/signal/non_text.rsrename to crates/core/src/ranking/core/non_text.rsindex 35be0344..2a3b77db 100644--- a/crates/core/src/ranking/signal/non_text.rs+++ b/crates/core/src/ranking/core/non_text.rs@@ -699,3 +699,52 @@ impl Signal for LinkDensity {Some(score_link_density(val))}}++#[derive(+ Debug,+ Clone,+ Copy,+ PartialEq,+ Eq,+ Hash,+ serde::Serialize,+ serde::Deserialize,+ bincode::Encode,+ bincode::Decode,+)]+pub struct HasAds;+impl Signal for HasAds {+ fn default_coefficient(&self) -> f64 {+ 0.01+ }++ fn as_field(&self) -> Option<Field> {+ Some(Field::Numerical(+ schema::numerical_field::LikelyHasAds.into(),+ ))+ }++ fn precompute(self, webpage: &Webpage, _: &SignalComputer) -> Option<f64> {+ if !webpage.html.likely_has_ads() {+ Some(1.0)+ } else {+ Some(0.0)+ }+ }++ fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {+ let seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();+ let numericalfield_reader = seg_reader.numericalfield_reader().get_field_reader(doc);++ let has_ads = numericalfield_reader+ .get(self.as_numericalfield().unwrap())+ .and_then(|v| v.as_bool())+ .unwrap();++ if !has_ads {+ Some(1.0)+ } else {+ Some(0.0)+ }+ }+}diff --git a/crates/core/src/ranking/signal/prelude.rs b/crates/core/src/ranking/core/prelude.rssimilarity index 99%rename from crates/core/src/ranking/signal/prelude.rsrename to crates/core/src/ranking/core/prelude.rsindex 8c375afb..825c0878 100644--- a/crates/core/src/ranking/signal/prelude.rs+++ b/crates/core/src/ranking/core/prelude.rs@@ -119,6 +119,7 @@ pub enum SignalEnum {LinkDensity,TitleEmbeddingSimilarity,KeywordEmbeddingSimilarity,+ HasAds,}enum_dispatch_from_discriminant!(SignalEnumDiscriminants => SignalEnum,@@ -164,6 +165,7 @@ enum_dispatch_from_discriminant!(SignalEnumDiscriminants => SignalEnum,LinkDensity,TitleEmbeddingSimilarity,KeywordEmbeddingSimilarity,+ HasAds,]);impl SignalEnum {diff --git a/crates/core/src/ranking/signal/text.rs b/crates/core/src/ranking/core/text.rssimilarity index 100%rename from crates/core/src/ranking/signal/text.rsrename to crates/core/src/ranking/core/text.rsdiff --git a/crates/core/src/ranking/mod.rs b/crates/core/src/ranking/mod.rsindex fcc31d64..3bf70d88 100644--- a/crates/core/src/ranking/mod.rs+++ b/crates/core/src/ranking/mod.rs@@ -17,12 +17,12 @@pub mod bitvec_similarity;pub mod bm25;pub mod bm25f;+pub mod core;pub mod inbound_similarity;pub mod initial;pub mod models;pub mod optics;pub mod pipeline;-pub mod signal;use initial::InitialScoreTweaker;@@ -34,7 +34,7 @@ use crate::{searcher::NUM_RESULTS_PER_PAGE,};-pub use self::signal::*;+pub use self::core::*;#[derive(Clone)]pub struct Ranker {@@ -331,7 +331,7 @@ mod tests {query: "title".to_string(),return_ranking_signals: true,signal_coefficients: crate::enum_map! {- crate::ranking::SignalEnum::from(crate::ranking::signal::UpdateTimestamp) => 100_000.0,+ crate::ranking::SignalEnum::from(crate::ranking::core::UpdateTimestamp) => 100_000.0,}.into(),..Default::default()})@@ -556,8 +556,8 @@ mod tests {query: "example".to_string(),signal_coefficients: crate::enum_map! {- crate::ranking::SignalEnum::from(crate::ranking::signal::Bm25Title) => 20_000_000.0,- crate::ranking::SignalEnum::from(crate::ranking::signal::HostCentrality) => 0.0,+ crate::ranking::SignalEnum::from(crate::ranking::core::Bm25Title) => 20_000_000.0,+ crate::ranking::SignalEnum::from(crate::ranking::core::HostCentrality) => 0.0,}.into(),..Default::default()@@ -572,7 +572,7 @@ mod tests {query: "example".to_string(),signal_coefficients: crate::enum_map! {- crate::ranking::SignalEnum::from(crate::ranking::signal::HostCentrality) => 2_000_000.0,+ crate::ranking::SignalEnum::from(crate::ranking::core::HostCentrality) => 2_000_000.0,}.into(),..Default::default()@@ -641,7 +641,7 @@ mod tests {.search(&SearchQuery {query: "test".to_string(),signal_coefficients: crate::enum_map! {- crate::ranking::SignalEnum::from(crate::ranking::signal::FetchTimeMs) => 100_000.0,+ crate::ranking::SignalEnum::from(crate::ranking::core::FetchTimeMs) => 100_000.0,}.into(),..Default::default()})@@ -735,9 +735,10 @@ mod tests {query: "test".to_string(),signal_coefficients: crate::enum_map! {- crate::ranking::SignalEnum::from(crate::ranking::signal::UrlSlashes) => 100_000.0,- crate::ranking::SignalEnum::from(crate::ranking::signal::UrlDigits) => 100_000.0,- }.into(),+ crate::ranking::SignalEnum::from(crate::ranking::core::UrlSlashes) => 100_000.0,+ crate::ranking::SignalEnum::from(crate::ranking::core::UrlDigits) => 100_000.0,+ }+ .into(),..Default::default()})@@ -841,7 +842,7 @@ mod tests {query: "best chocolate cake".to_string(),signal_coefficients: crate::enum_map! {- crate::ranking::SignalEnum::from(crate::ranking::signal::TitleEmbeddingSimilarity) => 100_000.0,+ crate::ranking::SignalEnum::from(crate::ranking::core::TitleEmbeddingSimilarity) => 100_000.0,}.into(),..Default::default()@@ -929,7 +930,7 @@ mod tests {query: "best chocolate cake".to_string(),signal_coefficients: crate::enum_map! {- crate::ranking::SignalEnum::from(crate::ranking::signal::KeywordEmbeddingSimilarity) => 100_000.0,+ crate::ranking::SignalEnum::from(crate::ranking::core::KeywordEmbeddingSimilarity) => 100_000.0,}.into(),..Default::default()diff --git a/crates/core/src/ranking/models/lambdamart.rs b/crates/core/src/ranking/models/lambdamart.rsindex aac7b839..1632fc8d 100644--- a/crates/core/src/ranking/models/lambdamart.rs+++ b/crates/core/src/ranking/models/lambdamart.rs@@ -16,11 +16,11 @@use std::{path::Path, str::FromStr};-use signal::SignalEnumDiscriminants;+use core::SignalEnumDiscriminants;use crate::{enum_map::EnumMap,- ranking::{signal, SignalEnum},+ ranking::{core, SignalEnum},};type Result<T> = std::result::Result<T, Error>;@@ -36,7 +36,7 @@ pub enum Error {NoEndOfTrees,#[error("Signal error: {0}")]- Signal(#[from] signal::Error),+ Signal(#[from] core::Error),#[error("ParseInt error: {0}")]ParseInt(#[from] std::num::ParseIntError),@@ -314,44 +314,41 @@ mod tests {assert!(!model.trees.is_empty());let mut features = EnumMap::new();- features.insert(ranking::signal::Bm25BacklinkText.into(), 85.7750244140625);- features.insert(ranking::signal::Bm25CleanBody.into(), 67.41311645507812);- features.insert(ranking::signal::Bm25CleanBodyBigrams.into(), 0.0);- features.insert(ranking::signal::Bm25CleanBodyTrigrams.into(), 0.0);- features.insert(ranking::signal::IdfSumDomain.into(), 43.332096099853516);- features.insert(ranking::signal::IdfSumDomainIfHomepage.into(), 0.0);+ features.insert(ranking::core::Bm25BacklinkText.into(), 85.7750244140625);+ features.insert(ranking::core::Bm25CleanBody.into(), 67.41311645507812);+ features.insert(ranking::core::Bm25CleanBodyBigrams.into(), 0.0);+ features.insert(ranking::core::Bm25CleanBodyTrigrams.into(), 0.0);+ features.insert(ranking::core::IdfSumDomain.into(), 43.332096099853516);+ features.insert(ranking::core::IdfSumDomainIfHomepage.into(), 0.0);+ features.insert(ranking::core::IdfSumDomainIfHomepageNoTokenizer.into(), 0.0);features.insert(- ranking::signal::IdfSumDomainIfHomepageNoTokenizer.into(),+ ranking::core::IdfSumDomainNameIfHomepageNoTokenizer.into(),0.0,);+ features.insert(ranking::core::IdfSumDomainNameNoTokenizer.into(), 0.0);+ features.insert(ranking::core::IdfSumDomainNoTokenizer.into(), 0.0);+ features.insert(ranking::core::IdfSumSite.into(), 61.47410202026367);+ features.insert(ranking::core::IdfSumSiteNoTokenizer.into(), 0.0);features.insert(- ranking::signal::IdfSumDomainNameIfHomepageNoTokenizer.into(),- 0.0,- );- features.insert(ranking::signal::IdfSumDomainNameNoTokenizer.into(), 0.0);- features.insert(ranking::signal::IdfSumDomainNoTokenizer.into(), 0.0);- features.insert(ranking::signal::IdfSumSite.into(), 61.47410202026367);- features.insert(ranking::signal::IdfSumSiteNoTokenizer.into(), 0.0);- features.insert(- ranking::signal::Bm25StemmedCleanBody.into(),+ ranking::core::Bm25StemmedCleanBody.into(),65.94627380371094,);- features.insert(ranking::signal::Bm25StemmedTitle.into(), 0.0);- features.insert(ranking::signal::Bm25Title.into(), 59.817813873291016);- features.insert(ranking::signal::Bm25TitleBigrams.into(), 0.0);- features.insert(ranking::signal::IdfSumTitleIfHomepage.into(), 0.0);- features.insert(ranking::signal::Bm25TitleTrigrams.into(), 0.0);- features.insert(ranking::signal::IdfSumUrl.into(), 57.07925033569336);- features.insert(ranking::signal::FetchTimeMs.into(), 0.023255813953488372);- features.insert(ranking::signal::HostCentrality.into(), 0.017958538);- features.insert(ranking::signal::InboundSimilarity.into(), 0.0);- features.insert(ranking::signal::IsHomepage.into(), 0.0);- features.insert(ranking::signal::PageCentrality.into(), 0.008253236);- features.insert(ranking::signal::Region.into(), 0.16622349570454012);- features.insert(ranking::signal::TrackerScore.into(), 0.07692307692307693);- features.insert(ranking::signal::UpdateTimestamp.into(), 0.0);- features.insert(ranking::signal::UrlDigits.into(), 0.25);- features.insert(ranking::signal::UrlSlashes.into(), 0.3333333333333333);+ features.insert(ranking::core::Bm25StemmedTitle.into(), 0.0);+ features.insert(ranking::core::Bm25Title.into(), 59.817813873291016);+ features.insert(ranking::core::Bm25TitleBigrams.into(), 0.0);+ features.insert(ranking::core::IdfSumTitleIfHomepage.into(), 0.0);+ features.insert(ranking::core::Bm25TitleTrigrams.into(), 0.0);+ features.insert(ranking::core::IdfSumUrl.into(), 57.07925033569336);+ features.insert(ranking::core::FetchTimeMs.into(), 0.023255813953488372);+ features.insert(ranking::core::HostCentrality.into(), 0.017958538);+ features.insert(ranking::core::InboundSimilarity.into(), 0.0);+ features.insert(ranking::core::IsHomepage.into(), 0.0);+ features.insert(ranking::core::PageCentrality.into(), 0.008253236);+ features.insert(ranking::core::Region.into(), 0.16622349570454012);+ features.insert(ranking::core::TrackerScore.into(), 0.07692307692307693);+ features.insert(ranking::core::UpdateTimestamp.into(), 0.0);+ features.insert(ranking::core::UrlDigits.into(), 0.25);+ features.insert(ranking::core::UrlSlashes.into(), 0.3333333333333333);assert_eq!((model.predict(&features) * 1000.0) as u64, 1050);}diff --git a/crates/core/src/ranking/optics.rs b/crates/core/src/ranking/optics.rsindex 6f1494c5..56e3f812 100644--- a/crates/core/src/ranking/optics.rs+++ b/crates/core/src/ranking/optics.rs@@ -185,7 +185,7 @@ mod tests {}),signal_coefficients: crate::enum_map! {- crate::ranking::SignalEnum::from(crate::ranking::signal::InboundSimilarity) => 100_000_000.0,+ crate::ranking::SignalEnum::from(crate::ranking::core::InboundSimilarity) => 100_000_000.0,}.into(),..Default::default()@@ -209,7 +209,7 @@ mod tests {}),signal_coefficients: crate::enum_map! {- crate::ranking::SignalEnum::from(crate::ranking::signal::InboundSimilarity) => 100_000_000.0,+ crate::ranking::SignalEnum::from(crate::ranking::core::InboundSimilarity) => 100_000_000.0,}.into(),return_ranking_signals: true,@@ -233,7 +233,7 @@ mod tests {}),signal_coefficients: crate::enum_map! {- crate::ranking::SignalEnum::from(crate::ranking::signal::InboundSimilarity) => 100_000_000.0,+ crate::ranking::SignalEnum::from(crate::ranking::core::InboundSimilarity) => 100_000_000.0,}.into(),return_ranking_signals: true,diff --git a/crates/core/src/ranking/pipeline/mod.rs b/crates/core/src/ranking/pipeline/mod.rsindex e65b64a6..0062165d 100644--- a/crates/core/src/ranking/pipeline/mod.rs+++ b/crates/core/src/ranking/pipeline/mod.rs@@ -100,7 +100,7 @@ impl<T: RankableWebpage> RankingStage<T> {fn calculate_score(&self, signals: &EnumMap<SignalEnum, f64>) -> f64 {match self.model.as_ref() {Some(model) => {- let coeff = self.coefficients.get(&super::signal::LambdaMart.into());+ let coeff = self.coefficients.get(&super::core::LambdaMart.into());if coeff == 0.0 {signals.iter()@@ -205,7 +205,7 @@ mod tests {};let mut signals = EnumMap::new();- signals.insert(ranking::signal::HostCentrality.into(), 1.0 / i as f64);+ signals.insert(ranking::core::HostCentrality.into(), 1.0 / i as f64);LocalRecallRankingWebpage::new_testing(pointer, signals, 1.0 / i as f64)}).collect()diff --git a/crates/core/src/ranking/pipeline/scorers/embedding.rs b/crates/core/src/ranking/pipeline/scorers/embedding.rsindex f68527e5..7b448063 100644--- a/crates/core/src/ranking/pipeline/scorers/embedding.rs+++ b/crates/core/src/ranking/pipeline/scorers/embedding.rs@@ -164,7 +164,7 @@ pub trait EmbeddingSignal<W>: Send + Sync {impl EmbeddingSignal<ScoredWebpagePointer> for TitleEmbeddings {fn signal() -> SignalEnum {- ranking::signal::TitleEmbeddingSimilarity.into()+ ranking::core::TitleEmbeddingSimilarity.into()}fn has_embedding(webpage: &ScoredWebpagePointer) -> bool {@@ -183,7 +183,7 @@ impl EmbeddingSignal<ScoredWebpagePointer> for TitleEmbeddings {impl EmbeddingSignal<LocalRecallRankingWebpage> for TitleEmbeddings {fn signal() -> SignalEnum {- ranking::signal::TitleEmbeddingSimilarity.into()+ ranking::core::TitleEmbeddingSimilarity.into()}fn has_embedding(webpage: &LocalRecallRankingWebpage) -> bool {@@ -203,7 +203,7 @@ impl EmbeddingSignal<LocalRecallRankingWebpage> for TitleEmbeddings {impl EmbeddingSignal<ScoredWebpagePointer> for KeywordEmbeddings {fn signal() -> SignalEnum {- ranking::signal::KeywordEmbeddingSimilarity.into()+ ranking::core::KeywordEmbeddingSimilarity.into()}fn has_embedding(webpage: &ScoredWebpagePointer) -> bool {@@ -222,7 +222,7 @@ impl EmbeddingSignal<ScoredWebpagePointer> for KeywordEmbeddings {impl EmbeddingSignal<LocalRecallRankingWebpage> for KeywordEmbeddings {fn signal() -> SignalEnum {- ranking::signal::KeywordEmbeddingSimilarity.into()+ ranking::core::KeywordEmbeddingSimilarity.into()}fn has_embedding(webpage: &LocalRecallRankingWebpage) -> bool {diff --git a/crates/core/src/ranking/pipeline/scorers/inbound_similarity.rs b/crates/core/src/ranking/pipeline/scorers/inbound_similarity.rsindex 19ac0852..0ad1e977 100644--- a/crates/core/src/ranking/pipeline/scorers/inbound_similarity.rs+++ b/crates/core/src/ranking/pipeline/scorers/inbound_similarity.rs@@ -17,7 +17,7 @@use std::sync::Mutex;use crate::{- ranking::{inbound_similarity, signal},+ ranking::{core, inbound_similarity},searcher::api::ScoredWebpagePointer,};@@ -47,7 +47,7 @@ impl Scorer<ScoredWebpagePointer> for InboundScorer {webpage.as_ranking_mut().signals_mut()- .insert(signal::InboundSimilarity.into(), score);+ .insert(core::InboundSimilarity.into(), score);}}}diff --git a/crates/core/src/ranking/pipeline/scorers/reranker.rs b/crates/core/src/ranking/pipeline/scorers/reranker.rsindex f5f79176..5eefd5ca 100644--- a/crates/core/src/ranking/pipeline/scorers/reranker.rs+++ b/crates/core/src/ranking/pipeline/scorers/reranker.rs@@ -54,12 +54,12 @@ impl<M: CrossEncoder> ReRanker<M> {webpage.ranking_mut().signals_mut()- .insert(ranking::signal::CrossEncoderSnippet.into(), snippet);+ .insert(ranking::core::CrossEncoderSnippet.into(), snippet);webpage.ranking_mut().signals_mut()- .insert(ranking::signal::CrossEncoderTitle.into(), title);+ .insert(ranking::core::CrossEncoderTitle.into(), title);}}}