Skip to content

web search done right

specific likely_has_ads ranking signal in addition to the num_trackers signal

Mikkel Denker8ec40ab9a195
diff --git a/crates/core/src/api/docs.rs b/crates/core/src/api/docs.rs
index cffb5767..514e52fc 100644
--- a/crates/core/src/api/docs.rs
+++ b/crates/core/src/api/docs.rs
@@ -77,8 +77,8 @@ use utoipa_swagger_ui::SwaggerUi;
crate::widgets::thesaurus::PartOfSpeech,
crate::widgets::thesaurus::PartOfSpeechMeaning,
- crate::ranking::signal::SignalEnumDiscriminants,
- crate::ranking::signal::SignalScore,
+ crate::ranking::core::SignalEnumDiscriminants,
+ crate::ranking::core::SignalScore,
crate::bangs::BangHit,
crate::bangs::Bang,
diff --git a/crates/core/src/index.rs b/crates/core/src/index.rs
index e61b771a..d39056ba 100644
--- a/crates/core/src/index.rs
+++ b/crates/core/src/index.rs
@@ -246,7 +246,7 @@ mod tests {
.ranking_signals
.as_ref()
.unwrap()
- .get(&crate::ranking::SignalEnum::from(ranking::signal::Bm25Title).into())
+ .get(&crate::ranking::SignalEnum::from(ranking::core::Bm25Title).into())
.unwrap())
.all(|&v| v.value > 0.0));
}
diff --git a/crates/core/src/query/optic.rs b/crates/core/src/query/optic.rs
index 03f1dd39..0b9be805 100644
--- a/crates/core/src/query/optic.rs
+++ b/crates/core/src/query/optic.rs
@@ -351,10 +351,11 @@ mod tests {
let res = searcher
.search(&SearchQuery {
query: "website".to_string(),
- signal_coefficients: enum_map!{
- crate::ranking::SignalEnum::from(crate::ranking::signal::Bm25Title) => 1_000_000.0
-
- }.into(),
+ signal_coefficients: enum_map! {
+ crate::ranking::SignalEnum::from(crate::ranking::core::Bm25Title) => 1_000_000.0
+
+ }
+ .into(),
..Default::default()
})
.unwrap()
@@ -640,33 +641,33 @@ mod tests {
Node::from("https://www.e.com").into_host(),
Node::from("https://www.a.com").into_host(),
String::new(),
- RelFlags::default()
+ RelFlags::default(),
);
writer.insert(
Node::from("https://www.a.com").into_host(),
Node::from("https://www.e.com").into_host(),
String::new(),
- RelFlags::default()
+ RelFlags::default(),
);
writer.insert(
Node::from("https://www.c.com").into_host(),
Node::from("https://www.c.com").into_host(),
String::new(),
- RelFlags::default()
+ RelFlags::default(),
);
writer.insert(
Node::from("https://www.b.com").into_host(),
Node::from("https://www.e.com").into_host(),
String::new(),
- RelFlags::default()
+ RelFlags::default(),
);
writer.insert(
Node::from("https://www.e.com").into_host(),
Node::from("https://www.b.com").into_host(),
String::new(),
- RelFlags::default()
+ RelFlags::default(),
);
let graph = writer.finalize();
@@ -759,7 +760,7 @@ mod tests {
.search(&SearchQuery {
query: "website".to_string(),
signal_coefficients: crate::enum_map! {
- crate::ranking::SignalEnum::from(crate::ranking::signal::InboundSimilarity) => 100_000.0
+ crate::ranking::SignalEnum::from(crate::ranking::core::InboundSimilarity) => 100_000.0
}.into(),
optic: Some(Optic {
diff --git a/crates/core/src/ranking/signal/computer/mod.rs b/crates/core/src/ranking/core/computer/mod.rs
similarity index 100%
rename from crates/core/src/ranking/signal/computer/mod.rs
rename to crates/core/src/ranking/core/computer/mod.rs
diff --git a/crates/core/src/ranking/signal/computer/order.rs b/crates/core/src/ranking/core/computer/order.rs
similarity index 100%
rename from crates/core/src/ranking/signal/computer/order.rs
rename to crates/core/src/ranking/core/computer/order.rs
diff --git a/crates/core/src/ranking/signal/mod.rs b/crates/core/src/ranking/core/mod.rs
similarity index 100%
rename from crates/core/src/ranking/signal/mod.rs
rename to crates/core/src/ranking/core/mod.rs
diff --git a/crates/core/src/ranking/signal/non_text.rs b/crates/core/src/ranking/core/non_text.rs
similarity index 93%
rename from crates/core/src/ranking/signal/non_text.rs
rename to crates/core/src/ranking/core/non_text.rs
index 35be0344..2a3b77db 100644
--- a/crates/core/src/ranking/signal/non_text.rs
+++ b/crates/core/src/ranking/core/non_text.rs
@@ -699,3 +699,52 @@ impl Signal for LinkDensity {
Some(score_link_density(val))
}
}
+
+#[derive(
+ Debug,
+ Clone,
+ Copy,
+ PartialEq,
+ Eq,
+ Hash,
+ serde::Serialize,
+ serde::Deserialize,
+ bincode::Encode,
+ bincode::Decode,
+)]
+pub struct HasAds;
+impl Signal for HasAds {
+ fn default_coefficient(&self) -> f64 {
+ 0.01
+ }
+
+ fn as_field(&self) -> Option<Field> {
+ Some(Field::Numerical(
+ schema::numerical_field::LikelyHasAds.into(),
+ ))
+ }
+
+ fn precompute(self, webpage: &Webpage, _: &SignalComputer) -> Option<f64> {
+ if !webpage.html.likely_has_ads() {
+ Some(1.0)
+ } else {
+ Some(0.0)
+ }
+ }
+
+ fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {
+ let seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();
+ let numericalfield_reader = seg_reader.numericalfield_reader().get_field_reader(doc);
+
+ let has_ads = numericalfield_reader
+ .get(self.as_numericalfield().unwrap())
+ .and_then(|v| v.as_bool())
+ .unwrap();
+
+ if !has_ads {
+ Some(1.0)
+ } else {
+ Some(0.0)
+ }
+ }
+}
diff --git a/crates/core/src/ranking/signal/prelude.rs b/crates/core/src/ranking/core/prelude.rs
similarity index 99%
rename from crates/core/src/ranking/signal/prelude.rs
rename to crates/core/src/ranking/core/prelude.rs
index 8c375afb..825c0878 100644
--- a/crates/core/src/ranking/signal/prelude.rs
+++ b/crates/core/src/ranking/core/prelude.rs
@@ -119,6 +119,7 @@ pub enum SignalEnum {
LinkDensity,
TitleEmbeddingSimilarity,
KeywordEmbeddingSimilarity,
+ HasAds,
}
enum_dispatch_from_discriminant!(SignalEnumDiscriminants => SignalEnum,
@@ -164,6 +165,7 @@ enum_dispatch_from_discriminant!(SignalEnumDiscriminants => SignalEnum,
LinkDensity,
TitleEmbeddingSimilarity,
KeywordEmbeddingSimilarity,
+ HasAds,
]);
impl SignalEnum {
diff --git a/crates/core/src/ranking/signal/text.rs b/crates/core/src/ranking/core/text.rs
similarity index 100%
rename from crates/core/src/ranking/signal/text.rs
rename to crates/core/src/ranking/core/text.rs
diff --git a/crates/core/src/ranking/mod.rs b/crates/core/src/ranking/mod.rs
index fcc31d64..3bf70d88 100644
--- a/crates/core/src/ranking/mod.rs
+++ b/crates/core/src/ranking/mod.rs
@@ -17,12 +17,12 @@
pub mod bitvec_similarity;
pub mod bm25;
pub mod bm25f;
+pub mod core;
pub mod inbound_similarity;
pub mod initial;
pub mod models;
pub mod optics;
pub mod pipeline;
-pub mod signal;
use initial::InitialScoreTweaker;
@@ -34,7 +34,7 @@ use crate::{
searcher::NUM_RESULTS_PER_PAGE,
};
-pub use self::signal::*;
+pub use self::core::*;
#[derive(Clone)]
pub struct Ranker {
@@ -331,7 +331,7 @@ mod tests {
query: "title".to_string(),
return_ranking_signals: true,
signal_coefficients: crate::enum_map! {
- crate::ranking::SignalEnum::from(crate::ranking::signal::UpdateTimestamp) => 100_000.0,
+ crate::ranking::SignalEnum::from(crate::ranking::core::UpdateTimestamp) => 100_000.0,
}.into(),
..Default::default()
})
@@ -556,8 +556,8 @@ mod tests {
query: "example".to_string(),
signal_coefficients: crate::enum_map! {
- crate::ranking::SignalEnum::from(crate::ranking::signal::Bm25Title) => 20_000_000.0,
- crate::ranking::SignalEnum::from(crate::ranking::signal::HostCentrality) => 0.0,
+ crate::ranking::SignalEnum::from(crate::ranking::core::Bm25Title) => 20_000_000.0,
+ crate::ranking::SignalEnum::from(crate::ranking::core::HostCentrality) => 0.0,
}.into(),
..Default::default()
@@ -572,7 +572,7 @@ mod tests {
query: "example".to_string(),
signal_coefficients: crate::enum_map! {
- crate::ranking::SignalEnum::from(crate::ranking::signal::HostCentrality) => 2_000_000.0,
+ crate::ranking::SignalEnum::from(crate::ranking::core::HostCentrality) => 2_000_000.0,
}.into(),
..Default::default()
@@ -641,7 +641,7 @@ mod tests {
.search(&SearchQuery {
query: "test".to_string(),
signal_coefficients: crate::enum_map! {
- crate::ranking::SignalEnum::from(crate::ranking::signal::FetchTimeMs) => 100_000.0,
+ crate::ranking::SignalEnum::from(crate::ranking::core::FetchTimeMs) => 100_000.0,
}.into(),
..Default::default()
})
@@ -735,9 +735,10 @@ mod tests {
query: "test".to_string(),
signal_coefficients: crate::enum_map! {
- crate::ranking::SignalEnum::from(crate::ranking::signal::UrlSlashes) => 100_000.0,
- crate::ranking::SignalEnum::from(crate::ranking::signal::UrlDigits) => 100_000.0,
- }.into(),
+ crate::ranking::SignalEnum::from(crate::ranking::core::UrlSlashes) => 100_000.0,
+ crate::ranking::SignalEnum::from(crate::ranking::core::UrlDigits) => 100_000.0,
+ }
+ .into(),
..Default::default()
})
@@ -841,7 +842,7 @@ mod tests {
query: "best chocolate cake".to_string(),
signal_coefficients: crate::enum_map! {
- crate::ranking::SignalEnum::from(crate::ranking::signal::TitleEmbeddingSimilarity) => 100_000.0,
+ crate::ranking::SignalEnum::from(crate::ranking::core::TitleEmbeddingSimilarity) => 100_000.0,
}.into(),
..Default::default()
@@ -929,7 +930,7 @@ mod tests {
query: "best chocolate cake".to_string(),
signal_coefficients: crate::enum_map! {
- crate::ranking::SignalEnum::from(crate::ranking::signal::KeywordEmbeddingSimilarity) => 100_000.0,
+ crate::ranking::SignalEnum::from(crate::ranking::core::KeywordEmbeddingSimilarity) => 100_000.0,
}.into(),
..Default::default()
diff --git a/crates/core/src/ranking/models/lambdamart.rs b/crates/core/src/ranking/models/lambdamart.rs
index aac7b839..1632fc8d 100644
--- a/crates/core/src/ranking/models/lambdamart.rs
+++ b/crates/core/src/ranking/models/lambdamart.rs
@@ -16,11 +16,11 @@
use std::{path::Path, str::FromStr};
-use signal::SignalEnumDiscriminants;
+use core::SignalEnumDiscriminants;
use crate::{
enum_map::EnumMap,
- ranking::{signal, SignalEnum},
+ ranking::{core, SignalEnum},
};
type Result<T> = std::result::Result<T, Error>;
@@ -36,7 +36,7 @@ pub enum Error {
NoEndOfTrees,
#[error("Signal error: {0}")]
- Signal(#[from] signal::Error),
+ Signal(#[from] core::Error),
#[error("ParseInt error: {0}")]
ParseInt(#[from] std::num::ParseIntError),
@@ -314,44 +314,41 @@ mod tests {
assert!(!model.trees.is_empty());
let mut features = EnumMap::new();
- features.insert(ranking::signal::Bm25BacklinkText.into(), 85.7750244140625);
- features.insert(ranking::signal::Bm25CleanBody.into(), 67.41311645507812);
- features.insert(ranking::signal::Bm25CleanBodyBigrams.into(), 0.0);
- features.insert(ranking::signal::Bm25CleanBodyTrigrams.into(), 0.0);
- features.insert(ranking::signal::IdfSumDomain.into(), 43.332096099853516);
- features.insert(ranking::signal::IdfSumDomainIfHomepage.into(), 0.0);
+ features.insert(ranking::core::Bm25BacklinkText.into(), 85.7750244140625);
+ features.insert(ranking::core::Bm25CleanBody.into(), 67.41311645507812);
+ features.insert(ranking::core::Bm25CleanBodyBigrams.into(), 0.0);
+ features.insert(ranking::core::Bm25CleanBodyTrigrams.into(), 0.0);
+ features.insert(ranking::core::IdfSumDomain.into(), 43.332096099853516);
+ features.insert(ranking::core::IdfSumDomainIfHomepage.into(), 0.0);
+ features.insert(ranking::core::IdfSumDomainIfHomepageNoTokenizer.into(), 0.0);
features.insert(
- ranking::signal::IdfSumDomainIfHomepageNoTokenizer.into(),
+ ranking::core::IdfSumDomainNameIfHomepageNoTokenizer.into(),
0.0,
);
+ features.insert(ranking::core::IdfSumDomainNameNoTokenizer.into(), 0.0);
+ features.insert(ranking::core::IdfSumDomainNoTokenizer.into(), 0.0);
+ features.insert(ranking::core::IdfSumSite.into(), 61.47410202026367);
+ features.insert(ranking::core::IdfSumSiteNoTokenizer.into(), 0.0);
features.insert(
- ranking::signal::IdfSumDomainNameIfHomepageNoTokenizer.into(),
- 0.0,
- );
- features.insert(ranking::signal::IdfSumDomainNameNoTokenizer.into(), 0.0);
- features.insert(ranking::signal::IdfSumDomainNoTokenizer.into(), 0.0);
- features.insert(ranking::signal::IdfSumSite.into(), 61.47410202026367);
- features.insert(ranking::signal::IdfSumSiteNoTokenizer.into(), 0.0);
- features.insert(
- ranking::signal::Bm25StemmedCleanBody.into(),
+ ranking::core::Bm25StemmedCleanBody.into(),
65.94627380371094,
);
- features.insert(ranking::signal::Bm25StemmedTitle.into(), 0.0);
- features.insert(ranking::signal::Bm25Title.into(), 59.817813873291016);
- features.insert(ranking::signal::Bm25TitleBigrams.into(), 0.0);
- features.insert(ranking::signal::IdfSumTitleIfHomepage.into(), 0.0);
- features.insert(ranking::signal::Bm25TitleTrigrams.into(), 0.0);
- features.insert(ranking::signal::IdfSumUrl.into(), 57.07925033569336);
- features.insert(ranking::signal::FetchTimeMs.into(), 0.023255813953488372);
- features.insert(ranking::signal::HostCentrality.into(), 0.017958538);
- features.insert(ranking::signal::InboundSimilarity.into(), 0.0);
- features.insert(ranking::signal::IsHomepage.into(), 0.0);
- features.insert(ranking::signal::PageCentrality.into(), 0.008253236);
- features.insert(ranking::signal::Region.into(), 0.16622349570454012);
- features.insert(ranking::signal::TrackerScore.into(), 0.07692307692307693);
- features.insert(ranking::signal::UpdateTimestamp.into(), 0.0);
- features.insert(ranking::signal::UrlDigits.into(), 0.25);
- features.insert(ranking::signal::UrlSlashes.into(), 0.3333333333333333);
+ features.insert(ranking::core::Bm25StemmedTitle.into(), 0.0);
+ features.insert(ranking::core::Bm25Title.into(), 59.817813873291016);
+ features.insert(ranking::core::Bm25TitleBigrams.into(), 0.0);
+ features.insert(ranking::core::IdfSumTitleIfHomepage.into(), 0.0);
+ features.insert(ranking::core::Bm25TitleTrigrams.into(), 0.0);
+ features.insert(ranking::core::IdfSumUrl.into(), 57.07925033569336);
+ features.insert(ranking::core::FetchTimeMs.into(), 0.023255813953488372);
+ features.insert(ranking::core::HostCentrality.into(), 0.017958538);
+ features.insert(ranking::core::InboundSimilarity.into(), 0.0);
+ features.insert(ranking::core::IsHomepage.into(), 0.0);
+ features.insert(ranking::core::PageCentrality.into(), 0.008253236);
+ features.insert(ranking::core::Region.into(), 0.16622349570454012);
+ features.insert(ranking::core::TrackerScore.into(), 0.07692307692307693);
+ features.insert(ranking::core::UpdateTimestamp.into(), 0.0);
+ features.insert(ranking::core::UrlDigits.into(), 0.25);
+ features.insert(ranking::core::UrlSlashes.into(), 0.3333333333333333);
assert_eq!((model.predict(&features) * 1000.0) as u64, 1050);
}
diff --git a/crates/core/src/ranking/optics.rs b/crates/core/src/ranking/optics.rs
index 6f1494c5..56e3f812 100644
--- a/crates/core/src/ranking/optics.rs
+++ b/crates/core/src/ranking/optics.rs
@@ -185,7 +185,7 @@ mod tests {
}),
signal_coefficients: crate::enum_map! {
- crate::ranking::SignalEnum::from(crate::ranking::signal::InboundSimilarity) => 100_000_000.0,
+ crate::ranking::SignalEnum::from(crate::ranking::core::InboundSimilarity) => 100_000_000.0,
}.into(),
..Default::default()
@@ -209,7 +209,7 @@ mod tests {
}),
signal_coefficients: crate::enum_map! {
- crate::ranking::SignalEnum::from(crate::ranking::signal::InboundSimilarity) => 100_000_000.0,
+ crate::ranking::SignalEnum::from(crate::ranking::core::InboundSimilarity) => 100_000_000.0,
}.into(),
return_ranking_signals: true,
@@ -233,7 +233,7 @@ mod tests {
}),
signal_coefficients: crate::enum_map! {
- crate::ranking::SignalEnum::from(crate::ranking::signal::InboundSimilarity) => 100_000_000.0,
+ crate::ranking::SignalEnum::from(crate::ranking::core::InboundSimilarity) => 100_000_000.0,
}.into(),
return_ranking_signals: true,
diff --git a/crates/core/src/ranking/pipeline/mod.rs b/crates/core/src/ranking/pipeline/mod.rs
index e65b64a6..0062165d 100644
--- a/crates/core/src/ranking/pipeline/mod.rs
+++ b/crates/core/src/ranking/pipeline/mod.rs
@@ -100,7 +100,7 @@ impl<T: RankableWebpage> RankingStage<T> {
fn calculate_score(&self, signals: &EnumMap<SignalEnum, f64>) -> f64 {
match self.model.as_ref() {
Some(model) => {
- let coeff = self.coefficients.get(&super::signal::LambdaMart.into());
+ let coeff = self.coefficients.get(&super::core::LambdaMart.into());
if coeff == 0.0 {
signals
.iter()
@@ -205,7 +205,7 @@ mod tests {
};
let mut signals = EnumMap::new();
- signals.insert(ranking::signal::HostCentrality.into(), 1.0 / i as f64);
+ signals.insert(ranking::core::HostCentrality.into(), 1.0 / i as f64);
LocalRecallRankingWebpage::new_testing(pointer, signals, 1.0 / i as f64)
})
.collect()
diff --git a/crates/core/src/ranking/pipeline/scorers/embedding.rs b/crates/core/src/ranking/pipeline/scorers/embedding.rs
index f68527e5..7b448063 100644
--- a/crates/core/src/ranking/pipeline/scorers/embedding.rs
+++ b/crates/core/src/ranking/pipeline/scorers/embedding.rs
@@ -164,7 +164,7 @@ pub trait EmbeddingSignal<W>: Send + Sync {
impl EmbeddingSignal<ScoredWebpagePointer> for TitleEmbeddings {
fn signal() -> SignalEnum {
- ranking::signal::TitleEmbeddingSimilarity.into()
+ ranking::core::TitleEmbeddingSimilarity.into()
}
fn has_embedding(webpage: &ScoredWebpagePointer) -> bool {
@@ -183,7 +183,7 @@ impl EmbeddingSignal<ScoredWebpagePointer> for TitleEmbeddings {
impl EmbeddingSignal<LocalRecallRankingWebpage> for TitleEmbeddings {
fn signal() -> SignalEnum {
- ranking::signal::TitleEmbeddingSimilarity.into()
+ ranking::core::TitleEmbeddingSimilarity.into()
}
fn has_embedding(webpage: &LocalRecallRankingWebpage) -> bool {
@@ -203,7 +203,7 @@ impl EmbeddingSignal<LocalRecallRankingWebpage> for TitleEmbeddings {
impl EmbeddingSignal<ScoredWebpagePointer> for KeywordEmbeddings {
fn signal() -> SignalEnum {
- ranking::signal::KeywordEmbeddingSimilarity.into()
+ ranking::core::KeywordEmbeddingSimilarity.into()
}
fn has_embedding(webpage: &ScoredWebpagePointer) -> bool {
@@ -222,7 +222,7 @@ impl EmbeddingSignal<ScoredWebpagePointer> for KeywordEmbeddings {
impl EmbeddingSignal<LocalRecallRankingWebpage> for KeywordEmbeddings {
fn signal() -> SignalEnum {
- ranking::signal::KeywordEmbeddingSimilarity.into()
+ ranking::core::KeywordEmbeddingSimilarity.into()
}
fn has_embedding(webpage: &LocalRecallRankingWebpage) -> bool {
diff --git a/crates/core/src/ranking/pipeline/scorers/inbound_similarity.rs b/crates/core/src/ranking/pipeline/scorers/inbound_similarity.rs
index 19ac0852..0ad1e977 100644
--- a/crates/core/src/ranking/pipeline/scorers/inbound_similarity.rs
+++ b/crates/core/src/ranking/pipeline/scorers/inbound_similarity.rs
@@ -17,7 +17,7 @@
use std::sync::Mutex;
use crate::{
- ranking::{inbound_similarity, signal},
+ ranking::{core, inbound_similarity},
searcher::api::ScoredWebpagePointer,
};
@@ -47,7 +47,7 @@ impl Scorer<ScoredWebpagePointer> for InboundScorer {
webpage
.as_ranking_mut()
.signals_mut()
- .insert(signal::InboundSimilarity.into(), score);
+ .insert(core::InboundSimilarity.into(), score);
}
}
}
diff --git a/crates/core/src/ranking/pipeline/scorers/reranker.rs b/crates/core/src/ranking/pipeline/scorers/reranker.rs
index f5f79176..5eefd5ca 100644
--- a/crates/core/src/ranking/pipeline/scorers/reranker.rs
+++ b/crates/core/src/ranking/pipeline/scorers/reranker.rs
@@ -54,12 +54,12 @@ impl<M: CrossEncoder> ReRanker<M> {
webpage
.ranking_mut()
.signals_mut()
- .insert(ranking::signal::CrossEncoderSnippet.into(), snippet);
+ .insert(ranking::core::CrossEncoderSnippet.into(), snippet);
webpage
.ranking_mut()
.signals_mut()
- .insert(ranking::signal::CrossEncoderTitle.into(), title);
+ .insert(ranking::core::CrossEncoderTitle.into(), title);
}
}
}