Skip to content

web search done right

update signal coefficients

Mikkel Denker24f0ab4cf1b1
diff --git a/crates/core/src/ranking/signal/non_text.rs b/crates/core/src/ranking/signal/non_text.rs
index 35be0344..2c0d5bb9 100644
--- a/crates/core/src/ranking/signal/non_text.rs
+++ b/crates/core/src/ranking/signal/non_text.rs
@@ -108,7 +108,7 @@ fn score_region(webpage_region: crate::webpage::Region, computer: &SignalCompute
pub struct HostCentrality;
impl Signal for HostCentrality {
fn default_coefficient(&self) -> f64 {
- 2.0
+ 2.9659120948746986
}
fn as_field(&self) -> Option<Field> {
@@ -148,7 +148,7 @@ impl Signal for HostCentrality {
pub struct HostCentralityRank;
impl Signal for HostCentralityRank {
fn default_coefficient(&self) -> f64 {
- 0.02
+ 9.143798189215966
}
fn as_field(&self) -> Option<Field> {
@@ -188,7 +188,7 @@ impl Signal for HostCentralityRank {
pub struct PageCentrality;
impl Signal for PageCentrality {
fn default_coefficient(&self) -> f64 {
- 2.0
+ 0.7649287808734062
}
fn as_field(&self) -> Option<Field> {
@@ -228,7 +228,7 @@ impl Signal for PageCentrality {
pub struct PageCentralityRank;
impl Signal for PageCentralityRank {
fn default_coefficient(&self) -> f64 {
- 0.02
+ 5.21833938812728
}
fn as_field(&self) -> Option<Field> {
@@ -268,7 +268,7 @@ impl Signal for PageCentralityRank {
pub struct IsHomepage;
impl Signal for IsHomepage {
fn default_coefficient(&self) -> f64 {
- 0.01
+ 9.266952696093686
}
fn as_field(&self) -> Option<Field> {
@@ -311,7 +311,7 @@ impl Signal for IsHomepage {
pub struct FetchTimeMs;
impl Signal for FetchTimeMs {
fn default_coefficient(&self) -> f64 {
- 0.001
+ 5.116256200778887
}
fn as_field(&self) -> Option<Field> {
@@ -361,7 +361,7 @@ impl Signal for FetchTimeMs {
pub struct UpdateTimestamp;
impl Signal for UpdateTimestamp {
fn default_coefficient(&self) -> f64 {
- 0.001
+ 7.287059491944903
}
fn as_field(&self) -> Option<Field> {
@@ -408,7 +408,7 @@ impl Signal for UpdateTimestamp {
pub struct TrackerScore;
impl Signal for TrackerScore {
fn default_coefficient(&self) -> f64 {
- 0.1
+ 6.849809317789405
}
fn as_field(&self) -> Option<Field> {
@@ -449,7 +449,7 @@ impl Signal for TrackerScore {
pub struct Region;
impl Signal for Region {
fn default_coefficient(&self) -> f64 {
- 0.15
+ 2.1051568804700302
}
fn as_field(&self) -> Option<Field> {
@@ -571,7 +571,7 @@ impl Signal for LambdaMart {
pub struct UrlDigits;
impl Signal for UrlDigits {
fn default_coefficient(&self) -> f64 {
- 0.01
+ 3.3068818007086835
}
fn as_field(&self) -> Option<Field> {
@@ -627,7 +627,7 @@ impl Signal for UrlDigits {
pub struct UrlSlashes;
impl Signal for UrlSlashes {
fn default_coefficient(&self) -> f64 {
- 0.1
+ 7.059663870068346
}
fn as_field(&self) -> Option<Field> {
diff --git a/crates/core/src/ranking/signal/text.rs b/crates/core/src/ranking/signal/text.rs
index 6ffb05bf..0fbd2beb 100644
--- a/crates/core/src/ranking/signal/text.rs
+++ b/crates/core/src/ranking/signal/text.rs
@@ -35,7 +35,7 @@ use super::{Signal, SignalComputer};
pub struct Bm25F;
impl Signal for Bm25F {
fn default_coefficient(&self) -> f64 {
- 0.1
+ 0.1820132194220756
}
fn as_field(&self) -> Option<Field> {
@@ -102,7 +102,7 @@ impl Signal for Bm25Title {
pub struct Bm25TitleBigrams;
impl Signal for Bm25TitleBigrams {
fn default_coefficient(&self) -> f64 {
- 0.005
+ 8.891498184948324
}
fn as_field(&self) -> Option<Field> {
@@ -134,7 +134,7 @@ impl Signal for Bm25TitleBigrams {
pub struct Bm25TitleTrigrams;
impl Signal for Bm25TitleTrigrams {
fn default_coefficient(&self) -> f64 {
- 0.005
+ 7.271941728505791
}
fn as_field(&self) -> Option<Field> {
@@ -166,7 +166,7 @@ impl Signal for Bm25TitleTrigrams {
pub struct Bm25CleanBody;
impl Signal for Bm25CleanBody {
fn default_coefficient(&self) -> f64 {
- 0.005
+ 6.176695902218496
}
fn as_field(&self) -> Option<Field> {
@@ -198,7 +198,7 @@ impl Signal for Bm25CleanBody {
pub struct Bm25CleanBodyBigrams;
impl Signal for Bm25CleanBodyBigrams {
fn default_coefficient(&self) -> f64 {
- 0.005
+ 2.2352412543414357
}
fn as_field(&self) -> Option<Field> {
@@ -230,7 +230,7 @@ impl Signal for Bm25CleanBodyBigrams {
pub struct Bm25CleanBodyTrigrams;
impl Signal for Bm25CleanBodyTrigrams {
fn default_coefficient(&self) -> f64 {
- 0.005
+ 6.800453858962986
}
fn as_field(&self) -> Option<Field> {
@@ -262,7 +262,7 @@ impl Signal for Bm25CleanBodyTrigrams {
pub struct Bm25StemmedTitle;
impl Signal for Bm25StemmedTitle {
fn default_coefficient(&self) -> f64 {
- 0.003
+ 5.747253223706083
}
fn as_field(&self) -> Option<Field> {
@@ -294,7 +294,7 @@ impl Signal for Bm25StemmedTitle {
pub struct Bm25StemmedCleanBody;
impl Signal for Bm25StemmedCleanBody {
fn default_coefficient(&self) -> f64 {
- 0.001
+ 2.77296645471509
}
fn as_field(&self) -> Option<Field> {
@@ -358,7 +358,7 @@ impl Signal for Bm25AllBody {
pub struct Bm25Keywords;
impl Signal for Bm25Keywords {
fn default_coefficient(&self) -> f64 {
- 0.001
+ 2.385257402981021
}
fn as_field(&self) -> Option<Field> {
@@ -390,7 +390,7 @@ impl Signal for Bm25Keywords {
pub struct Bm25BacklinkText;
impl Signal for Bm25BacklinkText {
fn default_coefficient(&self) -> f64 {
- 0.003
+ 0.15795085970840717
}
fn as_field(&self) -> Option<Field> {
@@ -422,7 +422,7 @@ impl Signal for Bm25BacklinkText {
pub struct IdfSumUrl;
impl Signal for IdfSumUrl {
fn default_coefficient(&self) -> f64 {
- 0.0006
+ 4.829485047194098
}
fn as_field(&self) -> Option<Field> {
@@ -454,7 +454,7 @@ impl Signal for IdfSumUrl {
pub struct IdfSumSite;
impl Signal for IdfSumSite {
fn default_coefficient(&self) -> f64 {
- 0.00015
+ 7.160439598002364
}
fn as_field(&self) -> Option<Field> {
@@ -486,7 +486,7 @@ impl Signal for IdfSumSite {
pub struct IdfSumDomain;
impl Signal for IdfSumDomain {
fn default_coefficient(&self) -> f64 {
- 0.0003
+ 1.9285185409464924
}
fn as_field(&self) -> Option<Field> {
@@ -518,7 +518,7 @@ impl Signal for IdfSumDomain {
pub struct IdfSumSiteNoTokenizer;
impl Signal for IdfSumSiteNoTokenizer {
fn default_coefficient(&self) -> f64 {
- 0.00015
+ 1.2430639379064399
}
fn as_field(&self) -> Option<Field> {
@@ -550,7 +550,7 @@ impl Signal for IdfSumSiteNoTokenizer {
pub struct IdfSumDomainNoTokenizer;
impl Signal for IdfSumDomainNoTokenizer {
fn default_coefficient(&self) -> f64 {
- 0.0036
+ 0.7742207284655196
}
fn as_field(&self) -> Option<Field> {
@@ -582,7 +582,7 @@ impl Signal for IdfSumDomainNoTokenizer {
pub struct IdfSumDomainNameNoTokenizer;
impl Signal for IdfSumDomainNameNoTokenizer {
fn default_coefficient(&self) -> f64 {
- 0.0002
+ 0.49281735909508484
}
fn as_field(&self) -> Option<Field> {
@@ -616,7 +616,7 @@ impl Signal for IdfSumDomainNameNoTokenizer {
pub struct IdfSumDomainIfHomepage;
impl Signal for IdfSumDomainIfHomepage {
fn default_coefficient(&self) -> f64 {
- 0.0004
+ 4.849842977684016
}
fn as_field(&self) -> Option<Field> {
@@ -648,7 +648,7 @@ impl Signal for IdfSumDomainIfHomepage {
pub struct IdfSumDomainNameIfHomepageNoTokenizer;
impl Signal for IdfSumDomainNameIfHomepageNoTokenizer {
fn default_coefficient(&self) -> f64 {
- 0.0036
+ 5.122026248358028
}
fn as_field(&self) -> Option<Field> {
@@ -682,7 +682,7 @@ impl Signal for IdfSumDomainNameIfHomepageNoTokenizer {
pub struct IdfSumDomainIfHomepageNoTokenizer;
impl Signal for IdfSumDomainIfHomepageNoTokenizer {
fn default_coefficient(&self) -> f64 {
- 0.0036
+ 7.974638346224731
}
fn as_field(&self) -> Option<Field> {
@@ -716,7 +716,7 @@ impl Signal for IdfSumDomainIfHomepageNoTokenizer {
pub struct IdfSumTitleIfHomepage;
impl Signal for IdfSumTitleIfHomepage {
fn default_coefficient(&self) -> f64 {
- 0.001
+ 9.299807604028546
}
fn as_field(&self) -> Option<Field> {
diff --git a/ltr/lambdamart.py b/ltr/lambdamart.py
index c22b4834..cca2022f 100644
--- a/ltr/lambdamart.py
+++ b/ltr/lambdamart.py
@@ -18,7 +18,7 @@ param_grid = {
# "max_depth": [-1, 2, 4, 8],
"max_depth": [-1],
# "num_leaves": [7, 15, 31],
- "num_leaves": [63],
+ "num_leaves": [31, 63],
"lambda_l2": [2.5],
"linear_tree": [False],
}
diff --git a/ltr/leechy_annotate.py b/ltr/leechy_annotate.py
index 45d5e8bf..4d93b54c 100644
--- a/ltr/leechy_annotate.py
+++ b/ltr/leechy_annotate.py
@@ -76,7 +76,7 @@ for qid, query in tqdm(unannotated_queries.items()):
db.annotate(qid, result, label)
for page in range(2, 2 + 1):
- bad_results = stract.search(query, page)
+ bad_results = stract.search(query, page=page)
for i, result in enumerate(bad_results):
if result["url"] in leechy_results: