web search done right
update signal coefficients
Mikkel Denker24f0ab4cf1b1
diff --git a/crates/core/src/ranking/signal/non_text.rs b/crates/core/src/ranking/signal/non_text.rsindex 35be0344..2c0d5bb9 100644--- a/crates/core/src/ranking/signal/non_text.rs+++ b/crates/core/src/ranking/signal/non_text.rs@@ -108,7 +108,7 @@ fn score_region(webpage_region: crate::webpage::Region, computer: &SignalComputepub struct HostCentrality;impl Signal for HostCentrality {fn default_coefficient(&self) -> f64 {- 2.0+ 2.9659120948746986}fn as_field(&self) -> Option<Field> {@@ -148,7 +148,7 @@ impl Signal for HostCentrality {pub struct HostCentralityRank;impl Signal for HostCentralityRank {fn default_coefficient(&self) -> f64 {- 0.02+ 9.143798189215966}fn as_field(&self) -> Option<Field> {@@ -188,7 +188,7 @@ impl Signal for HostCentralityRank {pub struct PageCentrality;impl Signal for PageCentrality {fn default_coefficient(&self) -> f64 {- 2.0+ 0.7649287808734062}fn as_field(&self) -> Option<Field> {@@ -228,7 +228,7 @@ impl Signal for PageCentrality {pub struct PageCentralityRank;impl Signal for PageCentralityRank {fn default_coefficient(&self) -> f64 {- 0.02+ 5.21833938812728}fn as_field(&self) -> Option<Field> {@@ -268,7 +268,7 @@ impl Signal for PageCentralityRank {pub struct IsHomepage;impl Signal for IsHomepage {fn default_coefficient(&self) -> f64 {- 0.01+ 9.266952696093686}fn as_field(&self) -> Option<Field> {@@ -311,7 +311,7 @@ impl Signal for IsHomepage {pub struct FetchTimeMs;impl Signal for FetchTimeMs {fn default_coefficient(&self) -> f64 {- 0.001+ 5.116256200778887}fn as_field(&self) -> Option<Field> {@@ -361,7 +361,7 @@ impl Signal for FetchTimeMs {pub struct UpdateTimestamp;impl Signal for UpdateTimestamp {fn default_coefficient(&self) -> f64 {- 0.001+ 7.287059491944903}fn as_field(&self) -> Option<Field> {@@ -408,7 +408,7 @@ impl Signal for UpdateTimestamp {pub struct TrackerScore;impl Signal for TrackerScore {fn default_coefficient(&self) -> f64 {- 0.1+ 6.849809317789405}fn as_field(&self) -> Option<Field> {@@ -449,7 +449,7 @@ impl Signal for TrackerScore {pub struct Region;impl Signal for Region {fn default_coefficient(&self) -> f64 {- 0.15+ 2.1051568804700302}fn as_field(&self) -> Option<Field> {@@ -571,7 +571,7 @@ impl Signal for LambdaMart {pub struct UrlDigits;impl Signal for UrlDigits {fn default_coefficient(&self) -> f64 {- 0.01+ 3.3068818007086835}fn as_field(&self) -> Option<Field> {@@ -627,7 +627,7 @@ impl Signal for UrlDigits {pub struct UrlSlashes;impl Signal for UrlSlashes {fn default_coefficient(&self) -> f64 {- 0.1+ 7.059663870068346}fn as_field(&self) -> Option<Field> {diff --git a/crates/core/src/ranking/signal/text.rs b/crates/core/src/ranking/signal/text.rsindex 6ffb05bf..0fbd2beb 100644--- a/crates/core/src/ranking/signal/text.rs+++ b/crates/core/src/ranking/signal/text.rs@@ -35,7 +35,7 @@ use super::{Signal, SignalComputer};pub struct Bm25F;impl Signal for Bm25F {fn default_coefficient(&self) -> f64 {- 0.1+ 0.1820132194220756}fn as_field(&self) -> Option<Field> {@@ -102,7 +102,7 @@ impl Signal for Bm25Title {pub struct Bm25TitleBigrams;impl Signal for Bm25TitleBigrams {fn default_coefficient(&self) -> f64 {- 0.005+ 8.891498184948324}fn as_field(&self) -> Option<Field> {@@ -134,7 +134,7 @@ impl Signal for Bm25TitleBigrams {pub struct Bm25TitleTrigrams;impl Signal for Bm25TitleTrigrams {fn default_coefficient(&self) -> f64 {- 0.005+ 7.271941728505791}fn as_field(&self) -> Option<Field> {@@ -166,7 +166,7 @@ impl Signal for Bm25TitleTrigrams {pub struct Bm25CleanBody;impl Signal for Bm25CleanBody {fn default_coefficient(&self) -> f64 {- 0.005+ 6.176695902218496}fn as_field(&self) -> Option<Field> {@@ -198,7 +198,7 @@ impl Signal for Bm25CleanBody {pub struct Bm25CleanBodyBigrams;impl Signal for Bm25CleanBodyBigrams {fn default_coefficient(&self) -> f64 {- 0.005+ 2.2352412543414357}fn as_field(&self) -> Option<Field> {@@ -230,7 +230,7 @@ impl Signal for Bm25CleanBodyBigrams {pub struct Bm25CleanBodyTrigrams;impl Signal for Bm25CleanBodyTrigrams {fn default_coefficient(&self) -> f64 {- 0.005+ 6.800453858962986}fn as_field(&self) -> Option<Field> {@@ -262,7 +262,7 @@ impl Signal for Bm25CleanBodyTrigrams {pub struct Bm25StemmedTitle;impl Signal for Bm25StemmedTitle {fn default_coefficient(&self) -> f64 {- 0.003+ 5.747253223706083}fn as_field(&self) -> Option<Field> {@@ -294,7 +294,7 @@ impl Signal for Bm25StemmedTitle {pub struct Bm25StemmedCleanBody;impl Signal for Bm25StemmedCleanBody {fn default_coefficient(&self) -> f64 {- 0.001+ 2.77296645471509}fn as_field(&self) -> Option<Field> {@@ -358,7 +358,7 @@ impl Signal for Bm25AllBody {pub struct Bm25Keywords;impl Signal for Bm25Keywords {fn default_coefficient(&self) -> f64 {- 0.001+ 2.385257402981021}fn as_field(&self) -> Option<Field> {@@ -390,7 +390,7 @@ impl Signal for Bm25Keywords {pub struct Bm25BacklinkText;impl Signal for Bm25BacklinkText {fn default_coefficient(&self) -> f64 {- 0.003+ 0.15795085970840717}fn as_field(&self) -> Option<Field> {@@ -422,7 +422,7 @@ impl Signal for Bm25BacklinkText {pub struct IdfSumUrl;impl Signal for IdfSumUrl {fn default_coefficient(&self) -> f64 {- 0.0006+ 4.829485047194098}fn as_field(&self) -> Option<Field> {@@ -454,7 +454,7 @@ impl Signal for IdfSumUrl {pub struct IdfSumSite;impl Signal for IdfSumSite {fn default_coefficient(&self) -> f64 {- 0.00015+ 7.160439598002364}fn as_field(&self) -> Option<Field> {@@ -486,7 +486,7 @@ impl Signal for IdfSumSite {pub struct IdfSumDomain;impl Signal for IdfSumDomain {fn default_coefficient(&self) -> f64 {- 0.0003+ 1.9285185409464924}fn as_field(&self) -> Option<Field> {@@ -518,7 +518,7 @@ impl Signal for IdfSumDomain {pub struct IdfSumSiteNoTokenizer;impl Signal for IdfSumSiteNoTokenizer {fn default_coefficient(&self) -> f64 {- 0.00015+ 1.2430639379064399}fn as_field(&self) -> Option<Field> {@@ -550,7 +550,7 @@ impl Signal for IdfSumSiteNoTokenizer {pub struct IdfSumDomainNoTokenizer;impl Signal for IdfSumDomainNoTokenizer {fn default_coefficient(&self) -> f64 {- 0.0036+ 0.7742207284655196}fn as_field(&self) -> Option<Field> {@@ -582,7 +582,7 @@ impl Signal for IdfSumDomainNoTokenizer {pub struct IdfSumDomainNameNoTokenizer;impl Signal for IdfSumDomainNameNoTokenizer {fn default_coefficient(&self) -> f64 {- 0.0002+ 0.49281735909508484}fn as_field(&self) -> Option<Field> {@@ -616,7 +616,7 @@ impl Signal for IdfSumDomainNameNoTokenizer {pub struct IdfSumDomainIfHomepage;impl Signal for IdfSumDomainIfHomepage {fn default_coefficient(&self) -> f64 {- 0.0004+ 4.849842977684016}fn as_field(&self) -> Option<Field> {@@ -648,7 +648,7 @@ impl Signal for IdfSumDomainIfHomepage {pub struct IdfSumDomainNameIfHomepageNoTokenizer;impl Signal for IdfSumDomainNameIfHomepageNoTokenizer {fn default_coefficient(&self) -> f64 {- 0.0036+ 5.122026248358028}fn as_field(&self) -> Option<Field> {@@ -682,7 +682,7 @@ impl Signal for IdfSumDomainNameIfHomepageNoTokenizer {pub struct IdfSumDomainIfHomepageNoTokenizer;impl Signal for IdfSumDomainIfHomepageNoTokenizer {fn default_coefficient(&self) -> f64 {- 0.0036+ 7.974638346224731}fn as_field(&self) -> Option<Field> {@@ -716,7 +716,7 @@ impl Signal for IdfSumDomainIfHomepageNoTokenizer {pub struct IdfSumTitleIfHomepage;impl Signal for IdfSumTitleIfHomepage {fn default_coefficient(&self) -> f64 {- 0.001+ 9.299807604028546}fn as_field(&self) -> Option<Field> {diff --git a/ltr/lambdamart.py b/ltr/lambdamart.pyindex c22b4834..cca2022f 100644--- a/ltr/lambdamart.py+++ b/ltr/lambdamart.py@@ -18,7 +18,7 @@ param_grid = {# "max_depth": [-1, 2, 4, 8],"max_depth": [-1],# "num_leaves": [7, 15, 31],- "num_leaves": [63],+ "num_leaves": [31, 63],"lambda_l2": [2.5],"linear_tree": [False],}diff --git a/ltr/leechy_annotate.py b/ltr/leechy_annotate.pyindex 45d5e8bf..4d93b54c 100644--- a/ltr/leechy_annotate.py+++ b/ltr/leechy_annotate.py@@ -76,7 +76,7 @@ for qid, query in tqdm(unannotated_queries.items()):db.annotate(qid, result, label)for page in range(2, 2 + 1):- bad_results = stract.search(query, page)+ bad_results = stract.search(query, page=page)for i, result in enumerate(bad_results):if result["url"] in leechy_results: