web search done right
update ranking signal defaults
Mikkel Denker23e1f95914f2
diff --git a/core/src/ranking/signal.rs b/core/src/ranking/signal.rsindex da448e69..d3799524 100644--- a/core/src/ranking/signal.rs+++ b/core/src/ranking/signal.rs@@ -240,33 +240,33 @@ impl Signal {pub fn default_coefficient(&self) -> f64 {match self {Signal::Bm25Title => 0.0063,- Signal::Bm25TitleBigrams => 0.0054,- Signal::Bm25TitleTrigrams => 0.0054,+ Signal::Bm25TitleBigrams => 0.01,+ Signal::Bm25TitleTrigrams => 0.01,Signal::Bm25CleanBody => 0.0063,- Signal::Bm25CleanBodyBigrams => 0.0025,- Signal::Bm25CleanBodyTrigrams => 0.0025,+ Signal::Bm25CleanBodyBigrams => 0.005,+ Signal::Bm25CleanBodyTrigrams => 0.005,Signal::Bm25StemmedTitle => 0.003,Signal::Bm25StemmedCleanBody => 0.001,Signal::Bm25AllBody => 0.0,- Signal::Bm25Url => 0.00015,+ Signal::Bm25Url => 0.0003,Signal::Bm25Site => 0.00015,- Signal::Bm25Domain => 0.00015,+ Signal::Bm25Domain => 0.0003,Signal::Bm25SiteNoTokenizer => 0.00015,- Signal::Bm25DomainNoTokenizer => 0.0001,- Signal::Bm25DomainNameNoTokenizer => 0.0001,- Signal::Bm25DomainIfHomepage => 0.0002,- Signal::Bm25DomainNameIfHomepageNoTokenizer => 0.0018,- Signal::Bm25DomainIfHomepageNoTokenizer => 0.0018,+ Signal::Bm25DomainNoTokenizer => 0.0002,+ Signal::Bm25DomainNameNoTokenizer => 0.0002,+ Signal::Bm25DomainIfHomepage => 0.0004,+ Signal::Bm25DomainNameIfHomepageNoTokenizer => 0.0036,+ Signal::Bm25DomainIfHomepageNoTokenizer => 0.0036,Signal::Bm25TitleIfHomepage => 0.00022,- Signal::Bm25BacklinkText => 0.004,+ Signal::Bm25BacklinkText => 0.003,Signal::CrossEncoder => 0.17,- Signal::HostCentrality => 2.5,- Signal::PageCentrality => 1.0,+ Signal::HostCentrality => 0.5,+ Signal::PageCentrality => 0.25,Signal::QueryCentrality => 0.0,Signal::IsHomepage => 0.0005,Signal::FetchTimeMs => 0.001,Signal::UpdateTimestamp => 0.001,- Signal::TrackerScore => 0.02,+ Signal::TrackerScore => 0.05,Signal::Region => 0.01,Signal::InboundSimilarity => 0.25,Signal::LambdaMART => 10.0,diff --git a/core/src/web_spell/stupid_backoff.rs b/core/src/web_spell/stupid_backoff.rsindex f65df34d..ecb07c82 100644--- a/core/src/web_spell/stupid_backoff.rs+++ b/core/src/web_spell/stupid_backoff.rs@@ -353,3 +353,29 @@ impl NextWordsStrategy for IntoMiddle {res}}++#[cfg(test)]+mod tests {+ use super::*;+ use crate::gen_temp_path;++ #[test]+ fn test_contexts() {+ let mut trainer = StupidBackoffTrainer::new(3);++ trainer.train(&tokenize(+ "a b c d e f g h i j k l m n o p q r s t u v w x y z",+ ));++ let path = gen_temp_path();++ trainer.build(&path).unwrap();++ let model = StupidBackoff::open(&path).unwrap();++ assert_eq!(+ model.contexts("b"),+ vec![(vec!["a".to_string(), "b".to_string(), "c".to_string()], 1)]+ );+ }+}diff --git a/core/src/web_spell/trainer.rs b/core/src/web_spell/trainer.rsindex 26b18f43..91865b99 100644--- a/core/src/web_spell/trainer.rs+++ b/core/src/web_spell/trainer.rs@@ -138,7 +138,7 @@ impl SecondTrainer {.filter(|correction| {correction != &term&& 10 * self.term_dict.freq(&term).unwrap_or_default()- > self.term_dict.freq(correction).unwrap_or_default()+ < self.term_dict.freq(correction).unwrap_or_default()}).collect::<Vec<_>>();diff --git a/ltr/auto_annotate.py b/ltr/auto_annotate.pyindex 510c0c63..3097f2db 100644--- a/ltr/auto_annotate.py+++ b/ltr/auto_annotate.py@@ -7,12 +7,12 @@ import jsonimport timeimport requests-NUM_RESULTS_PER_QUERY = 20+NUM_RESULTS_PER_QUERY = 50NUM_ENSEMBLE_PREDS = 1PROMPT = """<|im_start|>systemPerform the task to the best of your ability.<|im_end|><|im_start|>user-Think about this step-by-step. You task is to evaluate a single search result on an integer scale of 0-4, where a higher score means the result is more relevant. A relevant result should come from a trust-worthy source and answer the users query. The final relevancy should be on a new line on the form "Relevancy: {{}}". It is very important that you think about the steps before giving the final "Relevancy: {{}}" score.+Think about this step-by-step. You task is to evaluate a single search result on an integer scale of 0-10, where a higher score means the result is more relevant. A relevant result should come from a trust-worthy source and answer the users query. The final relevancy should be on a new line on the form "Relevancy: {{}}". It is very important that you think about the steps before giving the final "Relevancy: {{}}" score.Query: "{}"Url: "{}"@@ -128,15 +128,13 @@ def get_search_results(query):]-# add search results to db-print("Adding search results to db")-for qid, query in tqdm(unannotated_queries.items()):+def add_results(query):has_results = (cur.execute("SELECT 1 FROM search_results WHERE qid = ?", (qid,)).fetchone()is not None)if has_results:- continue+ returnresults = get_search_results(query)time.sleep(1)@@ -171,6 +169,7 @@ def get_relevancy(res):for qid, query in tqdm(unannotated_queries.items()):+ add_results(query)unnanotated_results = cur.execute("""SELECT url, orig_rank, webpage_json@@ -189,7 +188,7 @@ for qid, query in tqdm(unannotated_queries.items()):for _ in range(NUM_ENSEMBLE_PREDS):output = llm.create_completion(prompt,- max_tokens=-1,+ max_tokens=1024,echo=False,temperature=0.4,stop=["<|im_start|>", "<|im_end|>"],@@ -205,9 +204,13 @@ for qid, query in tqdm(unannotated_queries.items()):if n == 0:print("No relevancy annotation for", query, url)+ print(output)continue+relevancy = np.round(res / n).astype(int)- relevancy = max(0, min(4, int(relevancy)))+ relevancy = max(0, min(10, int(relevancy)))++ print(f'relevancy={relevancy} query="{query}" url={url}')cur.execute("""diff --git a/ltr/lambdamart.py b/ltr/lambdamart.pyindex 6b2bdcc6..8680f440 100644--- a/ltr/lambdamart.py+++ b/ltr/lambdamart.py@@ -67,6 +67,9 @@ queries = new_queries# Create datasetitems = list(queries.items())+# shuffle items+np.random.shuffle(items)+train_size = int(len(items) * 0.8)X_train = []y_train = []@@ -105,10 +108,10 @@ model = lgb.LGBMRanker(objective="lambdarank",metric="ndcg",importance_type="gain",- num_leaves=15,+ num_leaves=50,n_estimators=n_estimators,- max_depth=3,- learning_rate=0.1,+ max_depth=10,+ learning_rate=0.01,label_gain=[i for i in range(max(y_train.max(), y_test.max()) + 1)],)model.fit(diff --git a/ltr/linear_model.py b/ltr/linear_model.pyindex b35319b0..bf913b1f 100644--- a/ltr/linear_model.py+++ b/ltr/linear_model.py@@ -3,54 +3,121 @@ import numpy as npimport jsonimport randomfrom sklearn.linear_model import LinearRegression+from pprint import pprintTRAIN_PERCENT = 0.8-with open("ltr_scores.json", "r") as f:- scores = json.load(f)+import sqlite3++con = sqlite3.connect("data/auto-ranking-annotation.sqlite")+cur = con.cursor()++res = cur.execute(+ """+ SELECT qid, query+ FROM queries+ WHERE EXISTS (+ SELECT 1 FROM search_results WHERE search_results.qid = queries.qid AND search_results.annotation IS NOT NULL+ )+"""+)+queries = {qid: {"query": query} for qid, query in res.fetchall()}++for qid in queries:+ res = cur.execute(+ """+ SELECT qid, url, annotation, webpage_json+ FROM search_results+ WHERE qid = ?+ """,+ (qid,),+ )+ urls = {+ url: {"label": label, "signals": json.loads(page)["rankingSignals"]}+ for _, url, label, page in res.fetchall()+ }+ urls = [+ (url, w["label"], w["signals"])+ for url, w in urls.items()+ if w["label"] is not None+ ]++ urls = sorted(urls, key=lambda x: int(x[1]), reverse=True)+ queries[qid]["urls"] = urls+++feature2id = {}+id2feature = {}++for qid, data in queries.items():+ for url, score, signals in data["urls"]:+ for feature, value in signals.items():+ if feature not in feature2id:+ id = len(feature2id)+ feature2id[feature] = id+ id2feature[id] = feature++# convert to [{ranking_signals, score}]+scores = []random.shuffle(scores)-feature2id = {k: i for i, (k, _) in enumerate(- scores[0]['ranking_signals'].items())}-id2feature = {v: k for k, v in feature2id.items()}+sorted_features = sorted(feature2id.items(), key=lambda x: x[1])++for qid, data in queries.items():+ for url, score, signals in data["urls"]:+ signals = {feature2id[k]: v for k, v in signals.items()}++ signals = [signals[k] for _, k in sorted_features]++ scores.append({"ranking_signals": signals, "score": score})-train = scores[:int(len(scores) * TRAIN_PERCENT)]-test = scores[int(len(scores) * TRAIN_PERCENT):]++for score in scores:+ score["score"] = np.power(2, score["score"])++train = scores[: int(len(scores) * TRAIN_PERCENT)]+test = scores[int(len(scores) * TRAIN_PERCENT) :]X_train = []y_train = []for score in train:- X_train.append([score['ranking_signals'][k] for k in feature2id])- y_train.append(score['score'])+ X_train.append(score["ranking_signals"])+ y_train.append(score["score"])X_test = []y_test = []for score in test:- X_test.append([score['ranking_signals'][k] for k in feature2id])- y_test.append(score['score'])+ X_test.append(score["ranking_signals"])+ y_test.append(score["score"])X_train = np.array(X_train)y_train = np.array(y_train)X_test = np.array(X_test)y_test = np.array(y_test)-model = LinearRegression(- fit_intercept=False, positive=True)+model = LinearRegression(fit_intercept=False, positive=True)+# model = SVR(kernel="linear")+# model = DecisionTreeRegressor()model.fit(X_train, y_train)-weights = {id2feature[i]: v for (i, v) in enumerate(model.coef_)}-linear_model = {'weights': weights}-with open("data/linear_model.json", "w") as f:- json.dump(linear_model, f, indent=2)-print("TRAIN")print("score: ", model.score(X_train, y_train))-print("Mean squared error: ", sklearn.metrics.mean_squared_error(- y_train, model.predict(X_train)))+print(+ "Mean squared error: ",+ sklearn.metrics.mean_squared_error(y_train, model.predict(X_train)),+)print()print("TEST")print("score: ", model.score(X_test, y_test))-print("Mean squared error: ", sklearn.metrics.mean_squared_error(- y_test, model.predict(X_test)))+print(+ "Mean squared error: ",+ sklearn.metrics.mean_squared_error(y_test, model.predict(X_test)),+)++weights = {id2feature[i]: v for (i, v) in enumerate(model.coef_)}+linear_model = {"weights": weights}+pprint(linear_model)+with open("data/linear_model.json", "w") as f:+ json.dump(linear_model, f, indent=2)