Skip to content

web search done right

update ranking signal defaults

Mikkel Denker23e1f95914f2
diff --git a/core/src/ranking/signal.rs b/core/src/ranking/signal.rs
index da448e69..d3799524 100644
--- a/core/src/ranking/signal.rs
+++ b/core/src/ranking/signal.rs
@@ -240,33 +240,33 @@ impl Signal {
pub fn default_coefficient(&self) -> f64 {
match self {
Signal::Bm25Title => 0.0063,
- Signal::Bm25TitleBigrams => 0.0054,
- Signal::Bm25TitleTrigrams => 0.0054,
+ Signal::Bm25TitleBigrams => 0.01,
+ Signal::Bm25TitleTrigrams => 0.01,
Signal::Bm25CleanBody => 0.0063,
- Signal::Bm25CleanBodyBigrams => 0.0025,
- Signal::Bm25CleanBodyTrigrams => 0.0025,
+ Signal::Bm25CleanBodyBigrams => 0.005,
+ Signal::Bm25CleanBodyTrigrams => 0.005,
Signal::Bm25StemmedTitle => 0.003,
Signal::Bm25StemmedCleanBody => 0.001,
Signal::Bm25AllBody => 0.0,
- Signal::Bm25Url => 0.00015,
+ Signal::Bm25Url => 0.0003,
Signal::Bm25Site => 0.00015,
- Signal::Bm25Domain => 0.00015,
+ Signal::Bm25Domain => 0.0003,
Signal::Bm25SiteNoTokenizer => 0.00015,
- Signal::Bm25DomainNoTokenizer => 0.0001,
- Signal::Bm25DomainNameNoTokenizer => 0.0001,
- Signal::Bm25DomainIfHomepage => 0.0002,
- Signal::Bm25DomainNameIfHomepageNoTokenizer => 0.0018,
- Signal::Bm25DomainIfHomepageNoTokenizer => 0.0018,
+ Signal::Bm25DomainNoTokenizer => 0.0002,
+ Signal::Bm25DomainNameNoTokenizer => 0.0002,
+ Signal::Bm25DomainIfHomepage => 0.0004,
+ Signal::Bm25DomainNameIfHomepageNoTokenizer => 0.0036,
+ Signal::Bm25DomainIfHomepageNoTokenizer => 0.0036,
Signal::Bm25TitleIfHomepage => 0.00022,
- Signal::Bm25BacklinkText => 0.004,
+ Signal::Bm25BacklinkText => 0.003,
Signal::CrossEncoder => 0.17,
- Signal::HostCentrality => 2.5,
- Signal::PageCentrality => 1.0,
+ Signal::HostCentrality => 0.5,
+ Signal::PageCentrality => 0.25,
Signal::QueryCentrality => 0.0,
Signal::IsHomepage => 0.0005,
Signal::FetchTimeMs => 0.001,
Signal::UpdateTimestamp => 0.001,
- Signal::TrackerScore => 0.02,
+ Signal::TrackerScore => 0.05,
Signal::Region => 0.01,
Signal::InboundSimilarity => 0.25,
Signal::LambdaMART => 10.0,
diff --git a/core/src/web_spell/stupid_backoff.rs b/core/src/web_spell/stupid_backoff.rs
index f65df34d..ecb07c82 100644
--- a/core/src/web_spell/stupid_backoff.rs
+++ b/core/src/web_spell/stupid_backoff.rs
@@ -353,3 +353,29 @@ impl NextWordsStrategy for IntoMiddle {
res
}
}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use crate::gen_temp_path;
+
+ #[test]
+ fn test_contexts() {
+ let mut trainer = StupidBackoffTrainer::new(3);
+
+ trainer.train(&tokenize(
+ "a b c d e f g h i j k l m n o p q r s t u v w x y z",
+ ));
+
+ let path = gen_temp_path();
+
+ trainer.build(&path).unwrap();
+
+ let model = StupidBackoff::open(&path).unwrap();
+
+ assert_eq!(
+ model.contexts("b"),
+ vec![(vec!["a".to_string(), "b".to_string(), "c".to_string()], 1)]
+ );
+ }
+}
diff --git a/core/src/web_spell/trainer.rs b/core/src/web_spell/trainer.rs
index 26b18f43..91865b99 100644
--- a/core/src/web_spell/trainer.rs
+++ b/core/src/web_spell/trainer.rs
@@ -138,7 +138,7 @@ impl SecondTrainer {
.filter(|correction| {
correction != &term
&& 10 * self.term_dict.freq(&term).unwrap_or_default()
- > self.term_dict.freq(correction).unwrap_or_default()
+ < self.term_dict.freq(correction).unwrap_or_default()
})
.collect::<Vec<_>>();
diff --git a/ltr/auto_annotate.py b/ltr/auto_annotate.py
index 510c0c63..3097f2db 100644
--- a/ltr/auto_annotate.py
+++ b/ltr/auto_annotate.py
@@ -7,12 +7,12 @@ import json
import time
import requests
-NUM_RESULTS_PER_QUERY = 20
+NUM_RESULTS_PER_QUERY = 50
NUM_ENSEMBLE_PREDS = 1
PROMPT = """<|im_start|>system
Perform the task to the best of your ability.<|im_end|>
<|im_start|>user
-Think about this step-by-step. You task is to evaluate a single search result on an integer scale of 0-4, where a higher score means the result is more relevant. A relevant result should come from a trust-worthy source and answer the users query. The final relevancy should be on a new line on the form "Relevancy: {{}}". It is very important that you think about the steps before giving the final "Relevancy: {{}}" score.
+Think about this step-by-step. You task is to evaluate a single search result on an integer scale of 0-10, where a higher score means the result is more relevant. A relevant result should come from a trust-worthy source and answer the users query. The final relevancy should be on a new line on the form "Relevancy: {{}}". It is very important that you think about the steps before giving the final "Relevancy: {{}}" score.
Query: "{}"
Url: "{}"
@@ -128,15 +128,13 @@ def get_search_results(query):
]
-# add search results to db
-print("Adding search results to db")
-for qid, query in tqdm(unannotated_queries.items()):
+def add_results(query):
has_results = (
cur.execute("SELECT 1 FROM search_results WHERE qid = ?", (qid,)).fetchone()
is not None
)
if has_results:
- continue
+ return
results = get_search_results(query)
time.sleep(1)
@@ -171,6 +169,7 @@ def get_relevancy(res):
for qid, query in tqdm(unannotated_queries.items()):
+ add_results(query)
unnanotated_results = cur.execute(
"""
SELECT url, orig_rank, webpage_json
@@ -189,7 +188,7 @@ for qid, query in tqdm(unannotated_queries.items()):
for _ in range(NUM_ENSEMBLE_PREDS):
output = llm.create_completion(
prompt,
- max_tokens=-1,
+ max_tokens=1024,
echo=False,
temperature=0.4,
stop=["<|im_start|>", "<|im_end|>"],
@@ -205,9 +204,13 @@ for qid, query in tqdm(unannotated_queries.items()):
if n == 0:
print("No relevancy annotation for", query, url)
+ print(output)
continue
+
relevancy = np.round(res / n).astype(int)
- relevancy = max(0, min(4, int(relevancy)))
+ relevancy = max(0, min(10, int(relevancy)))
+
+ print(f'relevancy={relevancy} query="{query}" url={url}')
cur.execute(
"""
diff --git a/ltr/lambdamart.py b/ltr/lambdamart.py
index 6b2bdcc6..8680f440 100644
--- a/ltr/lambdamart.py
+++ b/ltr/lambdamart.py
@@ -67,6 +67,9 @@ queries = new_queries
# Create dataset
items = list(queries.items())
+# shuffle items
+np.random.shuffle(items)
+
train_size = int(len(items) * 0.8)
X_train = []
y_train = []
@@ -105,10 +108,10 @@ model = lgb.LGBMRanker(
objective="lambdarank",
metric="ndcg",
importance_type="gain",
- num_leaves=15,
+ num_leaves=50,
n_estimators=n_estimators,
- max_depth=3,
- learning_rate=0.1,
+ max_depth=10,
+ learning_rate=0.01,
label_gain=[i for i in range(max(y_train.max(), y_test.max()) + 1)],
)
model.fit(
diff --git a/ltr/linear_model.py b/ltr/linear_model.py
index b35319b0..bf913b1f 100644
--- a/ltr/linear_model.py
+++ b/ltr/linear_model.py
@@ -3,54 +3,121 @@ import numpy as np
import json
import random
from sklearn.linear_model import LinearRegression
+from pprint import pprint
TRAIN_PERCENT = 0.8
-with open("ltr_scores.json", "r") as f:
- scores = json.load(f)
+import sqlite3
+
+con = sqlite3.connect("data/auto-ranking-annotation.sqlite")
+cur = con.cursor()
+
+res = cur.execute(
+ """
+ SELECT qid, query
+ FROM queries
+ WHERE EXISTS (
+ SELECT 1 FROM search_results WHERE search_results.qid = queries.qid AND search_results.annotation IS NOT NULL
+ )
+"""
+)
+queries = {qid: {"query": query} for qid, query in res.fetchall()}
+
+for qid in queries:
+ res = cur.execute(
+ """
+ SELECT qid, url, annotation, webpage_json
+ FROM search_results
+ WHERE qid = ?
+ """,
+ (qid,),
+ )
+ urls = {
+ url: {"label": label, "signals": json.loads(page)["rankingSignals"]}
+ for _, url, label, page in res.fetchall()
+ }
+ urls = [
+ (url, w["label"], w["signals"])
+ for url, w in urls.items()
+ if w["label"] is not None
+ ]
+
+ urls = sorted(urls, key=lambda x: int(x[1]), reverse=True)
+ queries[qid]["urls"] = urls
+
+
+feature2id = {}
+id2feature = {}
+
+for qid, data in queries.items():
+ for url, score, signals in data["urls"]:
+ for feature, value in signals.items():
+ if feature not in feature2id:
+ id = len(feature2id)
+ feature2id[feature] = id
+ id2feature[id] = feature
+
+# convert to [{ranking_signals, score}]
+scores = []
random.shuffle(scores)
-feature2id = {k: i for i, (k, _) in enumerate(
- scores[0]['ranking_signals'].items())}
-id2feature = {v: k for k, v in feature2id.items()}
+sorted_features = sorted(feature2id.items(), key=lambda x: x[1])
+
+for qid, data in queries.items():
+ for url, score, signals in data["urls"]:
+ signals = {feature2id[k]: v for k, v in signals.items()}
+
+ signals = [signals[k] for _, k in sorted_features]
+
+ scores.append({"ranking_signals": signals, "score": score})
-train = scores[:int(len(scores) * TRAIN_PERCENT)]
-test = scores[int(len(scores) * TRAIN_PERCENT):]
+
+for score in scores:
+ score["score"] = np.power(2, score["score"])
+
+train = scores[: int(len(scores) * TRAIN_PERCENT)]
+test = scores[int(len(scores) * TRAIN_PERCENT) :]
X_train = []
y_train = []
for score in train:
- X_train.append([score['ranking_signals'][k] for k in feature2id])
- y_train.append(score['score'])
+ X_train.append(score["ranking_signals"])
+ y_train.append(score["score"])
X_test = []
y_test = []
for score in test:
- X_test.append([score['ranking_signals'][k] for k in feature2id])
- y_test.append(score['score'])
+ X_test.append(score["ranking_signals"])
+ y_test.append(score["score"])
X_train = np.array(X_train)
y_train = np.array(y_train)
X_test = np.array(X_test)
y_test = np.array(y_test)
-model = LinearRegression(
- fit_intercept=False, positive=True)
+model = LinearRegression(fit_intercept=False, positive=True)
+# model = SVR(kernel="linear")
+# model = DecisionTreeRegressor()
model.fit(X_train, y_train)
-weights = {id2feature[i]: v for (i, v) in enumerate(model.coef_)}
-linear_model = {'weights': weights}
-with open("data/linear_model.json", "w") as f:
- json.dump(linear_model, f, indent=2)
-
print("TRAIN")
print("score: ", model.score(X_train, y_train))
-print("Mean squared error: ", sklearn.metrics.mean_squared_error(
- y_train, model.predict(X_train)))
+print(
+ "Mean squared error: ",
+ sklearn.metrics.mean_squared_error(y_train, model.predict(X_train)),
+)
print()
print("TEST")
print("score: ", model.score(X_test, y_test))
-print("Mean squared error: ", sklearn.metrics.mean_squared_error(
- y_test, model.predict(X_test)))
+print(
+ "Mean squared error: ",
+ sklearn.metrics.mean_squared_error(y_test, model.predict(X_test)),
+)
+
+weights = {id2feature[i]: v for (i, v) in enumerate(model.coef_)}
+linear_model = {"weights": weights}
+pprint(linear_model)
+with open("data/linear_model.json", "w") as f:
+ json.dump(linear_model, f, indent=2)