Skip to content

web search done right

update signal coefficients

Mikkel Denker08dc07c57592
diff --git a/crates/core/src/api/metrics.rs b/crates/core/src/api/metrics.rs
index 9ee20f89..2992cb87 100644
--- a/crates/core/src/api/metrics.rs
+++ b/crates/core/src/api/metrics.rs
@@ -24,5 +24,5 @@ use axum_macros::debug_handler;
pub async fn route(
extract::State(registry): extract::State<Arc<crate::metrics::PrometheusRegistry>>,
) -> impl IntoResponse {
- format!("{registry}");
+ registry.to_string()
}
diff --git a/crates/core/src/entrypoint/indexer/worker.rs b/crates/core/src/entrypoint/indexer/worker.rs
index 88599ff9..72b35193 100644
--- a/crates/core/src/entrypoint/indexer/worker.rs
+++ b/crates/core/src/entrypoint/indexer/worker.rs
@@ -38,7 +38,7 @@ use crate::ranking::SignalComputer;
use crate::webgraph::{self, Edge, EdgeLimit, Node, NodeID};
use crate::webpage::{safety_classifier, Html, Webpage};
-const MAX_BACKLINKS: EdgeLimit = EdgeLimit::Limit(512);
+const MAX_BACKLINKS: EdgeLimit = EdgeLimit::Limit(1024);
pub struct Config {
pub host_centrality_store_path: String,
diff --git a/crates/core/src/entrypoint/webgraph.rs b/crates/core/src/entrypoint/webgraph.rs
index 7713f405..946036ae 100644
--- a/crates/core/src/entrypoint/webgraph.rs
+++ b/crates/core/src/entrypoint/webgraph.rs
@@ -28,11 +28,6 @@ use std::{path::Path, sync::Arc};
use tokio::pin;
use tracing::{info, trace};
-#[derive(Debug, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, Clone)]
-struct GraphPointer {
- path: String,
-}
-
#[derive(Debug, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, Clone)]
pub enum JobConfig {
Http(config::HttpConfig),
diff --git a/crates/core/src/inverted_index/key_phrase.rs b/crates/core/src/inverted_index/key_phrase.rs
index 5b5ad38e..afab83ee 100644
--- a/crates/core/src/inverted_index/key_phrase.rs
+++ b/crates/core/src/inverted_index/key_phrase.rs
@@ -174,6 +174,13 @@ impl KeyPhrase {
continue;
}
+ let left_paren = term_str.chars().filter(|c| c == &'(').count();
+ let right_paren = term_str.chars().filter(|c| c == &')').count();
+
+ if left_paren != right_paren {
+ continue;
+ }
+
let words = term_str.split_whitespace().collect::<Vec<_>>();
let score = scorer.score(&words, info.doc_freq);
diff --git a/crates/core/src/models/bert.rs b/crates/core/src/models/bert.rs
index cff96a6a..4fdf3df0 100644
--- a/crates/core/src/models/bert.rs
+++ b/crates/core/src/models/bert.rs
@@ -431,7 +431,7 @@ struct BertEncoder {
impl BertEncoder {
fn load(vb: VarBuilder, config: &Config) -> Result<Self> {
let layers = (0..config.num_hidden_layers)
- .map(|index| BertLayer::load(vb.pp(&format!("layer.{index}")), config))
+ .map(|index| BertLayer::load(vb.pp(format!("layer.{index}")), config))
.collect::<Result<Vec<_>>>()?;
let span = tracing::span!(tracing::Level::TRACE, "encoder");
Ok(BertEncoder { layers, span })
@@ -488,8 +488,8 @@ impl BertModel {
(Err(err), _) | (_, Err(err)) => {
if let Some(model_type) = &config.model_type {
if let (Ok(embeddings), Ok(encoder)) = (
- BertEmbeddings::load(vb.pp(&format!("{model_type}.embeddings")), config),
- BertEncoder::load(vb.pp(&format!("{model_type}.encoder")), config),
+ BertEmbeddings::load(vb.pp(format!("{model_type}.embeddings")), config),
+ BertEncoder::load(vb.pp(format!("{model_type}.encoder")), config),
) {
(embeddings, encoder)
} else {
@@ -504,7 +504,7 @@ impl BertModel {
let pooler = BertPooler::load(vb.pp("pooler"), config)
.or_else(|e| {
if let Some(model_type) = &config.model_type {
- BertPooler::load(vb.pp(&format!("{model_type}.pooler")), config)
+ BertPooler::load(vb.pp(format!("{model_type}.pooler")), config)
} else {
Err(e)
}
diff --git a/crates/core/src/ranking/signal/non_text.rs b/crates/core/src/ranking/signal/non_text.rs
index cb66611f..92cfe930 100644
--- a/crates/core/src/ranking/signal/non_text.rs
+++ b/crates/core/src/ranking/signal/non_text.rs
@@ -39,7 +39,14 @@ fn score_timestamp(page_timestamp: usize, signal_computer: &SignalComputer) -> f
#[inline]
fn score_rank(rank: f64) -> f64 {
- 1.0 / (rank + 1.0)
+ // 10 groups with log base 8 gives us
+ // 1.1 billion ranks we can score without
+ // exceeding the groups.
+
+ const NUM_GROUPS: f64 = 10.0;
+ const BASE: f64 = 8.0;
+
+ (NUM_GROUPS - (1.0 + rank).log(BASE)).max(NUM_GROUPS)
}
#[inline]
@@ -101,7 +108,7 @@ fn score_region(webpage_region: crate::webpage::Region, computer: &SignalCompute
pub struct HostCentrality;
impl Signal for HostCentrality {
fn default_coefficient(&self) -> f64 {
- 0.5
+ 2.0
}
fn as_field(&self) -> Option<Field> {
@@ -141,7 +148,7 @@ impl Signal for HostCentrality {
pub struct HostCentralityRank;
impl Signal for HostCentralityRank {
fn default_coefficient(&self) -> f64 {
- 0.0
+ 0.4
}
fn as_field(&self) -> Option<Field> {
@@ -181,7 +188,7 @@ impl Signal for HostCentralityRank {
pub struct PageCentrality;
impl Signal for PageCentrality {
fn default_coefficient(&self) -> f64 {
- 0.25
+ 2.0
}
fn as_field(&self) -> Option<Field> {
@@ -221,7 +228,7 @@ impl Signal for PageCentrality {
pub struct PageCentralityRank;
impl Signal for PageCentralityRank {
fn default_coefficient(&self) -> f64 {
- 0.0
+ 0.4
}
fn as_field(&self) -> Option<Field> {
@@ -261,7 +268,7 @@ impl Signal for PageCentralityRank {
pub struct IsHomepage;
impl Signal for IsHomepage {
fn default_coefficient(&self) -> f64 {
- 0.0005
+ 0.01
}
fn as_field(&self) -> Option<Field> {
@@ -401,7 +408,7 @@ impl Signal for UpdateTimestamp {
pub struct TrackerScore;
impl Signal for TrackerScore {
fn default_coefficient(&self) -> f64 {
- 0.05
+ 0.1
}
fn as_field(&self) -> Option<Field> {
@@ -620,7 +627,7 @@ impl Signal for UrlDigits {
pub struct UrlSlashes;
impl Signal for UrlSlashes {
fn default_coefficient(&self) -> f64 {
- 0.01
+ 0.1
}
fn as_field(&self) -> Option<Field> {
diff --git a/crates/core/src/ranking/signal/text.rs b/crates/core/src/ranking/signal/text.rs
index f9e9884f..6ffb05bf 100644
--- a/crates/core/src/ranking/signal/text.rs
+++ b/crates/core/src/ranking/signal/text.rs
@@ -102,7 +102,7 @@ impl Signal for Bm25Title {
pub struct Bm25TitleBigrams;
impl Signal for Bm25TitleBigrams {
fn default_coefficient(&self) -> f64 {
- 0.01
+ 0.005
}
fn as_field(&self) -> Option<Field> {
@@ -134,7 +134,7 @@ impl Signal for Bm25TitleBigrams {
pub struct Bm25TitleTrigrams;
impl Signal for Bm25TitleTrigrams {
fn default_coefficient(&self) -> f64 {
- 0.01
+ 0.005
}
fn as_field(&self) -> Option<Field> {
@@ -166,7 +166,7 @@ impl Signal for Bm25TitleTrigrams {
pub struct Bm25CleanBody;
impl Signal for Bm25CleanBody {
fn default_coefficient(&self) -> f64 {
- 0.0063
+ 0.005
}
fn as_field(&self) -> Option<Field> {
@@ -422,7 +422,7 @@ impl Signal for Bm25BacklinkText {
pub struct IdfSumUrl;
impl Signal for IdfSumUrl {
fn default_coefficient(&self) -> f64 {
- 0.0003
+ 0.0006
}
fn as_field(&self) -> Option<Field> {
@@ -550,7 +550,7 @@ impl Signal for IdfSumSiteNoTokenizer {
pub struct IdfSumDomainNoTokenizer;
impl Signal for IdfSumDomainNoTokenizer {
fn default_coefficient(&self) -> f64 {
- 0.0002
+ 0.0036
}
fn as_field(&self) -> Option<Field> {
@@ -716,7 +716,7 @@ impl Signal for IdfSumDomainIfHomepageNoTokenizer {
pub struct IdfSumTitleIfHomepage;
impl Signal for IdfSumTitleIfHomepage {
fn default_coefficient(&self) -> f64 {
- 0.00022
+ 0.001
}
fn as_field(&self) -> Option<Field> {
diff --git a/crates/core/src/webgraph/store.rs b/crates/core/src/webgraph/store.rs
index bb31caa0..2c140643 100644
--- a/crates/core/src/webgraph/store.rs
+++ b/crates/core/src/webgraph/store.rs
@@ -37,23 +37,6 @@ use super::{
Compression, EdgeLimit, FullNodeID, NodeDatum, NodeID, SegmentEdge, StoredEdge,
};
-#[derive(
- Debug,
- Clone,
- PartialEq,
- Eq,
- Hash,
- serde::Serialize,
- serde::Deserialize,
- bincode::Encode,
- bincode::Decode,
-)]
-struct SerializedEdge {
- from_host: NodeID,
- to_host: NodeID,
- label: Vec<u8>,
-}
-
pub struct HostDb {
db: speedy_kv::Db<Vec<u8>, ()>,
}
diff --git a/crates/kuchiki/src/iter.rs b/crates/kuchiki/src/iter.rs
index 83279d90..fa838386 100644
--- a/crates/kuchiki/src/iter.rs
+++ b/crates/kuchiki/src/iter.rs
@@ -231,9 +231,8 @@ impl Iterator for Ancestors {
#[inline]
fn next(&mut self) -> Option<NodeRef> {
- self.0.take().map(|node| {
+ self.0.take().inspect(|node| {
self.0 = node.parent();
- node
})
}
}
diff --git a/crates/tantivy/src/tokenizer/ngram_tokenizer.rs b/crates/tantivy/src/tokenizer/ngram_tokenizer.rs
index 691a6ed0..286e26c1 100644
--- a/crates/tantivy/src/tokenizer/ngram_tokenizer.rs
+++ b/crates/tantivy/src/tokenizer/ngram_tokenizer.rs
@@ -290,7 +290,7 @@ impl<'a> Iterator for CodepointFrontiers<'a> {
type Item = usize;
fn next(&mut self) -> Option<usize> {
- self.next_el.map(|offset| {
+ self.next_el.inspect(|offset| {
if self.s.is_empty() {
self.next_el = None;
} else {
@@ -298,7 +298,6 @@ impl<'a> Iterator for CodepointFrontiers<'a> {
self.s = &self.s[first_codepoint_width..];
self.next_el = Some(offset + first_codepoint_width);
}
- offset
})
}
}
diff --git a/ltr/auto_annotate.py b/ltr/auto_annotate.py
index c8a0132a..61c1d054 100644
--- a/ltr/auto_annotate.py
+++ b/ltr/auto_annotate.py
@@ -1,29 +1,71 @@
import numpy as np
-import re
+from pprint import pprint
from llama_cpp import Llama
import sqlite3
from tqdm import tqdm
import json
import time
import requests
-
-NUM_RESULTS_PER_QUERY = 100
-NUM_ENSEMBLE_PREDS = 1
-PROMPT = """<|im_start|>system
-Perform the task to the best of your ability.<|im_end|>
-<|im_start|>user
-Think about this step-by-step. You task is to evaluate a single search result on an integer scale of 0-10, where a higher score means the result is more relevant. A relevant result should come from a trust-worthy source and answer the users query. The final relevancy should be on a new line on the form "Relevancy: {{}}". It is very important that you think about the steps before giving the final "Relevancy: {{}}" score.
+import random
+
+API = "https://stract.com/beta/api/search"
+NUM_RESULTS_PER_QUERY = 20
+
+ELO_K = 32
+ELO_SCALE = 400
+ELO_ROUNDS_MULT = 5
+NUM_LABELS = 4
+
+# PROMPT = """<|begin_of_text|><|start_header_id|>system<|end_header_id|>
+#
+# You are a helpful, smart, kind, and efficient AI assistant. You always fulfill the user's requests to the best of your ability.<|eot_id|><|start_header_id|>user<|end_header_id|>
+#
+# Think about this step-by-step. You are a search engine evaluator and your task is to evaluate search results based on how well the result matches the query
+# You will be shown two results for each query and most choose which result is best for the users query. A good result most answer the users query and come from an authoritative source.
+# To choose the best result, write "Best: RESULT_A" or "Best: RESULT_B". Before choosing the best result, you should first evaluate the relevance of each result to the query.
+#
+# Query: "{}"
+#
+# RESULT_A:
+# Url: "{}"
+# Title: "{}"
+# Snippet: "{}"
+#
+# RESULT_B:
+# Url: "{}"
+# Title: "{}"
+# Snippet: "{}"
+#
+# Evaluation:<|eot_id|><|start_header_id|>assistant<|end_header_id|>
+#
+# """
+
+PROMPT = """<start_of_turn>user
+Think about this step-by-step. You are a search engine evaluator and your task is to evaluate search results based on how well the result matches the query
+You will be shown two results for each query and most choose which result is best for the users query. A good result most answer the users query and come from an authoritative source.
+To choose the best result, write "Best: RESULT_A" or "Best: RESULT_B". Before choosing the best result, you should first evaluate the relevance of each result to the query.
Query: "{}"
+
+RESULT_A:
+Url: "{}"
+Title: "{}"
+Snippet: "{}"
+
+RESULT_B:
Url: "{}"
Title: "{}"
-Snippet: "{}"<|im_end|>
-<|im_start|>assistant
+Snippet: "{}"<end_of_turn>
+<start_of_turn>model
"""
llm = Llama(
n_gpu_layers=-1,
n_ctx=8000,
- model_path="data/neuralhermes-2.5-mistral-7b.Q5_K_M.gguf",
+ # model_path="data/Meta-Llama-3.1-8B-Instruct-Q8_0.gguf",
+ model_path="data/gemma-2-2b-it-abliterated-Q6_K_L.gguf",
+ repeat_penalty=False,
+ no_penalize_nl=True,
+ verbose=False,
)
@@ -54,7 +96,7 @@ def setup_db():
url TEXT NOT NULL,
orig_rank INTEGER NOT NULL,
webpage_json TEXT NOT NULL,
- annotation INTEGER,
+ annotation INTEGER DEFAULT NULL,
PRIMARY KEY (qid, url)
);"""
)
@@ -114,8 +156,6 @@ def simplify_snippet(snippet):
def get_search_results(query):
- url = "https://stract.com/beta/api/search"
-
payload = {
"query": query,
"numResults": NUM_RESULTS_PER_QUERY,
@@ -129,7 +169,7 @@ def get_search_results(query):
"rankingSignals": {s: v["value"] for (s, v) in w["rankingSignals"].items()},
"snippet": simplify_snippet(w["snippet"]),
}
- for w in requests.post(url, json=payload).json()["webpages"][
+ for w in requests.post(API, json=payload).json()["webpages"][
:NUM_RESULTS_PER_QUERY
]
]
@@ -158,21 +198,33 @@ def add_results(query):
db.commit()
-def get_prompt(query, url, title, snippet):
+def get_prompt(query, url_a, title_a, snippet_a, url_b, title_b, snippet_b):
return PROMPT.format(
query,
- url,
- title,
- snippet,
+ url_a,
+ title_a,
+ snippet_a,
+ url_b,
+ title_b,
+ snippet_b,
)
-def get_relevancy(res):
- regex = r"Relevancy: (\d)"
- matches = re.findall(regex, res)
- if len(matches) == 0:
- return None
- return int(matches[0])
+def get_best(res):
+ if "Best: RESULT_A" in res:
+ return 1
+ if "Best: RESULT_B" in res:
+ return 0
+ return None
+
+def elo_update(winner, loser, elo):
+ p_winner = 1 / (1 + 10 ** ((elo[loser] - elo[winner]) / ELO_SCALE))
+ p_loser = 1 - p_winner
+
+ elo[winner] += ELO_K * (1 - p_winner)
+ elo[loser] += ELO_K * (0 - p_loser)
+
+ return elo
for qid, query in tqdm(unannotated_queries.items()):
@@ -187,38 +239,44 @@ for qid, query in tqdm(unannotated_queries.items()):
(qid,),
).fetchall()
- for url, orig_rank, webpage_json in tqdm(unnanotated_results):
- webpage = json.loads(webpage_json)
- prompt = get_prompt(query, url, webpage["title"], webpage["snippet"])
- res = 0
- n = 0
- for _ in range(NUM_ENSEMBLE_PREDS):
- output = llm.create_completion(
- prompt,
- max_tokens=1024,
- echo=False,
- temperature=0.4,
- stop=["<|im_start|>", "<|im_end|>"],
- )
- output = output["choices"][0]["text"]
-
- relevancy = get_relevancy(output)
- if relevancy is None:
- continue
-
- n += 1
- res += relevancy
-
- if n == 0:
- print("No relevancy annotation for", query, url)
- print(output)
- continue
-
- relevancy = np.round(res / n).astype(int)
- relevancy = max(0, min(10, int(relevancy)))
-
- print(f'relevancy={relevancy} query="{query}" url={url}')
+ elo = {url: ELO_SCALE // 2 for url, _, _ in unnanotated_results}
+ for _ in tqdm(range(0, ELO_ROUNDS_MULT * len(unnanotated_results))):
+ (url_a, _, json_a), (url_b, _, json_b)= random.sample(unnanotated_results, 2)
+
+ webpage_a = json.loads(json_a)
+ webpage_b = json.loads(json_b)
+ prompt = get_prompt(query, url_a, webpage_a["title"], webpage_a["snippet"], url_b, webpage_b["title"], webpage_b["snippet"])
+ output = llm.create_completion(
+ prompt,
+ max_tokens=1024,
+ echo=False,
+ temperature=0.4,
+ stop=["<|start_header_id|>", "<|eot_id|>"],
+ )
+ output = output["choices"][0]["text"]
+
+ relevancy = get_best(output)
+ if relevancy is not None:
+ if relevancy == 1:
+ elo = elo_update(url_a, url_b, elo)
+ else:
+ elo = elo_update(url_b, url_a, elo)
+
+ # sort resutls by elo and assign labels based on their rank
+ elo = {url: elo[url] for url, _, _ in unnanotated_results}
+ elo = sorted(elo.items(), key=lambda x: x[1], reverse=True)
+ elo = [{"url": url} for url, _ in elo]
+
+ for i in range(len(elo)):
+ # elo[i]['label'] = (NUM_LABELS - i // (len(elo) // NUM_LABELS))
+ elo[i]['label'] = NUM_LABELS - int(np.log2(i + 1))
+
+ print(query)
+ pprint(elo)
+ for website in elo:
+ url = website['url']
+ relevancy = website['label']
cur.execute(
"""
UPDATE search_results
@@ -227,5 +285,4 @@ for qid, query in tqdm(unannotated_queries.items()):
""",
(relevancy, qid, url),
)
-
db.commit()
diff --git a/ltr/lambdamart.py b/ltr/lambdamart.py
index 4de29c78..c819bda9 100644
--- a/ltr/lambdamart.py
+++ b/ltr/lambdamart.py
@@ -3,9 +3,23 @@ import numpy as np
import json
from pprint import pprint
import sqlite3
+from sklearn import metrics
+import itertools
CATEGORICAL_FEATURES = ["is_homepage"]
+param_grid = {
+ "objective": ["lambdarank"],
+ "verbosity": [-1],
+ "metric": ["ndcg"],
+ "ndcg_at": [[1, 2, 3, 5, 10]],
+ "learning_rate": [0.025],
+ "max_depth": [-1, 2, 4, 8],
+ "num_leaves": [7, 15, 31],
+ "lambda_l2": [0.1, 0.5, 1.0],
+}
+
+
con = sqlite3.connect("data/auto-ranking-annotation.sqlite")
cur = con.cursor()
@@ -105,54 +119,96 @@ q_test = np.array([len(queries[qid]) for qid in q_test])
print("Train size:", len(X_train))
print("Test size:", len(X_test))
+
+params = [
+ dict(zip(param_grid.keys(), values))
+ for values in itertools.product(*param_grid.values())
+]
+
+
+best_param = None
+best_score = 0
+
+for param in params:
+ dataset = lgb.Dataset(
+ X_train,
+ y_train,
+ group=q_train,
+ feature_name=[k for k in feature2id],
+ categorical_feature=[feature2id[k] for k in CATEGORICAL_FEATURES],
+ )
+
+ res = lgb.cv(
+ train_set=dataset,
+ params=param,
+ nfold=5,
+ return_cvbooster=True,
+ )
+
+ scores = []
+
+ for metric, vals in res.items():
+ if metric in ['cvbooster']:
+ continue
+
+ if 'stdv' in metric:
+ continue
+ scores.append(vals[0])
+
+ score = np.mean(scores)
+
+ if score > best_score:
+ best_score = score
+ best_param = param
+
+print("Best param:")
+pprint(best_param)
+
+
# Train model
-n_estimators = 50
-model = lgb.LGBMRanker(
- objective="lambdarank",
- metric="ndcg",
- importance_type="gain",
- num_leaves=50,
- n_estimators=n_estimators,
- categorical_features=[feature2id[k] for k in CATEGORICAL_FEATURES],
- max_depth=10,
- learning_rate=0.1,
- label_gain=[i for i in range(max(y_train.max(), y_test.max()) + 1)],
-)
-model.fit(
+dataset = lgb.Dataset(
X_train,
y_train,
group=q_train,
feature_name=[k for k in feature2id],
- eval_set=[(X_test, y_test)],
- eval_group=[q_test],
- eval_at=[1, 2, 3, 5, 10],
- eval_metric="ndcg",
+ categorical_feature=[feature2id[k] for k in CATEGORICAL_FEATURES],
+)
+booster = lgb.train(
+ best_param,
+ dataset,
)
# dump model
-model.booster_.save_model(
+booster.save_model(
"data/lambdamart.txt",
)
-
# print feature importance
+print()
+print("Feature importance:")
pprint(
sorted(
- [(id2feature[i], v) for i, v in enumerate(model.feature_importances_) if v > 0],
+ [(id2feature[i], v) for i, v in enumerate(booster.feature_importance()) if v > 0],
key=lambda x: x[1],
reverse=True,
)
)
+print()
+print("Test set:")
+for k in [1, 2, 3, 5, 10]:
+ print(f"NDCG@{k}: {metrics.ndcg_score([y_test], [booster.predict(X_test)], k=k)}")
# verify that the saved model outputs the same scores
# for the same input
saved_model = lgb.Booster(model_file="data/lambdamart.txt")
for i in range(len(X_test)):
- assert model.predict(X_test[i : i + 1]) == saved_model.predict(X_test[i : i + 1])
+ assert booster.predict(X_test[i : i + 1]) == saved_model.predict(X_test[i : i + 1])
+
# print an example
+# print()
# print("Example:")
# t = X_test[0]
# print("Features:")
# pprint({id2feature[i]: v for i, v in enumerate(t)})
-# print("Score:", model.predict(t.reshape(1, -1))[0])
+# print("Score:", booster.predict(t.reshape(1, -1))[0])