web search done right
update signal coefficients
Mikkel Denker08dc07c57592
diff --git a/crates/core/src/api/metrics.rs b/crates/core/src/api/metrics.rsindex 9ee20f89..2992cb87 100644--- a/crates/core/src/api/metrics.rs+++ b/crates/core/src/api/metrics.rs@@ -24,5 +24,5 @@ use axum_macros::debug_handler;pub async fn route(extract::State(registry): extract::State<Arc<crate::metrics::PrometheusRegistry>>,) -> impl IntoResponse {- format!("{registry}");+ registry.to_string()}diff --git a/crates/core/src/entrypoint/indexer/worker.rs b/crates/core/src/entrypoint/indexer/worker.rsindex 88599ff9..72b35193 100644--- a/crates/core/src/entrypoint/indexer/worker.rs+++ b/crates/core/src/entrypoint/indexer/worker.rs@@ -38,7 +38,7 @@ use crate::ranking::SignalComputer;use crate::webgraph::{self, Edge, EdgeLimit, Node, NodeID};use crate::webpage::{safety_classifier, Html, Webpage};-const MAX_BACKLINKS: EdgeLimit = EdgeLimit::Limit(512);+const MAX_BACKLINKS: EdgeLimit = EdgeLimit::Limit(1024);pub struct Config {pub host_centrality_store_path: String,diff --git a/crates/core/src/entrypoint/webgraph.rs b/crates/core/src/entrypoint/webgraph.rsindex 7713f405..946036ae 100644--- a/crates/core/src/entrypoint/webgraph.rs+++ b/crates/core/src/entrypoint/webgraph.rs@@ -28,11 +28,6 @@ use std::{path::Path, sync::Arc};use tokio::pin;use tracing::{info, trace};-#[derive(Debug, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, Clone)]-struct GraphPointer {- path: String,-}-#[derive(Debug, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, Clone)]pub enum JobConfig {Http(config::HttpConfig),diff --git a/crates/core/src/inverted_index/key_phrase.rs b/crates/core/src/inverted_index/key_phrase.rsindex 5b5ad38e..afab83ee 100644--- a/crates/core/src/inverted_index/key_phrase.rs+++ b/crates/core/src/inverted_index/key_phrase.rs@@ -174,6 +174,13 @@ impl KeyPhrase {continue;}+ let left_paren = term_str.chars().filter(|c| c == &'(').count();+ let right_paren = term_str.chars().filter(|c| c == &')').count();++ if left_paren != right_paren {+ continue;+ }+let words = term_str.split_whitespace().collect::<Vec<_>>();let score = scorer.score(&words, info.doc_freq);diff --git a/crates/core/src/models/bert.rs b/crates/core/src/models/bert.rsindex cff96a6a..4fdf3df0 100644--- a/crates/core/src/models/bert.rs+++ b/crates/core/src/models/bert.rs@@ -431,7 +431,7 @@ struct BertEncoder {impl BertEncoder {fn load(vb: VarBuilder, config: &Config) -> Result<Self> {let layers = (0..config.num_hidden_layers)- .map(|index| BertLayer::load(vb.pp(&format!("layer.{index}")), config))+ .map(|index| BertLayer::load(vb.pp(format!("layer.{index}")), config)).collect::<Result<Vec<_>>>()?;let span = tracing::span!(tracing::Level::TRACE, "encoder");Ok(BertEncoder { layers, span })@@ -488,8 +488,8 @@ impl BertModel {(Err(err), _) | (_, Err(err)) => {if let Some(model_type) = &config.model_type {if let (Ok(embeddings), Ok(encoder)) = (- BertEmbeddings::load(vb.pp(&format!("{model_type}.embeddings")), config),- BertEncoder::load(vb.pp(&format!("{model_type}.encoder")), config),+ BertEmbeddings::load(vb.pp(format!("{model_type}.embeddings")), config),+ BertEncoder::load(vb.pp(format!("{model_type}.encoder")), config),) {(embeddings, encoder)} else {@@ -504,7 +504,7 @@ impl BertModel {let pooler = BertPooler::load(vb.pp("pooler"), config).or_else(|e| {if let Some(model_type) = &config.model_type {- BertPooler::load(vb.pp(&format!("{model_type}.pooler")), config)+ BertPooler::load(vb.pp(format!("{model_type}.pooler")), config)} else {Err(e)}diff --git a/crates/core/src/ranking/signal/non_text.rs b/crates/core/src/ranking/signal/non_text.rsindex cb66611f..92cfe930 100644--- a/crates/core/src/ranking/signal/non_text.rs+++ b/crates/core/src/ranking/signal/non_text.rs@@ -39,7 +39,14 @@ fn score_timestamp(page_timestamp: usize, signal_computer: &SignalComputer) -> f#[inline]fn score_rank(rank: f64) -> f64 {- 1.0 / (rank + 1.0)+ // 10 groups with log base 8 gives us+ // 1.1 billion ranks we can score without+ // exceeding the groups.++ const NUM_GROUPS: f64 = 10.0;+ const BASE: f64 = 8.0;++ (NUM_GROUPS - (1.0 + rank).log(BASE)).max(NUM_GROUPS)}#[inline]@@ -101,7 +108,7 @@ fn score_region(webpage_region: crate::webpage::Region, computer: &SignalComputepub struct HostCentrality;impl Signal for HostCentrality {fn default_coefficient(&self) -> f64 {- 0.5+ 2.0}fn as_field(&self) -> Option<Field> {@@ -141,7 +148,7 @@ impl Signal for HostCentrality {pub struct HostCentralityRank;impl Signal for HostCentralityRank {fn default_coefficient(&self) -> f64 {- 0.0+ 0.4}fn as_field(&self) -> Option<Field> {@@ -181,7 +188,7 @@ impl Signal for HostCentralityRank {pub struct PageCentrality;impl Signal for PageCentrality {fn default_coefficient(&self) -> f64 {- 0.25+ 2.0}fn as_field(&self) -> Option<Field> {@@ -221,7 +228,7 @@ impl Signal for PageCentrality {pub struct PageCentralityRank;impl Signal for PageCentralityRank {fn default_coefficient(&self) -> f64 {- 0.0+ 0.4}fn as_field(&self) -> Option<Field> {@@ -261,7 +268,7 @@ impl Signal for PageCentralityRank {pub struct IsHomepage;impl Signal for IsHomepage {fn default_coefficient(&self) -> f64 {- 0.0005+ 0.01}fn as_field(&self) -> Option<Field> {@@ -401,7 +408,7 @@ impl Signal for UpdateTimestamp {pub struct TrackerScore;impl Signal for TrackerScore {fn default_coefficient(&self) -> f64 {- 0.05+ 0.1}fn as_field(&self) -> Option<Field> {@@ -620,7 +627,7 @@ impl Signal for UrlDigits {pub struct UrlSlashes;impl Signal for UrlSlashes {fn default_coefficient(&self) -> f64 {- 0.01+ 0.1}fn as_field(&self) -> Option<Field> {diff --git a/crates/core/src/ranking/signal/text.rs b/crates/core/src/ranking/signal/text.rsindex f9e9884f..6ffb05bf 100644--- a/crates/core/src/ranking/signal/text.rs+++ b/crates/core/src/ranking/signal/text.rs@@ -102,7 +102,7 @@ impl Signal for Bm25Title {pub struct Bm25TitleBigrams;impl Signal for Bm25TitleBigrams {fn default_coefficient(&self) -> f64 {- 0.01+ 0.005}fn as_field(&self) -> Option<Field> {@@ -134,7 +134,7 @@ impl Signal for Bm25TitleBigrams {pub struct Bm25TitleTrigrams;impl Signal for Bm25TitleTrigrams {fn default_coefficient(&self) -> f64 {- 0.01+ 0.005}fn as_field(&self) -> Option<Field> {@@ -166,7 +166,7 @@ impl Signal for Bm25TitleTrigrams {pub struct Bm25CleanBody;impl Signal for Bm25CleanBody {fn default_coefficient(&self) -> f64 {- 0.0063+ 0.005}fn as_field(&self) -> Option<Field> {@@ -422,7 +422,7 @@ impl Signal for Bm25BacklinkText {pub struct IdfSumUrl;impl Signal for IdfSumUrl {fn default_coefficient(&self) -> f64 {- 0.0003+ 0.0006}fn as_field(&self) -> Option<Field> {@@ -550,7 +550,7 @@ impl Signal for IdfSumSiteNoTokenizer {pub struct IdfSumDomainNoTokenizer;impl Signal for IdfSumDomainNoTokenizer {fn default_coefficient(&self) -> f64 {- 0.0002+ 0.0036}fn as_field(&self) -> Option<Field> {@@ -716,7 +716,7 @@ impl Signal for IdfSumDomainIfHomepageNoTokenizer {pub struct IdfSumTitleIfHomepage;impl Signal for IdfSumTitleIfHomepage {fn default_coefficient(&self) -> f64 {- 0.00022+ 0.001}fn as_field(&self) -> Option<Field> {diff --git a/crates/core/src/webgraph/store.rs b/crates/core/src/webgraph/store.rsindex bb31caa0..2c140643 100644--- a/crates/core/src/webgraph/store.rs+++ b/crates/core/src/webgraph/store.rs@@ -37,23 +37,6 @@ use super::{Compression, EdgeLimit, FullNodeID, NodeDatum, NodeID, SegmentEdge, StoredEdge,};-#[derive(- Debug,- Clone,- PartialEq,- Eq,- Hash,- serde::Serialize,- serde::Deserialize,- bincode::Encode,- bincode::Decode,-)]-struct SerializedEdge {- from_host: NodeID,- to_host: NodeID,- label: Vec<u8>,-}-pub struct HostDb {db: speedy_kv::Db<Vec<u8>, ()>,}diff --git a/crates/kuchiki/src/iter.rs b/crates/kuchiki/src/iter.rsindex 83279d90..fa838386 100644--- a/crates/kuchiki/src/iter.rs+++ b/crates/kuchiki/src/iter.rs@@ -231,9 +231,8 @@ impl Iterator for Ancestors {#[inline]fn next(&mut self) -> Option<NodeRef> {- self.0.take().map(|node| {+ self.0.take().inspect(|node| {self.0 = node.parent();- node})}}diff --git a/crates/tantivy/src/tokenizer/ngram_tokenizer.rs b/crates/tantivy/src/tokenizer/ngram_tokenizer.rsindex 691a6ed0..286e26c1 100644--- a/crates/tantivy/src/tokenizer/ngram_tokenizer.rs+++ b/crates/tantivy/src/tokenizer/ngram_tokenizer.rs@@ -290,7 +290,7 @@ impl<'a> Iterator for CodepointFrontiers<'a> {type Item = usize;fn next(&mut self) -> Option<usize> {- self.next_el.map(|offset| {+ self.next_el.inspect(|offset| {if self.s.is_empty() {self.next_el = None;} else {@@ -298,7 +298,6 @@ impl<'a> Iterator for CodepointFrontiers<'a> {self.s = &self.s[first_codepoint_width..];self.next_el = Some(offset + first_codepoint_width);}- offset})}}diff --git a/ltr/auto_annotate.py b/ltr/auto_annotate.pyindex c8a0132a..61c1d054 100644--- a/ltr/auto_annotate.py+++ b/ltr/auto_annotate.py@@ -1,29 +1,71 @@import numpy as np-import re+from pprint import pprintfrom llama_cpp import Llamaimport sqlite3from tqdm import tqdmimport jsonimport timeimport requests--NUM_RESULTS_PER_QUERY = 100-NUM_ENSEMBLE_PREDS = 1-PROMPT = """<|im_start|>system-Perform the task to the best of your ability.<|im_end|>-<|im_start|>user-Think about this step-by-step. You task is to evaluate a single search result on an integer scale of 0-10, where a higher score means the result is more relevant. A relevant result should come from a trust-worthy source and answer the users query. The final relevancy should be on a new line on the form "Relevancy: {{}}". It is very important that you think about the steps before giving the final "Relevancy: {{}}" score.+import random++API = "https://stract.com/beta/api/search"+NUM_RESULTS_PER_QUERY = 20++ELO_K = 32+ELO_SCALE = 400+ELO_ROUNDS_MULT = 5+NUM_LABELS = 4++# PROMPT = """<|begin_of_text|><|start_header_id|>system<|end_header_id|>+#+# You are a helpful, smart, kind, and efficient AI assistant. You always fulfill the user's requests to the best of your ability.<|eot_id|><|start_header_id|>user<|end_header_id|>+#+# Think about this step-by-step. You are a search engine evaluator and your task is to evaluate search results based on how well the result matches the query+# You will be shown two results for each query and most choose which result is best for the users query. A good result most answer the users query and come from an authoritative source.+# To choose the best result, write "Best: RESULT_A" or "Best: RESULT_B". Before choosing the best result, you should first evaluate the relevance of each result to the query.+#+# Query: "{}"+#+# RESULT_A:+# Url: "{}"+# Title: "{}"+# Snippet: "{}"+#+# RESULT_B:+# Url: "{}"+# Title: "{}"+# Snippet: "{}"+#+# Evaluation:<|eot_id|><|start_header_id|>assistant<|end_header_id|>+#+# """++PROMPT = """<start_of_turn>user+Think about this step-by-step. You are a search engine evaluator and your task is to evaluate search results based on how well the result matches the query+You will be shown two results for each query and most choose which result is best for the users query. A good result most answer the users query and come from an authoritative source.+To choose the best result, write "Best: RESULT_A" or "Best: RESULT_B". Before choosing the best result, you should first evaluate the relevance of each result to the query.Query: "{}"++RESULT_A:+Url: "{}"+Title: "{}"+Snippet: "{}"++RESULT_B:Url: "{}"Title: "{}"-Snippet: "{}"<|im_end|>-<|im_start|>assistant+Snippet: "{}"<end_of_turn>+<start_of_turn>model"""llm = Llama(n_gpu_layers=-1,n_ctx=8000,- model_path="data/neuralhermes-2.5-mistral-7b.Q5_K_M.gguf",+ # model_path="data/Meta-Llama-3.1-8B-Instruct-Q8_0.gguf",+ model_path="data/gemma-2-2b-it-abliterated-Q6_K_L.gguf",+ repeat_penalty=False,+ no_penalize_nl=True,+ verbose=False,)@@ -54,7 +96,7 @@ def setup_db():url TEXT NOT NULL,orig_rank INTEGER NOT NULL,webpage_json TEXT NOT NULL,- annotation INTEGER,+ annotation INTEGER DEFAULT NULL,PRIMARY KEY (qid, url));""")@@ -114,8 +156,6 @@ def simplify_snippet(snippet):def get_search_results(query):- url = "https://stract.com/beta/api/search"-payload = {"query": query,"numResults": NUM_RESULTS_PER_QUERY,@@ -129,7 +169,7 @@ def get_search_results(query):"rankingSignals": {s: v["value"] for (s, v) in w["rankingSignals"].items()},"snippet": simplify_snippet(w["snippet"]),}- for w in requests.post(url, json=payload).json()["webpages"][+ for w in requests.post(API, json=payload).json()["webpages"][:NUM_RESULTS_PER_QUERY]]@@ -158,21 +198,33 @@ def add_results(query):db.commit()-def get_prompt(query, url, title, snippet):+def get_prompt(query, url_a, title_a, snippet_a, url_b, title_b, snippet_b):return PROMPT.format(query,- url,- title,- snippet,+ url_a,+ title_a,+ snippet_a,+ url_b,+ title_b,+ snippet_b,)-def get_relevancy(res):- regex = r"Relevancy: (\d)"- matches = re.findall(regex, res)- if len(matches) == 0:- return None- return int(matches[0])+def get_best(res):+ if "Best: RESULT_A" in res:+ return 1+ if "Best: RESULT_B" in res:+ return 0+ return None++def elo_update(winner, loser, elo):+ p_winner = 1 / (1 + 10 ** ((elo[loser] - elo[winner]) / ELO_SCALE))+ p_loser = 1 - p_winner++ elo[winner] += ELO_K * (1 - p_winner)+ elo[loser] += ELO_K * (0 - p_loser)++ return elofor qid, query in tqdm(unannotated_queries.items()):@@ -187,38 +239,44 @@ for qid, query in tqdm(unannotated_queries.items()):(qid,),).fetchall()- for url, orig_rank, webpage_json in tqdm(unnanotated_results):- webpage = json.loads(webpage_json)- prompt = get_prompt(query, url, webpage["title"], webpage["snippet"])- res = 0- n = 0- for _ in range(NUM_ENSEMBLE_PREDS):- output = llm.create_completion(- prompt,- max_tokens=1024,- echo=False,- temperature=0.4,- stop=["<|im_start|>", "<|im_end|>"],- )- output = output["choices"][0]["text"]-- relevancy = get_relevancy(output)- if relevancy is None:- continue-- n += 1- res += relevancy-- if n == 0:- print("No relevancy annotation for", query, url)- print(output)- continue-- relevancy = np.round(res / n).astype(int)- relevancy = max(0, min(10, int(relevancy)))-- print(f'relevancy={relevancy} query="{query}" url={url}')+ elo = {url: ELO_SCALE // 2 for url, _, _ in unnanotated_results}+ for _ in tqdm(range(0, ELO_ROUNDS_MULT * len(unnanotated_results))):+ (url_a, _, json_a), (url_b, _, json_b)= random.sample(unnanotated_results, 2)++ webpage_a = json.loads(json_a)+ webpage_b = json.loads(json_b)+ prompt = get_prompt(query, url_a, webpage_a["title"], webpage_a["snippet"], url_b, webpage_b["title"], webpage_b["snippet"])+ output = llm.create_completion(+ prompt,+ max_tokens=1024,+ echo=False,+ temperature=0.4,+ stop=["<|start_header_id|>", "<|eot_id|>"],+ )+ output = output["choices"][0]["text"]++ relevancy = get_best(output)+ if relevancy is not None:+ if relevancy == 1:+ elo = elo_update(url_a, url_b, elo)+ else:+ elo = elo_update(url_b, url_a, elo)++ # sort resutls by elo and assign labels based on their rank+ elo = {url: elo[url] for url, _, _ in unnanotated_results}+ elo = sorted(elo.items(), key=lambda x: x[1], reverse=True)+ elo = [{"url": url} for url, _ in elo]++ for i in range(len(elo)):+ # elo[i]['label'] = (NUM_LABELS - i // (len(elo) // NUM_LABELS))+ elo[i]['label'] = NUM_LABELS - int(np.log2(i + 1))++ print(query)+ pprint(elo)+ for website in elo:+ url = website['url']+ relevancy = website['label']cur.execute("""UPDATE search_results@@ -227,5 +285,4 @@ for qid, query in tqdm(unannotated_queries.items()):""",(relevancy, qid, url),)-db.commit()diff --git a/ltr/lambdamart.py b/ltr/lambdamart.pyindex 4de29c78..c819bda9 100644--- a/ltr/lambdamart.py+++ b/ltr/lambdamart.py@@ -3,9 +3,23 @@ import numpy as npimport jsonfrom pprint import pprintimport sqlite3+from sklearn import metrics+import itertoolsCATEGORICAL_FEATURES = ["is_homepage"]+param_grid = {+ "objective": ["lambdarank"],+ "verbosity": [-1],+ "metric": ["ndcg"],+ "ndcg_at": [[1, 2, 3, 5, 10]],+ "learning_rate": [0.025],+ "max_depth": [-1, 2, 4, 8],+ "num_leaves": [7, 15, 31],+ "lambda_l2": [0.1, 0.5, 1.0],+}++con = sqlite3.connect("data/auto-ranking-annotation.sqlite")cur = con.cursor()@@ -105,54 +119,96 @@ q_test = np.array([len(queries[qid]) for qid in q_test])print("Train size:", len(X_train))print("Test size:", len(X_test))++params = [+ dict(zip(param_grid.keys(), values))+ for values in itertools.product(*param_grid.values())+]+++best_param = None+best_score = 0++for param in params:+ dataset = lgb.Dataset(+ X_train,+ y_train,+ group=q_train,+ feature_name=[k for k in feature2id],+ categorical_feature=[feature2id[k] for k in CATEGORICAL_FEATURES],+ )++ res = lgb.cv(+ train_set=dataset,+ params=param,+ nfold=5,+ return_cvbooster=True,+ )++ scores = []++ for metric, vals in res.items():+ if metric in ['cvbooster']:+ continue++ if 'stdv' in metric:+ continue+ scores.append(vals[0])++ score = np.mean(scores)++ if score > best_score:+ best_score = score+ best_param = param++print("Best param:")+pprint(best_param)++# Train model-n_estimators = 50-model = lgb.LGBMRanker(- objective="lambdarank",- metric="ndcg",- importance_type="gain",- num_leaves=50,- n_estimators=n_estimators,- categorical_features=[feature2id[k] for k in CATEGORICAL_FEATURES],- max_depth=10,- learning_rate=0.1,- label_gain=[i for i in range(max(y_train.max(), y_test.max()) + 1)],-)-model.fit(+dataset = lgb.Dataset(X_train,y_train,group=q_train,feature_name=[k for k in feature2id],- eval_set=[(X_test, y_test)],- eval_group=[q_test],- eval_at=[1, 2, 3, 5, 10],- eval_metric="ndcg",+ categorical_feature=[feature2id[k] for k in CATEGORICAL_FEATURES],+)+booster = lgb.train(+ best_param,+ dataset,)# dump model-model.booster_.save_model(+booster.save_model("data/lambdamart.txt",)-# print feature importance+print()+print("Feature importance:")pprint(sorted(- [(id2feature[i], v) for i, v in enumerate(model.feature_importances_) if v > 0],+ [(id2feature[i], v) for i, v in enumerate(booster.feature_importance()) if v > 0],key=lambda x: x[1],reverse=True,))+print()+print("Test set:")+for k in [1, 2, 3, 5, 10]:+ print(f"NDCG@{k}: {metrics.ndcg_score([y_test], [booster.predict(X_test)], k=k)}")# verify that the saved model outputs the same scores# for the same inputsaved_model = lgb.Booster(model_file="data/lambdamart.txt")for i in range(len(X_test)):- assert model.predict(X_test[i : i + 1]) == saved_model.predict(X_test[i : i + 1])+ assert booster.predict(X_test[i : i + 1]) == saved_model.predict(X_test[i : i + 1])+# print an example+# print()# print("Example:")# t = X_test[0]# print("Features:")# pprint({id2feature[i]: v for i, v in enumerate(t)})-# print("Score:", model.predict(t.reshape(1, -1))[0])+# print("Score:", booster.predict(t.reshape(1, -1))[0])