web search done right
ltr experiment
Mikkel Denker1f09a4247d61
diff --git a/crates/core/src/main.rs b/crates/core/src/main.rsindex f2e02c34..7a37dc06 100644--- a/crates/core/src/main.rs+++ b/crates/core/src/main.rs@@ -210,13 +210,13 @@ enum SafetyClassifierOptions {#[derive(Subcommand)]enum CentralityMode {- /// Calculate metrics for the host webgraph.- Host {+ /// Calculate harmonic centrality for the webgraph.+ Harmonic {webgraph_path: String,output_path: String,},- /// Calculate metrics for the page webgraph.- Page {+ /// Calculate approximate harmonic centrality for the page webgraph.+ ApproxHarmonic {webgraph_path: String,output_path: String,},@@ -324,13 +324,13 @@ fn main() -> Result<()> {},Commands::Centrality { mode } => {match mode {- CentralityMode::Host {+ CentralityMode::Harmonic {webgraph_path,output_path,} => {entrypoint::Centrality::build_harmonic(&webgraph_path, &output_path);}- CentralityMode::Page {+ CentralityMode::ApproxHarmonic {webgraph_path,output_path,} => entrypoint::Centrality::build_approx_harmonic(webgraph_path, output_path)?,diff --git a/frontend/src/routes/search/Result.svelte b/frontend/src/routes/search/Result.svelteindex c3bac6db..d0165b6a 100644--- a/frontend/src/routes/search/Result.svelte+++ b/frontend/src/routes/search/Result.svelte@@ -82,6 +82,7 @@{/if}<buttonclass="noscript:hidden flex w-5 min-w-fit items-center justify-center bg-transparent text-neutral hover:cursor-pointer hover:text-neutral-focus"+ aria-label="Open modal for result number: {resultIndex}"bind:this={button}on:click|stopPropagation={() => dispatch('modal', button)}>diff --git a/ltr/linear_model.py b/ltr/linear_model.pyindex 4a701db9..bd18bba6 100644--- a/ltr/linear_model.py+++ b/ltr/linear_model.py@@ -1,16 +1,14 @@-import sklearnimport numpy as npimport json+import sqlite3+import stract+from scipy.optimize import differential_evolutionimport random-from sklearn.linear_model import LinearRegression, LogisticRegressionfrom pprint import pprint-import jax-import jax.numpy as jnp-import rax-TRAIN_PERCENT = 0.8-import sqlite3+MAX_WEIGHT = 10+NUM_QUERIES_FOR_EVAL = 40con = sqlite3.connect("data/auto-ranking-annotation.sqlite")cur = con.cursor()@@ -24,7 +22,18 @@ res = cur.execute()""")-queries = {qid: {"query": query} for qid, query in res.fetchall()}++accepted_queries = set()+with open("data/queries_us.csv") as f:+ for query in f.readlines():+ if len(query.strip()) > 1:+ accepted_queries.add(query.strip().lower())++queries = {+ qid: {"query": query}+ for qid, query in res.fetchall()+ if query.lower() in accepted_queries+}for qid in queries:res = cur.execute(@@ -57,7 +66,6 @@ for qid in queries:urls = sorted(urls, key=sorting_key)queries[qid]["urls"] = urls-feature2id = {}id2feature = {}@@ -69,95 +77,67 @@ for qid, data in queries.items():feature2id[feature] = idid2feature[id] = feature-# convert to [{ranking_signals, score}]-scores = []+queries = [+ (q["query"], [u[0] for u in q["urls"] if u[1] > 0]) for q in queries.values()+]+queries = random.sample(queries, NUM_QUERIES_FOR_EVAL)+pprint([q for q, _ in queries])-random.shuffle(scores)+bounds = [(0, MAX_WEIGHT) for _ in range(len(feature2id))]-sorted_features = sorted(feature2id.items(), key=lambda x: x[1])-for qid, data in queries.items():- sum_scores = sum([score for _, score, _, _ in data["urls"]])- for i, (url, score, _, signals) in enumerate(data["urls"]):- signals = {feature2id[k]: v for k, v in signals.items()}-- signals = [signals.get(k, 0) for _, k in sorted_features]+def eval_query(query, expected_urls, weights):+ coeffs = None- scores.append({"ranking_signals": signals, "score": score})+ if len(weights) != 0:+ coeffs = {id2feature[i]: w for i, w in enumerate(weights)}+ coeffs["lambda_mart"] = 10.0-train = scores[: int(len(scores) * TRAIN_PERCENT)]-test = scores[int(len(scores) * TRAIN_PERCENT) :]+ res = stract.search(query, signal_coefficients=coeffs)+ return sum([1 for r in res if r["url"] in expected_urls])-X_train = []-y_train = []-for score in train:- X_train.append(score["ranking_signals"])- y_train.append(score["score"])-X_test = []-y_test = []-for score in test:- X_test.append(score["ranking_signals"])- y_test.append(score["score"])+cache = {}-X_train = jnp.array(X_train)-y_train = jnp.array(y_train)-X_test = jnp.array(X_test)-y_test = jnp.array(y_test)-# jax model-w = jnp.zeros(X_train.shape[1])+def eval_weights(weights):+ if tuple(weights) in cache:+ return cache[tuple(weights)]+ _queries = queries+ total = sum([len(urls) for _, urls in _queries])-def model(w, X):- return jnp.dot(X, w)+ res = sum([eval_query(q, urls, weights) for q, urls in _queries]) / total+ cache[tuple(weights)] = res-def loss(w, batch):- features, labels, mask = batch- # scores = model(w, features)- # return rax.approx_t12n(rax.mrr_metric)(scores, labels)+ return res- return ((w * features - jnp.ones_like(w)) ** 2).sum()+def optim(weights):+ res = -eval_weights(weights)+ print("Score:", res)+ return res-grad_fn = jax.jit(jax.grad(loss))-# train-for i in range(128):- batch_size = 20- for j in range(0, len(X_train), batch_size):- batch = (- X_train[j : j + batch_size],- y_train[j : j + batch_size],- jnp.ones(batch_size),- )- w = w - grad_fn(w, batch) * 3e-4- w = w.clip(0, 10)+def callback(intermediate_result):+ pprint({id2feature[i]: w for i, w in enumerate(intermediate_result.x)})+ print("Score:", -intermediate_result.fun)-# model = LinearRegression(fit_intercept=False, positive=True)-# model = SVR(kernel="linear")-# model = DecisionTreeRegressor()-# model.fit(X_train, y_train)+print("baseline", eval_weights([]))-print("TRAIN")-for k in [1, 2, 5, 10]:- print(- f"NDCG@{k}: ", sklearn.metrics.ndcg_score([y_train], [model(w, X_train)], k=k)- )--print()-print("TEST")-for k in [1, 2, 5, 10]:- print(f"NDCG@{k}: ", sklearn.metrics.ndcg_score([y_test], [model(w, X_test)], k=k))+result = differential_evolution(+ optim,+ bounds,+ maxiter=100,+ popsize=1,+ disp=False,+ polish=False,+ callback=callback,+)+weights = {id2feature[i]: w for i, w in enumerate(result.x)}-weights = {id2feature[i]: float(v) for (i, v) in enumerate(w)}-print()-print("Weights:")-for k, v in sorted(weights.items(), key=lambda x: -x[1]):- print(k, v)-linear_model = {"weights": weights}-with open("data/linear_model.json", "w") as f:- json.dump(linear_model, f, indent=2)+print("Best weights")+pprint(weights)diff --git a/ltr/stract.py b/ltr/stract.pyindex c9698e14..ad1f9831 100644--- a/ltr/stract.py+++ b/ltr/stract.py@@ -12,12 +12,13 @@ def simplify_snippet(snippet):return "".join([f["text"] for f in snippet["text"]["fragments"]])-def search(query, page=0):+def search(query, num_resuts=NUM_RESULTS_PER_QUERY, page=0, signal_coefficients=None):payload = {"query": query,- "numResults": NUM_RESULTS_PER_QUERY,+ "numResults": num_resuts,"page": page,"returnRankingSignals": True,+ "signalCoefficients": signal_coefficients,}return [