Skip to content

web search done right

ltr experiment

Mikkel Denker1f09a4247d61
diff --git a/crates/core/src/main.rs b/crates/core/src/main.rs
index f2e02c34..7a37dc06 100644
--- a/crates/core/src/main.rs
+++ b/crates/core/src/main.rs
@@ -210,13 +210,13 @@ enum SafetyClassifierOptions {
#[derive(Subcommand)]
enum CentralityMode {
- /// Calculate metrics for the host webgraph.
- Host {
+ /// Calculate harmonic centrality for the webgraph.
+ Harmonic {
webgraph_path: String,
output_path: String,
},
- /// Calculate metrics for the page webgraph.
- Page {
+ /// Calculate approximate harmonic centrality for the page webgraph.
+ ApproxHarmonic {
webgraph_path: String,
output_path: String,
},
@@ -324,13 +324,13 @@ fn main() -> Result<()> {
},
Commands::Centrality { mode } => {
match mode {
- CentralityMode::Host {
+ CentralityMode::Harmonic {
webgraph_path,
output_path,
} => {
entrypoint::Centrality::build_harmonic(&webgraph_path, &output_path);
}
- CentralityMode::Page {
+ CentralityMode::ApproxHarmonic {
webgraph_path,
output_path,
} => entrypoint::Centrality::build_approx_harmonic(webgraph_path, output_path)?,
diff --git a/frontend/src/routes/search/Result.svelte b/frontend/src/routes/search/Result.svelte
index c3bac6db..d0165b6a 100644
--- a/frontend/src/routes/search/Result.svelte
+++ b/frontend/src/routes/search/Result.svelte
@@ -82,6 +82,7 @@
{/if}
<button
class="noscript:hidden flex w-5 min-w-fit items-center justify-center bg-transparent text-neutral hover:cursor-pointer hover:text-neutral-focus"
+ aria-label="Open modal for result number: {resultIndex}"
bind:this={button}
on:click|stopPropagation={() => dispatch('modal', button)}
>
diff --git a/ltr/linear_model.py b/ltr/linear_model.py
index 4a701db9..bd18bba6 100644
--- a/ltr/linear_model.py
+++ b/ltr/linear_model.py
@@ -1,16 +1,14 @@
-import sklearn
import numpy as np
import json
+import sqlite3
+import stract
+from scipy.optimize import differential_evolution
import random
-from sklearn.linear_model import LinearRegression, LogisticRegression
from pprint import pprint
-import jax
-import jax.numpy as jnp
-import rax
-TRAIN_PERCENT = 0.8
-import sqlite3
+MAX_WEIGHT = 10
+NUM_QUERIES_FOR_EVAL = 40
con = sqlite3.connect("data/auto-ranking-annotation.sqlite")
cur = con.cursor()
@@ -24,7 +22,18 @@ res = cur.execute(
)
"""
)
-queries = {qid: {"query": query} for qid, query in res.fetchall()}
+
+accepted_queries = set()
+with open("data/queries_us.csv") as f:
+ for query in f.readlines():
+ if len(query.strip()) > 1:
+ accepted_queries.add(query.strip().lower())
+
+queries = {
+ qid: {"query": query}
+ for qid, query in res.fetchall()
+ if query.lower() in accepted_queries
+}
for qid in queries:
res = cur.execute(
@@ -57,7 +66,6 @@ for qid in queries:
urls = sorted(urls, key=sorting_key)
queries[qid]["urls"] = urls
-
feature2id = {}
id2feature = {}
@@ -69,95 +77,67 @@ for qid, data in queries.items():
feature2id[feature] = id
id2feature[id] = feature
-# convert to [{ranking_signals, score}]
-scores = []
+queries = [
+ (q["query"], [u[0] for u in q["urls"] if u[1] > 0]) for q in queries.values()
+]
+queries = random.sample(queries, NUM_QUERIES_FOR_EVAL)
+pprint([q for q, _ in queries])
-random.shuffle(scores)
+bounds = [(0, MAX_WEIGHT) for _ in range(len(feature2id))]
-sorted_features = sorted(feature2id.items(), key=lambda x: x[1])
-for qid, data in queries.items():
- sum_scores = sum([score for _, score, _, _ in data["urls"]])
- for i, (url, score, _, signals) in enumerate(data["urls"]):
- signals = {feature2id[k]: v for k, v in signals.items()}
-
- signals = [signals.get(k, 0) for _, k in sorted_features]
+def eval_query(query, expected_urls, weights):
+ coeffs = None
- scores.append({"ranking_signals": signals, "score": score})
+ if len(weights) != 0:
+ coeffs = {id2feature[i]: w for i, w in enumerate(weights)}
+ coeffs["lambda_mart"] = 10.0
-train = scores[: int(len(scores) * TRAIN_PERCENT)]
-test = scores[int(len(scores) * TRAIN_PERCENT) :]
+ res = stract.search(query, signal_coefficients=coeffs)
+ return sum([1 for r in res if r["url"] in expected_urls])
-X_train = []
-y_train = []
-for score in train:
- X_train.append(score["ranking_signals"])
- y_train.append(score["score"])
-X_test = []
-y_test = []
-for score in test:
- X_test.append(score["ranking_signals"])
- y_test.append(score["score"])
+cache = {}
-X_train = jnp.array(X_train)
-y_train = jnp.array(y_train)
-X_test = jnp.array(X_test)
-y_test = jnp.array(y_test)
-# jax model
-w = jnp.zeros(X_train.shape[1])
+def eval_weights(weights):
+ if tuple(weights) in cache:
+ return cache[tuple(weights)]
+ _queries = queries
+ total = sum([len(urls) for _, urls in _queries])
-def model(w, X):
- return jnp.dot(X, w)
+ res = sum([eval_query(q, urls, weights) for q, urls in _queries]) / total
+ cache[tuple(weights)] = res
-def loss(w, batch):
- features, labels, mask = batch
- # scores = model(w, features)
- # return rax.approx_t12n(rax.mrr_metric)(scores, labels)
+ return res
- return ((w * features - jnp.ones_like(w)) ** 2).sum()
+def optim(weights):
+ res = -eval_weights(weights)
+ print("Score:", res)
+ return res
-grad_fn = jax.jit(jax.grad(loss))
-# train
-for i in range(128):
- batch_size = 20
- for j in range(0, len(X_train), batch_size):
- batch = (
- X_train[j : j + batch_size],
- y_train[j : j + batch_size],
- jnp.ones(batch_size),
- )
- w = w - grad_fn(w, batch) * 3e-4
- w = w.clip(0, 10)
+def callback(intermediate_result):
+ pprint({id2feature[i]: w for i, w in enumerate(intermediate_result.x)})
+ print("Score:", -intermediate_result.fun)
-# model = LinearRegression(fit_intercept=False, positive=True)
-# model = SVR(kernel="linear")
-# model = DecisionTreeRegressor()
-# model.fit(X_train, y_train)
+print("baseline", eval_weights([]))
-print("TRAIN")
-for k in [1, 2, 5, 10]:
- print(
- f"NDCG@{k}: ", sklearn.metrics.ndcg_score([y_train], [model(w, X_train)], k=k)
- )
-
-print()
-print("TEST")
-for k in [1, 2, 5, 10]:
- print(f"NDCG@{k}: ", sklearn.metrics.ndcg_score([y_test], [model(w, X_test)], k=k))
+result = differential_evolution(
+ optim,
+ bounds,
+ maxiter=100,
+ popsize=1,
+ disp=False,
+ polish=False,
+ callback=callback,
+)
+weights = {id2feature[i]: w for i, w in enumerate(result.x)}
-weights = {id2feature[i]: float(v) for (i, v) in enumerate(w)}
-print()
-print("Weights:")
-for k, v in sorted(weights.items(), key=lambda x: -x[1]):
- print(k, v)
-linear_model = {"weights": weights}
-with open("data/linear_model.json", "w") as f:
- json.dump(linear_model, f, indent=2)
+print("Best weights")
+pprint(weights)
diff --git a/ltr/stract.py b/ltr/stract.py
index c9698e14..ad1f9831 100644
--- a/ltr/stract.py
+++ b/ltr/stract.py
@@ -12,12 +12,13 @@ def simplify_snippet(snippet):
return "".join([f["text"] for f in snippet["text"]["fragments"]])
-def search(query, page=0):
+def search(query, num_resuts=NUM_RESULTS_PER_QUERY, page=0, signal_coefficients=None):
payload = {
"query": query,
- "numResults": NUM_RESULTS_PER_QUERY,
+ "numResults": num_resuts,
"page": page,
"returnRankingSignals": True,
+ "signalCoefficients": signal_coefficients,
}
return [