Skip to content

armgit / mwmbl

Public Imported from GitHub
0

An open source, non-profit web search engine

Offline eval for the xgb source model: judge rewards, holdout, replay

Daoud Clarke99097cf76fe4
diff --git a/mwmbl/tinysearchengine/super_search_select/evaluation.py b/mwmbl/tinysearchengine/super_search_select/evaluation.py
index 3f8a6c1..bdc4388 100644
--- a/mwmbl/tinysearchengine/super_search_select/evaluation.py
+++ b/mwmbl/tinysearchengine/super_search_select/evaluation.py
@@ -8,9 +8,20 @@ top-K), and a ``mask`` of which sources actually returned anything.
With every cell filled we can, without any online traffic:
* select features — fit an XGBoost reward model (grouped CV by query) and
rank/ablate features by held-out recall@k;
- * simulate the policy — replay linear-Gaussian Thompson sampling over the
- matrix, sweep the exploration scale ``nu``, and compare against baselines
- (random / popularity / cosine / oracle).
+ * simulate the policy — replay linear-Gaussian Thompson sampling
+ (``simulate_ts``) or the epsilon-greedy XGBoost contextual bandit
+ (``simulate_xgb``) over the matrix and compare against baselines
+ (random / popularity / cosine / oracle);
+ * holdout-evaluate the xgb source model (``evaluate_holdout``): split
+ queries into train/test (stratified by each query's home source), fit on
+ train, and report test coverage@k / home-source recall@k against the same
+ baselines plus a train-then-greedy LinTS comparator.
+
+Off-policy note: replay over a *dense* matrix is unbiased (every arm's
+counterfactual reward is known). The online impression log, by contrast, is
+conditioned on whatever policy was live; the mitigations are warm-starting
+from a dense offline matrix and the epsilon slots continuously injecting
+randomized pairs.
This module is pure (numpy/sklearn/xgboost only); building the matrix from live
sources lives in ``scripts/super_search_eval.py``.
@@ -212,3 +223,208 @@ def sweep_explore_scale(matrix: RewardMatrix, k: int, nus: list[float],
sigma2: float = 0.25, lam: float = 1.0, seed: int = 0) -> dict[float, float]:
"""Mean captured reward for each candidate exploration scale ``nu``."""
return {nu: simulate_ts(matrix, k, nu, sigma2, lam, seed) for nu in nus}
+
+
+def simulate_xgb(matrix: RewardMatrix, k: int, epsilon: float,
+ refit_every: int = 200, min_rows: int = 300, seed: int = 0,
+ params: dict | None = None) -> float:
+ """Replay the epsilon-greedy XGBoost contextual bandit over the matrix.
+
+ Mirrors ``simulate_ts``'s contract (mean per-query captured reward).
+ Queries are visited in a seeded shuffled order; until ``min_rows`` chosen
+ (source, reward) pairs have accumulated, selection uses the cosine
+ baseline (matching how real impression logs bootstrap), after which the
+ model is fit and refit every ``refit_every`` queries.
+ """
+ from mwmbl.tinysearchengine.super_search_select import xgb_model
+
+ rng = np.random.default_rng(seed)
+ Q = matrix.X.shape[0]
+ vocab = xgb_model.build_vocab(matrix.sources)
+ vocab_index = {name: i for i, name in enumerate(vocab)}
+ cos_i = matrix.feature_names.index("cos_bow")
+
+ model = None
+ xs: list[np.ndarray] = []
+ ys: list[float] = []
+ captured = 0.0
+ since_fit = 0
+ for q in rng.permutation(Q):
+ avail = np.where(matrix.mask[q])[0]
+ if avail.size == 0:
+ continue
+ k_q = min(k, avail.size)
+ if model is None:
+ chosen = avail[np.argsort(matrix.X[q, avail, cos_i])[::-1][:k_q]]
+ else:
+ enc = np.stack([xgb_model.encode(matrix.X[q, s], matrix.sources[s], vocab_index)
+ for s in avail])
+ ranked = avail[np.argsort(model.predict(enc))[::-1]]
+ n_explore = int(rng.binomial(k_q, epsilon))
+ chosen = list(ranked[:k_q - n_explore])
+ rest = ranked[k_q - n_explore:]
+ if n_explore and rest.size:
+ chosen += list(rng.choice(rest, size=min(n_explore, rest.size), replace=False))
+ chosen = np.asarray(chosen)
+ for s in chosen:
+ xs.append(xgb_model.encode(matrix.X[q, s], matrix.sources[s], vocab_index))
+ ys.append(float(matrix.R[q, s]))
+ captured += matrix.R[q, s]
+ since_fit += 1
+ if len(ys) >= min_rows and (model is None or since_fit >= refit_every):
+ model = xgb_model.train(np.stack(xs), np.asarray(ys), params=params)
+ since_fit = 0
+ return captured / Q
+
+
+def sweep_epsilon(matrix: RewardMatrix, k: int, epsilons: list[float],
+ refit_every: int = 200, min_rows: int = 300,
+ seed: int = 0) -> dict[float, float]:
+ """Mean captured reward for each candidate exploration rate ``epsilon``."""
+ return {eps: simulate_xgb(matrix, k, eps, refit_every, min_rows, seed)
+ for eps in epsilons}
+
+
+# ---------------------------------------------------------------------------
+# Holdout evaluation of the xgb source model
+# ---------------------------------------------------------------------------
+
+def _holdout_split(matrix: RewardMatrix, home: list[str | None], test_frac: float,
+ seed: int) -> tuple[np.ndarray, np.ndarray]:
+ """Split query indices into (train, test), stratified by home source so
+ every source's home queries appear in both splits where possible."""
+ rng = np.random.default_rng(seed)
+ by_home: dict[str | None, list[int]] = {}
+ for q, h in enumerate(home):
+ by_home.setdefault(h, []).append(q)
+ test: list[int] = []
+ for qs in by_home.values():
+ qs = list(rng.permutation(qs))
+ n_test = round(len(qs) * test_frac)
+ if len(qs) >= 2:
+ n_test = max(n_test, 1)
+ test.extend(qs[:n_test])
+ test_idx = np.array(sorted(test), dtype=int)
+ train_idx = np.array(sorted(set(range(len(home))) - set(test)), dtype=int)
+ return train_idx, test_idx
+
+
+def _home_recall_at_k(scores: np.ndarray, matrix: RewardMatrix,
+ home: list[str | None], rows: np.ndarray, k: int) -> float:
+ """Fraction of queries whose home source lands in the top-k scored sources."""
+ s_index = {name: i for i, name in enumerate(matrix.sources)}
+ hits, n = 0, 0
+ for q in rows:
+ h = home[q]
+ if h is None or h not in s_index:
+ continue
+ avail = np.where(matrix.mask[q])[0]
+ if avail.size == 0 or s_index[h] not in avail:
+ continue
+ top = avail[np.argsort(scores[q, avail])[::-1][:k]]
+ hits += int(s_index[h] in top)
+ n += 1
+ return hits / n if n else 0.0
+
+
+def _lints_posterior_means(matrix: RewardMatrix, train_idx: np.ndarray, k: int,
+ nu: float = 0.05, sigma2: float = 0.25,
+ lam: float = 1.0, seed: int = 0) -> np.ndarray:
+ """Replay LinTS over the train queries; return per-arm posterior means for
+ greedy scoring of held-out queries."""
+ rng = np.random.default_rng(seed)
+ S, F = matrix.X.shape[1], matrix.X.shape[2]
+ A = np.stack([lam * np.eye(F) for _ in range(S)])
+ b = np.zeros((S, F))
+ for q in train_idx:
+ avail = np.where(matrix.mask[q])[0]
+ if avail.size == 0:
+ continue
+ scores = np.full(S, -np.inf)
+ for s in avail:
+ A_inv = np.linalg.inv(A[s])
+ mean = A_inv @ b[s]
+ cov = (nu * nu * sigma2) * A_inv
+ theta = rng.multivariate_normal(mean, 0.5 * (cov + cov.T))
+ scores[s] = theta @ matrix.X[q, s]
+ for s in avail[np.argsort(scores[avail])[::-1][:k]]:
+ x = matrix.X[q, s]
+ A[s] += np.outer(x, x)
+ b[s] += matrix.R[q, s] * x
+ return np.stack([np.linalg.inv(A[s]) @ b[s] for s in range(S)])
+
+
+def evaluate_holdout(matrix: RewardMatrix, k: int = 10, test_frac: float = 0.2,
+ seed: int = 0, home_by_query: dict[str, str] | None = None,
+ params: dict | None = None) -> dict:
+ """Train the xgb source model on a query split, evaluate on the rest.
+
+ ``home_by_query`` maps a query to the source it was written for (the
+ synthetic per-source query set); if omitted, each query's oracle-best
+ source stands in. Returns test coverage@k, RMSE, home-source recall@k,
+ and the same metrics for random/popularity/cosine baselines plus a
+ train-then-greedy LinTS comparator.
+ """
+ from mwmbl.tinysearchengine.super_search_select import xgb_model
+
+ if home_by_query is not None:
+ home = [home_by_query.get(q) for q in matrix.queries]
+ else:
+ home = [matrix.sources[int(np.argmax(np.where(matrix.mask[q], matrix.R[q], -np.inf)))]
+ if matrix.mask[q].any() else None
+ for q in range(len(matrix.queries))]
+ train_idx, test_idx = _holdout_split(matrix, home, test_frac, seed)
+
+ vocab = xgb_model.build_vocab(matrix.sources)
+ vocab_index = {name: i for i, name in enumerate(vocab)}
+
+ def rows_for(idx: np.ndarray):
+ xs, ys, cells = [], [], []
+ for q in idx:
+ for s in np.where(matrix.mask[q])[0]:
+ xs.append(xgb_model.encode(matrix.X[q, s], matrix.sources[s], vocab_index))
+ ys.append(float(matrix.R[q, s]))
+ cells.append((q, s))
+ return np.stack(xs), np.asarray(ys), cells
+
+ X_train, y_train, _ = rows_for(train_idx)
+ X_test, y_test, test_cells = rows_for(test_idx)
+ model = xgb_model.train(X_train, y_train, params=params)
+ preds = model.predict(X_test)
+
+ scores = np.full(matrix.R.shape, -np.inf)
+ for (q, s), p in zip(test_cells, preds):
+ scores[q, s] = p
+
+ R_t, m_t = matrix.R[test_idx], matrix.mask[test_idx]
+
+ def coverage(score_grid: np.ndarray) -> float:
+ return coverage_at_k(score_grid[test_idx], R_t, m_t, k)
+
+ rng = np.random.default_rng(seed)
+ rand_scores = rng.random(matrix.R.shape)
+ pop_i = matrix.feature_names.index("popularity")
+ cos_i = matrix.feature_names.index("cos_bow")
+ theta = _lints_posterior_means(matrix, train_idx, k, seed=seed)
+ lints_scores = np.einsum("qsf,sf->qs", matrix.X, theta)
+
+ out = {
+ "n_train_queries": int(train_idx.size),
+ "n_test_queries": int(test_idx.size),
+ "rmse": float(np.sqrt(np.mean((preds - y_test) ** 2))),
+ "coverage_at_k": {
+ "xgb": coverage(scores),
+ "lints_greedy": coverage(lints_scores),
+ "cosine": coverage(matrix.X[:, :, cos_i]),
+ "popularity": coverage(matrix.X[:, :, pop_i]),
+ "random": coverage(rand_scores),
+ },
+ "home_recall_at_k": {
+ "xgb": _home_recall_at_k(scores, matrix, home, test_idx, k),
+ "lints_greedy": _home_recall_at_k(lints_scores, matrix, home, test_idx, k),
+ "cosine": _home_recall_at_k(matrix.X[:, :, cos_i], matrix, home, test_idx, k),
+ "popularity": _home_recall_at_k(matrix.X[:, :, pop_i], matrix, home, test_idx, k),
+ "random": _home_recall_at_k(rand_scores, matrix, home, test_idx, k),
+ },
+ }
+ return out
diff --git a/scripts/super_search_eval.py b/scripts/super_search_eval.py
index a1ebb99..db08ee6 100644
--- a/scripts/super_search_eval.py
+++ b/scripts/super_search_eval.py
@@ -3,9 +3,12 @@
Subcommands:
- build-matrix - query every source for each query in a query file, rank the
- union with the LTR model, and write a dense (queries x sources)
- reward + feature matrix. Requires network + Django.
+ build-matrix - query every source for each query in a query file and write a
+ dense (queries x sources) reward + feature matrix. Rewards are
+ either LTR-top-K survival (--reward survival, default) or the
+ fine-tuned relevance judge's mean per-source doc score
+ (--reward judge). The network fetch is checkpointed per query
+ (JSONL next to --out) and resumable. Requires network + Django.
build-gold-matrix
- build the same kind of matrix offline from the LTR dataset's
@@ -22,6 +25,15 @@ Subcommands:
exploration scale nu, and print it against random / popularity
/ cosine / oracle baselines.
+ simulate-xgb - replay the epsilon-greedy XGBoost contextual bandit over the
+ matrix for each candidate epsilon, printed against the same
+ baselines and the best LinTS nu for a direct comparison.
+
+ holdout - standard offline evaluation of the xgb source model: split
+ queries train/test (stratified by home source), fit on train,
+ report test coverage@k and home-source recall@k vs baselines.
+ --queries-by-source supplies the {source: [queries]} home map.
+
Usage:
DJANGO_SETTINGS_MODULE=mwmbl.settings_dev uv run python scripts/super_search_eval.py \
build-matrix --queries queries.txt --out eval_matrix
@@ -30,6 +42,7 @@ Usage:
"""
import argparse
import asyncio
+import json
import math
import os
import sys
@@ -76,12 +89,82 @@ async def _query_all_sources(query: str, limit: int):
return dict(results)
-def build_matrix(queries: list[str], out: str):
- from django.conf import settings
+def _fetch_all_queries(queries: list[str], checkpoint: Path, limit: int) -> dict[str, dict]:
+ """Pass 1 with a resumable per-query JSONL checkpoint.
+
+ Returns ``{query: {source: [[url, title, extract], ...]}}``. Queries already
+ in the checkpoint are not re-fetched, so a multi-hour network run can be
+ interrupted and resumed.
+ """
+ from mwmbl.tinysearchengine.super_search_sources import SOURCES
+
+ fetched: dict[str, dict] = {}
+ if checkpoint.exists():
+ for line in checkpoint.read_text().splitlines():
+ rec = json.loads(line)
+ fetched[rec["query"]] = rec["docs"]
+ print(f"checkpoint {checkpoint}: {len(fetched)} queries already fetched")
+ todo = [q for q in queries if q not in fetched]
+ with checkpoint.open("a") as fh:
+ for qi, query in enumerate(todo):
+ docs_by_source = asyncio.run(_query_all_sources(query, limit))
+ docs = {name: [[d.url, d.title, d.extract] for d in ds]
+ for name, ds in docs_by_source.items() if ds}
+ fetched[query] = docs
+ fh.write(json.dumps({"query": query, "docs": docs}) + "\n")
+ fh.flush()
+ print(f" [{qi + 1}/{len(todo)}] {query!r}: "
+ f"{sum(len(d) for d in docs.values())} docs")
+ return fetched
+
+
+def _survival_rewards(query: str, docs_by_source: dict, s_index: dict, R, qi: int,
+ limit: int, top_k: int) -> None:
+ """Reward = fraction of a source's results surviving the LTR model's top-K."""
from mwmbl.search_setup import ltr_model
+ from mwmbl.tinysearchengine.indexer import Document
from mwmbl.tinysearchengine.ltr_rank import score_documents
+
+ source_by_url, all_docs = {}, []
+ for name, docs in docs_by_source.items():
+ for url, title, extract in docs:
+ if url and title:
+ source_by_url.setdefault(url, name)
+ all_docs.append(Document(title=title, url=url, extract=extract or ""))
+ if not all_docs:
+ return
+ scores = score_documents(ltr_model, query, all_docs)
+ ranked = [d.url for d, _ in sorted(zip(all_docs, scores), key=lambda x: -x[1])][:top_k]
+ counts = {}
+ for url in ranked:
+ src = source_by_url.get(url)
+ if src:
+ counts[src] = counts.get(src, 0) + 1
+ for name, c in counts.items():
+ R[qi, s_index[name]] = min(c / max(limit, 1), 1.0)
+
+
+def _judge_rewards(query: str, docs_by_source: dict, s_index: dict, R, qi: int) -> None:
+ """Reward = mean relevance-judge score of the source's returned docs — the
+ offline analog of rewards.compute_judge_rewards."""
+ from mwmbl.tinysearchengine.super_search_select.judge import doc_text, get_judge
+
+ judge = get_judge()
+ if judge is None:
+ raise RuntimeError(
+ "--reward judge requires the relevance judge artifact "
+ "(settings.SUPER_SEARCH_JUDGE_MODEL_DIR)")
+ for name, docs in docs_by_source.items():
+ texts = [doc_text(title, extract) for _, title, extract in docs]
+ if texts:
+ R[qi, s_index[name]] = float(np.mean(judge.score(query, texts)))
+
+
+def build_matrix(queries: list[str], out: str, reward: str = "survival",
+ checkpoint: str | None = None):
+ from django.conf import settings
from mwmbl.tinysearchengine.super_search_sources import SOURCES
- from mwmbl.tinysearchengine.super_search_select import profiles, vectors
+ from mwmbl.tinysearchengine.super_search_select import vectors
from mwmbl.tinysearchengine.super_search_select.features import (
FEATURE_NAMES, QueryContext, feature_vector,
)
@@ -95,59 +178,45 @@ def build_matrix(queries: list[str], out: str):
dim = settings.SUPER_SEARCH_PROJECTION_DIM
F = len(FEATURE_NAMES)
- # Pass 1: query everything, recording per-query docs and accumulating each
- # site's content profile (mean of its projected result samples).
- per_query_docs = [] # list[dict[source -> list[Document]]]
+ # Pass 1: query everything (resumable), then accumulate each site's content
+ # profile (mean of its projected result samples).
+ cp = Path(checkpoint) if checkpoint else Path(f"{out}.fetch.jsonl")
+ fetched = _fetch_all_queries(queries, cp, limit)
prof_bow = {n: np.zeros(dim) for n in sources}
prof_cng = {n: np.zeros(dim) for n in sources}
- for qi, query in enumerate(queries):
- docs_by_source = asyncio.run(_query_all_sources(query, limit))
- per_query_docs.append(docs_by_source)
- for name, docs in docs_by_source.items():
- if docs:
- text = profiles.sample_text(docs)
- prof_bow[name] += vectors.project_bow(text, dim)
- prof_cng[name] += vectors.project_char_ngrams(text, dim)
- print(f" [{qi + 1}/{len(queries)}] {query!r}: "
- f"{sum(len(d) for d in docs_by_source.values())} docs")
+ for query in queries:
+ for name, docs in fetched[query].items():
+ text = " ".join(f"{title or ''} {extract or ''}" for _, title, extract in docs)
+ prof_bow[name] += vectors.project_bow(text, dim)
+ prof_cng[name] += vectors.project_char_ngrams(text, dim)
profile = {n: (vectors._l2_normalise(prof_bow[n]), vectors._l2_normalise(prof_cng[n]))
for n in sources}
- # Pass 2: features + implicit-contribution reward against the LTR top-K.
+ # Pass 2: features + rewards.
Q, S = len(queries), len(sources)
X = np.zeros((Q, S, F))
R = np.zeros((Q, S))
mask = np.zeros((Q, S), dtype=bool)
- for qi, (query, docs_by_source) in enumerate(zip(queries, per_query_docs)):
+ for qi, query in enumerate(queries):
+ docs_by_source = fetched[query]
bow = vectors.project_bow(query, dim)
cng = vectors.project_char_ngrams(query, dim)
qctx = QueryContext.build(query, bow, cng)
-
- source_by_url, all_docs = {}, []
- for name, docs in docs_by_source.items():
- si = s_index[name]
- X[qi, si] = feature_vector(qctx, get_meta(name), profile[name])
- if docs:
- mask[qi, si] = True
- for d in docs:
- if d.url and d.title:
- source_by_url.setdefault(d.url, name)
- all_docs.append(d)
- if all_docs:
- scores = score_documents(ltr_model, query, all_docs)
- ranked = [d.url for d, _ in sorted(zip(all_docs, scores), key=lambda x: -x[1])][:top_k]
- counts = {}
- for url in ranked:
- src = source_by_url.get(url)
- if src:
- counts[src] = counts.get(src, 0) + 1
- for name, c in counts.items():
- R[qi, s_index[name]] = min(c / max(limit, 1), 1.0)
+ for name in sources:
+ X[qi, s_index[name]] = feature_vector(qctx, get_meta(name), profile[name])
+ for name in docs_by_source:
+ mask[qi, s_index[name]] = True
+ if reward == "judge":
+ _judge_rewards(query, docs_by_source, s_index, R, qi)
+ else:
+ _survival_rewards(query, docs_by_source, s_index, R, qi, limit, top_k)
+ if (qi + 1) % 50 == 0:
+ print(f" scored [{qi + 1}/{Q}]")
matrix = RewardMatrix(queries=queries, sources=sources,
feature_names=list(FEATURE_NAMES), X=X, R=R, mask=mask)
matrix.save(out)
- print(f"Wrote matrix {out}.npz/.json: {Q} queries x {S} sources, "
+ print(f"Wrote matrix {out}.npz/.json ({reward} rewards): {Q} queries x {S} sources, "
f"{int(mask.sum())} filled cells.")
@@ -310,6 +379,50 @@ def cmd_simulate(matrix_path: str, k: int):
print(f" nu={nu:<4} {val:.4f}{marker}")
+def cmd_simulate_xgb(matrix_path: str, k: int, epsilons: list[float],
+ refit_every: int, min_rows: int):
+ from mwmbl.tinysearchengine.super_search_select.evaluation import (
+ RewardMatrix, simulate_baselines, sweep_epsilon, sweep_explore_scale,
+ )
+ m = RewardMatrix.load(matrix_path)
+ base = simulate_baselines(m, k=k)
+ print("baselines (mean captured reward per query):")
+ for name, val in sorted(base.items(), key=lambda kv: -kv[1]):
+ print(f" {name:12} {val:.4f}")
+ ts = sweep_explore_scale(m, k=k, nus=[0.05, 0.25, 1.0])
+ best_nu = max(ts, key=ts.get)
+ print(f"\nbest LinTS: nu={best_nu} {ts[best_nu]:.4f}")
+ print("\nXGB contextual bandit by epsilon:")
+ sweep = sweep_epsilon(m, k=k, epsilons=epsilons,
+ refit_every=refit_every, min_rows=min_rows)
+ best = max(sweep, key=sweep.get)
+ for eps, val in sweep.items():
+ marker = " <- best" if eps == best else ""
+ print(f" eps={eps:<5} {val:.4f}{marker}")
+
+
+def cmd_holdout(matrix_path: str, k: int, test_frac: float,
+ queries_by_source: str | None, seed: int):
+ from mwmbl.tinysearchengine.super_search_select.evaluation import (
+ RewardMatrix, evaluate_holdout,
+ )
+ m = RewardMatrix.load(matrix_path)
+ home_by_query = None
+ if queries_by_source:
+ by_source = json.loads(Path(queries_by_source).read_text())
+ home_by_query = {q: source for source, qs in by_source.items() for q in qs}
+ result = evaluate_holdout(m, k=k, test_frac=test_frac, seed=seed,
+ home_by_query=home_by_query)
+ print(f"train queries: {result['n_train_queries']}, "
+ f"test queries: {result['n_test_queries']}")
+ print(f"xgb test RMSE: {result['rmse']:.4f}\n")
+ for metric in ("coverage_at_k", "home_recall_at_k"):
+ print(f"{metric} (k={k}):")
+ for name, val in sorted(result[metric].items(), key=lambda kv: -kv[1]):
+ print(f" {name:14} {val:.4f}")
+ print()
+
+
def main():
parser = argparse.ArgumentParser(description=__doc__,
formatter_class=argparse.RawDescriptionHelpFormatter)
@@ -318,6 +431,9 @@ def main():
p_build = sub.add_parser("build-matrix")
p_build.add_argument("--queries", required=True, help="file with one query per line")
p_build.add_argument("--out", default="eval_matrix")
+ p_build.add_argument("--reward", choices=["survival", "judge"], default="survival")
+ p_build.add_argument("--checkpoint", default=None,
+ help="fetch checkpoint JSONL (default: <out>.fetch.jsonl)")
p_gold = sub.add_parser("build-gold-matrix")
p_gold.add_argument("--out", default="devdata/ss_gold_matrix")
@@ -330,18 +446,40 @@ def main():
p_sim.add_argument("--matrix", default="eval_matrix")
p_sim.add_argument("--k", type=int, default=10)
+ p_sxgb = sub.add_parser("simulate-xgb")
+ p_sxgb.add_argument("--matrix", default="eval_matrix")
+ p_sxgb.add_argument("--k", type=int, default=10)
+ p_sxgb.add_argument("--epsilons", type=float, nargs="+",
+ default=[0.0, 0.05, 0.1, 0.2])
+ p_sxgb.add_argument("--refit-every", type=int, default=200)
+ p_sxgb.add_argument("--min-rows", type=int, default=300)
+
+ p_hold = sub.add_parser("holdout")
+ p_hold.add_argument("--matrix", default="eval_matrix")
+ p_hold.add_argument("--k", type=int, default=10)
+ p_hold.add_argument("--test-frac", type=float, default=0.2)
+ p_hold.add_argument("--queries-by-source", default=None,
+ help="JSON {source: [queries]} home map (ss_source_queries.json)")
+ p_hold.add_argument("--seed", type=int, default=0)
+
args = parser.parse_args()
_bootstrap_django()
if args.command == "build-matrix":
queries = [ln.strip() for ln in Path(args.queries).read_text().splitlines() if ln.strip()]
- build_matrix(queries, args.out)
+ build_matrix(queries, args.out, reward=args.reward, checkpoint=args.checkpoint)
elif args.command == "build-gold-matrix":
build_gold_matrix(args.out)
elif args.command == "select":
cmd_select(args.matrix, args.k)
elif args.command == "simulate":
cmd_simulate(args.matrix, args.k)
+ elif args.command == "simulate-xgb":
+ cmd_simulate_xgb(args.matrix, args.k, args.epsilons,
+ args.refit_every, args.min_rows)
+ elif args.command == "holdout":
+ cmd_holdout(args.matrix, args.k, args.test_frac,
+ args.queries_by_source, args.seed)
if __name__ == "__main__":
diff --git a/test/test_super_search_eval.py b/test/test_super_search_eval.py
index 52ac0a0..6c68011 100644
--- a/test/test_super_search_eval.py
+++ b/test/test_super_search_eval.py
@@ -99,6 +99,77 @@ def test_feature_selection_flags_cos_bow(tmp_path):
assert drops["cos_bow"] == max(drops.values())
+# ---------------------------------------------------------------------------
+# XGB contextual bandit: replay + holdout evaluation
+# ---------------------------------------------------------------------------
+
+FAST_XGB = {"n_estimators": 50, "max_depth": 6, "learning_rate": 0.1,
+ "subsample": 0.8, "random_state": 0}
+
+
+def _routing_matrix(Q=160, S=16, seed=0):
+ """Planted intent -> source routing that only identity x intent explains:
+ each source is affine to one intent, each query carries one intent, and the
+ reward is high iff they match. Shared features alone can't predict it.
+ Returns (matrix, home_by_query) where home is the designated matched source.
+ """
+ rng = np.random.default_rng(seed)
+ F = len(FEATURE_NAMES)
+ n_intents = 8
+ first_intent = FEATURE_NAMES.index("intent_code")
+ X = np.zeros((Q, S, F))
+ X[:, :, 0] = 1.0 # bias
+ R = np.zeros((Q, S))
+ home = {}
+ for q in range(Q):
+ intent = q % n_intents
+ X[q, :, first_intent + intent] = 1.0
+ matched = [s for s in range(S) if s % n_intents == intent]
+ for s in range(S):
+ base = 0.9 if s % n_intents == intent else 0.1
+ R[q, s] = np.clip(base + 0.02 * rng.standard_normal(), 0.0, 1.0)
+ home[f"q{q}"] = f"s{matched[q % len(matched)]}"
+ mask = np.ones((Q, S), dtype=bool)
+ matrix = RewardMatrix(
+ queries=[f"q{i}" for i in range(Q)],
+ sources=[f"s{i}" for i in range(S)],
+ feature_names=list(FEATURE_NAMES), X=X, R=R, mask=mask,
+ )
+ return matrix, home
+
+
+def test_simulate_xgb_beats_random():
+ pytest.importorskip("xgboost")
+ m = _synthetic_matrix()
+ base = evaluation.simulate_baselines(m, k=10)
+ captured = evaluation.simulate_xgb(m, k=10, epsilon=0.1, refit_every=30,
+ min_rows=100, params=FAST_XGB)
+ assert captured > base["random"]
+ assert captured <= base["oracle"] + 1e-9
+
+
+def test_evaluate_holdout_learns_identity_intent_routing():
+ pytest.importorskip("xgboost")
+ m, home = _routing_matrix()
+ result = evaluation.evaluate_holdout(m, k=3, test_frac=0.25,
+ home_by_query=home, params=FAST_XGB)
+ cov, recall = result["coverage_at_k"], result["home_recall_at_k"]
+ # Only identity x intent explains the reward: xgb should crush the
+ # identity-blind baselines on both metrics.
+ assert cov["xgb"] > cov["random"] + 0.1
+ assert cov["xgb"] > cov["popularity"]
+ assert recall["xgb"] > recall["random"] + 0.2
+ assert result["rmse"] < 0.2
+
+
+def test_evaluate_holdout_default_home_is_oracle_best():
+ pytest.importorskip("xgboost")
+ m, _ = _routing_matrix(Q=80, S=8)
+ result = evaluation.evaluate_holdout(m, k=3, test_frac=0.25, params=FAST_XGB)
+ assert result["n_train_queries"] + result["n_test_queries"] == 80
+ assert 0.0 <= result["home_recall_at_k"]["xgb"] <= 1.0
+
+
# ---------------------------------------------------------------------------
# Domain matching (shared helpers behind the gold-grounded matrix)
# ---------------------------------------------------------------------------