An open source, non-profit web search engine
Offline eval for the xgb source model: judge rewards, holdout, replay
Daoud Clarke99097cf76fe4
diff --git a/mwmbl/tinysearchengine/super_search_select/evaluation.py b/mwmbl/tinysearchengine/super_search_select/evaluation.pyindex 3f8a6c1..bdc4388 100644--- a/mwmbl/tinysearchengine/super_search_select/evaluation.py+++ b/mwmbl/tinysearchengine/super_search_select/evaluation.py@@ -8,9 +8,20 @@ top-K), and a ``mask`` of which sources actually returned anything.With every cell filled we can, without any online traffic:* select features — fit an XGBoost reward model (grouped CV by query) andrank/ablate features by held-out recall@k;- * simulate the policy — replay linear-Gaussian Thompson sampling over the- matrix, sweep the exploration scale ``nu``, and compare against baselines- (random / popularity / cosine / oracle).+ * simulate the policy — replay linear-Gaussian Thompson sampling+ (``simulate_ts``) or the epsilon-greedy XGBoost contextual bandit+ (``simulate_xgb``) over the matrix and compare against baselines+ (random / popularity / cosine / oracle);+ * holdout-evaluate the xgb source model (``evaluate_holdout``): split+ queries into train/test (stratified by each query's home source), fit on+ train, and report test coverage@k / home-source recall@k against the same+ baselines plus a train-then-greedy LinTS comparator.++Off-policy note: replay over a *dense* matrix is unbiased (every arm's+counterfactual reward is known). The online impression log, by contrast, is+conditioned on whatever policy was live; the mitigations are warm-starting+from a dense offline matrix and the epsilon slots continuously injecting+randomized pairs.This module is pure (numpy/sklearn/xgboost only); building the matrix from livesources lives in ``scripts/super_search_eval.py``.@@ -212,3 +223,208 @@ def sweep_explore_scale(matrix: RewardMatrix, k: int, nus: list[float],sigma2: float = 0.25, lam: float = 1.0, seed: int = 0) -> dict[float, float]:"""Mean captured reward for each candidate exploration scale ``nu``."""return {nu: simulate_ts(matrix, k, nu, sigma2, lam, seed) for nu in nus}+++def simulate_xgb(matrix: RewardMatrix, k: int, epsilon: float,+ refit_every: int = 200, min_rows: int = 300, seed: int = 0,+ params: dict | None = None) -> float:+ """Replay the epsilon-greedy XGBoost contextual bandit over the matrix.++ Mirrors ``simulate_ts``'s contract (mean per-query captured reward).+ Queries are visited in a seeded shuffled order; until ``min_rows`` chosen+ (source, reward) pairs have accumulated, selection uses the cosine+ baseline (matching how real impression logs bootstrap), after which the+ model is fit and refit every ``refit_every`` queries.+ """+ from mwmbl.tinysearchengine.super_search_select import xgb_model++ rng = np.random.default_rng(seed)+ Q = matrix.X.shape[0]+ vocab = xgb_model.build_vocab(matrix.sources)+ vocab_index = {name: i for i, name in enumerate(vocab)}+ cos_i = matrix.feature_names.index("cos_bow")++ model = None+ xs: list[np.ndarray] = []+ ys: list[float] = []+ captured = 0.0+ since_fit = 0+ for q in rng.permutation(Q):+ avail = np.where(matrix.mask[q])[0]+ if avail.size == 0:+ continue+ k_q = min(k, avail.size)+ if model is None:+ chosen = avail[np.argsort(matrix.X[q, avail, cos_i])[::-1][:k_q]]+ else:+ enc = np.stack([xgb_model.encode(matrix.X[q, s], matrix.sources[s], vocab_index)+ for s in avail])+ ranked = avail[np.argsort(model.predict(enc))[::-1]]+ n_explore = int(rng.binomial(k_q, epsilon))+ chosen = list(ranked[:k_q - n_explore])+ rest = ranked[k_q - n_explore:]+ if n_explore and rest.size:+ chosen += list(rng.choice(rest, size=min(n_explore, rest.size), replace=False))+ chosen = np.asarray(chosen)+ for s in chosen:+ xs.append(xgb_model.encode(matrix.X[q, s], matrix.sources[s], vocab_index))+ ys.append(float(matrix.R[q, s]))+ captured += matrix.R[q, s]+ since_fit += 1+ if len(ys) >= min_rows and (model is None or since_fit >= refit_every):+ model = xgb_model.train(np.stack(xs), np.asarray(ys), params=params)+ since_fit = 0+ return captured / Q+++def sweep_epsilon(matrix: RewardMatrix, k: int, epsilons: list[float],+ refit_every: int = 200, min_rows: int = 300,+ seed: int = 0) -> dict[float, float]:+ """Mean captured reward for each candidate exploration rate ``epsilon``."""+ return {eps: simulate_xgb(matrix, k, eps, refit_every, min_rows, seed)+ for eps in epsilons}+++# ---------------------------------------------------------------------------+# Holdout evaluation of the xgb source model+# ---------------------------------------------------------------------------++def _holdout_split(matrix: RewardMatrix, home: list[str | None], test_frac: float,+ seed: int) -> tuple[np.ndarray, np.ndarray]:+ """Split query indices into (train, test), stratified by home source so+ every source's home queries appear in both splits where possible."""+ rng = np.random.default_rng(seed)+ by_home: dict[str | None, list[int]] = {}+ for q, h in enumerate(home):+ by_home.setdefault(h, []).append(q)+ test: list[int] = []+ for qs in by_home.values():+ qs = list(rng.permutation(qs))+ n_test = round(len(qs) * test_frac)+ if len(qs) >= 2:+ n_test = max(n_test, 1)+ test.extend(qs[:n_test])+ test_idx = np.array(sorted(test), dtype=int)+ train_idx = np.array(sorted(set(range(len(home))) - set(test)), dtype=int)+ return train_idx, test_idx+++def _home_recall_at_k(scores: np.ndarray, matrix: RewardMatrix,+ home: list[str | None], rows: np.ndarray, k: int) -> float:+ """Fraction of queries whose home source lands in the top-k scored sources."""+ s_index = {name: i for i, name in enumerate(matrix.sources)}+ hits, n = 0, 0+ for q in rows:+ h = home[q]+ if h is None or h not in s_index:+ continue+ avail = np.where(matrix.mask[q])[0]+ if avail.size == 0 or s_index[h] not in avail:+ continue+ top = avail[np.argsort(scores[q, avail])[::-1][:k]]+ hits += int(s_index[h] in top)+ n += 1+ return hits / n if n else 0.0+++def _lints_posterior_means(matrix: RewardMatrix, train_idx: np.ndarray, k: int,+ nu: float = 0.05, sigma2: float = 0.25,+ lam: float = 1.0, seed: int = 0) -> np.ndarray:+ """Replay LinTS over the train queries; return per-arm posterior means for+ greedy scoring of held-out queries."""+ rng = np.random.default_rng(seed)+ S, F = matrix.X.shape[1], matrix.X.shape[2]+ A = np.stack([lam * np.eye(F) for _ in range(S)])+ b = np.zeros((S, F))+ for q in train_idx:+ avail = np.where(matrix.mask[q])[0]+ if avail.size == 0:+ continue+ scores = np.full(S, -np.inf)+ for s in avail:+ A_inv = np.linalg.inv(A[s])+ mean = A_inv @ b[s]+ cov = (nu * nu * sigma2) * A_inv+ theta = rng.multivariate_normal(mean, 0.5 * (cov + cov.T))+ scores[s] = theta @ matrix.X[q, s]+ for s in avail[np.argsort(scores[avail])[::-1][:k]]:+ x = matrix.X[q, s]+ A[s] += np.outer(x, x)+ b[s] += matrix.R[q, s] * x+ return np.stack([np.linalg.inv(A[s]) @ b[s] for s in range(S)])+++def evaluate_holdout(matrix: RewardMatrix, k: int = 10, test_frac: float = 0.2,+ seed: int = 0, home_by_query: dict[str, str] | None = None,+ params: dict | None = None) -> dict:+ """Train the xgb source model on a query split, evaluate on the rest.++ ``home_by_query`` maps a query to the source it was written for (the+ synthetic per-source query set); if omitted, each query's oracle-best+ source stands in. Returns test coverage@k, RMSE, home-source recall@k,+ and the same metrics for random/popularity/cosine baselines plus a+ train-then-greedy LinTS comparator.+ """+ from mwmbl.tinysearchengine.super_search_select import xgb_model++ if home_by_query is not None:+ home = [home_by_query.get(q) for q in matrix.queries]+ else:+ home = [matrix.sources[int(np.argmax(np.where(matrix.mask[q], matrix.R[q], -np.inf)))]+ if matrix.mask[q].any() else None+ for q in range(len(matrix.queries))]+ train_idx, test_idx = _holdout_split(matrix, home, test_frac, seed)++ vocab = xgb_model.build_vocab(matrix.sources)+ vocab_index = {name: i for i, name in enumerate(vocab)}++ def rows_for(idx: np.ndarray):+ xs, ys, cells = [], [], []+ for q in idx:+ for s in np.where(matrix.mask[q])[0]:+ xs.append(xgb_model.encode(matrix.X[q, s], matrix.sources[s], vocab_index))+ ys.append(float(matrix.R[q, s]))+ cells.append((q, s))+ return np.stack(xs), np.asarray(ys), cells++ X_train, y_train, _ = rows_for(train_idx)+ X_test, y_test, test_cells = rows_for(test_idx)+ model = xgb_model.train(X_train, y_train, params=params)+ preds = model.predict(X_test)++ scores = np.full(matrix.R.shape, -np.inf)+ for (q, s), p in zip(test_cells, preds):+ scores[q, s] = p++ R_t, m_t = matrix.R[test_idx], matrix.mask[test_idx]++ def coverage(score_grid: np.ndarray) -> float:+ return coverage_at_k(score_grid[test_idx], R_t, m_t, k)++ rng = np.random.default_rng(seed)+ rand_scores = rng.random(matrix.R.shape)+ pop_i = matrix.feature_names.index("popularity")+ cos_i = matrix.feature_names.index("cos_bow")+ theta = _lints_posterior_means(matrix, train_idx, k, seed=seed)+ lints_scores = np.einsum("qsf,sf->qs", matrix.X, theta)++ out = {+ "n_train_queries": int(train_idx.size),+ "n_test_queries": int(test_idx.size),+ "rmse": float(np.sqrt(np.mean((preds - y_test) ** 2))),+ "coverage_at_k": {+ "xgb": coverage(scores),+ "lints_greedy": coverage(lints_scores),+ "cosine": coverage(matrix.X[:, :, cos_i]),+ "popularity": coverage(matrix.X[:, :, pop_i]),+ "random": coverage(rand_scores),+ },+ "home_recall_at_k": {+ "xgb": _home_recall_at_k(scores, matrix, home, test_idx, k),+ "lints_greedy": _home_recall_at_k(lints_scores, matrix, home, test_idx, k),+ "cosine": _home_recall_at_k(matrix.X[:, :, cos_i], matrix, home, test_idx, k),+ "popularity": _home_recall_at_k(matrix.X[:, :, pop_i], matrix, home, test_idx, k),+ "random": _home_recall_at_k(rand_scores, matrix, home, test_idx, k),+ },+ }+ return outdiff --git a/scripts/super_search_eval.py b/scripts/super_search_eval.pyindex a1ebb99..db08ee6 100644--- a/scripts/super_search_eval.py+++ b/scripts/super_search_eval.py@@ -3,9 +3,12 @@Subcommands:- build-matrix - query every source for each query in a query file, rank the- union with the LTR model, and write a dense (queries x sources)- reward + feature matrix. Requires network + Django.+ build-matrix - query every source for each query in a query file and write a+ dense (queries x sources) reward + feature matrix. Rewards are+ either LTR-top-K survival (--reward survival, default) or the+ fine-tuned relevance judge's mean per-source doc score+ (--reward judge). The network fetch is checkpointed per query+ (JSONL next to --out) and resumable. Requires network + Django.build-gold-matrix- build the same kind of matrix offline from the LTR dataset's@@ -22,6 +25,15 @@ Subcommands:exploration scale nu, and print it against random / popularity/ cosine / oracle baselines.+ simulate-xgb - replay the epsilon-greedy XGBoost contextual bandit over the+ matrix for each candidate epsilon, printed against the same+ baselines and the best LinTS nu for a direct comparison.++ holdout - standard offline evaluation of the xgb source model: split+ queries train/test (stratified by home source), fit on train,+ report test coverage@k and home-source recall@k vs baselines.+ --queries-by-source supplies the {source: [queries]} home map.+Usage:DJANGO_SETTINGS_MODULE=mwmbl.settings_dev uv run python scripts/super_search_eval.py \build-matrix --queries queries.txt --out eval_matrix@@ -30,6 +42,7 @@ Usage:"""import argparseimport asyncio+import jsonimport mathimport osimport sys@@ -76,12 +89,82 @@ async def _query_all_sources(query: str, limit: int):return dict(results)-def build_matrix(queries: list[str], out: str):- from django.conf import settings+def _fetch_all_queries(queries: list[str], checkpoint: Path, limit: int) -> dict[str, dict]:+ """Pass 1 with a resumable per-query JSONL checkpoint.++ Returns ``{query: {source: [[url, title, extract], ...]}}``. Queries already+ in the checkpoint are not re-fetched, so a multi-hour network run can be+ interrupted and resumed.+ """+ from mwmbl.tinysearchengine.super_search_sources import SOURCES++ fetched: dict[str, dict] = {}+ if checkpoint.exists():+ for line in checkpoint.read_text().splitlines():+ rec = json.loads(line)+ fetched[rec["query"]] = rec["docs"]+ print(f"checkpoint {checkpoint}: {len(fetched)} queries already fetched")+ todo = [q for q in queries if q not in fetched]+ with checkpoint.open("a") as fh:+ for qi, query in enumerate(todo):+ docs_by_source = asyncio.run(_query_all_sources(query, limit))+ docs = {name: [[d.url, d.title, d.extract] for d in ds]+ for name, ds in docs_by_source.items() if ds}+ fetched[query] = docs+ fh.write(json.dumps({"query": query, "docs": docs}) + "\n")+ fh.flush()+ print(f" [{qi + 1}/{len(todo)}] {query!r}: "+ f"{sum(len(d) for d in docs.values())} docs")+ return fetched+++def _survival_rewards(query: str, docs_by_source: dict, s_index: dict, R, qi: int,+ limit: int, top_k: int) -> None:+ """Reward = fraction of a source's results surviving the LTR model's top-K."""from mwmbl.search_setup import ltr_model+ from mwmbl.tinysearchengine.indexer import Documentfrom mwmbl.tinysearchengine.ltr_rank import score_documents++ source_by_url, all_docs = {}, []+ for name, docs in docs_by_source.items():+ for url, title, extract in docs:+ if url and title:+ source_by_url.setdefault(url, name)+ all_docs.append(Document(title=title, url=url, extract=extract or ""))+ if not all_docs:+ return+ scores = score_documents(ltr_model, query, all_docs)+ ranked = [d.url for d, _ in sorted(zip(all_docs, scores), key=lambda x: -x[1])][:top_k]+ counts = {}+ for url in ranked:+ src = source_by_url.get(url)+ if src:+ counts[src] = counts.get(src, 0) + 1+ for name, c in counts.items():+ R[qi, s_index[name]] = min(c / max(limit, 1), 1.0)+++def _judge_rewards(query: str, docs_by_source: dict, s_index: dict, R, qi: int) -> None:+ """Reward = mean relevance-judge score of the source's returned docs — the+ offline analog of rewards.compute_judge_rewards."""+ from mwmbl.tinysearchengine.super_search_select.judge import doc_text, get_judge++ judge = get_judge()+ if judge is None:+ raise RuntimeError(+ "--reward judge requires the relevance judge artifact "+ "(settings.SUPER_SEARCH_JUDGE_MODEL_DIR)")+ for name, docs in docs_by_source.items():+ texts = [doc_text(title, extract) for _, title, extract in docs]+ if texts:+ R[qi, s_index[name]] = float(np.mean(judge.score(query, texts)))+++def build_matrix(queries: list[str], out: str, reward: str = "survival",+ checkpoint: str | None = None):+ from django.conf import settingsfrom mwmbl.tinysearchengine.super_search_sources import SOURCES- from mwmbl.tinysearchengine.super_search_select import profiles, vectors+ from mwmbl.tinysearchengine.super_search_select import vectorsfrom mwmbl.tinysearchengine.super_search_select.features import (FEATURE_NAMES, QueryContext, feature_vector,)@@ -95,59 +178,45 @@ def build_matrix(queries: list[str], out: str):dim = settings.SUPER_SEARCH_PROJECTION_DIMF = len(FEATURE_NAMES)- # Pass 1: query everything, recording per-query docs and accumulating each- # site's content profile (mean of its projected result samples).- per_query_docs = [] # list[dict[source -> list[Document]]]+ # Pass 1: query everything (resumable), then accumulate each site's content+ # profile (mean of its projected result samples).+ cp = Path(checkpoint) if checkpoint else Path(f"{out}.fetch.jsonl")+ fetched = _fetch_all_queries(queries, cp, limit)prof_bow = {n: np.zeros(dim) for n in sources}prof_cng = {n: np.zeros(dim) for n in sources}- for qi, query in enumerate(queries):- docs_by_source = asyncio.run(_query_all_sources(query, limit))- per_query_docs.append(docs_by_source)- for name, docs in docs_by_source.items():- if docs:- text = profiles.sample_text(docs)- prof_bow[name] += vectors.project_bow(text, dim)- prof_cng[name] += vectors.project_char_ngrams(text, dim)- print(f" [{qi + 1}/{len(queries)}] {query!r}: "- f"{sum(len(d) for d in docs_by_source.values())} docs")+ for query in queries:+ for name, docs in fetched[query].items():+ text = " ".join(f"{title or ''} {extract or ''}" for _, title, extract in docs)+ prof_bow[name] += vectors.project_bow(text, dim)+ prof_cng[name] += vectors.project_char_ngrams(text, dim)profile = {n: (vectors._l2_normalise(prof_bow[n]), vectors._l2_normalise(prof_cng[n]))for n in sources}- # Pass 2: features + implicit-contribution reward against the LTR top-K.+ # Pass 2: features + rewards.Q, S = len(queries), len(sources)X = np.zeros((Q, S, F))R = np.zeros((Q, S))mask = np.zeros((Q, S), dtype=bool)- for qi, (query, docs_by_source) in enumerate(zip(queries, per_query_docs)):+ for qi, query in enumerate(queries):+ docs_by_source = fetched[query]bow = vectors.project_bow(query, dim)cng = vectors.project_char_ngrams(query, dim)qctx = QueryContext.build(query, bow, cng)-- source_by_url, all_docs = {}, []- for name, docs in docs_by_source.items():- si = s_index[name]- X[qi, si] = feature_vector(qctx, get_meta(name), profile[name])- if docs:- mask[qi, si] = True- for d in docs:- if d.url and d.title:- source_by_url.setdefault(d.url, name)- all_docs.append(d)- if all_docs:- scores = score_documents(ltr_model, query, all_docs)- ranked = [d.url for d, _ in sorted(zip(all_docs, scores), key=lambda x: -x[1])][:top_k]- counts = {}- for url in ranked:- src = source_by_url.get(url)- if src:- counts[src] = counts.get(src, 0) + 1- for name, c in counts.items():- R[qi, s_index[name]] = min(c / max(limit, 1), 1.0)+ for name in sources:+ X[qi, s_index[name]] = feature_vector(qctx, get_meta(name), profile[name])+ for name in docs_by_source:+ mask[qi, s_index[name]] = True+ if reward == "judge":+ _judge_rewards(query, docs_by_source, s_index, R, qi)+ else:+ _survival_rewards(query, docs_by_source, s_index, R, qi, limit, top_k)+ if (qi + 1) % 50 == 0:+ print(f" scored [{qi + 1}/{Q}]")matrix = RewardMatrix(queries=queries, sources=sources,feature_names=list(FEATURE_NAMES), X=X, R=R, mask=mask)matrix.save(out)- print(f"Wrote matrix {out}.npz/.json: {Q} queries x {S} sources, "+ print(f"Wrote matrix {out}.npz/.json ({reward} rewards): {Q} queries x {S} sources, "f"{int(mask.sum())} filled cells.")@@ -310,6 +379,50 @@ def cmd_simulate(matrix_path: str, k: int):print(f" nu={nu:<4} {val:.4f}{marker}")+def cmd_simulate_xgb(matrix_path: str, k: int, epsilons: list[float],+ refit_every: int, min_rows: int):+ from mwmbl.tinysearchengine.super_search_select.evaluation import (+ RewardMatrix, simulate_baselines, sweep_epsilon, sweep_explore_scale,+ )+ m = RewardMatrix.load(matrix_path)+ base = simulate_baselines(m, k=k)+ print("baselines (mean captured reward per query):")+ for name, val in sorted(base.items(), key=lambda kv: -kv[1]):+ print(f" {name:12} {val:.4f}")+ ts = sweep_explore_scale(m, k=k, nus=[0.05, 0.25, 1.0])+ best_nu = max(ts, key=ts.get)+ print(f"\nbest LinTS: nu={best_nu} {ts[best_nu]:.4f}")+ print("\nXGB contextual bandit by epsilon:")+ sweep = sweep_epsilon(m, k=k, epsilons=epsilons,+ refit_every=refit_every, min_rows=min_rows)+ best = max(sweep, key=sweep.get)+ for eps, val in sweep.items():+ marker = " <- best" if eps == best else ""+ print(f" eps={eps:<5} {val:.4f}{marker}")+++def cmd_holdout(matrix_path: str, k: int, test_frac: float,+ queries_by_source: str | None, seed: int):+ from mwmbl.tinysearchengine.super_search_select.evaluation import (+ RewardMatrix, evaluate_holdout,+ )+ m = RewardMatrix.load(matrix_path)+ home_by_query = None+ if queries_by_source:+ by_source = json.loads(Path(queries_by_source).read_text())+ home_by_query = {q: source for source, qs in by_source.items() for q in qs}+ result = evaluate_holdout(m, k=k, test_frac=test_frac, seed=seed,+ home_by_query=home_by_query)+ print(f"train queries: {result['n_train_queries']}, "+ f"test queries: {result['n_test_queries']}")+ print(f"xgb test RMSE: {result['rmse']:.4f}\n")+ for metric in ("coverage_at_k", "home_recall_at_k"):+ print(f"{metric} (k={k}):")+ for name, val in sorted(result[metric].items(), key=lambda kv: -kv[1]):+ print(f" {name:14} {val:.4f}")+ print()++def main():parser = argparse.ArgumentParser(description=__doc__,formatter_class=argparse.RawDescriptionHelpFormatter)@@ -318,6 +431,9 @@ def main():p_build = sub.add_parser("build-matrix")p_build.add_argument("--queries", required=True, help="file with one query per line")p_build.add_argument("--out", default="eval_matrix")+ p_build.add_argument("--reward", choices=["survival", "judge"], default="survival")+ p_build.add_argument("--checkpoint", default=None,+ help="fetch checkpoint JSONL (default: <out>.fetch.jsonl)")p_gold = sub.add_parser("build-gold-matrix")p_gold.add_argument("--out", default="devdata/ss_gold_matrix")@@ -330,18 +446,40 @@ def main():p_sim.add_argument("--matrix", default="eval_matrix")p_sim.add_argument("--k", type=int, default=10)+ p_sxgb = sub.add_parser("simulate-xgb")+ p_sxgb.add_argument("--matrix", default="eval_matrix")+ p_sxgb.add_argument("--k", type=int, default=10)+ p_sxgb.add_argument("--epsilons", type=float, nargs="+",+ default=[0.0, 0.05, 0.1, 0.2])+ p_sxgb.add_argument("--refit-every", type=int, default=200)+ p_sxgb.add_argument("--min-rows", type=int, default=300)++ p_hold = sub.add_parser("holdout")+ p_hold.add_argument("--matrix", default="eval_matrix")+ p_hold.add_argument("--k", type=int, default=10)+ p_hold.add_argument("--test-frac", type=float, default=0.2)+ p_hold.add_argument("--queries-by-source", default=None,+ help="JSON {source: [queries]} home map (ss_source_queries.json)")+ p_hold.add_argument("--seed", type=int, default=0)+args = parser.parse_args()_bootstrap_django()if args.command == "build-matrix":queries = [ln.strip() for ln in Path(args.queries).read_text().splitlines() if ln.strip()]- build_matrix(queries, args.out)+ build_matrix(queries, args.out, reward=args.reward, checkpoint=args.checkpoint)elif args.command == "build-gold-matrix":build_gold_matrix(args.out)elif args.command == "select":cmd_select(args.matrix, args.k)elif args.command == "simulate":cmd_simulate(args.matrix, args.k)+ elif args.command == "simulate-xgb":+ cmd_simulate_xgb(args.matrix, args.k, args.epsilons,+ args.refit_every, args.min_rows)+ elif args.command == "holdout":+ cmd_holdout(args.matrix, args.k, args.test_frac,+ args.queries_by_source, args.seed)if __name__ == "__main__":diff --git a/test/test_super_search_eval.py b/test/test_super_search_eval.pyindex 52ac0a0..6c68011 100644--- a/test/test_super_search_eval.py+++ b/test/test_super_search_eval.py@@ -99,6 +99,77 @@ def test_feature_selection_flags_cos_bow(tmp_path):assert drops["cos_bow"] == max(drops.values())+# ---------------------------------------------------------------------------+# XGB contextual bandit: replay + holdout evaluation+# ---------------------------------------------------------------------------++FAST_XGB = {"n_estimators": 50, "max_depth": 6, "learning_rate": 0.1,+ "subsample": 0.8, "random_state": 0}+++def _routing_matrix(Q=160, S=16, seed=0):+ """Planted intent -> source routing that only identity x intent explains:+ each source is affine to one intent, each query carries one intent, and the+ reward is high iff they match. Shared features alone can't predict it.+ Returns (matrix, home_by_query) where home is the designated matched source.+ """+ rng = np.random.default_rng(seed)+ F = len(FEATURE_NAMES)+ n_intents = 8+ first_intent = FEATURE_NAMES.index("intent_code")+ X = np.zeros((Q, S, F))+ X[:, :, 0] = 1.0 # bias+ R = np.zeros((Q, S))+ home = {}+ for q in range(Q):+ intent = q % n_intents+ X[q, :, first_intent + intent] = 1.0+ matched = [s for s in range(S) if s % n_intents == intent]+ for s in range(S):+ base = 0.9 if s % n_intents == intent else 0.1+ R[q, s] = np.clip(base + 0.02 * rng.standard_normal(), 0.0, 1.0)+ home[f"q{q}"] = f"s{matched[q % len(matched)]}"+ mask = np.ones((Q, S), dtype=bool)+ matrix = RewardMatrix(+ queries=[f"q{i}" for i in range(Q)],+ sources=[f"s{i}" for i in range(S)],+ feature_names=list(FEATURE_NAMES), X=X, R=R, mask=mask,+ )+ return matrix, home+++def test_simulate_xgb_beats_random():+ pytest.importorskip("xgboost")+ m = _synthetic_matrix()+ base = evaluation.simulate_baselines(m, k=10)+ captured = evaluation.simulate_xgb(m, k=10, epsilon=0.1, refit_every=30,+ min_rows=100, params=FAST_XGB)+ assert captured > base["random"]+ assert captured <= base["oracle"] + 1e-9+++def test_evaluate_holdout_learns_identity_intent_routing():+ pytest.importorskip("xgboost")+ m, home = _routing_matrix()+ result = evaluation.evaluate_holdout(m, k=3, test_frac=0.25,+ home_by_query=home, params=FAST_XGB)+ cov, recall = result["coverage_at_k"], result["home_recall_at_k"]+ # Only identity x intent explains the reward: xgb should crush the+ # identity-blind baselines on both metrics.+ assert cov["xgb"] > cov["random"] + 0.1+ assert cov["xgb"] > cov["popularity"]+ assert recall["xgb"] > recall["random"] + 0.2+ assert result["rmse"] < 0.2+++def test_evaluate_holdout_default_home_is_oracle_best():+ pytest.importorskip("xgboost")+ m, _ = _routing_matrix(Q=80, S=8)+ result = evaluation.evaluate_holdout(m, k=3, test_frac=0.25, params=FAST_XGB)+ assert result["n_train_queries"] + result["n_test_queries"] == 80+ assert 0.0 <= result["home_recall_at_k"]["xgb"] <= 1.0++# ---------------------------------------------------------------------------# Domain matching (shared helpers behind the gold-grounded matrix)# ---------------------------------------------------------------------------