Skip to content

armgit / mwmbl

Public Imported from GitHub
0

An open source, non-profit web search engine

Add XGBoost contextual-bandit source model and xgb selection mode

Daoud Clarkef5b20d283abd
diff --git a/.gitignore b/.gitignore
index c38f96c..284d324 100644
--- a/.gitignore
+++ b/.gitignore
@@ -196,6 +196,8 @@ super-search-targets.json
prescreen.json
devdata/ss_eval_matrix*
devdata/ss_gold_matrix*
+devdata/ss_judge_matrix*
+devdata/super_search_xgb/
# AI planning documents
plans/
diff --git a/mwmbl/rankeval/evaluation/evaluate_super_search.py b/mwmbl/rankeval/evaluation/evaluate_super_search.py
index bd6d400..d47781b 100644
--- a/mwmbl/rankeval/evaluation/evaluate_super_search.py
+++ b/mwmbl/rankeval/evaluation/evaluate_super_search.py
@@ -2,7 +2,7 @@
Evaluate the Super Search pipeline as a RankingModel against the gold dataset.
Runs the real Super Search ranking core offline: select sources (cosine baseline
-by default — see ``SUPER_SEARCH_USE_BANDIT``), fan out to them, re-rank the union
+by default — see ``SUPER_SEARCH_SELECTION_MODE``), fan out to them, re-rank the union
with the LTR model, crawl promoted pages, follow outbound links, and MMR-diversify
into a final ranking. Only the SSE/auth/quota/indexing/reward machinery of the
HTTP endpoint is skipped; the ranking logic is exactly what production serves.
diff --git a/mwmbl/settings_common.py b/mwmbl/settings_common.py
index b50cdf2..371fd8f 100644
--- a/mwmbl/settings_common.py
+++ b/mwmbl/settings_common.py
@@ -305,6 +305,20 @@ SUPER_SEARCH_TS_EXPLORE_SCALE = 1.0 # Thompson-sampling posterior-covariance mu
SUPER_SEARCH_TS_PRIOR_PRECISION = 1.0 # ridge prior precision (lambda); A0 = lambda*I
SUPER_SEARCH_TS_NOISE_VARIANCE = 0.25 # reward noise variance (sigma^2); reward in [0,1]
SUPER_SEARCH_QVEC_CACHE_TTL = 3600 # seconds to cache a query's projected vectors
-SUPER_SEARCH_USE_BANDIT = False # False = cosine-top-10 baseline; True = Thompson sampling
+# Selection strategy: "cosine" (greedy top-10 baseline), "lints" (per-arm
+# Thompson sampling), or "xgb" (contextual bandit: XGBoost reward model with
+# epsilon-greedy exploration). Transitional — cosine/lints are removed once the
+# xgb policy passes the offline gate.
+SUPER_SEARCH_SELECTION_MODE = os.environ.get("SUPER_SEARCH_SELECTION_MODE", "cosine")
SUPER_SEARCH_FORCE_INCLUDE = [] # source names always queried (high-value sources), like always_on
+
+# XGBoost contextual-bandit source model. The runtime dir receives online
+# retrains (background task); before the first retrain, serving uses the
+# repo-bundled warm-start artifact in super_search_select/artifacts/xgb.
+SUPER_SEARCH_XGB_MODEL_DIR = os.environ.get(
+ "SUPER_SEARCH_XGB_MODEL_DIR",
+ str(Path(__file__).parent.parent / "devdata" / "super_search_xgb"))
+SUPER_SEARCH_XGB_EPSILON = 0.1 # epsilon-greedy exploration rate
+SUPER_SEARCH_XGB_MIN_TRAIN_ROWS = 2000 # (source, reward) pairs required before an online retrain
+SUPER_SEARCH_XGB_TRAIN_WINDOW_DAYS = 90 # impression window for online retrains
SOURCE_PROVENANCE_MAX_DEPTH = 3 # max crawl hops a Super Search source propagates to descendant pages
diff --git a/mwmbl/tinysearchengine/super_search.py b/mwmbl/tinysearchengine/super_search.py
index 57e5540..6b7a40a 100644
--- a/mwmbl/tinysearchengine/super_search.py
+++ b/mwmbl/tinysearchengine/super_search.py
@@ -560,7 +560,9 @@ async def _record_rewards(ctx: SelectionContext, last_results_key: list) -> None
await asyncio.to_thread(_enqueue_discovered_urls, ctx)
except Exception:
logger.exception("super-search failed to enqueue discovered urls")
- if getattr(settings, "SUPER_SEARCH_USE_BANDIT", False):
+ # Only the per-arm LinTS bandit needs a per-request update; the xgb policy
+ # learns from the impression log in batch.
+ if getattr(settings, "SUPER_SEARCH_SELECTION_MODE", "cosine") == "lints":
try:
await asyncio.to_thread(ss_bandit.update, ctx.features, rewards)
except Exception:
diff --git a/mwmbl/tinysearchengine/super_search_select/policy.py b/mwmbl/tinysearchengine/super_search_select/policy.py
index 9a42923..52d2e83 100644
--- a/mwmbl/tinysearchengine/super_search_select/policy.py
+++ b/mwmbl/tinysearchengine/super_search_select/policy.py
@@ -1,17 +1,20 @@
"""Source-selection policy: pick which ~10 of ~100 sources to query.
-Two interchangeable scorings over the same context features:
-
-- **cosine baseline** (default): greedy query-vs-profile cosine, with a few
- reserved slots for cold sites so new sources are discovered.
-- **Thompson sampling** (``SUPER_SEARCH_USE_BANDIT``): per-arm linear-Gaussian
- posterior sampling, which explores cold arms automatically via their wide
- posterior.
-
-Either way the always-on global sources (own index, HN) are included for free,
-and the feature vectors used for the decision are stashed on the
-``SelectionContext`` so the bandit update at request completion is consistent
-with the action taken.
+Three interchangeable scorings over the same context features, chosen by
+``SUPER_SEARCH_SELECTION_MODE``:
+
+- **cosine** (default): greedy query-vs-profile cosine, with a few reserved
+ slots for cold sites so new sources are discovered.
+- **lints**: per-arm linear-Gaussian Thompson sampling, which explores cold
+ arms automatically via their wide posterior.
+- **xgb**: contextual bandit — an XGBoost model predicts each source's judge
+ reward from the context features plus a source-identity one-hot, with
+ epsilon-greedy exploration. Learns in batch from the impression log.
+
+Whichever runs, the always-on global sources (own index, HN) are included for
+free, and the feature vectors used for the decision are stashed on the
+``SelectionContext`` so the reward attribution at request completion is
+consistent with the action taken.
"""
from __future__ import annotations
@@ -20,7 +23,7 @@ import random
import numpy as np
from django.conf import settings
-from mwmbl.tinysearchengine.super_search_select import bandit, profiles
+from mwmbl.tinysearchengine.super_search_select import bandit, profiles, xgb_model
from mwmbl.tinysearchengine.super_search_select.features import (
QueryContext,
SiteStats,
@@ -68,10 +71,15 @@ def select_sources(
profs = profiles.get_profiles(selectable)
feats = {n: feature_vector(qctx, get_meta(n), profs[n]) for n in selectable}
- if settings.SUPER_SEARCH_USE_BANDIT:
+ mode = settings.SUPER_SEARCH_SELECTION_MODE
+ if mode == "xgb":
+ chosen = _select_xgb(selectable, feats, budget)
+ elif mode == "lints":
chosen = _select_bandit(selectable, feats, budget)
- else:
+ elif mode == "cosine":
chosen = _select_cosine(qctx, selectable, profs, budget)
+ else:
+ raise ValueError(f"unknown SUPER_SEARCH_SELECTION_MODE {mode!r}")
if ctx is not None:
for name in pinned + chosen:
@@ -100,6 +108,25 @@ def _select_cosine(qctx, selectable, profs, budget) -> list[str]:
return chosen
+def _select_xgb(selectable, feats, budget) -> list[str]:
+ """Epsilon-greedy over the XGBoost model's predicted rewards.
+
+ Each of the ``budget`` slots independently explores with probability
+ epsilon: the non-explore slots take the top-scored sources, the explore
+ slots are filled uniformly at random from the rest (cold sources included,
+ so this subsumes the cosine path's explore floor and keeps randomized
+ pairs flowing into the training log).
+ """
+ scores = xgb_model.get_model().score(feats)
+ ranked = sorted(selectable, key=lambda n: scores[n], reverse=True)
+ epsilon = settings.SUPER_SEARCH_XGB_EPSILON
+ n_explore = sum(random.random() < epsilon for _ in range(budget))
+ chosen = ranked[:budget - n_explore]
+ rest = ranked[budget - n_explore:]
+ chosen += random.sample(rest, min(n_explore, len(rest)))
+ return chosen
+
+
def _select_bandit(selectable, feats, budget) -> list[str]:
rng = np.random.default_rng()
states = bandit.get_states(selectable)
diff --git a/mwmbl/tinysearchengine/super_search_select/xgb_model.py b/mwmbl/tinysearchengine/super_search_select/xgb_model.py
new file mode 100644
index 0000000..70fd66f
--- /dev/null
+++ b/mwmbl/tinysearchengine/super_search_select/xgb_model.py
@@ -0,0 +1,285 @@
+"""XGBoost contextual-bandit model for source selection.
+
+A pointwise ``XGBRegressor`` predicts the judge reward of querying a source
+for the current query. Its input is the shared context feature vector
+(``features.FEATURE_NAMES``) concatenated with a per-source identity one-hot
+block (``src_{name}``): with source identity in the trees, the model can learn
+interactions like intent_code x github or cos_bow x arxiv that a within-query
+ranker is otherwise structurally blind to. Unknown sources get an all-zero
+identity block and back off to the shared features.
+
+There is no per-request update: the model is retrained in batch from
+``SuperSearchImpression`` rows (which store per-source shared vectors keyed by
+source name, plus judge rewards, and never the query text) or from an offline
+``RewardMatrix``. Artifacts are a directory holding ``model.json`` (XGBoost's
+native, version-stable format) and ``meta.json`` (source vocabulary, feature
+names, provenance); writes are atomic (tmp + ``os.replace``, meta last) so
+serving processes can hot-reload safely.
+
+Loading is fail-fast by design: a corrupt artifact, a feature-set mismatch
+with the running code, or no artifact anywhere raises a plain exception that
+propagates out of source selection. The only quiet path is the designed one —
+no artifact in the runtime dir yet (normal before the first online retrain),
+in which case the repo-bundled warm-start artifact is used.
+"""
+from __future__ import annotations
+
+import json
+import logging
+import os
+import threading
+import time
+from dataclasses import dataclass
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Iterable, Sequence
+
+import numpy as np
+from django.conf import settings
+
+from mwmbl.tinysearchengine.super_search_select.features import (
+ FEATURE_NAMES,
+ NUM_FEATURES,
+)
+
+logger = logging.getLogger(__name__)
+
+FORMAT_VERSION = 1
+MODEL_FILE = "model.json"
+META_FILE = "meta.json"
+
+# Repo-bundled warm-start artifact, committed inside the package so it ships
+# with every deployment (unlike devdata, which may not be mounted).
+BUNDLED_DIR = Path(__file__).parent / "artifacts" / "xgb"
+
+# How often get_model() re-stats meta.json to pick up a retrain.
+RELOAD_CHECK_SECONDS = 60.0
+
+# Tuned offline via evaluation.evaluate_holdout; change only with offline evidence.
+XGB_PARAMS = {
+ "n_estimators": 200,
+ "max_depth": 6,
+ "learning_rate": 0.1,
+ "subsample": 0.8,
+ "random_state": 0,
+}
+
+
+# ---------------------------------------------------------------------------
+# Encoding
+# ---------------------------------------------------------------------------
+
+def build_vocab(names: Iterable[str]) -> list[str]:
+ """Frozen, sorted source vocabulary for the identity one-hot block."""
+ return sorted(set(names))
+
+
+def full_feature_names(vocab: Sequence[str]) -> list[str]:
+ return list(FEATURE_NAMES) + [f"src_{name}" for name in vocab]
+
+
+def encode(shared: Sequence[float], source: str, vocab_index: dict[str, int]) -> np.ndarray:
+ """shared context vector ++ source identity one-hot.
+
+ Shared vectors shorter than ``NUM_FEATURES`` are zero-padded: features are
+ only ever appended to ``FEATURE_NAMES``, so older logged vectors align
+ exactly with zeros in the newer slots. A *longer* vector means the data
+ came from a newer feature set than this code and is an error.
+ """
+ v = np.asarray(shared, dtype=np.float64)
+ if v.ndim != 1 or v.size > NUM_FEATURES:
+ raise ValueError(
+ f"shared feature vector has {v.shape} shape; expected <= {NUM_FEATURES} values")
+ x = np.zeros(NUM_FEATURES + len(vocab_index), dtype=np.float64)
+ x[:v.size] = v
+ idx = vocab_index.get(source)
+ if idx is not None:
+ x[NUM_FEATURES + idx] = 1.0
+ return x
+
+
+# ---------------------------------------------------------------------------
+# Training data
+# ---------------------------------------------------------------------------
+
+def build_training_data(
+ rows: Iterable[tuple[dict[str, Sequence[float]], dict[str, float]]],
+ vocab: Sequence[str],
+) -> tuple[np.ndarray, np.ndarray]:
+ """(X, y) from impression rows: each row is (features by source, rewards by source).
+
+ One training pair per source with both a stored feature vector and a
+ reward — exactly what ``SuperSearchImpression`` logs per request.
+ """
+ vocab_index = {name: i for i, name in enumerate(vocab)}
+ xs, ys = [], []
+ for features, rewards in rows:
+ for name, reward in rewards.items():
+ shared = features.get(name)
+ if shared is None:
+ continue
+ xs.append(encode(shared, name, vocab_index))
+ ys.append(float(reward))
+ if not xs:
+ return (np.zeros((0, NUM_FEATURES + len(vocab))), np.zeros(0))
+ return np.stack(xs), np.asarray(ys)
+
+
+def build_training_data_from_matrix(matrix) -> tuple[np.ndarray, np.ndarray, list[str]]:
+ """(X, y, vocab) from a dense offline ``evaluation.RewardMatrix`` (masked cells)."""
+ if list(matrix.feature_names) != list(FEATURE_NAMES):
+ raise ValueError(
+ f"matrix feature names {matrix.feature_names} do not match the "
+ f"running code's FEATURE_NAMES {list(FEATURE_NAMES)}; rebuild the matrix")
+ vocab = build_vocab(matrix.sources)
+ vocab_index = {name: i for i, name in enumerate(vocab)}
+ xs, ys = [], []
+ for q in range(matrix.X.shape[0]):
+ for s in range(matrix.X.shape[1]):
+ if matrix.mask[q, s]:
+ xs.append(encode(matrix.X[q, s], matrix.sources[s], vocab_index))
+ ys.append(float(matrix.R[q, s]))
+ return np.stack(xs), np.asarray(ys), vocab
+
+
+def train(X: np.ndarray, y: np.ndarray, params: dict | None = None):
+ from xgboost import XGBRegressor
+
+ model = XGBRegressor(**(params or XGB_PARAMS))
+ model.fit(X, y)
+ return model
+
+
+# ---------------------------------------------------------------------------
+# Artifact save / load
+# ---------------------------------------------------------------------------
+
+def save_artifact(
+ model,
+ vocab: Sequence[str],
+ model_dir: str | Path,
+ reward_kind: str,
+ n_rows: int,
+ metrics: dict | None = None,
+) -> None:
+ """Atomically write ``model.json`` + ``meta.json`` (meta last: its presence
+ and mtime are the load/reload signal)."""
+ model_dir = Path(model_dir)
+ model_dir.mkdir(parents=True, exist_ok=True)
+ # tmp name keeps the .json extension so xgboost writes the JSON format
+ tmp_model = model_dir / ("tmp." + MODEL_FILE)
+ model.save_model(tmp_model)
+ os.replace(tmp_model, model_dir / MODEL_FILE)
+
+ meta = {
+ "format_version": FORMAT_VERSION,
+ "shared_feature_names": list(FEATURE_NAMES),
+ "source_vocab": list(vocab),
+ "reward_kind": reward_kind,
+ "trained_at": datetime.now(timezone.utc).isoformat(),
+ "n_rows": int(n_rows),
+ "metrics": metrics or {},
+ }
+ tmp_meta = model_dir / (META_FILE + ".tmp")
+ tmp_meta.write_text(json.dumps(meta, indent=2))
+ os.replace(tmp_meta, model_dir / META_FILE)
+ logger.info("saved super-search xgb artifact to %s (%d rows, reward=%s)",
+ model_dir, n_rows, reward_kind)
+
+
+@dataclass
+class XgbSourceModel:
+ model: object # fitted XGBRegressor
+ vocab: list[str]
+ vocab_index: dict[str, int]
+ meta: dict
+
+ def score(self, feats: dict[str, Sequence[float]]) -> dict[str, float]:
+ """Predicted reward for each source given its shared feature vector."""
+ if not feats:
+ return {}
+ names = list(feats)
+ X = np.stack([encode(feats[n], n, self.vocab_index) for n in names])
+ preds = self.model.predict(X)
+ return {name: float(p) for name, p in zip(names, preds)}
+
+
+def load_artifact(model_dir: str | Path) -> XgbSourceModel:
+ """Load and validate an artifact directory. Raises on anything unexpected:
+ missing files, unreadable JSON, format or feature-set mismatch."""
+ from xgboost import XGBRegressor
+
+ model_dir = Path(model_dir)
+ meta = json.loads((model_dir / META_FILE).read_text())
+ if meta["format_version"] != FORMAT_VERSION:
+ raise ValueError(
+ f"artifact {model_dir} has format_version {meta['format_version']}, "
+ f"expected {FORMAT_VERSION}")
+ if meta["shared_feature_names"] != list(FEATURE_NAMES):
+ raise ValueError(
+ f"artifact {model_dir} was trained on features "
+ f"{meta['shared_feature_names']} but the running code has "
+ f"{list(FEATURE_NAMES)}; retrain the artifact")
+ model = XGBRegressor()
+ model.load_model(model_dir / MODEL_FILE)
+ vocab = list(meta["source_vocab"])
+ return XgbSourceModel(
+ model=model,
+ vocab=vocab,
+ vocab_index={name: i for i, name in enumerate(vocab)},
+ meta=meta,
+ )
+
+
+# ---------------------------------------------------------------------------
+# Serving singleton
+# ---------------------------------------------------------------------------
+
+_cached: XgbSourceModel | None = None
+_cached_key: tuple[Path, float] | None = None
+_next_check: float = 0.0
+_lock = threading.Lock()
+
+
+def _artifact_dir() -> Path:
+ """Runtime dir if it holds an artifact (i.e. an online retrain has run),
+ else the repo-bundled warm-start artifact."""
+ runtime = Path(settings.SUPER_SEARCH_XGB_MODEL_DIR)
+ if (runtime / META_FILE).exists():
+ return runtime
+ return BUNDLED_DIR
+
+
+def get_model() -> XgbSourceModel:
+ """Shared per-process model, hot-reloaded when a retrain replaces the artifact.
+
+ Fail-fast: if no artifact exists anywhere or the artifact is invalid this
+ raises (FileNotFoundError / ValueError / json errors) rather than falling
+ back — a missing model is a deployment bug to fix, not a state to mask.
+ """
+ global _cached, _cached_key, _next_check
+ now = time.monotonic()
+ if _cached is not None and now < _next_check:
+ return _cached
+ with _lock:
+ if _cached is not None and time.monotonic() < _next_check:
+ return _cached
+ model_dir = _artifact_dir()
+ mtime = (model_dir / META_FILE).stat().st_mtime
+ key = (model_dir, mtime)
+ if key != _cached_key:
+ _cached = load_artifact(model_dir)
+ _cached_key = key
+ logger.info("super-search xgb model loaded from %s (trained %s, %d sources)",
+ model_dir, _cached.meta.get("trained_at"), len(_cached.vocab))
+ _next_check = time.monotonic() + RELOAD_CHECK_SECONDS
+ return _cached
+
+
+def reset_model_cache() -> None:
+ """Drop the cached model (tests / after an in-process retrain)."""
+ global _cached, _cached_key, _next_check
+ with _lock:
+ _cached = None
+ _cached_key = None
+ _next_check = 0.0
diff --git a/test/test_super_search_bandit.py b/test/test_super_search_bandit.py
index 951e5f9..1b1a28a 100644
--- a/test/test_super_search_bandit.py
+++ b/test/test_super_search_bandit.py
@@ -106,11 +106,11 @@ def test_thompson_prefers_learned_good_arm(fake_bandit_redis):
def test_policy_bandit_branch_selects_and_records_features(monkeypatch):
- """End-to-end: USE_BANDIT selection populates ctx.features and respects k."""
+ """End-to-end: lints selection populates ctx.features and respects k."""
r = fakeredis.FakeRedis()
monkeypatch.setattr(bandit, "_redis", r)
monkeypatch.setattr(profiles, "_redis", r)
- monkeypatch.setattr("django.conf.settings.SUPER_SEARCH_USE_BANDIT", True)
+ monkeypatch.setattr("django.conf.settings.SUPER_SEARCH_SELECTION_MODE", "lints")
names = ["mwmbl", "hn"] + [f"site{i}" for i in range(20)]
ctx = SelectionContext()
diff --git a/test/test_super_search_xgb.py b/test/test_super_search_xgb.py
new file mode 100644
index 0000000..c02e816
--- /dev/null
+++ b/test/test_super_search_xgb.py
@@ -0,0 +1,266 @@
+"""Tests for the XGBoost contextual-bandit source model and the xgb policy."""
+import json
+
+import fakeredis
+import numpy as np
+import pytest
+
+from mwmbl.tinysearchengine.super_search_select import policy, profiles, xgb_model
+from mwmbl.tinysearchengine.super_search_select.evaluation import RewardMatrix
+from mwmbl.tinysearchengine.super_search_select.features import (
+ FEATURE_NAMES,
+ NUM_FEATURES,
+)
+from mwmbl.tinysearchengine.super_search_select.rewards import SelectionContext
+
+INTENT_CODE = FEATURE_NAMES.index("intent_code")
+
+FAST_PARAMS = {**xgb_model.XGB_PARAMS, "n_estimators": 50}
+
+
[email protected](autouse=True)
+def clean_model_cache():
+ xgb_model.reset_model_cache()
+ yield
+ xgb_model.reset_model_cache()
+
+
+def _shared(intent_code: float = 0.0) -> list[float]:
+ x = [0.0] * NUM_FEATURES
+ x[0] = 1.0 # bias
+ x[INTENT_CODE] = intent_code
+ return x
+
+
+def _interaction_rows(n: int = 60):
+ """Planted structure: 'github' pays off iff intent_code, 'recipes' iff not."""
+ rows = []
+ for i in range(n):
+ code = float(i % 2)
+ features = {"github": _shared(code), "recipes": _shared(code)}
+ rewards = {"github": code, "recipes": 1.0 - code}
+ rows.append((features, rewards))
+ return rows
+
+
+def _trained_model(tmp_path, sources=("github", "recipes"), reward_kind="test"):
+ vocab = xgb_model.build_vocab(sources)
+ X, y = xgb_model.build_training_data(_interaction_rows(), vocab)
+ model = xgb_model.train(X, y, params=FAST_PARAMS)
+ xgb_model.save_artifact(model, vocab, tmp_path, reward_kind=reward_kind, n_rows=len(y))
+ return vocab
+
+
+# ---------------------------------------------------------------------------
+# Encoding / training data
+# ---------------------------------------------------------------------------
+
+def test_encode_shape_and_identity_block():
+ vocab_index = {"a": 0, "b": 1}
+ x = xgb_model.encode(_shared(), "b", vocab_index)
+ assert x.shape == (NUM_FEATURES + 2,)
+ assert x[NUM_FEATURES] == 0.0 and x[NUM_FEATURES + 1] == 1.0
+
+
+def test_encode_unknown_source_gets_zero_block():
+ x = xgb_model.encode(_shared(), "never_seen", {"a": 0})
+ assert np.all(x[NUM_FEATURES:] == 0.0)
+
+
+def test_encode_zero_pads_old_short_vectors():
+ # Pre-intent impressions stored 10-dim vectors; the intent block was
+ # appended last, so zero-padding aligns exactly.
+ old = [1.0] * 10
+ x = xgb_model.encode(old, "a", {"a": 0})
+ assert np.all(x[:10] == 1.0)
+ assert np.all(x[10:NUM_FEATURES] == 0.0)
+
+
+def test_encode_rejects_vectors_from_a_newer_feature_set():
+ with pytest.raises(ValueError):
+ xgb_model.encode([0.0] * (NUM_FEATURES + 1), "a", {"a": 0})
+
+
+def test_build_training_data_skips_rewards_without_features():
+ rows = [({"a": _shared()}, {"a": 1.0, "phantom": 0.5})]
+ X, y = xgb_model.build_training_data(rows, ["a", "phantom"])
+ assert X.shape == (1, NUM_FEATURES + 2)
+ assert y.tolist() == [1.0]
+
+
+def test_build_training_data_from_matrix():
+ Q, S = 3, 2
+ X = np.zeros((Q, S, NUM_FEATURES))
+ X[:, :, 0] = 1.0
+ R = np.array([[1.0, 0.0], [0.5, 0.2], [0.0, 0.9]])
+ mask = np.ones((Q, S), dtype=bool)
+ mask[2, 0] = False
+ matrix = RewardMatrix(queries=["q1", "q2", "q3"], sources=["b", "a"],
+ feature_names=list(FEATURE_NAMES), X=X, R=R, mask=mask)
+ Xf, y, vocab = xgb_model.build_training_data_from_matrix(matrix)
+ assert vocab == ["a", "b"]
+ assert Xf.shape == (5, NUM_FEATURES + 2)
+ assert set(y.tolist()) == {1.0, 0.0, 0.5, 0.2, 0.9}
+
+
+def test_matrix_with_mismatched_features_raises():
+ matrix = RewardMatrix(queries=["q"], sources=["a"], feature_names=["bias"],
+ X=np.zeros((1, 1, 1)), R=np.zeros((1, 1)),
+ mask=np.ones((1, 1), dtype=bool))
+ with pytest.raises(ValueError, match="feature names"):
+ xgb_model.build_training_data_from_matrix(matrix)
+
+
+# ---------------------------------------------------------------------------
+# Model learns identity x context interactions
+# ---------------------------------------------------------------------------
+
+def test_model_learns_intent_source_interaction(tmp_path):
+ _trained_model(tmp_path)
+ loaded = xgb_model.load_artifact(tmp_path)
+ code_scores = loaded.score({"github": _shared(1.0), "recipes": _shared(1.0)})
+ plain_scores = loaded.score({"github": _shared(0.0), "recipes": _shared(0.0)})
+ assert code_scores["github"] > code_scores["recipes"]
+ assert plain_scores["recipes"] > plain_scores["github"]
+
+
+# ---------------------------------------------------------------------------
+# Artifact save / load / get_model
+# ---------------------------------------------------------------------------
+
+def test_artifact_roundtrip_meta(tmp_path):
+ vocab = _trained_model(tmp_path, reward_kind="judge")
+ loaded = xgb_model.load_artifact(tmp_path)
+ assert loaded.vocab == list(vocab)
+ assert loaded.meta["reward_kind"] == "judge"
+ assert loaded.meta["shared_feature_names"] == list(FEATURE_NAMES)
+ assert loaded.meta["n_rows"] == 120
+
+
+def test_load_artifact_missing_dir_raises(tmp_path):
+ with pytest.raises(FileNotFoundError):
+ xgb_model.load_artifact(tmp_path / "nothing_here")
+
+
+def test_load_artifact_feature_mismatch_raises(tmp_path):
+ _trained_model(tmp_path)
+ meta_path = tmp_path / xgb_model.META_FILE
+ meta = json.loads(meta_path.read_text())
+ meta["shared_feature_names"] = meta["shared_feature_names"][:-1]
+ meta_path.write_text(json.dumps(meta))
+ with pytest.raises(ValueError, match="retrain"):
+ xgb_model.load_artifact(tmp_path)
+
+
+def test_load_artifact_bad_format_version_raises(tmp_path):
+ _trained_model(tmp_path)
+ meta_path = tmp_path / xgb_model.META_FILE
+ meta = json.loads(meta_path.read_text())
+ meta["format_version"] = 999
+ meta_path.write_text(json.dumps(meta))
+ with pytest.raises(ValueError, match="format_version"):
+ xgb_model.load_artifact(tmp_path)
+
+
+def test_get_model_uses_runtime_dir_over_bundle(tmp_path, monkeypatch):
+ runtime = tmp_path / "runtime"
+ _trained_model(runtime)
+ monkeypatch.setattr("django.conf.settings.SUPER_SEARCH_XGB_MODEL_DIR", str(runtime))
+ model = xgb_model.get_model()
+ assert model.vocab == ["github", "recipes"]
+
+
+def test_get_model_falls_back_to_bundle_when_runtime_empty(tmp_path, monkeypatch):
+ bundle = tmp_path / "bundle"
+ _trained_model(bundle, sources=("bundled_source",))
+ monkeypatch.setattr(xgb_model, "BUNDLED_DIR", bundle)
+ monkeypatch.setattr("django.conf.settings.SUPER_SEARCH_XGB_MODEL_DIR",
+ str(tmp_path / "empty_runtime"))
+ assert xgb_model.get_model().vocab == ["bundled_source"]
+
+
+def test_get_model_no_artifact_anywhere_raises(tmp_path, monkeypatch):
+ monkeypatch.setattr(xgb_model, "BUNDLED_DIR", tmp_path / "no_bundle")
+ monkeypatch.setattr("django.conf.settings.SUPER_SEARCH_XGB_MODEL_DIR",
+ str(tmp_path / "no_runtime"))
+ with pytest.raises(FileNotFoundError):
+ xgb_model.get_model()
+
+
+def test_get_model_hot_reloads_after_retrain(tmp_path, monkeypatch):
+ runtime = tmp_path / "runtime"
+ _trained_model(runtime)
+ monkeypatch.setattr("django.conf.settings.SUPER_SEARCH_XGB_MODEL_DIR", str(runtime))
+ monkeypatch.setattr(xgb_model, "RELOAD_CHECK_SECONDS", 0.0)
+ assert xgb_model.get_model().vocab == ["github", "recipes"]
+
+ _trained_model(runtime, sources=("github", "recipes", "arxiv"))
+ meta_path = runtime / xgb_model.META_FILE
+ # ensure the mtime moves even on coarse-grained filesystems
+ stat = meta_path.stat()
+ import os
+ os.utime(meta_path, (stat.st_atime, stat.st_mtime + 1))
+ assert xgb_model.get_model().vocab == ["arxiv", "github", "recipes"]
+
+
+# ---------------------------------------------------------------------------
+# xgb policy
+# ---------------------------------------------------------------------------
+
+def xgb_policy_env(tmp_path, monkeypatch):
+ """xgb selection mode with a toy artifact trained over the policy's sources."""
+ monkeypatch.setattr(profiles, "_redis", fakeredis.FakeRedis())
+ monkeypatch.setattr("django.conf.settings.SUPER_SEARCH_SELECTION_MODE", "xgb")
+ monkeypatch.setattr("django.conf.settings.SUPER_SEARCH_XGB_MODEL_DIR", str(tmp_path))
+ names = ["mwmbl", "hn"] + [f"site{i}" for i in range(20)]
+ _trained_model(tmp_path, sources=names)
+ return names
+
+
+def test_policy_xgb_selects_and_records_features(xgb_policy_env, monkeypatch):
+ monkeypatch.setattr("django.conf.settings.SUPER_SEARCH_XGB_EPSILON", 0.0)
+ ctx = SelectionContext()
+ chosen = policy.select_sources("python testing tools", xgb_policy_env, k=5, ctx=ctx)
+ assert len(chosen) == 5
+ assert "mwmbl" in chosen and "hn" in chosen # always-on included
+ assert set(chosen) <= set(ctx.features)
+ assert all(len(v) == NUM_FEATURES for v in ctx.features.values())
+
+
+def test_policy_xgb_greedy_is_deterministic(xgb_policy_env, monkeypatch):
+ monkeypatch.setattr("django.conf.settings.SUPER_SEARCH_XGB_EPSILON", 0.0)
+ runs = {tuple(policy.select_sources("some query", xgb_policy_env, k=6))
+ for _ in range(5)}
+ assert len(runs) == 1
+
+
+def test_policy_xgb_full_explore_still_valid(xgb_policy_env, monkeypatch):
+ monkeypatch.setattr("django.conf.settings.SUPER_SEARCH_XGB_EPSILON", 1.0)
+ seen = set()
+ for _ in range(10):
+ chosen = policy.select_sources("some query", xgb_policy_env, k=5)
+ assert len(chosen) == 5
+ assert len(set(chosen)) == 5
+ assert "mwmbl" in chosen and "hn" in chosen
+ seen.add(tuple(chosen))
+ assert len(seen) > 1 # actually exploring
+
+
+def test_policy_xgb_missing_artifact_raises(monkeypatch, tmp_path):
+ monkeypatch.setattr(profiles, "_redis", fakeredis.FakeRedis())
+ monkeypatch.setattr("django.conf.settings.SUPER_SEARCH_SELECTION_MODE", "xgb")
+ monkeypatch.setattr("django.conf.settings.SUPER_SEARCH_XGB_MODEL_DIR",
+ str(tmp_path / "empty"))
+ monkeypatch.setattr(xgb_model, "BUNDLED_DIR", tmp_path / "no_bundle")
+ names = ["mwmbl", "hn"] + [f"site{i}" for i in range(20)]
+ with pytest.raises(FileNotFoundError):
+ policy.select_sources("some query", names, k=5)
+
+
+def test_policy_unknown_mode_raises(monkeypatch):
+ monkeypatch.setattr(profiles, "_redis", fakeredis.FakeRedis())
+ monkeypatch.setattr("django.conf.settings.SUPER_SEARCH_SELECTION_MODE", "bogus")
+ names = ["mwmbl", "hn"] + [f"site{i}" for i in range(20)]
+ with pytest.raises(ValueError, match="SUPER_SEARCH_SELECTION_MODE"):
+ policy.select_sources("some query", names, k=5)