An open source, non-profit web search engine
Add XGBoost contextual-bandit source model and xgb selection mode
Daoud Clarkef5b20d283abd
diff --git a/.gitignore b/.gitignoreindex c38f96c..284d324 100644--- a/.gitignore+++ b/.gitignore@@ -196,6 +196,8 @@ super-search-targets.jsonprescreen.jsondevdata/ss_eval_matrix*devdata/ss_gold_matrix*+devdata/ss_judge_matrix*+devdata/super_search_xgb/# AI planning documentsplans/diff --git a/mwmbl/rankeval/evaluation/evaluate_super_search.py b/mwmbl/rankeval/evaluation/evaluate_super_search.pyindex bd6d400..d47781b 100644--- a/mwmbl/rankeval/evaluation/evaluate_super_search.py+++ b/mwmbl/rankeval/evaluation/evaluate_super_search.py@@ -2,7 +2,7 @@Evaluate the Super Search pipeline as a RankingModel against the gold dataset.Runs the real Super Search ranking core offline: select sources (cosine baseline-by default — see ``SUPER_SEARCH_USE_BANDIT``), fan out to them, re-rank the union+by default — see ``SUPER_SEARCH_SELECTION_MODE``), fan out to them, re-rank the unionwith the LTR model, crawl promoted pages, follow outbound links, and MMR-diversifyinto a final ranking. Only the SSE/auth/quota/indexing/reward machinery of theHTTP endpoint is skipped; the ranking logic is exactly what production serves.diff --git a/mwmbl/settings_common.py b/mwmbl/settings_common.pyindex b50cdf2..371fd8f 100644--- a/mwmbl/settings_common.py+++ b/mwmbl/settings_common.py@@ -305,6 +305,20 @@ SUPER_SEARCH_TS_EXPLORE_SCALE = 1.0 # Thompson-sampling posterior-covariance muSUPER_SEARCH_TS_PRIOR_PRECISION = 1.0 # ridge prior precision (lambda); A0 = lambda*ISUPER_SEARCH_TS_NOISE_VARIANCE = 0.25 # reward noise variance (sigma^2); reward in [0,1]SUPER_SEARCH_QVEC_CACHE_TTL = 3600 # seconds to cache a query's projected vectors-SUPER_SEARCH_USE_BANDIT = False # False = cosine-top-10 baseline; True = Thompson sampling+# Selection strategy: "cosine" (greedy top-10 baseline), "lints" (per-arm+# Thompson sampling), or "xgb" (contextual bandit: XGBoost reward model with+# epsilon-greedy exploration). Transitional — cosine/lints are removed once the+# xgb policy passes the offline gate.+SUPER_SEARCH_SELECTION_MODE = os.environ.get("SUPER_SEARCH_SELECTION_MODE", "cosine")SUPER_SEARCH_FORCE_INCLUDE = [] # source names always queried (high-value sources), like always_on++# XGBoost contextual-bandit source model. The runtime dir receives online+# retrains (background task); before the first retrain, serving uses the+# repo-bundled warm-start artifact in super_search_select/artifacts/xgb.+SUPER_SEARCH_XGB_MODEL_DIR = os.environ.get(+ "SUPER_SEARCH_XGB_MODEL_DIR",+ str(Path(__file__).parent.parent / "devdata" / "super_search_xgb"))+SUPER_SEARCH_XGB_EPSILON = 0.1 # epsilon-greedy exploration rate+SUPER_SEARCH_XGB_MIN_TRAIN_ROWS = 2000 # (source, reward) pairs required before an online retrain+SUPER_SEARCH_XGB_TRAIN_WINDOW_DAYS = 90 # impression window for online retrainsSOURCE_PROVENANCE_MAX_DEPTH = 3 # max crawl hops a Super Search source propagates to descendant pagesdiff --git a/mwmbl/tinysearchengine/super_search.py b/mwmbl/tinysearchengine/super_search.pyindex 57e5540..6b7a40a 100644--- a/mwmbl/tinysearchengine/super_search.py+++ b/mwmbl/tinysearchengine/super_search.py@@ -560,7 +560,9 @@ async def _record_rewards(ctx: SelectionContext, last_results_key: list) -> Noneawait asyncio.to_thread(_enqueue_discovered_urls, ctx)except Exception:logger.exception("super-search failed to enqueue discovered urls")- if getattr(settings, "SUPER_SEARCH_USE_BANDIT", False):+ # Only the per-arm LinTS bandit needs a per-request update; the xgb policy+ # learns from the impression log in batch.+ if getattr(settings, "SUPER_SEARCH_SELECTION_MODE", "cosine") == "lints":try:await asyncio.to_thread(ss_bandit.update, ctx.features, rewards)except Exception:diff --git a/mwmbl/tinysearchengine/super_search_select/policy.py b/mwmbl/tinysearchengine/super_search_select/policy.pyindex 9a42923..52d2e83 100644--- a/mwmbl/tinysearchengine/super_search_select/policy.py+++ b/mwmbl/tinysearchengine/super_search_select/policy.py@@ -1,17 +1,20 @@"""Source-selection policy: pick which ~10 of ~100 sources to query.-Two interchangeable scorings over the same context features:--- **cosine baseline** (default): greedy query-vs-profile cosine, with a few- reserved slots for cold sites so new sources are discovered.-- **Thompson sampling** (``SUPER_SEARCH_USE_BANDIT``): per-arm linear-Gaussian- posterior sampling, which explores cold arms automatically via their wide- posterior.--Either way the always-on global sources (own index, HN) are included for free,-and the feature vectors used for the decision are stashed on the-``SelectionContext`` so the bandit update at request completion is consistent-with the action taken.+Three interchangeable scorings over the same context features, chosen by+``SUPER_SEARCH_SELECTION_MODE``:++- **cosine** (default): greedy query-vs-profile cosine, with a few reserved+ slots for cold sites so new sources are discovered.+- **lints**: per-arm linear-Gaussian Thompson sampling, which explores cold+ arms automatically via their wide posterior.+- **xgb**: contextual bandit — an XGBoost model predicts each source's judge+ reward from the context features plus a source-identity one-hot, with+ epsilon-greedy exploration. Learns in batch from the impression log.++Whichever runs, the always-on global sources (own index, HN) are included for+free, and the feature vectors used for the decision are stashed on the+``SelectionContext`` so the reward attribution at request completion is+consistent with the action taken."""from __future__ import annotations@@ -20,7 +23,7 @@ import randomimport numpy as npfrom django.conf import settings-from mwmbl.tinysearchengine.super_search_select import bandit, profiles+from mwmbl.tinysearchengine.super_search_select import bandit, profiles, xgb_modelfrom mwmbl.tinysearchengine.super_search_select.features import (QueryContext,SiteStats,@@ -68,10 +71,15 @@ def select_sources(profs = profiles.get_profiles(selectable)feats = {n: feature_vector(qctx, get_meta(n), profs[n]) for n in selectable}- if settings.SUPER_SEARCH_USE_BANDIT:+ mode = settings.SUPER_SEARCH_SELECTION_MODE+ if mode == "xgb":+ chosen = _select_xgb(selectable, feats, budget)+ elif mode == "lints":chosen = _select_bandit(selectable, feats, budget)- else:+ elif mode == "cosine":chosen = _select_cosine(qctx, selectable, profs, budget)+ else:+ raise ValueError(f"unknown SUPER_SEARCH_SELECTION_MODE {mode!r}")if ctx is not None:for name in pinned + chosen:@@ -100,6 +108,25 @@ def _select_cosine(qctx, selectable, profs, budget) -> list[str]:return chosen+def _select_xgb(selectable, feats, budget) -> list[str]:+ """Epsilon-greedy over the XGBoost model's predicted rewards.++ Each of the ``budget`` slots independently explores with probability+ epsilon: the non-explore slots take the top-scored sources, the explore+ slots are filled uniformly at random from the rest (cold sources included,+ so this subsumes the cosine path's explore floor and keeps randomized+ pairs flowing into the training log).+ """+ scores = xgb_model.get_model().score(feats)+ ranked = sorted(selectable, key=lambda n: scores[n], reverse=True)+ epsilon = settings.SUPER_SEARCH_XGB_EPSILON+ n_explore = sum(random.random() < epsilon for _ in range(budget))+ chosen = ranked[:budget - n_explore]+ rest = ranked[budget - n_explore:]+ chosen += random.sample(rest, min(n_explore, len(rest)))+ return chosen++def _select_bandit(selectable, feats, budget) -> list[str]:rng = np.random.default_rng()states = bandit.get_states(selectable)diff --git a/mwmbl/tinysearchengine/super_search_select/xgb_model.py b/mwmbl/tinysearchengine/super_search_select/xgb_model.pynew file mode 100644index 0000000..70fd66f--- /dev/null+++ b/mwmbl/tinysearchengine/super_search_select/xgb_model.py@@ -0,0 +1,285 @@+"""XGBoost contextual-bandit model for source selection.++A pointwise ``XGBRegressor`` predicts the judge reward of querying a source+for the current query. Its input is the shared context feature vector+(``features.FEATURE_NAMES``) concatenated with a per-source identity one-hot+block (``src_{name}``): with source identity in the trees, the model can learn+interactions like intent_code x github or cos_bow x arxiv that a within-query+ranker is otherwise structurally blind to. Unknown sources get an all-zero+identity block and back off to the shared features.++There is no per-request update: the model is retrained in batch from+``SuperSearchImpression`` rows (which store per-source shared vectors keyed by+source name, plus judge rewards, and never the query text) or from an offline+``RewardMatrix``. Artifacts are a directory holding ``model.json`` (XGBoost's+native, version-stable format) and ``meta.json`` (source vocabulary, feature+names, provenance); writes are atomic (tmp + ``os.replace``, meta last) so+serving processes can hot-reload safely.++Loading is fail-fast by design: a corrupt artifact, a feature-set mismatch+with the running code, or no artifact anywhere raises a plain exception that+propagates out of source selection. The only quiet path is the designed one —+no artifact in the runtime dir yet (normal before the first online retrain),+in which case the repo-bundled warm-start artifact is used.+"""+from __future__ import annotations++import json+import logging+import os+import threading+import time+from dataclasses import dataclass+from datetime import datetime, timezone+from pathlib import Path+from typing import Iterable, Sequence++import numpy as np+from django.conf import settings++from mwmbl.tinysearchengine.super_search_select.features import (+ FEATURE_NAMES,+ NUM_FEATURES,+)++logger = logging.getLogger(__name__)++FORMAT_VERSION = 1+MODEL_FILE = "model.json"+META_FILE = "meta.json"++# Repo-bundled warm-start artifact, committed inside the package so it ships+# with every deployment (unlike devdata, which may not be mounted).+BUNDLED_DIR = Path(__file__).parent / "artifacts" / "xgb"++# How often get_model() re-stats meta.json to pick up a retrain.+RELOAD_CHECK_SECONDS = 60.0++# Tuned offline via evaluation.evaluate_holdout; change only with offline evidence.+XGB_PARAMS = {+ "n_estimators": 200,+ "max_depth": 6,+ "learning_rate": 0.1,+ "subsample": 0.8,+ "random_state": 0,+}+++# ---------------------------------------------------------------------------+# Encoding+# ---------------------------------------------------------------------------++def build_vocab(names: Iterable[str]) -> list[str]:+ """Frozen, sorted source vocabulary for the identity one-hot block."""+ return sorted(set(names))+++def full_feature_names(vocab: Sequence[str]) -> list[str]:+ return list(FEATURE_NAMES) + [f"src_{name}" for name in vocab]+++def encode(shared: Sequence[float], source: str, vocab_index: dict[str, int]) -> np.ndarray:+ """shared context vector ++ source identity one-hot.++ Shared vectors shorter than ``NUM_FEATURES`` are zero-padded: features are+ only ever appended to ``FEATURE_NAMES``, so older logged vectors align+ exactly with zeros in the newer slots. A *longer* vector means the data+ came from a newer feature set than this code and is an error.+ """+ v = np.asarray(shared, dtype=np.float64)+ if v.ndim != 1 or v.size > NUM_FEATURES:+ raise ValueError(+ f"shared feature vector has {v.shape} shape; expected <= {NUM_FEATURES} values")+ x = np.zeros(NUM_FEATURES + len(vocab_index), dtype=np.float64)+ x[:v.size] = v+ idx = vocab_index.get(source)+ if idx is not None:+ x[NUM_FEATURES + idx] = 1.0+ return x+++# ---------------------------------------------------------------------------+# Training data+# ---------------------------------------------------------------------------++def build_training_data(+ rows: Iterable[tuple[dict[str, Sequence[float]], dict[str, float]]],+ vocab: Sequence[str],+) -> tuple[np.ndarray, np.ndarray]:+ """(X, y) from impression rows: each row is (features by source, rewards by source).++ One training pair per source with both a stored feature vector and a+ reward — exactly what ``SuperSearchImpression`` logs per request.+ """+ vocab_index = {name: i for i, name in enumerate(vocab)}+ xs, ys = [], []+ for features, rewards in rows:+ for name, reward in rewards.items():+ shared = features.get(name)+ if shared is None:+ continue+ xs.append(encode(shared, name, vocab_index))+ ys.append(float(reward))+ if not xs:+ return (np.zeros((0, NUM_FEATURES + len(vocab))), np.zeros(0))+ return np.stack(xs), np.asarray(ys)+++def build_training_data_from_matrix(matrix) -> tuple[np.ndarray, np.ndarray, list[str]]:+ """(X, y, vocab) from a dense offline ``evaluation.RewardMatrix`` (masked cells)."""+ if list(matrix.feature_names) != list(FEATURE_NAMES):+ raise ValueError(+ f"matrix feature names {matrix.feature_names} do not match the "+ f"running code's FEATURE_NAMES {list(FEATURE_NAMES)}; rebuild the matrix")+ vocab = build_vocab(matrix.sources)+ vocab_index = {name: i for i, name in enumerate(vocab)}+ xs, ys = [], []+ for q in range(matrix.X.shape[0]):+ for s in range(matrix.X.shape[1]):+ if matrix.mask[q, s]:+ xs.append(encode(matrix.X[q, s], matrix.sources[s], vocab_index))+ ys.append(float(matrix.R[q, s]))+ return np.stack(xs), np.asarray(ys), vocab+++def train(X: np.ndarray, y: np.ndarray, params: dict | None = None):+ from xgboost import XGBRegressor++ model = XGBRegressor(**(params or XGB_PARAMS))+ model.fit(X, y)+ return model+++# ---------------------------------------------------------------------------+# Artifact save / load+# ---------------------------------------------------------------------------++def save_artifact(+ model,+ vocab: Sequence[str],+ model_dir: str | Path,+ reward_kind: str,+ n_rows: int,+ metrics: dict | None = None,+) -> None:+ """Atomically write ``model.json`` + ``meta.json`` (meta last: its presence+ and mtime are the load/reload signal)."""+ model_dir = Path(model_dir)+ model_dir.mkdir(parents=True, exist_ok=True)+ # tmp name keeps the .json extension so xgboost writes the JSON format+ tmp_model = model_dir / ("tmp." + MODEL_FILE)+ model.save_model(tmp_model)+ os.replace(tmp_model, model_dir / MODEL_FILE)++ meta = {+ "format_version": FORMAT_VERSION,+ "shared_feature_names": list(FEATURE_NAMES),+ "source_vocab": list(vocab),+ "reward_kind": reward_kind,+ "trained_at": datetime.now(timezone.utc).isoformat(),+ "n_rows": int(n_rows),+ "metrics": metrics or {},+ }+ tmp_meta = model_dir / (META_FILE + ".tmp")+ tmp_meta.write_text(json.dumps(meta, indent=2))+ os.replace(tmp_meta, model_dir / META_FILE)+ logger.info("saved super-search xgb artifact to %s (%d rows, reward=%s)",+ model_dir, n_rows, reward_kind)+++@dataclass+class XgbSourceModel:+ model: object # fitted XGBRegressor+ vocab: list[str]+ vocab_index: dict[str, int]+ meta: dict++ def score(self, feats: dict[str, Sequence[float]]) -> dict[str, float]:+ """Predicted reward for each source given its shared feature vector."""+ if not feats:+ return {}+ names = list(feats)+ X = np.stack([encode(feats[n], n, self.vocab_index) for n in names])+ preds = self.model.predict(X)+ return {name: float(p) for name, p in zip(names, preds)}+++def load_artifact(model_dir: str | Path) -> XgbSourceModel:+ """Load and validate an artifact directory. Raises on anything unexpected:+ missing files, unreadable JSON, format or feature-set mismatch."""+ from xgboost import XGBRegressor++ model_dir = Path(model_dir)+ meta = json.loads((model_dir / META_FILE).read_text())+ if meta["format_version"] != FORMAT_VERSION:+ raise ValueError(+ f"artifact {model_dir} has format_version {meta['format_version']}, "+ f"expected {FORMAT_VERSION}")+ if meta["shared_feature_names"] != list(FEATURE_NAMES):+ raise ValueError(+ f"artifact {model_dir} was trained on features "+ f"{meta['shared_feature_names']} but the running code has "+ f"{list(FEATURE_NAMES)}; retrain the artifact")+ model = XGBRegressor()+ model.load_model(model_dir / MODEL_FILE)+ vocab = list(meta["source_vocab"])+ return XgbSourceModel(+ model=model,+ vocab=vocab,+ vocab_index={name: i for i, name in enumerate(vocab)},+ meta=meta,+ )+++# ---------------------------------------------------------------------------+# Serving singleton+# ---------------------------------------------------------------------------++_cached: XgbSourceModel | None = None+_cached_key: tuple[Path, float] | None = None+_next_check: float = 0.0+_lock = threading.Lock()+++def _artifact_dir() -> Path:+ """Runtime dir if it holds an artifact (i.e. an online retrain has run),+ else the repo-bundled warm-start artifact."""+ runtime = Path(settings.SUPER_SEARCH_XGB_MODEL_DIR)+ if (runtime / META_FILE).exists():+ return runtime+ return BUNDLED_DIR+++def get_model() -> XgbSourceModel:+ """Shared per-process model, hot-reloaded when a retrain replaces the artifact.++ Fail-fast: if no artifact exists anywhere or the artifact is invalid this+ raises (FileNotFoundError / ValueError / json errors) rather than falling+ back — a missing model is a deployment bug to fix, not a state to mask.+ """+ global _cached, _cached_key, _next_check+ now = time.monotonic()+ if _cached is not None and now < _next_check:+ return _cached+ with _lock:+ if _cached is not None and time.monotonic() < _next_check:+ return _cached+ model_dir = _artifact_dir()+ mtime = (model_dir / META_FILE).stat().st_mtime+ key = (model_dir, mtime)+ if key != _cached_key:+ _cached = load_artifact(model_dir)+ _cached_key = key+ logger.info("super-search xgb model loaded from %s (trained %s, %d sources)",+ model_dir, _cached.meta.get("trained_at"), len(_cached.vocab))+ _next_check = time.monotonic() + RELOAD_CHECK_SECONDS+ return _cached+++def reset_model_cache() -> None:+ """Drop the cached model (tests / after an in-process retrain)."""+ global _cached, _cached_key, _next_check+ with _lock:+ _cached = None+ _cached_key = None+ _next_check = 0.0diff --git a/test/test_super_search_bandit.py b/test/test_super_search_bandit.pyindex 951e5f9..1b1a28a 100644--- a/test/test_super_search_bandit.py+++ b/test/test_super_search_bandit.py@@ -106,11 +106,11 @@ def test_thompson_prefers_learned_good_arm(fake_bandit_redis):def test_policy_bandit_branch_selects_and_records_features(monkeypatch):- """End-to-end: USE_BANDIT selection populates ctx.features and respects k."""+ """End-to-end: lints selection populates ctx.features and respects k."""r = fakeredis.FakeRedis()monkeypatch.setattr(bandit, "_redis", r)monkeypatch.setattr(profiles, "_redis", r)- monkeypatch.setattr("django.conf.settings.SUPER_SEARCH_USE_BANDIT", True)+ monkeypatch.setattr("django.conf.settings.SUPER_SEARCH_SELECTION_MODE", "lints")names = ["mwmbl", "hn"] + [f"site{i}" for i in range(20)]ctx = SelectionContext()diff --git a/test/test_super_search_xgb.py b/test/test_super_search_xgb.pynew file mode 100644index 0000000..c02e816--- /dev/null+++ b/test/test_super_search_xgb.py@@ -0,0 +1,266 @@+"""Tests for the XGBoost contextual-bandit source model and the xgb policy."""+import json++import fakeredis+import numpy as np+import pytest++from mwmbl.tinysearchengine.super_search_select import policy, profiles, xgb_model+from mwmbl.tinysearchengine.super_search_select.evaluation import RewardMatrix+from mwmbl.tinysearchengine.super_search_select.features import (+ FEATURE_NAMES,+ NUM_FEATURES,+)+from mwmbl.tinysearchengine.super_search_select.rewards import SelectionContext++INTENT_CODE = FEATURE_NAMES.index("intent_code")++FAST_PARAMS = {**xgb_model.XGB_PARAMS, "n_estimators": 50}++[email protected](autouse=True)+def clean_model_cache():+ xgb_model.reset_model_cache()+ yield+ xgb_model.reset_model_cache()+++def _shared(intent_code: float = 0.0) -> list[float]:+ x = [0.0] * NUM_FEATURES+ x[0] = 1.0 # bias+ x[INTENT_CODE] = intent_code+ return x+++def _interaction_rows(n: int = 60):+ """Planted structure: 'github' pays off iff intent_code, 'recipes' iff not."""+ rows = []+ for i in range(n):+ code = float(i % 2)+ features = {"github": _shared(code), "recipes": _shared(code)}+ rewards = {"github": code, "recipes": 1.0 - code}+ rows.append((features, rewards))+ return rows+++def _trained_model(tmp_path, sources=("github", "recipes"), reward_kind="test"):+ vocab = xgb_model.build_vocab(sources)+ X, y = xgb_model.build_training_data(_interaction_rows(), vocab)+ model = xgb_model.train(X, y, params=FAST_PARAMS)+ xgb_model.save_artifact(model, vocab, tmp_path, reward_kind=reward_kind, n_rows=len(y))+ return vocab+++# ---------------------------------------------------------------------------+# Encoding / training data+# ---------------------------------------------------------------------------++def test_encode_shape_and_identity_block():+ vocab_index = {"a": 0, "b": 1}+ x = xgb_model.encode(_shared(), "b", vocab_index)+ assert x.shape == (NUM_FEATURES + 2,)+ assert x[NUM_FEATURES] == 0.0 and x[NUM_FEATURES + 1] == 1.0+++def test_encode_unknown_source_gets_zero_block():+ x = xgb_model.encode(_shared(), "never_seen", {"a": 0})+ assert np.all(x[NUM_FEATURES:] == 0.0)+++def test_encode_zero_pads_old_short_vectors():+ # Pre-intent impressions stored 10-dim vectors; the intent block was+ # appended last, so zero-padding aligns exactly.+ old = [1.0] * 10+ x = xgb_model.encode(old, "a", {"a": 0})+ assert np.all(x[:10] == 1.0)+ assert np.all(x[10:NUM_FEATURES] == 0.0)+++def test_encode_rejects_vectors_from_a_newer_feature_set():+ with pytest.raises(ValueError):+ xgb_model.encode([0.0] * (NUM_FEATURES + 1), "a", {"a": 0})+++def test_build_training_data_skips_rewards_without_features():+ rows = [({"a": _shared()}, {"a": 1.0, "phantom": 0.5})]+ X, y = xgb_model.build_training_data(rows, ["a", "phantom"])+ assert X.shape == (1, NUM_FEATURES + 2)+ assert y.tolist() == [1.0]+++def test_build_training_data_from_matrix():+ Q, S = 3, 2+ X = np.zeros((Q, S, NUM_FEATURES))+ X[:, :, 0] = 1.0+ R = np.array([[1.0, 0.0], [0.5, 0.2], [0.0, 0.9]])+ mask = np.ones((Q, S), dtype=bool)+ mask[2, 0] = False+ matrix = RewardMatrix(queries=["q1", "q2", "q3"], sources=["b", "a"],+ feature_names=list(FEATURE_NAMES), X=X, R=R, mask=mask)+ Xf, y, vocab = xgb_model.build_training_data_from_matrix(matrix)+ assert vocab == ["a", "b"]+ assert Xf.shape == (5, NUM_FEATURES + 2)+ assert set(y.tolist()) == {1.0, 0.0, 0.5, 0.2, 0.9}+++def test_matrix_with_mismatched_features_raises():+ matrix = RewardMatrix(queries=["q"], sources=["a"], feature_names=["bias"],+ X=np.zeros((1, 1, 1)), R=np.zeros((1, 1)),+ mask=np.ones((1, 1), dtype=bool))+ with pytest.raises(ValueError, match="feature names"):+ xgb_model.build_training_data_from_matrix(matrix)+++# ---------------------------------------------------------------------------+# Model learns identity x context interactions+# ---------------------------------------------------------------------------++def test_model_learns_intent_source_interaction(tmp_path):+ _trained_model(tmp_path)+ loaded = xgb_model.load_artifact(tmp_path)+ code_scores = loaded.score({"github": _shared(1.0), "recipes": _shared(1.0)})+ plain_scores = loaded.score({"github": _shared(0.0), "recipes": _shared(0.0)})+ assert code_scores["github"] > code_scores["recipes"]+ assert plain_scores["recipes"] > plain_scores["github"]+++# ---------------------------------------------------------------------------+# Artifact save / load / get_model+# ---------------------------------------------------------------------------++def test_artifact_roundtrip_meta(tmp_path):+ vocab = _trained_model(tmp_path, reward_kind="judge")+ loaded = xgb_model.load_artifact(tmp_path)+ assert loaded.vocab == list(vocab)+ assert loaded.meta["reward_kind"] == "judge"+ assert loaded.meta["shared_feature_names"] == list(FEATURE_NAMES)+ assert loaded.meta["n_rows"] == 120+++def test_load_artifact_missing_dir_raises(tmp_path):+ with pytest.raises(FileNotFoundError):+ xgb_model.load_artifact(tmp_path / "nothing_here")+++def test_load_artifact_feature_mismatch_raises(tmp_path):+ _trained_model(tmp_path)+ meta_path = tmp_path / xgb_model.META_FILE+ meta = json.loads(meta_path.read_text())+ meta["shared_feature_names"] = meta["shared_feature_names"][:-1]+ meta_path.write_text(json.dumps(meta))+ with pytest.raises(ValueError, match="retrain"):+ xgb_model.load_artifact(tmp_path)+++def test_load_artifact_bad_format_version_raises(tmp_path):+ _trained_model(tmp_path)+ meta_path = tmp_path / xgb_model.META_FILE+ meta = json.loads(meta_path.read_text())+ meta["format_version"] = 999+ meta_path.write_text(json.dumps(meta))+ with pytest.raises(ValueError, match="format_version"):+ xgb_model.load_artifact(tmp_path)+++def test_get_model_uses_runtime_dir_over_bundle(tmp_path, monkeypatch):+ runtime = tmp_path / "runtime"+ _trained_model(runtime)+ monkeypatch.setattr("django.conf.settings.SUPER_SEARCH_XGB_MODEL_DIR", str(runtime))+ model = xgb_model.get_model()+ assert model.vocab == ["github", "recipes"]+++def test_get_model_falls_back_to_bundle_when_runtime_empty(tmp_path, monkeypatch):+ bundle = tmp_path / "bundle"+ _trained_model(bundle, sources=("bundled_source",))+ monkeypatch.setattr(xgb_model, "BUNDLED_DIR", bundle)+ monkeypatch.setattr("django.conf.settings.SUPER_SEARCH_XGB_MODEL_DIR",+ str(tmp_path / "empty_runtime"))+ assert xgb_model.get_model().vocab == ["bundled_source"]+++def test_get_model_no_artifact_anywhere_raises(tmp_path, monkeypatch):+ monkeypatch.setattr(xgb_model, "BUNDLED_DIR", tmp_path / "no_bundle")+ monkeypatch.setattr("django.conf.settings.SUPER_SEARCH_XGB_MODEL_DIR",+ str(tmp_path / "no_runtime"))+ with pytest.raises(FileNotFoundError):+ xgb_model.get_model()+++def test_get_model_hot_reloads_after_retrain(tmp_path, monkeypatch):+ runtime = tmp_path / "runtime"+ _trained_model(runtime)+ monkeypatch.setattr("django.conf.settings.SUPER_SEARCH_XGB_MODEL_DIR", str(runtime))+ monkeypatch.setattr(xgb_model, "RELOAD_CHECK_SECONDS", 0.0)+ assert xgb_model.get_model().vocab == ["github", "recipes"]++ _trained_model(runtime, sources=("github", "recipes", "arxiv"))+ meta_path = runtime / xgb_model.META_FILE+ # ensure the mtime moves even on coarse-grained filesystems+ stat = meta_path.stat()+ import os+ os.utime(meta_path, (stat.st_atime, stat.st_mtime + 1))+ assert xgb_model.get_model().vocab == ["arxiv", "github", "recipes"]+++# ---------------------------------------------------------------------------+# xgb policy+# ---------------------------------------------------------------------------++def xgb_policy_env(tmp_path, monkeypatch):+ """xgb selection mode with a toy artifact trained over the policy's sources."""+ monkeypatch.setattr(profiles, "_redis", fakeredis.FakeRedis())+ monkeypatch.setattr("django.conf.settings.SUPER_SEARCH_SELECTION_MODE", "xgb")+ monkeypatch.setattr("django.conf.settings.SUPER_SEARCH_XGB_MODEL_DIR", str(tmp_path))+ names = ["mwmbl", "hn"] + [f"site{i}" for i in range(20)]+ _trained_model(tmp_path, sources=names)+ return names+++def test_policy_xgb_selects_and_records_features(xgb_policy_env, monkeypatch):+ monkeypatch.setattr("django.conf.settings.SUPER_SEARCH_XGB_EPSILON", 0.0)+ ctx = SelectionContext()+ chosen = policy.select_sources("python testing tools", xgb_policy_env, k=5, ctx=ctx)+ assert len(chosen) == 5+ assert "mwmbl" in chosen and "hn" in chosen # always-on included+ assert set(chosen) <= set(ctx.features)+ assert all(len(v) == NUM_FEATURES for v in ctx.features.values())+++def test_policy_xgb_greedy_is_deterministic(xgb_policy_env, monkeypatch):+ monkeypatch.setattr("django.conf.settings.SUPER_SEARCH_XGB_EPSILON", 0.0)+ runs = {tuple(policy.select_sources("some query", xgb_policy_env, k=6))+ for _ in range(5)}+ assert len(runs) == 1+++def test_policy_xgb_full_explore_still_valid(xgb_policy_env, monkeypatch):+ monkeypatch.setattr("django.conf.settings.SUPER_SEARCH_XGB_EPSILON", 1.0)+ seen = set()+ for _ in range(10):+ chosen = policy.select_sources("some query", xgb_policy_env, k=5)+ assert len(chosen) == 5+ assert len(set(chosen)) == 5+ assert "mwmbl" in chosen and "hn" in chosen+ seen.add(tuple(chosen))+ assert len(seen) > 1 # actually exploring+++def test_policy_xgb_missing_artifact_raises(monkeypatch, tmp_path):+ monkeypatch.setattr(profiles, "_redis", fakeredis.FakeRedis())+ monkeypatch.setattr("django.conf.settings.SUPER_SEARCH_SELECTION_MODE", "xgb")+ monkeypatch.setattr("django.conf.settings.SUPER_SEARCH_XGB_MODEL_DIR",+ str(tmp_path / "empty"))+ monkeypatch.setattr(xgb_model, "BUNDLED_DIR", tmp_path / "no_bundle")+ names = ["mwmbl", "hn"] + [f"site{i}" for i in range(20)]+ with pytest.raises(FileNotFoundError):+ policy.select_sources("some query", names, k=5)+++def test_policy_unknown_mode_raises(monkeypatch):+ monkeypatch.setattr(profiles, "_redis", fakeredis.FakeRedis())+ monkeypatch.setattr("django.conf.settings.SUPER_SEARCH_SELECTION_MODE", "bogus")+ names = ["mwmbl", "hn"] + [f"site{i}" for i in range(20)]+ with pytest.raises(ValueError, match="SUPER_SEARCH_SELECTION_MODE"):+ policy.select_sources("some query", names, k=5)