An open source, non-profit web search engine
Online retrain loop for the xgb source model + privacy tripwire
Daoud Clarke79f9be564bd8
diff --git a/.gitignore b/.gitignoreindex 284d324..bfdff88 100644--- a/.gitignore+++ b/.gitignore@@ -197,6 +197,7 @@ prescreen.jsondevdata/ss_eval_matrix*devdata/ss_gold_matrix*devdata/ss_judge_matrix*+devdata/ss_judge_queries.txtdevdata/super_search_xgb/# AI planning documentsdiff --git a/mwmbl/apps.py b/mwmbl/apps.pyindex ca5c60e..cdf4dc0 100644--- a/mwmbl/apps.py+++ b/mwmbl/apps.py@@ -67,7 +67,11 @@ class MwmblConfig(AppConfig):try:from background_task.models import Task- from mwmbl.background import sync_search_counts, sync_super_search_bandit+ from mwmbl.background import (+ retrain_super_search_xgb,+ sync_search_counts,+ sync_super_search_bandit,+ )SYNC_TASK = "mwmbl.background.sync_search_counts"@@ -80,6 +84,13 @@ class MwmblConfig(AppConfig):if not Task.objects.filter(task_name=BANDIT_TASK).exists():sync_super_search_bandit(repeat=3600, repeat_until=None)+ # Retrain the Super Search xgb source model daily from the+ # impression log (skips unless the xgb mode is live and enough+ # pairs have accumulated).+ XGB_TASK = "mwmbl.background.retrain_super_search_xgb"+ if not Task.objects.filter(task_name=XGB_TASK).exists():+ retrain_super_search_xgb(repeat=86400, repeat_until=None)+except Exception:# Don't prevent startup if background task scheduling failslog.exception("Failed to schedule background tasks")diff --git a/mwmbl/background.py b/mwmbl/background.pyindex 4439c64..abdd698 100644--- a/mwmbl/background.py+++ b/mwmbl/background.py@@ -124,6 +124,31 @@ def sync_super_search_bandit():)+@background(schedule=0)+def retrain_super_search_xgb():+ """Daily batch retrain of the Super Search xgb source model.++ The impression log is the contextual bandit's memory: this folds the last+ training window of logged (features, judge-reward) pairs into a fresh+ XGBoost artifact, which serving processes hot-reload via its mtime. Skips+ (with a log line) when the mode isn't xgb or too few pairs have+ accumulated; training/IO errors propagate so failures are visible.+ """+ from mwmbl.tinysearchengine.super_search_select import xgb_model++ if settings.SUPER_SEARCH_SELECTION_MODE != "xgb":+ logger.info("super-search xgb retrain skipped: selection mode is %s",+ settings.SUPER_SEARCH_SELECTION_MODE)+ return+ metrics = xgb_model.train_and_save_from_impressions(+ window_days=settings.SUPER_SEARCH_XGB_TRAIN_WINDOW_DAYS,+ min_rows=settings.SUPER_SEARCH_XGB_MIN_TRAIN_ROWS,+ out_dir=settings.SUPER_SEARCH_XGB_MODEL_DIR,+ )+ if metrics is not None:+ logger.info("super-search xgb retrained: %s", metrics)++@background(schedule=0)def sync_search_counts():"""diff --git a/mwmbl/management/commands/train_super_search_xgb.py b/mwmbl/management/commands/train_super_search_xgb.pynew file mode 100644index 0000000..05b5231--- /dev/null+++ b/mwmbl/management/commands/train_super_search_xgb.py@@ -0,0 +1,64 @@+"""Train the Super Search xgb source model and write its serving artifact.++Two data sources:++- ``--from-matrix PATH`` a dense offline ``RewardMatrix`` (``.npz`` ++ ``.json`` pair, e.g. ``devdata/ss_judge_matrix``) — used to build the+ repo-bundled warm-start artifact from the per-source home-query set.+- ``--days N`` logged ``SuperSearchImpression`` rows from the last+ N days — the same path the daily background retrain takes.++The artifact directory (``model.json`` + ``meta.json``) defaults to+``settings.SUPER_SEARCH_XGB_MODEL_DIR``; pass+``--out mwmbl/tinysearchengine/super_search_select/artifacts/xgb`` to refresh+the bundled warm-start model.+"""+from django.conf import settings+from django.core.management.base import BaseCommand, CommandError++from mwmbl.tinysearchengine.super_search_select import xgb_model+from mwmbl.tinysearchengine.super_search_select.evaluation import RewardMatrix+++class Command(BaseCommand):+ help = "Train the Super Search xgb source model (from a reward matrix or the impression log)"++ def add_arguments(self, parser):+ source = parser.add_mutually_exclusive_group(required=True)+ source.add_argument("--from-matrix", metavar="PATH",+ help="RewardMatrix path (without extension)")+ source.add_argument("--days", type=int,+ help="train from SuperSearchImpression rows of the last N days")+ parser.add_argument("--min-rows", type=int,+ default=settings.SUPER_SEARCH_XGB_MIN_TRAIN_ROWS)+ parser.add_argument("--out", default=settings.SUPER_SEARCH_XGB_MODEL_DIR,+ help="artifact directory (default: SUPER_SEARCH_XGB_MODEL_DIR)")+ parser.add_argument("--reward-kind", default="judge",+ help="provenance label stored in meta.json for --from-matrix")++ def handle(self, *args, **options):+ import numpy as np++ if options["from_matrix"]:+ matrix = RewardMatrix.load(options["from_matrix"])+ X, y, vocab = xgb_model.build_training_data_from_matrix(matrix)+ if len(y) < options["min_rows"]:+ raise CommandError(+ f"only {len(y)} training pairs in the matrix, "+ f"need {options['min_rows']} (--min-rows)")+ model = xgb_model.train(X, y)+ metrics = {"train_rmse": float(np.sqrt(np.mean((model.predict(X) - y) ** 2)))}+ xgb_model.save_artifact(model, vocab, options["out"],+ reward_kind=options["reward_kind"],+ n_rows=len(y), metrics=metrics)+ else:+ metrics = xgb_model.train_and_save_from_impressions(+ window_days=options["days"],+ min_rows=options["min_rows"],+ out_dir=options["out"],+ )+ if metrics is None:+ raise CommandError("not enough impression pairs to train "+ "(see --min-rows)")+ self.stdout.write(self.style.SUCCESS(+ f"trained xgb source model -> {options['out']} ({metrics})"))diff --git a/mwmbl/tinysearchengine/super_search_select/features.py b/mwmbl/tinysearchengine/super_search_select/features.pyindex 37f4351..e4d0243 100644--- a/mwmbl/tinysearchengine/super_search_select/features.py+++ b/mwmbl/tinysearchengine/super_search_select/features.py@@ -6,6 +6,14 @@query-vs-site-profile cosines (``cos_bow`` / ``cos_cng``); the rest are sitepriors, online per-site stats, and cheap query-shape signals. Which featuresactually earn their keep is decided by the offline evaluation (Phase 5).++PRIVACY INVARIANT: these vectors are persisted per impression+(``SuperSearchImpression.features``) while the query text never is. Every+feature must therefore be a scalar that does not permit reconstructing the+query — coarse counts, bits from public lexicons, cosines against known site+profiles are fine; raw query projections/embeddings are not and must never be+added here. ``test_persisted_features_are_scalar_and_reconstruction_safe``+trips if a feature name suggests otherwise."""from __future__ import annotationsdiff --git a/mwmbl/tinysearchengine/super_search_select/xgb_model.py b/mwmbl/tinysearchengine/super_search_select/xgb_model.pyindex 70fd66f..728d35f 100644--- a/mwmbl/tinysearchengine/super_search_select/xgb_model.py+++ b/mwmbl/tinysearchengine/super_search_select/xgb_model.py@@ -150,6 +150,41 @@ def train(X: np.ndarray, y: np.ndarray, params: dict | None = None):return model+def train_and_save_from_impressions(+ window_days: int, min_rows: int, out_dir: str | Path,+) -> dict | None:+ """Batch retrain from logged ``SuperSearchImpression`` rows.++ The impression log stores per-source shared feature vectors (keyed by+ source name — that key is the identity feature) and judge rewards, never+ the query text, so this is the privacy-safe online learning path. Returns+ the saved metrics, or None when fewer than ``min_rows`` (source, reward)+ pairs exist in the window (the designed not-enough-data-yet case; anything+ else that goes wrong raises).+ """+ from datetime import timedelta++ from mwmbl.models import SuperSearchImpression+ from mwmbl.tinysearchengine.super_search_sources import SOURCES++ cutoff = datetime.now(timezone.utc) - timedelta(days=window_days)+ rows = list(SuperSearchImpression.objects.filter(timestamp__gte=cutoff)+ .values_list("features", "rewards"))+ n_pairs = sum(len(set(f) & set(r)) for f, r in rows)+ if n_pairs < min_rows:+ logger.info("super-search xgb retrain skipped: %d pairs in %d days "+ "(need %d)", n_pairs, window_days, min_rows)+ return None+ names = {name for f, _ in rows for name in f} | set(SOURCES)+ vocab = build_vocab(names)+ X, y = build_training_data(rows, vocab)+ model = train(X, y)+ metrics = {"train_rmse": float(np.sqrt(np.mean((model.predict(X) - y) ** 2)))}+ save_artifact(model, vocab, out_dir, reward_kind="judge",+ n_rows=len(y), metrics=metrics)+ return metrics++# ---------------------------------------------------------------------------# Artifact save / load# ---------------------------------------------------------------------------diff --git a/test/test_super_search_xgb.py b/test/test_super_search_xgb.pyindex c02e816..f9fb64f 100644--- a/test/test_super_search_xgb.py+++ b/test/test_super_search_xgb.py@@ -264,3 +264,64 @@ def test_policy_unknown_mode_raises(monkeypatch):names = ["mwmbl", "hn"] + [f"site{i}" for i in range(20)]with pytest.raises(ValueError, match="SUPER_SEARCH_SELECTION_MODE"):policy.select_sources("some query", names, k=5)+++# ---------------------------------------------------------------------------+# Online retrain from the impression log+# ---------------------------------------------------------------------------++def test_retrain_from_impressions_writes_artifact(tmp_path, monkeypatch):+ from mwmbl.models import SuperSearchImpression++ monkeypatch.setattr(xgb_model, "XGB_PARAMS", FAST_PARAMS)+ for features, rewards in _interaction_rows(30):+ SuperSearchImpression.objects.create(+ candidates=list(features), selected=list(features),+ features={k: list(v) for k, v in features.items()}, rewards=rewards)+ # A pre-intent row with a short (10-dim) vector must zero-pad, not break.+ SuperSearchImpression.objects.create(+ candidates=["old"], selected=["old"],+ features={"old": [1.0] * 10}, rewards={"old": 0.5})++ metrics = xgb_model.train_and_save_from_impressions(+ window_days=7, min_rows=10, out_dir=tmp_path)+ assert metrics is not None and "train_rmse" in metrics+ loaded = xgb_model.load_artifact(tmp_path)+ assert {"github", "recipes", "old"} <= set(loaded.vocab)+ assert loaded.meta["reward_kind"] == "judge"+ assert loaded.meta["n_rows"] == 61+++def test_retrain_skips_below_min_rows(tmp_path):+ from mwmbl.models import SuperSearchImpression++ SuperSearchImpression.objects.create(+ candidates=["a"], selected=["a"],+ features={"a": [0.0] * NUM_FEATURES}, rewards={"a": 1.0})+ assert xgb_model.train_and_save_from_impressions(+ window_days=7, min_rows=100, out_dir=tmp_path) is None+ assert not (tmp_path / xgb_model.META_FILE).exists()+++# ---------------------------------------------------------------------------+# Privacy tripwire: nothing vector-valued may enter the persisted features+# ---------------------------------------------------------------------------++def test_persisted_features_are_scalar_and_reconstruction_safe():+ """Impressions persist the feature vector per source; every entry must be a+ named scalar that cannot reconstruct the query. A name matching the pattern+ below suggests someone is about to log a raw projection/embedding — that+ needs conscious review, hence this tripwire."""+ import re++ from mwmbl.tinysearchengine.super_search_select.features import (+ QueryContext, feature_vector,+ )+ from mwmbl.tinysearchengine.super_search_select.registry import get_meta++ assert not any(re.search(r"proj|embed|vec|hash", name) for name in FEATURE_NAMES)+ qctx = QueryContext.build("some test query", np.zeros(64), np.zeros(64))+ x = feature_vector(qctx, get_meta("github"), (None, None))+ assert x.shape == (len(FEATURE_NAMES),)