Skip to content

armgit / mwmbl

Public Imported from GitHub
0

An open source, non-profit web search engine

21.0 KB
Raw Blame
#!/usr/bin/env python3
"""Offline evaluation harness for Super Search source selection.

Subcommands:

  build-matrix  - query every source for each query in a query file and write a
                  dense (queries x sources) reward + feature matrix. Rewards are
                  either LTR-top-K survival (--reward survival, default) or the
                  fine-tuned relevance judge's mean per-source doc score
                  (--reward judge). The network fetch is checkpointed per query
                  (JSONL next to --out) and resumable. Requires network + Django.

  build-gold-matrix
                - build the same kind of matrix offline from the LTR dataset's
                  gold labels (no network): a (query x source) cell is "available"
                  (mask) if that source's domain appears in the query's LTR rows,
                  and gets reward 1.0 if any of those rows is gold-relevant. Lets
                  us learn/evaluate source selection against real gold relevance
                  instead of the LTR-top-K-survival proxy.

  select        - fit the XGBoost reward model on the matrix (grouped CV by
                  query) and print each feature's ablation drop in coverage@k.

  simulate      - replay Thompson sampling over the matrix, sweep the
                  exploration scale nu, and print it against random / popularity
                  / cosine / oracle baselines.

  simulate-xgb  - replay the epsilon-greedy XGBoost contextual bandit over the
                  matrix for each candidate epsilon, printed against the same
                  baselines and the best LinTS nu for a direct comparison.

  holdout       - standard offline evaluation of the xgb source model: split
                  queries train/test (stratified by home source), fit on train,
                  report test coverage@k and home-source recall@k vs baselines.
                  --queries-by-source supplies the {source: [queries]} home map.

Usage:
  DJANGO_SETTINGS_MODULE=mwmbl.settings_dev uv run python scripts/super_search_eval.py \
      build-matrix --queries queries.txt --out eval_matrix
  uv run python scripts/super_search_eval.py select   --matrix eval_matrix
  uv run python scripts/super_search_eval.py simulate --matrix eval_matrix
"""

import argparse
import asyncio
import json
import math
import os
import sys
from collections import defaultdict
from pathlib import Path

import django
import numpy as np

REPO_ROOT = Path(__file__).resolve().parent.parent
sys.path.insert(0, str(REPO_ROOT))
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "mwmbl.settings_dev")

# host_of/registrable are pure stdlib helpers (no Django) — safe to import at module
# load, before django.setup(); the registry-backed source_domain_map is loaded lazily.
from mwmbl.tinysearchengine.super_search_select.domains import (  # noqa: E402
    host_of,
    registrable,
)


def _bootstrap_django():
    django.setup()


# ---------------------------------------------------------------------------
# build-matrix (network + Django)
# ---------------------------------------------------------------------------


async def _query_all_sources(query: str, limit: int):
    import httpx
    from django.conf import settings

    from mwmbl.tinysearchengine.super_search_sources import SOURCES

    timeout = settings.SUPER_SEARCH_PER_SOURCE_TIMEOUT
    async with httpx.AsyncClient(
        follow_redirects=True, timeout=timeout, headers={"User-Agent": "mwmbl-super-search-eval/0.1"}
    ) as client:

        async def one(name, fn):
            try:
                docs = await asyncio.wait_for(fn(client, query, limit), timeout=timeout)
                return name, docs
            except Exception:
                return name, []

        results = await asyncio.gather(*[one(n, f) for n, f in SOURCES.items()])
    return dict(results)


def _fetch_all_queries(queries: list[str], checkpoint: Path, limit: int) -> dict[str, dict]:
    """Pass 1 with a resumable per-query JSONL checkpoint.

    Returns ``{query: {source: [[url, title, extract], ...]}}``. Queries already
    in the checkpoint are not re-fetched, so a multi-hour network run can be
    interrupted and resumed.
    """

    fetched: dict[str, dict] = {}
    if checkpoint.exists():
        for line in checkpoint.read_text().splitlines():
            rec = json.loads(line)
            fetched[rec["query"]] = rec["docs"]
        print(f"checkpoint {checkpoint}: {len(fetched)} queries already fetched")
    todo = [q for q in queries if q not in fetched]
    with checkpoint.open("a") as fh:
        for qi, query in enumerate(todo):
            docs_by_source = asyncio.run(_query_all_sources(query, limit))
            docs = {name: [[d.url, d.title, d.extract] for d in ds] for name, ds in docs_by_source.items() if ds}
            fetched[query] = docs
            fh.write(json.dumps({"query": query, "docs": docs}) + "\n")
            fh.flush()
            print(f"  [{qi + 1}/{len(todo)}] {query!r}: {sum(len(d) for d in docs.values())} docs")
    return fetched


def _survival_rewards(query: str, docs_by_source: dict, s_index: dict, R, qi: int, limit: int, top_k: int) -> None:
    """Reward = fraction of a source's results surviving the LTR model's top-K."""
    from mwmbl.search_setup import ltr_model
    from mwmbl.tinysearchengine.indexer import Document
    from mwmbl.tinysearchengine.ltr_rank import score_documents

    source_by_url, all_docs = {}, []
    for name, docs in docs_by_source.items():
        for url, title, extract in docs:
            if url and title:
                source_by_url.setdefault(url, name)
                all_docs.append(Document(title=title, url=url, extract=extract or ""))
    if not all_docs:
        return
    scores = score_documents(ltr_model, query, all_docs)
    ranked = [d.url for d, _ in sorted(zip(all_docs, scores), key=lambda x: -x[1])][:top_k]
    counts = {}
    for url in ranked:
        src = source_by_url.get(url)
        if src:
            counts[src] = counts.get(src, 0) + 1
    for name, c in counts.items():
        R[qi, s_index[name]] = min(c / max(limit, 1), 1.0)


def _judge_rewards(query: str, docs_by_source: dict, s_index: dict, R, qi: int) -> None:
    """Reward = mean relevance-judge score of the source's returned docs — the
    offline analog of rewards.compute_judge_rewards."""
    from mwmbl.tinysearchengine.super_search_select.judge import doc_text, get_judge

    judge = get_judge()
    if judge is None:
        raise RuntimeError(
            "--reward judge requires the relevance judge artifact (settings.SUPER_SEARCH_JUDGE_MODEL_DIR)"
        )
    for name, docs in docs_by_source.items():
        texts = [doc_text(title, extract) for _, title, extract in docs]
        if texts:
            R[qi, s_index[name]] = float(np.mean(judge.score(query, texts)))


def build_matrix(queries: list[str], out: str, reward: str = "survival", checkpoint: str | None = None):
    from django.conf import settings

    from mwmbl.tinysearchengine.super_search_select import vectors
    from mwmbl.tinysearchengine.super_search_select.evaluation import RewardMatrix
    from mwmbl.tinysearchengine.super_search_select.features import (
        FEATURE_NAMES,
        QueryContext,
        feature_vector,
    )
    from mwmbl.tinysearchengine.super_search_select.registry import get_meta
    from mwmbl.tinysearchengine.super_search_sources import SOURCES

    sources = list(SOURCES.keys())
    s_index = {name: i for i, name in enumerate(sources)}
    limit = settings.SUPER_SEARCH_RESULTS_PER_SOURCE
    top_k = settings.SUPER_SEARCH_TOP_K
    dim = settings.SUPER_SEARCH_PROJECTION_DIM
    F = len(FEATURE_NAMES)

    # Pass 1: query everything (resumable), then accumulate each site's content
    # profile (mean of its projected result samples).
    cp = Path(checkpoint) if checkpoint else Path(f"{out}.fetch.jsonl")
    fetched = _fetch_all_queries(queries, cp, limit)
    prof_bow = {n: np.zeros(dim) for n in sources}
    prof_cng = {n: np.zeros(dim) for n in sources}
    for query in queries:
        for name, docs in fetched[query].items():
            text = " ".join(f"{title or ''} {extract or ''}" for _, title, extract in docs)
            prof_bow[name] += vectors.project_bow(text, dim)
            prof_cng[name] += vectors.project_char_ngrams(text, dim)
    profile = {n: (vectors._l2_normalise(prof_bow[n]), vectors._l2_normalise(prof_cng[n])) for n in sources}

    # Pass 2: features + rewards.
    Q, S = len(queries), len(sources)
    X = np.zeros((Q, S, F))
    R = np.zeros((Q, S))
    mask = np.zeros((Q, S), dtype=bool)
    for qi, query in enumerate(queries):
        docs_by_source = fetched[query]
        bow = vectors.project_bow(query, dim)
        cng = vectors.project_char_ngrams(query, dim)
        qctx = QueryContext.build(query, bow, cng)
        for name in sources:
            X[qi, s_index[name]] = feature_vector(qctx, get_meta(name), profile[name])
        for name in docs_by_source:
            mask[qi, s_index[name]] = True
        if reward == "judge":
            _judge_rewards(query, docs_by_source, s_index, R, qi)
        else:
            _survival_rewards(query, docs_by_source, s_index, R, qi, limit, top_k)
        if (qi + 1) % 50 == 0:
            print(f"  scored [{qi + 1}/{Q}]")

    matrix = RewardMatrix(queries=queries, sources=sources, feature_names=list(FEATURE_NAMES), X=X, R=R, mask=mask)
    matrix.save(out)
    print(
        f"Wrote matrix {out}.npz/.json ({reward} rewards): {Q} queries x {S} sources, {int(mask.sum())} filled cells."
    )


# ---------------------------------------------------------------------------
# build-gold-matrix (offline: LTR dataset gold labels, no network)
# ---------------------------------------------------------------------------


def _is_gold(rank) -> bool:
    """True if ``rank`` is a real gold rank (not pandas NaN / None / blank)."""
    if rank is None:
        return False
    if isinstance(rank, float) and math.isnan(rank):
        return False
    if isinstance(rank, str) and not rank.strip():
        return False
    return True


def attribute_rows(rows, reg_map):
    """Attribute LTR rows to Super Search sources by registrable domain (pure).

    ``rows`` is an iterable of ``(query, url, title, extract, gold_standard_rank)``
    and ``reg_map`` maps a registrable domain to the source names on it. Returns
    ``(per_query, prof_text)`` where ``per_query[query][source]`` is True iff that
    source has a gold-relevant row for the query (else False = available-but-not-gold),
    and ``prof_text[source]`` accumulates that source's title/extract text.
    """
    per_query: dict[str, dict[str, bool]] = {}  # query -> {source -> has_gold}
    prof_text: dict[str, list[str]] = defaultdict(list)
    for query, url, title, extract, gold in rows:
        names = reg_map.get(registrable(host_of(str(url))))
        if not names:
            continue
        is_gold = _is_gold(gold)
        q_sources = per_query.setdefault(str(query), {})
        text = f"{title or ''} {extract or ''}"
        for name in names:
            q_sources[name] = q_sources.get(name, False) or is_gold
            prof_text[name].append(text)
    return per_query, prof_text


def build_gold_matrix(out: str):
    """Build a (query x source) reward matrix from the LTR dataset's gold labels.

    Unlike ``build-matrix`` (which queries live sources and rewards a source by how
    many of its results survive the LTR model's *own* top-K), this is fully offline
    and grounds the reward in real gold relevance:

      * mask[q, s] = source ``s``'s registrable domain appears in query ``q``'s LTR
        rows (the offline "this source contributed a candidate" signal);
      * R[q, s]    = 1.0 if any such row is gold-relevant (non-null
        ``gold_standard_rank``), else 0.0 (binary has-gold).

    Source content profiles for the cosine features are accumulated from the LTR
    rows' title/extract text per source, so no network access is needed.
    """
    import pandas as pd
    from django.conf import settings

    from mwmbl.rankeval.paths import LEARNING_TO_RANK_DATASET_PATH
    from mwmbl.tinysearchengine.super_search_select import vectors
    from mwmbl.tinysearchengine.super_search_select.domains import source_domain_map
    from mwmbl.tinysearchengine.super_search_select.evaluation import RewardMatrix
    from mwmbl.tinysearchengine.super_search_select.features import (
        FEATURE_NAMES,
        QueryContext,
        feature_vector,
    )
    from mwmbl.tinysearchengine.super_search_select.registry import get_meta, get_registry

    dim = settings.SUPER_SEARCH_PROJECTION_DIM
    F = len(FEATURE_NAMES)

    sources = list(get_registry().keys())
    s_index = {name: i for i, name in enumerate(sources)}
    reg_map = source_domain_map()  # registrable domain -> [source names]

    df = pd.read_csv(LEARNING_TO_RANK_DATASET_PATH, lineterminator="\n")
    print(f"Loaded LTR dataset: {len(df)} rows, {df['query'].nunique()} queries")

    # Pass 1: attribute every row to its source(s), recording per-(query, source)
    # availability + gold and accumulating per-source content text for the profiles.
    per_query, prof_text = attribute_rows(
        zip(df["query"], df["url"], df["title"], df["extract"], df["gold_standard_rank"]),
        reg_map,
    )

    # In-coverage queries only: those with >=1 in-source candidate row.
    in_cov_queries = list(per_query.keys())
    profile = {
        n: (
            vectors.project_bow(" ".join(prof_text[n]), dim) if prof_text[n] else None,
            vectors.project_char_ngrams(" ".join(prof_text[n]), dim) if prof_text[n] else None,
        )
        for n in sources
    }

    # Pass 2: features + binary has-gold reward.
    Q, S = len(in_cov_queries), len(sources)
    X = np.zeros((Q, S, F))
    R = np.zeros((Q, S))
    mask = np.zeros((Q, S), dtype=bool)
    for qi, query in enumerate(in_cov_queries):
        bow = vectors.project_bow(query, dim)
        cng = vectors.project_char_ngrams(query, dim)
        qctx = QueryContext.build(query, bow, cng)
        for name, has_gold in per_query[query].items():
            si = s_index[name]
            X[qi, si] = feature_vector(qctx, get_meta(name), profile[name])
            mask[qi, si] = True
            if has_gold:
                R[qi, si] = 1.0

    matrix = RewardMatrix(
        queries=in_cov_queries, sources=sources, feature_names=list(FEATURE_NAMES), X=X, R=R, mask=mask
    )
    matrix.save(out)

    # Diagnostics: does selection even have room to matter? If most queries have
    # few in-source candidates, "query them all" is as good as any learned policy.
    avail_per_q = mask.sum(axis=1)
    gold_per_q = (R > 0).sum(axis=1)
    n_gold_q = int((gold_per_q > 0).sum())
    print(f"\nWrote matrix {out}.npz/.json: {Q} in-coverage queries x {S} sources, {int(mask.sum())} filled cells.")
    print(f"  queries with >=1 gold source:   {n_gold_q} ({100 * n_gold_q / max(Q, 1):.1f}%)")
    print(
        f"  mean available sources / query: {avail_per_q.mean():.2f} "
        f"(median {int(np.median(avail_per_q))}, max {int(avail_per_q.max())})"
    )
    for k in (1, 2, 3, 5, 10):
        n = int((avail_per_q > k).sum())
        print(
            f"  queries with > {k:>2} available sources: {n:>4} "
            f"({100 * n / max(Q, 1):.1f}%)  <- room for top-{k} selection to matter"
        )


# ---------------------------------------------------------------------------
# select / simulate (pure, no network)
# ---------------------------------------------------------------------------


def cmd_select(matrix_path: str, k: int):
    from mwmbl.tinysearchengine.super_search_select.evaluation import RewardMatrix, select_features

    m = RewardMatrix.load(matrix_path)
    result = select_features(m, k=k)
    print(f"baseline coverage@{k}: {result['baseline_coverage']:.4f}\n")
    print("feature ablation (drop in coverage@k when removed; higher = more useful):")
    for name, drop in sorted(result["ablation_drop"].items(), key=lambda kv: -kv[1]):
        print(f"  {name:18} {drop:+.4f}")


def cmd_simulate(matrix_path: str, k: int):
    from mwmbl.tinysearchengine.super_search_select.evaluation import (
        RewardMatrix,
        simulate_baselines,
        sweep_explore_scale,
    )

    m = RewardMatrix.load(matrix_path)
    base = simulate_baselines(m, k=k)
    print("baselines (mean captured reward per query):")
    for name, val in sorted(base.items(), key=lambda kv: -kv[1]):
        print(f"  {name:12} {val:.4f}")
    print("\nThompson sampling by exploration scale nu:")
    sweep = sweep_explore_scale(m, k=k, nus=[0.0, 0.25, 0.5, 1.0, 2.0, 4.0])
    best = max(sweep, key=sweep.get)
    for nu, val in sweep.items():
        marker = "  <- best" if nu == best else ""
        print(f"  nu={nu:<4} {val:.4f}{marker}")


def cmd_simulate_xgb(matrix_path: str, k: int, epsilons: list[float], refit_every: int, min_rows: int):
    from mwmbl.tinysearchengine.super_search_select.evaluation import (
        RewardMatrix,
        simulate_baselines,
        sweep_epsilon,
        sweep_explore_scale,
    )

    m = RewardMatrix.load(matrix_path)
    base = simulate_baselines(m, k=k)
    print("baselines (mean captured reward per query):")
    for name, val in sorted(base.items(), key=lambda kv: -kv[1]):
        print(f"  {name:12} {val:.4f}")
    ts = sweep_explore_scale(m, k=k, nus=[0.05, 0.25, 1.0])
    best_nu = max(ts, key=ts.get)
    print(f"\nbest LinTS: nu={best_nu} {ts[best_nu]:.4f}")
    print("\nXGB contextual bandit by epsilon:")
    sweep = sweep_epsilon(m, k=k, epsilons=epsilons, refit_every=refit_every, min_rows=min_rows)
    best = max(sweep, key=sweep.get)
    for eps, val in sweep.items():
        marker = "  <- best" if eps == best else ""
        print(f"  eps={eps:<5} {val:.4f}{marker}")


def cmd_holdout(matrix_path: str, k: int, test_frac: float, queries_by_source: str | None, seed: int):
    from mwmbl.tinysearchengine.super_search_select.evaluation import (
        RewardMatrix,
        evaluate_holdout,
    )

    m = RewardMatrix.load(matrix_path)
    home_by_query = None
    if queries_by_source:
        by_source = json.loads(Path(queries_by_source).read_text())
        home_by_query = {q: source for source, qs in by_source.items() for q in qs}
    result = evaluate_holdout(m, k=k, test_frac=test_frac, seed=seed, home_by_query=home_by_query)
    print(f"train queries: {result['n_train_queries']}, test queries: {result['n_test_queries']}")
    print(f"xgb test RMSE: {result['rmse']:.4f}\n")
    for metric in ("coverage_at_k", "home_recall_at_k"):
        print(f"{metric} (k={k}):")
        for name, val in sorted(result[metric].items(), key=lambda kv: -kv[1]):
            print(f"  {name:14} {val:.4f}")
        print()


def main():
    parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
    sub = parser.add_subparsers(dest="command", required=True)

    p_build = sub.add_parser("build-matrix")
    p_build.add_argument("--queries", required=True, help="file with one query per line")
    p_build.add_argument("--out", default="eval_matrix")
    p_build.add_argument("--reward", choices=["survival", "judge"], default="survival")
    p_build.add_argument("--checkpoint", default=None, help="fetch checkpoint JSONL (default: <out>.fetch.jsonl)")

    p_gold = sub.add_parser("build-gold-matrix")
    p_gold.add_argument("--out", default="devdata/ss_gold_matrix")

    p_select = sub.add_parser("select")
    p_select.add_argument("--matrix", default="eval_matrix")
    p_select.add_argument("--k", type=int, default=10)

    p_sim = sub.add_parser("simulate")
    p_sim.add_argument("--matrix", default="eval_matrix")
    p_sim.add_argument("--k", type=int, default=10)

    p_sxgb = sub.add_parser("simulate-xgb")
    p_sxgb.add_argument("--matrix", default="eval_matrix")
    p_sxgb.add_argument("--k", type=int, default=10)
    p_sxgb.add_argument("--epsilons", type=float, nargs="+", default=[0.0, 0.05, 0.1, 0.2])
    p_sxgb.add_argument("--refit-every", type=int, default=200)
    p_sxgb.add_argument("--min-rows", type=int, default=300)

    p_hold = sub.add_parser("holdout")
    p_hold.add_argument("--matrix", default="eval_matrix")
    p_hold.add_argument("--k", type=int, default=10)
    p_hold.add_argument("--test-frac", type=float, default=0.2)
    p_hold.add_argument(
        "--queries-by-source", default=None, help="JSON {source: [queries]} home map (ss_source_queries.json)"
    )
    p_hold.add_argument("--seed", type=int, default=0)

    args = parser.parse_args()
    _bootstrap_django()

    if args.command == "build-matrix":
        queries = [ln.strip() for ln in Path(args.queries).read_text().splitlines() if ln.strip()]
        build_matrix(queries, args.out, reward=args.reward, checkpoint=args.checkpoint)
    elif args.command == "build-gold-matrix":
        build_gold_matrix(args.out)
    elif args.command == "select":
        cmd_select(args.matrix, args.k)
    elif args.command == "simulate":
        cmd_simulate(args.matrix, args.k)
    elif args.command == "simulate-xgb":
        cmd_simulate_xgb(args.matrix, args.k, args.epsilons, args.refit_every, args.min_rows)
    elif args.command == "holdout":
        cmd_holdout(args.matrix, args.k, args.test_frac, args.queries_by_source, args.seed)


if __name__ == "__main__":
    main()