An open source, non-profit web search engine
Pool Staan into the relabel dataset and evaluate Combined Search
Daoud Clarke5b3ffb7f4d17
diff --git a/mwmbl/rankeval/evaluation/compare_search_modes.py b/mwmbl/rankeval/evaluation/compare_search_modes.pyindex a2ed3bb..71df606 100644--- a/mwmbl/rankeval/evaluation/compare_search_modes.py+++ b/mwmbl/rankeval/evaluation/compare_search_modes.py@@ -1,7 +1,7 @@"""-Compare standard search vs Super Search v2 on the gold test set.+Compare standard search, Super Search v2 and Combined Search on the gold test set.-Both are scored against the same gold test split via ``rankeval.evaluation.evaluate``+Every arm is scored against the same gold test split via ``rankeval.evaluation.evaluate``on the same sampled queries (the evaluate module RNG is reseeded per model).Fairness note — the standard-search baseline uses the *same* local mwmbl index,@@ -12,6 +12,10 @@ PyPI, Stack Exchange, recipes), page crawling and outbound-link following. Aremote-index baseline (``evaluate_remote.py``) would be confounded by index size,because Super Search's mwmbl source queries the local index, not the production one.+Combined Search is the arm that is meant to replace Super Search: the same local index and+LTR machinery, plus Staan and Wikipedia, with no crawling and no link-following. Its ablation+arm pools the index and Wikipedia only, so the gap between the two is Staan's contribution.+Usage::DJANGO_SETTINGS_MODULE=mwmbl.settings_dev \@@ -32,6 +36,7 @@ from django.conf import settings # noqa: E402import mwmbl.rankeval.evaluation.evaluate as evaluate_module # noqa: E402import mwmbl.rankeval.evaluation.evaluate_super_search as ss_module # noqa: E402from mwmbl.rankeval.evaluation.evaluate import evaluate # noqa: E402+from mwmbl.rankeval.evaluation.evaluate_combined_search import CombinedSearchRankingModel # noqa: E402from mwmbl.rankeval.evaluation.evaluate_ranker import MwmblRankingModel # noqa: E402from mwmbl.rankeval.evaluation.evaluate_super_search import SuperSearchRankingModel # noqa: E402from mwmbl.search_setup import ranker # noqa: E402 (local mwmbl index + LTR + MMR + wiki)@@ -63,6 +68,8 @@ def run():("standard search (local index + wiki)", lambda: MwmblRankingModel(ranker)),("super search v2 (cosine baseline, no new sources)", lambda: ss_arm([])),("super search + new sources (gov.uk, imdb)", lambda: ss_arm(NEW_SOURCES)),+ ("combined search (index + wiki, no staan)", lambda: CombinedSearchRankingModel(include_staan=False)),+ ("combined search (index + staan + wiki)", lambda: CombinedSearchRankingModel()),]for label, make_model in models:print(f"\n{'=' * 70}\nEvaluating {label}\n{'=' * 70}")diff --git a/mwmbl/rankeval/evaluation/evaluate_combined_search.py b/mwmbl/rankeval/evaluation/evaluate_combined_search.pynew file mode 100644index 0000000..71b3bc5--- /dev/null+++ b/mwmbl/rankeval/evaluation/evaluate_combined_search.py@@ -0,0 +1,68 @@+"""Evaluate the Combined Search ranking core as a RankingModel against the gold dataset.++Runs what /api/v2/combined-search/ runs, minus the HTTP layer: pool the Mwmbl index, Staan+and Wikipedia, rank the union with the combined LTR model, diversify. Only auth, quota and+response formatting are skipped, so the ranking measured here is the ranking served.++Not to be confused with evaluate_combined.py, which combines two ranking *models* (Wikipedia+and Mwmbl) and predates this endpoint.++Staan and Wikipedia are both network calls, so both are cached: Wikipedia through its own+external results cache, and Staan through the same. A gold-set run therefore pays for each+provider once per query and is cheap to repeat while iterating on the ranking. Pass+``--no-staan`` to measure what Staan is actually contributing.++Usage::++ DJANGO_SETTINGS_MODULE=mwmbl.settings_dev \+ uv run python -m mwmbl.rankeval.evaluation.evaluate_combined_search --fraction 0.05+"""++import os+from argparse import ArgumentParser++import django++os.environ.setdefault("DJANGO_SETTINGS_MODULE", "mwmbl.settings_dev")+django.setup()++# Imported after django.setup(): search_setup opens the index and loads the models.+from mwmbl.rankeval.evaluation.evaluate import RankingModel, evaluate # noqa: E402+from mwmbl.search_setup import combined_ranker # noqa: E402+from mwmbl.tinysearchengine.rank import get_wiki_results # noqa: E402+from mwmbl.tinysearchengine.staan import get_staan_results # noqa: E402+++class CombinedSearchRankingModel(RankingModel):+ """The endpoint's ranking core: three sources pooled, one model, MMR.++ include_staan=False is the ablation arm - the same pool minus Staan, so the difference+ between the two arms is what Staan contributes and nothing else.+ """++ def __init__(self, ranker=combined_ranker, include_staan: bool = True):+ self.ranker = ranker+ self.include_staan = include_staan++ def predict(self, query: str) -> list[str]:+ additional = get_staan_results(query) if self.include_staan else []+ additional = additional + get_wiki_results(query)+ # use_external_search=False: Wikipedia is already in `additional`, exactly as the+ # endpoint arranges it.+ results = self.ranker.search(query, additional, False)+ return [result.url for result in results]+++def run():+ parser = ArgumentParser(description=__doc__)+ parser.add_argument("--fraction", type=float, default=0.05, help="Fraction of gold queries to sample.")+ parser.add_argument("--train", action="store_true", help="Evaluate on the train split instead of test.")+ parser.add_argument("--no-staan", action="store_true", help="Ablation: pool the index and Wikipedia only.")+ args = parser.parse_args()++ model = CombinedSearchRankingModel(include_staan=not args.no_staan)+ evaluate(model, fraction=args.fraction, use_test=not args.train)+++if __name__ == "__main__":+ run()diff --git a/scripts/_relabel_pool.py b/scripts/_relabel_pool.pynew file mode 100644index 0000000..917d6d3--- /dev/null+++ b/scripts/_relabel_pool.py@@ -0,0 +1,58 @@+"""The Pass-2 candidate pool merge rule, shared by the collectors.++Lives on its own so the rule cannot drift between the original three-pool collector and any+later pass that adds a source to an already-collected pool: two copies of "keep the richest+title, first score wins, accumulate provenance" would eventually stop agreeing, and the+disagreement would show up as a feature shift in the training data rather than as an error.++Kept free of django.setup() and of any ranker, so importing it costs nothing.+"""+++def add(+ pool: dict,+ url: str,+ title: str,+ extract: str,+ state,+ score,+ *,+ pool_tag: str,+ ss_source: str | None = None,+ gold_rank: int | None = None,+):+ """Merge one result into the per-url pool, keeping the richest title/extract.++ First writer wins for `score`, `state`, `ss_source` and `gold_rank`; `pools` accumulates.+ That the first score wins matters: a URL two sources both returned keeps the score of+ whichever was pooled first, which is how every row in the existing dataset was built.+ """+ if not url:+ return+ item = pool.setdefault(+ url,+ {+ "url": url,+ "title": "",+ "extract": "",+ "state": None,+ "score": None,+ "pools": [],+ "ss_source": None,+ "gold_rank": None,+ },+ )+ if title and len(title) > len(item["title"]):+ item["title"] = title+ if extract and len(extract) > len(item["extract"]):+ item["extract"] = extract+ if score is not None and item["score"] is None:+ item["score"] = float(score)+ if state is not None and item["state"] is None:+ item["state"] = state+ if pool_tag not in item["pools"]:+ item["pools"].append(pool_tag)+ if ss_source and not item["ss_source"]:+ item["ss_source"] = ss_source+ if gold_rank is not None and item["gold_rank"] is None:+ item["gold_rank"] = int(gold_rank)diff --git a/scripts/llm_relabel_pass2_augment_staan.py b/scripts/llm_relabel_pass2_augment_staan.pynew file mode 100644index 0000000..aa6795f--- /dev/null+++ b/scripts/llm_relabel_pass2_augment_staan.py@@ -0,0 +1,160 @@+"""LLM relabel — Pass 2 AUGMENT: add Staan's results to an already-collected pool.++Combined Search pools the Mwmbl index, Staan and Wikipedia, so the model that ranks that+union has to be trained on it. The existing pool already covers the index and Wikipedia (both+arrive in the `standard` pool, via HeuristicAndWikiRanker); what it has never seen is Staan.++Re-running llm_relabel_pass2_collect.py would redo the whole Super Search fan-out — crawls,+link-following and all — for 849 queries to add one source. This adds only the Staan call,+and merges it into the existing pool through the same rule the other pools were built with+(scripts/_relabel_pool.add), so a Staan row is comparable with an existing one.++Two steps, so a long network-bound run can never leave the pool half-written:++1. ``--collect`` (default) appends one record per query to a checkpoint of its own. Resumable:+ re-running skips queries already collected.+2. ``--merge`` folds the checkpoint into pass2_pool.jsonl, writing via a temporary file and+ renaming, so the pool is either the old one or the new one and never a partial one.++The score written for a Staan result is staan_score(rank) — the same constant the serving+path uses. `score` becomes the item_score feature, so a scale here that production never+produces is a model trained on a feature it will not see. See STAAN_TOP_SCORE.++Usage::+ DJANGO_SETTINGS_MODULE=mwmbl.settings_dev DATABASE_URL="postgres://daoud@" \+ uv run python scripts/llm_relabel_pass2_augment_staan.py --limit 5 # smoke+ ... uv run python scripts/llm_relabel_pass2_augment_staan.py # full run+ ... uv run python scripts/llm_relabel_pass2_augment_staan.py --status+ ... uv run python scripts/llm_relabel_pass2_augment_staan.py --merge+"""++import json+import os+from argparse import ArgumentParser+from collections import Counter++import django++os.environ.setdefault("DJANGO_SETTINGS_MODULE", "mwmbl.settings_dev")+django.setup()++from mwmbl.tinysearchengine.staan import get_staan_results # noqa: E402+from scripts._relabel_pool import add # noqa: E402++POOL = "devdata/llm_relabel/pass2_pool.jsonl"+CHECKPOINT = "devdata/llm_relabel/pass2_staan.jsonl"+POOL_TAG = "staan"+++def _read_jsonl(path: str) -> list[dict]:+ if not os.path.exists(path):+ return []+ return [json.loads(line) for line in open(path) if line.strip()]+++def _pool_queries() -> list[str]:+ return [record["query"] for record in _read_jsonl(POOL)]+++def _collected() -> set[str]:+ return {record["query"] for record in _read_jsonl(CHECKPOINT)}+++def collect_query(query: str) -> dict:+ """Staan's results for one query, in the shape the merge step expects."""+ results = get_staan_results(query)+ return {+ "query": query,+ "results": [+ {"url": document.url, "title": document.title, "extract": document.extract, "score": document.score}+ for document in results+ ],+ }+++def cmd_collect(limit: int | None):+ todo = [query for query in _pool_queries() if query not in _collected()]+ if limit:+ todo = todo[:limit]+ print(f"Collecting Staan results for {len(todo)} queries")++ with open(CHECKPOINT, "a") as checkpoint:+ for i, query in enumerate(todo, 1):+ record = collect_query(query)+ checkpoint.write(json.dumps(record) + "\n")+ checkpoint.flush()+ print(f"[{i}/{len(todo)}] {query!r}: {len(record['results'])} results")+++def cmd_merge():+ collected = {record["query"]: record["results"] for record in _read_jsonl(CHECKPOINT)}+ if not collected:+ print(f"Nothing to merge: {CHECKPOINT} is empty")+ return++ added = 0+ merged_into_existing = 0+ records = _read_jsonl(POOL)+ for record in records:+ results = collected.get(record["query"])+ if results is None:+ continue+ pool = {candidate["url"]: candidate for candidate in record["candidates"]}+ for result in results:+ existing = result["url"] in pool+ add(+ pool,+ result["url"],+ result["title"] or "",+ result["extract"] or "",+ None,+ result["score"],+ pool_tag=POOL_TAG,+ ss_source=POOL_TAG,+ )+ if existing:+ merged_into_existing += 1+ else:+ added += 1+ record["candidates"] = list(pool.values())++ temporary = POOL + ".tmp"+ with open(temporary, "w") as out:+ for record in records:+ out.write(json.dumps(record) + "\n")+ os.replace(temporary, POOL)++ print(f"Merged {len(collected)} queries into {POOL}")+ print(f" {added} new candidates, {merged_into_existing} already pooled by another source")+++def cmd_status():+ queries = _pool_queries()+ collected = _read_jsonl(CHECKPOINT)+ print(f"collected {len(collected)} / {len(queries)} pool queries ({CHECKPOINT})")+ if collected:+ counts = [len(record["results"]) for record in collected]+ print(f" {sum(counts)} Staan results, {sum(counts) / len(counts):.1f} avg/query")+ print(f" queries with no Staan results: {sum(1 for c in counts if c == 0)}")++ pools = Counter(tag for record in _read_jsonl(POOL) for c in record["candidates"] for tag in c["pools"])+ print(f" pool membership in {POOL}: {dict(pools)}")+++def run():+ parser = ArgumentParser(description=__doc__)+ parser.add_argument("--limit", type=int, default=None, help="Collect at most this many queries.")+ parser.add_argument("--merge", action="store_true", help="Fold the checkpoint into pass2_pool.jsonl.")+ parser.add_argument("--status", action="store_true", help="Report progress and pool membership.")+ args = parser.parse_args()++ if args.status:+ cmd_status()+ elif args.merge:+ cmd_merge()+ else:+ cmd_collect(args.limit)+++if __name__ == "__main__":+ run()diff --git a/scripts/llm_relabel_pass2_collect.py b/scripts/llm_relabel_pass2_collect.pyindex 83ca464..4265e9c 100644--- a/scripts/llm_relabel_pass2_collect.py+++ b/scripts/llm_relabel_pass2_collect.py@@ -45,6 +45,7 @@ from mwmbl.tinysearchengine.super_search import _run_pipeline # noqa: E402from mwmbl.tinysearchengine.super_search_select.registry import get_meta # noqa: E402from mwmbl.tinysearchengine.super_search_select.rewards import SelectionContext # noqa: E402from mwmbl.tinysearchengine.super_search_sources import SOURCES # noqa: E402+from scripts._relabel_pool import add # noqa: E402PASS1 = "devdata/llm_relabel/pass1_intents.jsonl"CHECKPOINT = "devdata/llm_relabel/pass2_pool.jsonl"@@ -83,50 +84,6 @@ async def _collect_ss(query: str, sources: list[str]) -> tuple[list[Document], dreturn all_docs, dict(ctx.source_by_url)-def _add(- pool: dict,- url: str,- title: str,- extract: str,- state,- score,- *,- pool_tag: str,- ss_source: str | None = None,- gold_rank: int | None = None,-):- """Merge one result into the per-url pool, keeping the richest title/extract."""- if not url:- return- item = pool.setdefault(- url,- {- "url": url,- "title": "",- "extract": "",- "state": None,- "score": None,- "pools": [],- "ss_source": None,- "gold_rank": None,- },- )- if title and len(title) > len(item["title"]):- item["title"] = title- if extract and len(extract) > len(item["extract"]):- item["extract"] = extract- if score is not None and item["score"] is None:- item["score"] = float(score)- if state is not None and item["state"] is None:- item["state"] = state- if pool_tag not in item["pools"]:- item["pools"].append(pool_tag)- if ss_source and not item["ss_source"]:- item["ss_source"] = ss_source- if gold_rank is not None and item["gold_rank"] is None:- item["gold_rank"] = int(gold_rank)--def collect_query(query: str, sources: list[str], gold: pd.DataFrame, std_top_k: int) -> dict:pool: dict = {}gold_urls = set(gold["url"].tolist())@@ -142,7 +99,7 @@ def collect_query(query: str, sources: list[str], gold: pd.DataFrame, std_top_k:)kept = std[:std_top_k] + [d for d in std[std_top_k:] if d.url in gold_urls]for d in kept:- _add(pool, d.url, d.title or "", d.extract or "", d.state, d.score, pool_tag="standard")+ add(pool, d.url, d.title or "", d.extract or "", d.state, d.score, pool_tag="standard")# super search (forced sources) + provenance. Drop results from always-on# sources (e.g. hn) that Pass-1 did NOT choose: the pipeline pins them@@ -154,11 +111,11 @@ def collect_query(query: str, sources: list[str], gold: pd.DataFrame, std_top_k:src = source_by_url.get(d.url)if src in stray:continue- _add(pool, d.url, d.title or "", d.extract or "", d.state, d.score, pool_tag="supersearch", ss_source=src)+ add(pool, d.url, d.title or "", d.extract or "", d.state, d.score, pool_tag="supersearch", ss_source=src)# google gold (snippet only)for _, row in gold.iterrows():- _add(pool, row["url"], "", str(row.get("snippet") or ""), None, None, pool_tag="google", gold_rank=row["rank"])+ add(pool, row["url"], "", str(row.get("snippet") or ""), None, None, pool_tag="google", gold_rank=row["rank"])return {"query": query, "sources": sources, "candidates": list(pool.values())}diff --git a/test/test_relabel_pool_augment.py b/test/test_relabel_pool_augment.pynew file mode 100644index 0000000..a894806--- /dev/null+++ b/test/test_relabel_pool_augment.py@@ -0,0 +1,228 @@+"""Tests for the Pass-2 pool merge rule and the Staan augmentation step.++The merge rule decides what a training row's features are for a URL more than one source+returned, so getting it wrong shifts features rather than raising - which is exactly the kind+of mistake that only shows up as a model that underperforms for no visible reason.+"""++import json+import sys+from pathlib import Path++import pytest++sys.path.insert(0, str(Path(__file__).resolve().parent.parent))++from scripts._relabel_pool import add # noqa: E402++# ---------------------------------------------------------------------------+# The merge rule+# ---------------------------------------------------------------------------+++def test_a_new_url_enters_the_pool():+ pool = {}++ add(pool, "https://tokio.rs/", "Tokio", "An async runtime.", None, 6.0, pool_tag="staan", ss_source="staan")++ assert pool["https://tokio.rs/"] == {+ "url": "https://tokio.rs/",+ "title": "Tokio",+ "extract": "An async runtime.",+ "state": None,+ "score": 6.0,+ "pools": ["staan"],+ "ss_source": "staan",+ "gold_rank": None,+ }+++def test_a_url_with_no_url_is_ignored():+ pool = {}++ add(pool, "", "Tokio", "", None, 6.0, pool_tag="staan")++ assert pool == {}+++def test_provenance_accumulates_across_sources():+ """`pools` is what the dataset's calibration readout groups by, so a URL two sources both+ returned has to say so rather than claim the first one only."""+ pool = {}++ add(pool, "https://tokio.rs/", "Tokio", "", None, 2.0, pool_tag="standard")+ add(pool, "https://tokio.rs/", "Tokio", "", None, 6.0, pool_tag="staan", ss_source="staan")++ assert pool["https://tokio.rs/"]["pools"] == ["standard", "staan"]+++def test_the_first_score_wins():+ """A URL two sources both returned keeps the score of whichever was pooled first. Every+ row in the existing dataset was built this way; changing it here would make the new Staan+ rows incomparable with the ones already judged."""+ pool = {}++ add(pool, "https://tokio.rs/", "Tokio", "", None, 2.0, pool_tag="standard")+ add(pool, "https://tokio.rs/", "Tokio", "", None, 6.0, pool_tag="staan")++ assert pool["https://tokio.rs/"]["score"] == 2.0+++def test_the_richest_title_and_extract_win():+ pool = {}++ add(pool, "https://tokio.rs/", "Tokio", "Short.", None, 1.0, pool_tag="standard")+ add(pool, "https://tokio.rs/", "Tokio - async Rust", "A much longer description.", None, 1.0, pool_tag="staan")++ assert pool["https://tokio.rs/"]["title"] == "Tokio - async Rust"+ assert pool["https://tokio.rs/"]["extract"] == "A much longer description."+++def test_a_repeated_pool_tag_is_not_duplicated():+ pool = {}++ add(pool, "https://tokio.rs/", "Tokio", "", None, 1.0, pool_tag="staan")+ add(pool, "https://tokio.rs/", "Tokio", "", None, 1.0, pool_tag="staan")++ assert pool["https://tokio.rs/"]["pools"] == ["staan"]+++# ---------------------------------------------------------------------------+# The augmentation step+# ---------------------------------------------------------------------------+++def relabel_files(tmp_path, monkeypatch):+ """Point the augment script at a throwaway pool and checkpoint."""+ augment = pytest.importorskip("scripts.llm_relabel_pass2_augment_staan")++ pool = tmp_path / "pass2_pool.jsonl"+ checkpoint = tmp_path / "pass2_staan.jsonl"+ monkeypatch.setattr(augment, "POOL", str(pool))+ monkeypatch.setattr(augment, "CHECKPOINT", str(checkpoint))+ return augment, pool, checkpoint+++def _write(path, records):+ path.write_text("".join(json.dumps(record) + "\n" for record in records))+++def test_merge_adds_staan_candidates_to_the_pool(relabel_files):+ augment, pool, checkpoint = relabel_files+ _write(+ pool,+ [+ {+ "query": "tokio",+ "sources": ["mwmbl"],+ "candidates": [+ {+ "url": "https://blog.example.com/tokio",+ "title": "A crawled page",+ "extract": "",+ "state": None,+ "score": 1.0,+ "pools": ["standard"],+ "ss_source": None,+ "gold_rank": None,+ }+ ],+ }+ ],+ )+ _write(+ checkpoint,+ [{"query": "tokio", "results": [{"url": "https://tokio.rs/", "title": "Tokio", "extract": "", "score": 6.0}]}],+ )++ augment.cmd_merge()++ candidates = json.loads(pool.read_text().strip())["candidates"]+ assert [c["url"] for c in candidates] == ["https://blog.example.com/tokio", "https://tokio.rs/"]+ assert candidates[1]["pools"] == ["staan"]+ assert candidates[1]["ss_source"] == "staan"+ assert candidates[1]["score"] == 6.0+++def test_merge_folds_a_url_another_source_already_returned(relabel_files):+ augment, pool, checkpoint = relabel_files+ _write(+ pool,+ [+ {+ "query": "tokio",+ "sources": ["mwmbl"],+ "candidates": [+ {+ "url": "https://tokio.rs/",+ "title": "Tokio",+ "extract": "",+ "state": None,+ "score": 2.0,+ "pools": ["standard"],+ "ss_source": None,+ "gold_rank": None,+ }+ ],+ }+ ],+ )+ _write(+ checkpoint,+ [{"query": "tokio", "results": [{"url": "https://tokio.rs/", "title": "Tokio", "extract": "", "score": 6.0}]}],+ )++ augment.cmd_merge()++ candidates = json.loads(pool.read_text().strip())["candidates"]+ assert len(candidates) == 1+ assert candidates[0]["pools"] == ["standard", "staan"]+ assert candidates[0]["score"] == 2.0+++def test_merge_leaves_queries_with_no_staan_record_alone(relabel_files):+ augment, pool, checkpoint = relabel_files+ original = {+ "query": "untouched",+ "sources": ["mwmbl"],+ "candidates": [+ {+ "url": "https://example.com/",+ "title": "T",+ "extract": "",+ "state": None,+ "score": 1.0,+ "pools": ["standard"],+ "ss_source": None,+ "gold_rank": None,+ }+ ],+ }+ _write(pool, [original])+ _write(checkpoint, [{"query": "something else", "results": []}])++ augment.cmd_merge()++ assert json.loads(pool.read_text().strip()) == original+++def test_merge_is_idempotent(relabel_files):+ """Re-running the merge must not double-count: the checkpoint stays on disk after a+ merge, and the obvious mistake is to run it twice."""+ augment, pool, checkpoint = relabel_files+ _write(+ pool,+ [{"query": "tokio", "sources": [], "candidates": []}],+ )+ _write(+ checkpoint,+ [{"query": "tokio", "results": [{"url": "https://tokio.rs/", "title": "Tokio", "extract": "", "score": 6.0}]}],+ )++ augment.cmd_merge()+ augment.cmd_merge()++ candidates = json.loads(pool.read_text().strip())["candidates"]+ assert [c["url"] for c in candidates] == ["https://tokio.rs/"]+ assert candidates[0]["pools"] == ["staan"]