Skip to content

armgit / mwmbl

Public Imported from GitHub
0

An open source, non-profit web search engine

Super Search relabel: Pass-2 candidate-pool collection

Daoud Clarkebc5dae8e25a9
diff --git a/scripts/llm_relabel_pass2_collect.py b/scripts/llm_relabel_pass2_collect.py
new file mode 100644
index 0000000..11cee79
--- /dev/null
+++ b/scripts/llm_relabel_pass2_collect.py
@@ -0,0 +1,204 @@
+"""
+LLM relabel — Pass 2 COLLECTION: build the candidate pool per query.
+
+For each Pass-1 query, pool results from three sources, dedup by URL, and record
+where each URL came from (provenance) so Pass-3 judging can stay blind:
+
+- standard : the production heuristic+LTR ranker (mwmbl.search_setup.ranker).
+- supersearch: the real Super Search pipeline forced to query exactly the
+ Pass-1-selected sources (fan-out + crawl + link-following). The
+ url->source map (incl. inherited source for followed links) comes
+ from the SelectionContext.
+- google : the scraped Google gold (rankings-train.csv), kept for blind
+ calibration — NOT used as a label.
+
+Output: append-only JSONL keyed by query (resumable), one record per query with
+its pooled candidates. Slow + network-bound; run in the background and resume by
+re-running (already-collected queries are skipped).
+
+Usage::
+ DJANGO_SETTINGS_MODULE=mwmbl.settings_dev DATABASE_URL="postgres://daoud@" \
+ uv run python scripts/llm_relabel_pass2_collect.py --limit 5 # smoke
+ ... uv run python scripts/llm_relabel_pass2_collect.py # full run
+ ... uv run python scripts/llm_relabel_pass2_collect.py --status
+"""
+import asyncio
+import json
+import os
+from argparse import ArgumentParser
+
+import django
+import pandas as pd
+
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "mwmbl.settings_dev")
+django.setup()
+
+from django.conf import settings # noqa: E402
+
+from mwmbl.rankeval.evaluation.evaluate_ranker import DummyCompleter # noqa: E402
+from mwmbl.rankeval.evaluation.remote_index import RemoteIndex # noqa: E402
+from mwmbl.rankeval.paths import RANKINGS_DATASET_TRAIN_PATH # noqa: E402
+from mwmbl.tinysearchengine.indexer import Document # noqa: E402
+from mwmbl.tinysearchengine.rank import HeuristicAndWikiRanker # noqa: E402
+from mwmbl.tinysearchengine.super_search import _run_pipeline # noqa: E402
+from mwmbl.tinysearchengine.super_search_select.registry import get_meta # noqa: E402
+from mwmbl.tinysearchengine.super_search_select.rewards import SelectionContext # noqa: E402
+from mwmbl.tinysearchengine.super_search_sources import SOURCES # noqa: E402
+
+PASS1 = "devdata/llm_relabel/pass1_intents.jsonl"
+CHECKPOINT = "devdata/llm_relabel/pass2_pool.jsonl"
+
+ALWAYS_ON = {name for name in SOURCES if get_meta(name).always_on}
+
+# Standard/mwmbl pool: the same heuristic ranker over the production RemoteIndex
+# that dataset.py uses to build the existing LTR data, so the relabel is a
+# drop-in replacement (score_threshold=-inf keeps all candidates for pooling).
+_std_ranker = HeuristicAndWikiRanker(
+ RemoteIndex(), DummyCompleter(),
+ return_none_if_no_mwmbl_results=True, score_threshold=float("-inf"),
+ max_wiki_results=5,
+)
+
+
+async def _noop_emit(event_type, data):
+ pass
+
+
+async def _collect_ss(query: str, sources: list[str]) -> tuple[list[Document], dict]:
+ """Run the SS pipeline forced to exactly ``sources`` (+ always-on); return
+ the full document pool and the url->source provenance map."""
+ settings.SUPER_SEARCH_FORCE_INCLUDE = list(sources)
+ settings.SUPER_SEARCH_SOURCES_TO_QUERY = len(ALWAYS_ON | set(sources))
+ all_docs: list[Document] = []
+ ctx = SelectionContext()
+ try:
+ await asyncio.wait_for(
+ _run_pipeline(query, _noop_emit, all_docs, [None], asyncio.Lock(), ctx),
+ timeout=settings.SUPER_SEARCH_DEADLINE_SECONDS + 2,
+ )
+ except asyncio.TimeoutError:
+ pass
+ return all_docs, dict(ctx.source_by_url)
+
+
+def _add(pool: dict, url: str, title: str, extract: str, state, score, *,
+ pool_tag: str, ss_source: str | None = None, gold_rank: int | None = None):
+ """Merge one result into the per-url pool, keeping the richest title/extract."""
+ if not url:
+ return
+ item = pool.setdefault(url, {
+ "url": url, "title": "", "extract": "", "state": None, "score": None,
+ "pools": [], "ss_source": None, "gold_rank": None,
+ })
+ if title and len(title) > len(item["title"]):
+ item["title"] = title
+ if extract and len(extract) > len(item["extract"]):
+ item["extract"] = extract
+ if score is not None and item["score"] is None:
+ item["score"] = float(score)
+ if state is not None and item["state"] is None:
+ item["state"] = state
+ if pool_tag not in item["pools"]:
+ item["pools"].append(pool_tag)
+ if ss_source and not item["ss_source"]:
+ item["ss_source"] = ss_source
+ if gold_rank is not None and item["gold_rank"] is None:
+ item["gold_rank"] = int(gold_rank)
+
+
+def collect_query(query: str, sources: list[str], gold: pd.DataFrame, std_top_k: int) -> dict:
+ pool: dict = {}
+ gold_urls = set(gold["url"].tolist())
+
+ # standard (production index, mirrors dataset.py: trailing space + " ").
+ # The heuristic ranker returns the whole recall set; TREC-style we pool only
+ # the top-K by score (what could realistically rank), but always keep any
+ # gold URL as a calibration anchor.
+ std = sorted((_std_ranker.search(query + " ", []) or []),
+ key=lambda d: d.score if d.score is not None else float("-inf"), reverse=True)
+ kept = std[:std_top_k] + [d for d in std[std_top_k:] if d.url in gold_urls]
+ for d in kept:
+ _add(pool, d.url, d.title or "", d.extract or "", d.state, d.score, pool_tag="standard")
+
+ # super search (forced sources) + provenance. Drop results from always-on
+ # sources (e.g. hn) that Pass-1 did NOT choose: the pipeline pins them
+ # regardless, but the pool should reflect only the chosen sources, so they
+ # don't flood non-tech queries with off-topic noise.
+ stray = ALWAYS_ON - set(sources)
+ ss_docs, source_by_url = asyncio.run(_collect_ss(query, sources))
+ for d in ss_docs:
+ src = source_by_url.get(d.url)
+ if src in stray:
+ continue
+ _add(pool, d.url, d.title or "", d.extract or "", d.state, d.score,
+ pool_tag="supersearch", ss_source=src)
+
+ # google gold (snippet only)
+ for _, row in gold.iterrows():
+ _add(pool, row["url"], "", str(row.get("snippet") or ""), None, None,
+ pool_tag="google", gold_rank=row["rank"])
+
+ return {"query": query, "sources": sources, "candidates": list(pool.values())}
+
+
+def _load_pass1() -> dict:
+ out = {}
+ for line in open(PASS1):
+ line = line.strip()
+ if line:
+ rec = json.loads(line)
+ out[rec["query"]] = rec
+ return out
+
+
+def _done() -> set:
+ if not os.path.exists(CHECKPOINT):
+ return set()
+ return {json.loads(l)["query"] for l in open(CHECKPOINT) if l.strip()}
+
+
+def cmd_status():
+ p1, done = _load_pass1(), _done()
+ print(f"collected {len(done)} / {len(p1)} pass-1 queries ({CHECKPOINT})")
+ if done:
+ recs = [json.loads(l) for l in open(CHECKPOINT) if l.strip()]
+ n_cand = sum(len(r["candidates"]) for r in recs)
+ from collections import Counter
+ pools = Counter(t for r in recs for c in r["candidates"] for t in c["pools"])
+ print(f" {n_cand} candidates total, {n_cand / max(len(recs),1):.1f} avg/query")
+ print(f" pool membership: {dict(pools)}")
+
+
+def run(limit: int | None, std_top_k: int):
+ p1 = _load_pass1()
+ done = _done()
+ todo = [q for q in p1 if q not in done]
+ if limit:
+ todo = todo[:limit]
+ gold_all = pd.read_csv(RANKINGS_DATASET_TRAIN_PATH, index_col=0)
+ by_query = {q: g for q, g in gold_all.groupby("query")}
+ print(f"collecting {len(todo)} queries (skipping {len(done)} done)")
+ with open(CHECKPOINT, "a") as out:
+ for i, query in enumerate(todo, 1):
+ rec = collect_query(query, p1[query]["sources"],
+ by_query.get(query, gold_all.iloc[0:0]), std_top_k)
+ out.write(json.dumps(rec) + "\n")
+ out.flush()
+ print(f"[{i}/{len(todo)}] {query!r}: {len(rec['candidates'])} candidates", flush=True)
+
+
+def main():
+ parser = ArgumentParser()
+ parser.add_argument("--limit", type=int, default=None)
+ parser.add_argument("--std-top-k", type=int, default=30,
+ help="Pool only the top-K standard candidates by score (+ any gold URL).")
+ parser.add_argument("--status", action="store_true")
+ args = parser.parse_args()
+ if args.status:
+ cmd_status()
+ else:
+ run(args.limit, args.std_top_k)
+
+
+if __name__ == "__main__":
+ main()