Skip to content

armgit / mwmbl

Public Imported from GitHub
0

An open source, non-profit web search engine

7.2 KB
Raw Blame
"""
LLM relabel — Pass 2 COLLECTION: build the candidate pool per query.

For each Pass-1 query, pool results from three sources, dedup by URL, and record
where each URL came from (provenance) so Pass-3 judging can stay blind:

- standard  : the production heuristic+LTR ranker (mwmbl.search_setup.ranker).
- supersearch: the real Super Search pipeline forced to query exactly the
               Pass-1-selected sources (fan-out + crawl + link-following). The
               url->source map (incl. inherited source for followed links) comes
               from the SelectionContext.
- google    : the scraped Google gold (rankings-train.csv), kept for blind
               calibration — NOT used as a label.

Output: append-only JSONL keyed by query (resumable), one record per query with
its pooled candidates. Slow + network-bound; run in the background and resume by
re-running (already-collected queries are skipped).

Usage::
    DJANGO_SETTINGS_MODULE=mwmbl.settings_dev DATABASE_URL="postgres://daoud@" \
        uv run python scripts/llm_relabel_pass2_collect.py --limit 5     # smoke
    ... uv run python scripts/llm_relabel_pass2_collect.py               # full run
    ... uv run python scripts/llm_relabel_pass2_collect.py --status
"""

import asyncio
import json
import os
from argparse import ArgumentParser

import django
import pandas as pd

os.environ.setdefault("DJANGO_SETTINGS_MODULE", "mwmbl.settings_dev")
django.setup()

from django.conf import settings  # noqa: E402

from mwmbl.rankeval.evaluation.evaluate_ranker import DummyCompleter  # noqa: E402
from mwmbl.rankeval.evaluation.remote_index import RemoteIndex  # noqa: E402
from mwmbl.rankeval.paths import RANKINGS_DATASET_TRAIN_PATH  # noqa: E402
from mwmbl.tinysearchengine.indexer import Document  # noqa: E402
from mwmbl.tinysearchengine.rank import HeuristicAndWikiRanker  # noqa: E402
from mwmbl.tinysearchengine.super_search import _run_pipeline  # noqa: E402
from mwmbl.tinysearchengine.super_search_select.registry import get_meta  # noqa: E402
from mwmbl.tinysearchengine.super_search_select.rewards import SelectionContext  # noqa: E402
from mwmbl.tinysearchengine.super_search_sources import SOURCES  # noqa: E402
from scripts._relabel_pool import add  # noqa: E402

PASS1 = "devdata/llm_relabel/pass1_intents.jsonl"
CHECKPOINT = "devdata/llm_relabel/pass2_pool.jsonl"

ALWAYS_ON = {name for name in SOURCES if get_meta(name).always_on}

# Standard/mwmbl pool: the same heuristic ranker over the production RemoteIndex
# that dataset.py uses to build the existing LTR data, so the relabel is a
# drop-in replacement (score_threshold=-inf keeps all candidates for pooling).
_std_ranker = HeuristicAndWikiRanker(
    RemoteIndex(),
    DummyCompleter(),
    return_none_if_no_mwmbl_results=True,
    score_threshold=float("-inf"),
)


async def _noop_emit(event_type, data):
    pass


async def _collect_ss(query: str, sources: list[str]) -> tuple[list[Document], dict]:
    """Run the SS pipeline forced to exactly ``sources`` (+ always-on); return
    the full document pool and the url->source provenance map."""
    settings.SUPER_SEARCH_FORCE_INCLUDE = list(sources)
    settings.SUPER_SEARCH_SOURCES_TO_QUERY = len(ALWAYS_ON | set(sources))
    all_docs: list[Document] = []
    ctx = SelectionContext()
    try:
        await asyncio.wait_for(
            _run_pipeline(query, _noop_emit, all_docs, [None], asyncio.Lock(), ctx),
            timeout=settings.SUPER_SEARCH_DEADLINE_SECONDS + 2,
        )
    except asyncio.TimeoutError:
        pass
    return all_docs, dict(ctx.source_by_url)


def collect_query(query: str, sources: list[str], gold: pd.DataFrame, std_top_k: int) -> dict:
    pool: dict = {}
    gold_urls = set(gold["url"].tolist())

    # standard (production index, mirrors dataset.py: trailing space + " ").
    # The heuristic ranker returns the whole recall set; TREC-style we pool only
    # the top-K by score (what could realistically rank), but always keep any
    # gold URL as a calibration anchor.
    std = sorted(
        (_std_ranker.search(query + " ", []) or []),
        key=lambda d: d.score if d.score is not None else float("-inf"),
        reverse=True,
    )
    kept = std[:std_top_k] + [d for d in std[std_top_k:] if d.url in gold_urls]
    for d in kept:
        add(pool, d.url, d.title or "", d.extract or "", d.state, d.score, pool_tag="standard")

    # super search (forced sources) + provenance. Drop results from always-on
    # sources (e.g. hn) that Pass-1 did NOT choose: the pipeline pins them
    # regardless, but the pool should reflect only the chosen sources, so they
    # don't flood non-tech queries with off-topic noise.
    stray = ALWAYS_ON - set(sources)
    ss_docs, source_by_url = asyncio.run(_collect_ss(query, sources))
    for d in ss_docs:
        src = source_by_url.get(d.url)
        if src in stray:
            continue
        add(pool, d.url, d.title or "", d.extract or "", d.state, d.score, pool_tag="supersearch", ss_source=src)

    # google gold (snippet only)
    for _, row in gold.iterrows():
        add(pool, row["url"], "", str(row.get("snippet") or ""), None, None, pool_tag="google", gold_rank=row["rank"])

    return {"query": query, "sources": sources, "candidates": list(pool.values())}


def _load_pass1() -> dict:
    out = {}
    for line in open(PASS1):
        line = line.strip()
        if line:
            rec = json.loads(line)
            out[rec["query"]] = rec
    return out


def _done() -> set:
    if not os.path.exists(CHECKPOINT):
        return set()
    return {json.loads(line)["query"] for line in open(CHECKPOINT) if line.strip()}


def cmd_status():
    p1, done = _load_pass1(), _done()
    print(f"collected {len(done)} / {len(p1)} pass-1 queries ({CHECKPOINT})")
    if done:
        recs = [json.loads(line) for line in open(CHECKPOINT) if line.strip()]
        n_cand = sum(len(r["candidates"]) for r in recs)
        from collections import Counter

        pools = Counter(t for r in recs for c in r["candidates"] for t in c["pools"])
        print(f"  {n_cand} candidates total, {n_cand / max(len(recs), 1):.1f} avg/query")
        print(f"  pool membership: {dict(pools)}")


def run(limit: int | None, std_top_k: int):
    p1 = _load_pass1()
    done = _done()
    todo = [q for q in p1 if q not in done]
    if limit:
        todo = todo[:limit]
    gold_all = pd.read_csv(RANKINGS_DATASET_TRAIN_PATH, index_col=0)
    by_query = {q: g for q, g in gold_all.groupby("query")}
    print(f"collecting {len(todo)} queries (skipping {len(done)} done)")
    with open(CHECKPOINT, "a") as out:
        for i, query in enumerate(todo, 1):
            rec = collect_query(query, p1[query]["sources"], by_query.get(query, gold_all.iloc[0:0]), std_top_k)
            out.write(json.dumps(rec) + "\n")
            out.flush()
            print(f"[{i}/{len(todo)}] {query!r}: {len(rec['candidates'])} candidates", flush=True)


def main():
    parser = ArgumentParser()
    parser.add_argument("--limit", type=int, default=None)
    parser.add_argument(
        "--std-top-k", type=int, default=30, help="Pool only the top-K standard candidates by score (+ any gold URL)."
    )
    parser.add_argument("--status", action="store_true")
    args = parser.parse_args()
    if args.status:
        cmd_status()
    else:
        run(args.limit, args.std_top_k)


if __name__ == "__main__":
    main()