Skip to content

armgit / mwmbl

Public Imported from GitHub
0

An open source, non-profit web search engine

4.9 KB
Raw Blame
"""MiniLM re-ranks of the combined LTR top N, with and without Wikipedia, graded by UK Haiku."""

import glob
import json
import os
import random
import sys

import numpy as np

S = os.environ.get("HAIKU_WORK_DIR", "devdata/combined_providers_eval/haiku_work")
P = "devdata/combined_providers_eval/engb/"
N = 10
rows = json.load(open(P + "rows-0.05-with-brave.json"))
deep = json.load(open(P + "combined_top30.json"))
wiki = json.load(open(P + "wiki_pool.json"))
text = json.load(open(P + "pool_text.json"))


def load_grades():
    G = {}
    for d in ("haiku_uk", "haiku_uk_deep"):
        try:
            man = {int(k): v for k, v in json.load(open(f"{S}/{d}/manifest.json")).items()}
        except FileNotFoundError:
            continue
        qmap = (
            {int(k): v for k, v in json.load(open(f"{S}/{d}/queries.json")).items()}
            if d != "haiku_uk"
            else {i: r["query"] for i, r in enumerate(rows)}
        )
        for f in glob.glob(f"{S}/{d}/out_*.jsonl"):
            for line in open(f):
                line = line.strip()
                if not line.startswith("{"):
                    continue
                o = json.loads(line)
                urls = man[o["qid"]]
                g = o["grades"]
                if isinstance(g, dict) and sorted(map(int, g)) == list(range(len(urls))):
                    G.setdefault(qmap[o["qid"]], {}).update({urls[int(k)]: v for k, v in g.items()})
    return G


def arms(r):
    q = r["query"]
    s = r["lists"]["staan"][:N]
    c = r["lists"]["combined"][:N]
    D = deep[q]
    W = wiki[q]
    score = {d["url"]: d["minilm"] for d in D + W}

    def mini(cands):
        return sorted(dict.fromkeys(cands), key=lambda u: -score.get(u, 0))[:N]

    a = {"combined (shipped)": c, "staan": s, "staan-first + fill": (s + [u for u in c if u not in s])[:N]}
    for k in (10, 20, 30):
        top = [d["url"] for d in D[:k]]
        a[f"minilm top{k}"] = mini(top)
        a[f"minilm top{k} + wiki"] = mini(top + [w["url"] for w in W])
    rest = mini([d["url"] for d in D] + [w["url"] for w in W])
    a["staan-first, fill minilm(top30+wiki)"] = (s + [u for u in rest if u not in s])[:N]
    a["brave"] = r["lists"]["brave"][:N]
    return a


def doc(q, u):
    for d in deep[q] + wiki[q]:
        if d["url"] == u:
            return d["title"], d["extract"]
    t = text[q].get(u)
    return (t[0], t[1]) if t else None


if sys.argv[1] == "dump":
    G = load_grades()
    todo = {}
    for r in rows:
        q = r["query"]
        g = G.get(q, {})
        need = {u for urls in arms(r).values() for u in urls} - set(g)
        if need:
            todo[q] = sorted(need)
    print("queries needing grades", len(todo), "urls", sum(map(len, todo.values())))
    exec(
        open(f"{S}/make_batches_uk.py").read().split("PROMPT = ")[1].split("manifest = {}")[0].join(["PROMPT = ", ""])
        if False
        else ""
    )
    prompt = open(f"{S}/haiku_uk/batch_00.txt").read().split("=== QUERIES ===")[0]
    import os

    os.makedirs(f"{S}/haiku_uk_deep", exist_ok=True)
    man = {}
    items = list(todo.items())
    NB = int(sys.argv[2])
    for qid, (q, urls) in enumerate(items):
        urls = urls[:]
        random.Random(qid).shuffle(urls)
        man[qid] = (q, urls)
    json.dump({k: v[1] for k, v in man.items()}, open(f"{S}/haiku_uk_deep/manifest.json", "w"))
    json.dump({k: v[0] for k, v in man.items()}, open(f"{S}/haiku_uk_deep/queries.json", "w"))
    for b in range(NB):
        parts = [prompt, "=== QUERIES ===\n"]
        for qid in list(man)[b::NB]:
            q, urls = man[qid]
            parts.append(f"\n## qid {qid} | query: {q}\n")
            for i, u in enumerate(urls):
                t, e = doc(q, u)
                parts.append(f"[{i}] {t[:150]}\n    {u[:200]}\n    {(e or '').replace(chr(10), ' ')[:300]}\n")
        open(f"{S}/haiku_uk_deep/batch_{b:02d}.txt", "w").write("".join(parts))

if sys.argv[1] == "score":
    G = load_grades()

    def dcg(g):
        return sum((2**x - 1) / np.log2(i + 2) for i, x in enumerate(g))

    res = {}
    used = 0
    for r in rows:
        q = r["query"]
        g = G.get(q, {})
        A = arms(r)
        if any(u not in g for urls in A.values() for u in urls):
            continue
        used += 1
        ideal = dcg(sorted(g.values(), reverse=True)[:N])
        for k, urls in A.items():
            res.setdefault(k, []).append(dcg([g[u] for u in urls]) / ideal if ideal else 0)
    rng = np.random.default_rng(0)
    base = np.array(res["staan-first + fill"])
    br = np.array(res["brave"])

    def ci(d):
        bs = [d[rng.integers(0, len(d), len(d))].mean() for _ in range(4000)]
        return f"{d.mean():+.3f} [{np.percentile(bs, 2.5):+.3f},{np.percentile(bs, 97.5):+.3f}]"

    print(f"fully graded queries: {used}/{len(rows)}")
    print(f"{'arm':38s} haiku   vs staan-first             vs brave")
    for k, v in res.items():
        v = np.array(v)
        print(f"{k:38s} {v.mean():.3f}   {ci(v - base)}   {ci(v - br)}")