Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

6.7 KB
Raw Blame
"""Many-option choice on three public intent sets: one question, a wider budget, the embedding
shortlist, and `predict_tournament`.

Every strategy answers one row per call, as a caller would. Needs `pyarrow` for the parquet files.

    python research/benchmarks/tournament/benchmark.py --dataset banking77 --split test \
        --strategies single,wide,shortlist,tournament --out banking77.json
"""
import argparse
import json
import platform
import random
import sys
import time
from pathlib import Path

from huggingface_hub import hf_hub_download

sys.path.insert(0, str(Path(__file__).resolve().parents[3]))

import laya  # noqa: E402
from laya.evals import ece  # noqa: E402

REVISIONS = {
    "mteb/banking77": "18072d2685ea682290f7b8924d94c62acc19c0b2",
    "clinc/clinc_oos": "155b9c710419136e17307b80d0a13e68cd46b4ec",
    "SetFit/amazon_massive_intent_en-US": "f7672a018e8ceb37fc0184dcfbb7e665155ffea6",
}


def fetch(repo, filename):
    return Path(hf_hub_download(repo, filename, repo_type="dataset", revision=REVISIONS[repo]))


def banking77(split):
    import pyarrow.parquet as pq

    def read(name):
        return pq.read_table(fetch("mteb/banking77", "data/%s-00000-of-00001.parquet" % name)).to_pylist()

    names = dict((r["label"], r["label_text"]) for r in read("test"))
    return [names[i] for i in sorted(names)], [(r["text"], r["label_text"]) for r in read(split)]


def clinc150(split):
    import pyarrow.parquet as pq

    path = fetch("clinc/clinc_oos", "plus/%s-00000-of-00001.parquet" % split)
    names = json.loads(pq.read_schema(path).metadata[b"huggingface"])["info"]["features"]["intent"]["names"]
    rows = [(r["text"], names[r["intent"]]) for r in pq.read_table(path).to_pylist()]
    # In-scope intents only: "oos" is not an intent a label can describe.
    return [n for n in names if n != "oos"], [(text, gold) for text, gold in rows if gold != "oos"]


def massive(split):
    def read(name):
        path = fetch("SetFit/amazon_massive_intent_en-US", "%s.jsonl" % name)
        return [json.loads(line) for line in path.read_text().splitlines()]

    names = dict((r["label"], r["label_text"]) for r in read("train"))
    return [names[i] for i in sorted(names)], [(r["text"], r["label_text"]) for r in read(split)]


DATASETS = {
    "banking77": (banking77, "Which banking intent does `message` express?"),
    "clinc150": (clinc150, "Which intent does `message` express?"),
    "massive": (massive, "Which intent does `message` express?"),
}


def strategy(name, agent):
    """`(state, question) -> (answer, model calls)` for one strategy name."""
    if name in ("single", "wide"):
        budget = {"head_max_len": 512, "max_len": 1024} if name == "wide" else {}
        return lambda state, q: (agent.predict(state, {"q": q}, **budget)["answers"]["q"], 1)
    if name == "shortlist":
        embed_fn = laya.cached_embed_fn(laya.embed_fn_from_agent(agent))
        return lambda state, q: (laya.predict_shortlist(agent, state, {"q": q}, embed_fn)["answers"]["q"], 1)
    if name.startswith("tournament"):
        size = {"group_size": int(name.split("-")[1])} if "-" in name else {}

        def run(state, q):
            out = laya.predict_tournament(agent, state, {"q": q}, **size)
            return out["answers"]["q"], out["tournament"]["q"]["rounds"] + 1
        return run
    raise SystemExit("unknown strategy %r" % name)


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--dataset", required=True, choices=sorted(DATASETS))
    ap.add_argument("--split", required=True)
    ap.add_argument("--limit", type=int, help="score a seeded sample of this many rows")
    ap.add_argument("--seed", type=int, default=0)
    ap.add_argument("--strategies", default="single,wide,shortlist,tournament",
                    help="comma list of single, wide, shortlist, tournament, tournament-<group size>")
    ap.add_argument("--model", default="english")
    ap.add_argument("--device")
    ap.add_argument("--out", required=True)
    args = ap.parse_args()

    load, instructions = DATASETS[args.dataset]
    names, rows = load(args.split)
    if args.limit and args.limit < len(rows):
        rows = random.Random(args.seed).sample(rows, args.limit)
    question = {"type": "choice", "instructions": instructions,
                "criteria": {name.replace("_", " "): None for name in names}}
    agent = laya.load(args.model, device=args.device)
    few = dict(question, criteria=dict(list(question["criteria"].items())[:8]))
    for text, _gold in rows[:3]:  # warm up before anything is timed
        agent.predict({"message": text}, {"q": few})

    report = {"dataset": args.dataset, "split": args.split, "rows": len(rows), "options": len(names),
              "revisions": REVISIONS, "model": args.model, "device": str(agent.device),
              "laya": laya.__version__, "machine": platform.machine(), "strategies": {}}
    first = None
    for name in args.strategies.split(","):
        answer_fn = strategy(name, agent)
        correct, pairs, ms, calls, errors = [], [], [], [], []
        for text, gold in rows:
            started = time.perf_counter()
            try:
                answer, n_calls = answer_fn({"message": text}, question)
            except ValueError as exc:  # the question does not fit the window at all
                errors.append(str(exc))
                correct.append(0)
                if len(errors) == len(correct) == 3:
                    break
                continue
            ms.append((time.perf_counter() - started) * 1000)
            correct.append(int(answer["choice"] == gold.replace("_", " ")))
            pairs.append((answer["answer_confidence"], bool(correct[-1])))
            calls.append(n_calls)
        entry = {"errors": len(errors), "first_error": errors[0] if errors else None}
        if len(correct) == len(rows):
            entry["accuracy"] = round(sum(correct) / len(rows), 4)
        if ms:
            ms.sort()
            entry.update(ece=round(ece([c for c, _ in pairs], [ok for _, ok in pairs]), 4),
                         mean_ms=round(sum(ms) / len(ms), 1), p50_ms=round(ms[len(ms) // 2], 1),
                         p95_ms=round(ms[int(len(ms) * 0.95)], 1),
                         calls_per_row=round(sum(calls) / len(calls), 3))
        if first is None:
            first = (name, correct)
        elif len(correct) == len(first[1]) == len(rows):
            entry["vs_" + first[0]] = {"gained": sum(a > b for a, b in zip(correct, first[1])),
                                       "lost": sum(a < b for a, b in zip(correct, first[1]))}
        report["strategies"][name] = entry
        print(args.dataset, name, json.dumps(entry), flush=True)
    Path(args.out).write_text(json.dumps(report, indent=1) + "\n")


if __name__ == "__main__":
    main()