Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
feat(shortlist): add predict_tournament for choice questions past the option budget
Rishit Sharma7c8043efc076
diff --git a/laya/shortlist.py b/laya/shortlist.pyindex 17a0b3b2..2a92c062 100644--- a/laya/shortlist.py+++ b/laya/shortlist.py@@ -1,13 +1,16 @@-"""Opt-in embedding shortlist for high-cardinality choice questions.+"""Opt-in embedding shortlist and tournament for high-cardinality choice questions.Choice options share one ``head_max_len`` budget, so a large label set leaves only a fewtokens per label. ``predict_shortlist`` embeds the state and each option with acaller-supplied ``embed_fn``, keeps the top ``k``, and runs a single ``predict`` (or-``system_one``) on that reduced criteria set.+``system_one``) on that reduced criteria set. ``predict_tournament`` needs no embedder: the+decision model answers the labels in groups small enough to keep their tokens, and the group+winners meet in one more ``predict``.``Agent.predict`` and ``Agent.system_one`` are separate: they still score every criterion-they are given. This module does not change ``DecisionModel.forward`` and does not add a-second decision-model pass.+they are given. This module does not change ``DecisionModel.forward``.+``predict_shortlist`` adds no second decision-model pass; ``predict_tournament`` adds one+per elimination round.The coarse-to-fine pattern is the one the README recommends and the one reported inhttps://github.com/NandhaKishorM/laya/issues/102. Ranking here is cosine similarity on@@ -24,6 +27,7 @@ import numpy as npfrom .common import encode_text, render_options, serialize_stateDEFAULT_SHORTLIST_K = 20+DEFAULT_TOURNAMENT_GROUP = 16def shortlist_choice(@@ -123,6 +127,84 @@ def predict_shortlist(return out+def predict_tournament(+ agent: Any,+ state: Any,+ questions: Dict[str, Dict[str, Any]],+ group_size: int = DEFAULT_TOURNAMENT_GROUP,+ **predict_kwargs: Any,+) -> Dict[str, Any]:+ """Narrow each large choice question by elimination, then call ``predict`` once more.++ A choice with more than ``group_size`` labels is cut, in criteria order, into groups of+ near-equal size and at most ``group_size``. One ``predict`` call answers every group of+ every such question -- the groups go in as separate questions, so they share one forward+ pass -- and each group's answer goes through to the next round. Rounds repeat until no+ choice has more than ``group_size`` labels left; at the default 16, up to 256 labels take+ one round. Unlike ``predict_shortlist`` this needs no embedder, and each label is read+ with the option budget of a ``group_size``-label question, not of the whole label set.++ The final call answers every question of the request, with each choice that went through+ a round cut to its finalists. Non-choice questions and choices of at most ``group_size``+ labels go to it unchanged, so when nothing needs a round it is the only call. The+ caller's ``questions`` dict is not mutated.++ The returned dict is the final call's result plus a ``tournament`` entry.+ ``tournament[qid]`` holds ``labels`` (the finalists, in criteria order), ``n`` (the+ label count) and ``rounds`` for each choice question. Probabilities, confidences and+ ``usage`` come from the final call, so a tournament choice's probabilities are over its+ finalists only.++ Extra keyword arguments are forwarded to every call (for example ``model=`` on a ``Router``).+ """+ if not isinstance(questions, dict):+ raise TypeError("questions must be a dict of question id -> definition")+ if isinstance(group_size, bool) or not isinstance(group_size, int) or group_size < 2:+ raise ValueError("group_size must be an integer of at least 2, got %r" % (group_size,))+ meta: Dict[str, Dict[str, Any]] = {}+ for qid, qdef in questions.items():+ if isinstance(qdef, dict) and qdef.get("type") == "choice":+ if "criteria" not in qdef:+ raise ValueError("question %r is a choice but has no criteria" % (qid,))+ labels = [key for key, _value in _criteria_items(qdef["criteria"])]+ meta[qid] = {"labels": labels, "n": len(labels), "rounds": 0}++ def cut(qid, labels):+ return dict(questions[qid], criteria=_subset_criteria(questions[qid]["criteria"], labels))++ while True:+ groups = []+ for qid, entry in meta.items():+ labels = entry["labels"]+ parts = -(-len(labels) // group_size)+ if parts > 1:+ groups += [(qid, labels[i * len(labels) // parts:(i + 1) * len(labels) // parts])+ for i in range(parts)]+ if not groups:+ break+ round_questions = {str(i): cut(qid, labels) for i, (qid, labels) in enumerate(groups)}+ answers = _call_predict(agent, state, round_questions, **predict_kwargs)["answers"]+ winners: Dict[str, List[Any]] = {}+ for i, (qid, _labels) in enumerate(groups):+ winners.setdefault(qid, []).append(answers[str(i)]["choice"])+ for qid, labels in winners.items():+ meta[qid]["labels"] = labels+ meta[qid]["rounds"] += 1++ final = dict(questions)+ for qid, entry in meta.items():+ if entry["rounds"]:+ final[qid] = cut(qid, entry["labels"])+ result = _call_predict(agent, state, final, **predict_kwargs)+ if not isinstance(result, dict):+ raise TypeError(+ "predict/system_one must return a dict, got %s" % type(result).__name__+ )+ out = dict(result)+ out["tournament"] = meta+ return out++def embed_fn_from_agent(agent: Any,max_length: int = 512,diff --git a/tests/test_shortlist.py b/tests/test_shortlist.pyindex dbb27ad3..4b4633f7 100644--- a/tests/test_shortlist.py+++ b/tests/test_shortlist.py@@ -19,6 +19,7 @@ from laya.shortlist import ( # noqa: E402cached_embed_fn,embed_fn_from_agent,predict_shortlist,+ predict_tournament,shortlist_choice,)@@ -777,6 +778,72 @@ swap_cached(["beta"])check("cache/dim clear then re-embed works", swap_cached.cache_info()["size"], 1)+# --------------------------------------------------------------- tournament+class Ranker:+ """Stand-in for Agent that always answers the label it ranks highest, in any group."""++ def __init__(self, order):+ self.rank = {label: i for i, label in enumerate(order)}+ self.calls = []++ def predict(self, state, questions, **kwargs):+ self.calls.append((questions, kwargs))+ return {"model": "fake", "answers": {+ qid: {"type": "choice", "choice": min(qdef["criteria"], key=self.rank.get)}+ for qid, qdef in questions.items() if qdef["type"] == "choice"}}+++# A ranker that is consistent across groups must see its best label win from any position.+for n, calls in ((77, 2), (257, 3)):+ labels = ["l%03d" % i for i in range(n)]+ for best in (labels[0], labels[n // 2], labels[-1]):+ ranker = Ranker([best] + [label for label in labels if label != best])+ out = predict_tournament(ranker, "s", {"q": {"type": "choice", "instructions": "pick",+ "criteria": list(labels)}})+ check("tournament/%d labels: %s wins" % (n, best), out["answers"]["q"]["choice"], best)+ check("tournament/%d labels: %s calls" % (n, best), len(ranker.calls), calls)+ if n == 77:+ check("tournament/77 labels in balanced groups",+ [len(qdef["criteria"]) for qdef in ranker.calls[0][0].values()], [15, 15, 16, 15, 16])++# Only the choice over the group size plays rounds; the final call carries the whole request.+labels = ["l%02d" % i for i in range(20)]+big = {"type": "choice", "instructions": "pick", "criteria": {label: "about " + label for label in labels}}+small = {"type": "choice", "instructions": "pick", "criteria": ["x", "y"]}+level = {"type": "score", "instructions": "rate", "criteria": ["low", "high"]}+questions = {"big": big, "small": small, "level": level}+before = repr(questions)+ranker = Ranker(labels[::-1] + ["y", "x"])+out = predict_tournament(ranker, "s", questions, group_size=8, model="english")+check("tournament/caller questions unchanged", repr(questions), before)+check("tournament/two calls", len(ranker.calls), 2)+check("tournament/kwargs reach every call", [kwargs for _q, kwargs in ranker.calls], [{"model": "english"}] * 2)+check("tournament/round holds only the big choice's groups",+ [q["criteria"] for q in ranker.calls[0][0].values()],+ [{label: "about " + label for label in part} for part in (labels[:6], labels[6:13], labels[13:])])+final = ranker.calls[1][0]+check("tournament/final keeps every question id", sorted(final), ["big", "level", "small"])+check("tournament/final cuts the big choice to its finalists",+ final["big"], dict(big, criteria={label: "about " + label for label in ("l05", "l12", "l19")}))+check_true("tournament/final passes the rest unchanged", final["small"] is small and final["level"] is level)+check("tournament/answers come from the final call", out["answers"]["big"]["choice"], "l19")+check("tournament/meta", out["tournament"], {+ "big": {"labels": ["l05", "l12", "l19"], "n": 20, "rounds": 1},+ "small": {"labels": ["x", "y"], "n": 2, "rounds": 0}})++ranker = Ranker(["y", "x"])+predict_tournament(ranker, "s", {"small": small, "level": level})+check("tournament/nothing to narrow: one call with the request itself",+ ranker.calls, [({"small": small, "level": level}, {})])++for bad in (1, 0, -3, True, 2.5, "16"):+ check_raises("tournament/group_size %r refused" % (bad,),+ lambda bad=bad: predict_tournament(Ranker([]), "s", {}, group_size=bad))+check_raises("tournament/questions must be a dict", lambda: predict_tournament(Ranker([]), "s", []), TypeError)+check_raises("tournament/choice without criteria",+ lambda: predict_tournament(Ranker([]), "s", {"q": {"type": "choice", "instructions": "pick"}}))++print("\n%d passed, %d failed" % (len(PASS), len(FAIL)))for f in FAIL:print(" FAIL " + f)