Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

feat(shortlist): add predict_tournament for choice questions past the option budget

Rishit Sharma7c8043efc076
diff --git a/laya/shortlist.py b/laya/shortlist.py
index 17a0b3b2..2a92c062 100644
--- a/laya/shortlist.py
+++ b/laya/shortlist.py
@@ -1,13 +1,16 @@
-"""Opt-in embedding shortlist for high-cardinality choice questions.
+"""Opt-in embedding shortlist and tournament for high-cardinality choice questions.
Choice options share one ``head_max_len`` budget, so a large label set leaves only a few
tokens per label. ``predict_shortlist`` embeds the state and each option with a
caller-supplied ``embed_fn``, keeps the top ``k``, and runs a single ``predict`` (or
-``system_one``) on that reduced criteria set.
+``system_one``) on that reduced criteria set. ``predict_tournament`` needs no embedder: the
+decision model answers the labels in groups small enough to keep their tokens, and the group
+winners meet in one more ``predict``.
``Agent.predict`` and ``Agent.system_one`` are separate: they still score every criterion
-they are given. This module does not change ``DecisionModel.forward`` and does not add a
-second decision-model pass.
+they are given. This module does not change ``DecisionModel.forward``.
+``predict_shortlist`` adds no second decision-model pass; ``predict_tournament`` adds one
+per elimination round.
The coarse-to-fine pattern is the one the README recommends and the one reported in
https://github.com/NandhaKishorM/laya/issues/102. Ranking here is cosine similarity on
@@ -24,6 +27,7 @@ import numpy as np
from .common import encode_text, render_options, serialize_state
DEFAULT_SHORTLIST_K = 20
+DEFAULT_TOURNAMENT_GROUP = 16
def shortlist_choice(
@@ -123,6 +127,84 @@ def predict_shortlist(
return out
+def predict_tournament(
+ agent: Any,
+ state: Any,
+ questions: Dict[str, Dict[str, Any]],
+ group_size: int = DEFAULT_TOURNAMENT_GROUP,
+ **predict_kwargs: Any,
+) -> Dict[str, Any]:
+ """Narrow each large choice question by elimination, then call ``predict`` once more.
+
+ A choice with more than ``group_size`` labels is cut, in criteria order, into groups of
+ near-equal size and at most ``group_size``. One ``predict`` call answers every group of
+ every such question -- the groups go in as separate questions, so they share one forward
+ pass -- and each group's answer goes through to the next round. Rounds repeat until no
+ choice has more than ``group_size`` labels left; at the default 16, up to 256 labels take
+ one round. Unlike ``predict_shortlist`` this needs no embedder, and each label is read
+ with the option budget of a ``group_size``-label question, not of the whole label set.
+
+ The final call answers every question of the request, with each choice that went through
+ a round cut to its finalists. Non-choice questions and choices of at most ``group_size``
+ labels go to it unchanged, so when nothing needs a round it is the only call. The
+ caller's ``questions`` dict is not mutated.
+
+ The returned dict is the final call's result plus a ``tournament`` entry.
+ ``tournament[qid]`` holds ``labels`` (the finalists, in criteria order), ``n`` (the
+ label count) and ``rounds`` for each choice question. Probabilities, confidences and
+ ``usage`` come from the final call, so a tournament choice's probabilities are over its
+ finalists only.
+
+ Extra keyword arguments are forwarded to every call (for example ``model=`` on a ``Router``).
+ """
+ if not isinstance(questions, dict):
+ raise TypeError("questions must be a dict of question id -> definition")
+ if isinstance(group_size, bool) or not isinstance(group_size, int) or group_size < 2:
+ raise ValueError("group_size must be an integer of at least 2, got %r" % (group_size,))
+ meta: Dict[str, Dict[str, Any]] = {}
+ for qid, qdef in questions.items():
+ if isinstance(qdef, dict) and qdef.get("type") == "choice":
+ if "criteria" not in qdef:
+ raise ValueError("question %r is a choice but has no criteria" % (qid,))
+ labels = [key for key, _value in _criteria_items(qdef["criteria"])]
+ meta[qid] = {"labels": labels, "n": len(labels), "rounds": 0}
+
+ def cut(qid, labels):
+ return dict(questions[qid], criteria=_subset_criteria(questions[qid]["criteria"], labels))
+
+ while True:
+ groups = []
+ for qid, entry in meta.items():
+ labels = entry["labels"]
+ parts = -(-len(labels) // group_size)
+ if parts > 1:
+ groups += [(qid, labels[i * len(labels) // parts:(i + 1) * len(labels) // parts])
+ for i in range(parts)]
+ if not groups:
+ break
+ round_questions = {str(i): cut(qid, labels) for i, (qid, labels) in enumerate(groups)}
+ answers = _call_predict(agent, state, round_questions, **predict_kwargs)["answers"]
+ winners: Dict[str, List[Any]] = {}
+ for i, (qid, _labels) in enumerate(groups):
+ winners.setdefault(qid, []).append(answers[str(i)]["choice"])
+ for qid, labels in winners.items():
+ meta[qid]["labels"] = labels
+ meta[qid]["rounds"] += 1
+
+ final = dict(questions)
+ for qid, entry in meta.items():
+ if entry["rounds"]:
+ final[qid] = cut(qid, entry["labels"])
+ result = _call_predict(agent, state, final, **predict_kwargs)
+ if not isinstance(result, dict):
+ raise TypeError(
+ "predict/system_one must return a dict, got %s" % type(result).__name__
+ )
+ out = dict(result)
+ out["tournament"] = meta
+ return out
+
+
def embed_fn_from_agent(
agent: Any,
max_length: int = 512,
diff --git a/tests/test_shortlist.py b/tests/test_shortlist.py
index dbb27ad3..4b4633f7 100644
--- a/tests/test_shortlist.py
+++ b/tests/test_shortlist.py
@@ -19,6 +19,7 @@ from laya.shortlist import ( # noqa: E402
cached_embed_fn,
embed_fn_from_agent,
predict_shortlist,
+ predict_tournament,
shortlist_choice,
)
@@ -777,6 +778,72 @@ swap_cached(["beta"])
check("cache/dim clear then re-embed works", swap_cached.cache_info()["size"], 1)
+# --------------------------------------------------------------- tournament
+class Ranker:
+ """Stand-in for Agent that always answers the label it ranks highest, in any group."""
+
+ def __init__(self, order):
+ self.rank = {label: i for i, label in enumerate(order)}
+ self.calls = []
+
+ def predict(self, state, questions, **kwargs):
+ self.calls.append((questions, kwargs))
+ return {"model": "fake", "answers": {
+ qid: {"type": "choice", "choice": min(qdef["criteria"], key=self.rank.get)}
+ for qid, qdef in questions.items() if qdef["type"] == "choice"}}
+
+
+# A ranker that is consistent across groups must see its best label win from any position.
+for n, calls in ((77, 2), (257, 3)):
+ labels = ["l%03d" % i for i in range(n)]
+ for best in (labels[0], labels[n // 2], labels[-1]):
+ ranker = Ranker([best] + [label for label in labels if label != best])
+ out = predict_tournament(ranker, "s", {"q": {"type": "choice", "instructions": "pick",
+ "criteria": list(labels)}})
+ check("tournament/%d labels: %s wins" % (n, best), out["answers"]["q"]["choice"], best)
+ check("tournament/%d labels: %s calls" % (n, best), len(ranker.calls), calls)
+ if n == 77:
+ check("tournament/77 labels in balanced groups",
+ [len(qdef["criteria"]) for qdef in ranker.calls[0][0].values()], [15, 15, 16, 15, 16])
+
+# Only the choice over the group size plays rounds; the final call carries the whole request.
+labels = ["l%02d" % i for i in range(20)]
+big = {"type": "choice", "instructions": "pick", "criteria": {label: "about " + label for label in labels}}
+small = {"type": "choice", "instructions": "pick", "criteria": ["x", "y"]}
+level = {"type": "score", "instructions": "rate", "criteria": ["low", "high"]}
+questions = {"big": big, "small": small, "level": level}
+before = repr(questions)
+ranker = Ranker(labels[::-1] + ["y", "x"])
+out = predict_tournament(ranker, "s", questions, group_size=8, model="english")
+check("tournament/caller questions unchanged", repr(questions), before)
+check("tournament/two calls", len(ranker.calls), 2)
+check("tournament/kwargs reach every call", [kwargs for _q, kwargs in ranker.calls], [{"model": "english"}] * 2)
+check("tournament/round holds only the big choice's groups",
+ [q["criteria"] for q in ranker.calls[0][0].values()],
+ [{label: "about " + label for label in part} for part in (labels[:6], labels[6:13], labels[13:])])
+final = ranker.calls[1][0]
+check("tournament/final keeps every question id", sorted(final), ["big", "level", "small"])
+check("tournament/final cuts the big choice to its finalists",
+ final["big"], dict(big, criteria={label: "about " + label for label in ("l05", "l12", "l19")}))
+check_true("tournament/final passes the rest unchanged", final["small"] is small and final["level"] is level)
+check("tournament/answers come from the final call", out["answers"]["big"]["choice"], "l19")
+check("tournament/meta", out["tournament"], {
+ "big": {"labels": ["l05", "l12", "l19"], "n": 20, "rounds": 1},
+ "small": {"labels": ["x", "y"], "n": 2, "rounds": 0}})
+
+ranker = Ranker(["y", "x"])
+predict_tournament(ranker, "s", {"small": small, "level": level})
+check("tournament/nothing to narrow: one call with the request itself",
+ ranker.calls, [({"small": small, "level": level}, {})])
+
+for bad in (1, 0, -3, True, 2.5, "16"):
+ check_raises("tournament/group_size %r refused" % (bad,),
+ lambda bad=bad: predict_tournament(Ranker([]), "s", {}, group_size=bad))
+check_raises("tournament/questions must be a dict", lambda: predict_tournament(Ranker([]), "s", []), TypeError)
+check_raises("tournament/choice without criteria",
+ lambda: predict_tournament(Ranker([]), "s", {"q": {"type": "choice", "instructions": "pick"}}))
+
+
print("\n%d passed, %d failed" % (len(PASS), len(FAIL)))
for f in FAIL:
print(" FAIL " + f)