Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
fix(shortlist): hold every tournament call to the reply shape it indexes
Aashishfd6b726fe235
diff --git a/laya/shortlist.py b/laya/shortlist.pyindex 4cf8a0e6..2aedb729 100644--- a/laya/shortlist.py+++ b/laya/shortlist.py@@ -131,10 +131,6 @@ def predict_shortlist(reduced[qid] = _drop_stale_order(qid, updated, len(labels), warned)result = _call_predict(agent, state, reduced, **predict_kwargs)- if not isinstance(result, dict):- raise TypeError(- "predict/system_one must return a dict, got %s" % type(result).__name__- )out = dict(result)out["shortlist"] = metareturn out@@ -178,6 +174,12 @@ def predict_tournament(come from the final call, so a tournament choice's probabilities are over itsfinalists only.+ Every call's reply is checked against the same shape, the elimination rounds included: a+ reply that is not a dict, one whose ``answers`` is not a dict, and a round whose answers+ leave one of the numbered group questions out each raise that contract by name. The group+ numbers are invented here, so an unanswered group would otherwise surface as a ``KeyError``+ on a question id no request contains.+Extra keyword arguments are forwarded to every call (for example ``model=`` on a ``Router``)."""if not isinstance(questions, dict):@@ -208,7 +210,8 @@ def predict_tournament(if not groups:breakround_questions = {str(i): cut(qid, labels) for i, (qid, labels) in enumerate(groups)}- answers = _call_predict(agent, state, round_questions, **predict_kwargs)["answers"]+ answers = _round_answers(_call_predict(agent, state, round_questions, **predict_kwargs),+ len(groups))winners: Dict[str, List[Any]] = {}for i, (qid, _labels) in enumerate(groups):winners.setdefault(qid, []).append(answers[str(i)]["choice"])@@ -221,10 +224,6 @@ def predict_tournament(if entry["rounds"]:final[qid] = cut(qid, entry["labels"])result = _call_predict(agent, state, final, **predict_kwargs)- if not isinstance(result, dict):- raise TypeError(- "predict/system_one must return a dict, got %s" % type(result).__name__- )out = dict(result)out["tournament"] = metareturn out@@ -523,12 +522,43 @@ def _cosine(query: np.ndarray, docs: np.ndarray) -> np.ndarray:def _call_predict(agent, state, questions, **predict_kwargs):+ """One ``predict``/``system_one`` call, checked against the shape every caller here reads.++ Both public functions return that reply to their caller, and ``predict_tournament`` indexes+ into it between rounds, so the check belongs to the call rather than to a call site: an agent+ that answers with a list or ``None`` fails here, naming the contract, instead of raising a+ bare indexing error from inside the module.+ """fn = getattr(agent, "predict", None)if fn is None:fn = getattr(agent, "system_one", None)if fn is None:raise TypeError("agent must provide predict or system_one")- return fn(state, questions, **predict_kwargs)+ result = fn(state, questions, **predict_kwargs)+ if not isinstance(result, dict):+ raise TypeError(+ "predict/system_one must return a dict, got %s" % type(result).__name__+ )+ return result+++def _round_answers(result, n_groups):+ """The answers of one elimination round, keyed by the group numbers this module invented.++ ``predict_tournament`` numbers its own group questions, so the keys it indexes here are keys+ the caller never wrote. An agent that answers fewer questions than it was handed would fail on+ a ``KeyError`` naming a number no request contains; the omitted ids are named instead.+ """+ answers = result.get("answers")+ if not isinstance(answers, dict):+ raise TypeError(+ "predict/system_one must return an answers dict, got %s" % type(answers).__name__+ )+ missing = [str(i) for i in range(n_groups) if str(i) not in answers]+ if missing:+ raise ValueError("the round answered %d of %d group questions, omitting %s"+ % (n_groups - len(missing), n_groups, ", ".join(missing)))+ return answersdef _hidden_size(encoder) -> int:diff --git a/tests/test_shortlist.py b/tests/test_shortlist.pyindex 06915cd5..893884dd 100644--- a/tests/test_shortlist.py+++ b/tests/test_shortlist.py@@ -971,6 +971,109 @@ check_raises("tournament/choice without criteria",lambda: predict_tournament(Ranker([]), "s", {"q": {"type": "choice", "instructions": "pick"}}))+# --------------------------------------------- every call's reply is held to the same contract+# `predict_shortlist` and the tournament's final call both refused a non-dict reply by naming that+# contract; the tournament's round call indexed the reply instead. The same agent fault therefore+# raised Python's indexing error from inside the module, and a reply that skipped one of the+# numbered group questions raised a `KeyError` on an id no request contains -- the groups are+# numbered here, so the caller has nothing to match that number against.+class Reply:+ """Answers every call with ``shape``, so only the reply's shape varies between these stubs.++ A callable ``shape`` is handed the questions, which is how a stub replies to the caller's own+ request and to the invented group questions differently.+ """++ def __init__(self, shape):+ self.shape = shape+ self.seen = []++ def predict(self, state, questions, **kwargs):+ self.seen.append(sorted(questions))+ return self.shape(questions) if callable(self.shape) else self.shape+++def _message(fn):+ try:+ fn()+ except Exception as exc:+ return "%s: %s" % (type(exc).__name__, exc)+ return "no exception"+++def _outcome(fn):+ """What a call returns, or the exception it raised.++ A guard that fires where it should not has to fail the check that owns that call site, not+ abort the suite before the checks below it run.+ """+ try:+ return ("returned", fn())+ except Exception as exc:+ return ("raised", "%s: %s" % (type(exc).__name__, exc))+++# 20 labels at group_size 4 is two rounds -- five groups, then two -- and then the final call over+# the finalists. `_round` hands one reply shape to every call of such a tournament, as a thunk.+_TOUR_Q = {"q": {"type": "choice", "criteria": ["l%02d" % i for i in range(20)]}}+_DICT_MSG = "TypeError: predict/system_one must return a dict, got list"+++def _round(shape):+ return lambda: predict_tournament(Reply(shape), "s", _TOUR_Q, group_size=4)+++def _group_answers(questions):+ return {"model": "fake", "answers": {qid: {"choice": next(iter(qdef["criteria"]))}+ for qid, qdef in questions.items()}}+++check("tournament/round reply must be a dict, said by name", _message(_round(["not", "a", "dict"])),+ _DICT_MSG)+check("shortlist/reply must be a dict, said by name", _message(+ lambda: predict_shortlist(Reply(["not", "a", "dict"]), "s", _TOUR_Q,+ embed_fn=lambda texts: [[float(i)] for i in range(len(texts))], k=5)),+ _DICT_MSG)+check("tournament/round reply without an answers mapping is named too",+ _message(_round({"model": "fake"})),+ "TypeError: predict/system_one must return an answers dict, got NoneType")+check("tournament/a round that leaves groups out names the omitted ones",+ _message(_round(lambda q: {"model": "fake", "answers": {"0": {"choice": "l00"}}})),+ "ValueError: the round answered 1 of 5 group questions, omitting 1, 2, 3, 4")+# Each call site is guarded on its own: an agent can answer the caller's request and hand back a+# list for the invented groups, or the other way round. Either way the round has already run, so a+# guard only on the last call would not catch the first of these.+check("tournament/a bad round reply is refused before the final call",+ _message(_round(lambda q: ["not", "a", "dict"] if "q" not in q else _group_answers(q))),+ _DICT_MSG)+_bad_final = Reply(lambda q: ["not", "a", "dict"] if "q" in q else _group_answers(q))+check("tournament/a bad final reply is refused after the rounds ran",+ (_message(lambda: predict_tournament(_bad_final, "s", _TOUR_Q, group_size=4)), _bad_final.seen),+ (_DICT_MSG, [["0", "1", "2", "3", "4"], ["0", "1"], ["q"]]))+# And a reply that satisfies the contract reaches the same result it did before: the checks are on+# the shape this module indexes, not on an answer's contents, which the final dict passes through.+_clean = Reply(_group_answers)+check("tournament/a well-shaped reply still plays its rounds",+ _outcome(lambda: (predict_tournament(_clean, "s", _TOUR_Q, group_size=4)["tournament"]["q"]["rounds"],+ _clean.seen)),+ ("returned", (2, [["0", "1", "2", "3", "4"], ["0", "1"], ["q"]])))+check("tournament/an answer-less final reply is still returned as given",+ _outcome(lambda: predict_tournament(Reply({"model": "fake"}), "s",+ {"q": {"type": "choice", "criteria": ["a", "b"]}})),+ ("returned", {"model": "fake",+ "tournament": {"q": {"labels": ["a", "b"], "n": 2, "rounds": 0}}}))+# The contract belongs in the docstring, because a caller picking an agent for this function reads+# it there -- and it has to say the rounds are covered, since that is the call site that used to be+# unchecked.+_doc = " ".join(predict_tournament.__doc__.split())+_contract = (_doc.split("Every call's reply")[1].split("Extra keyword")[0]+ if "Every call's reply" in _doc else "")+check_true("tournament/docstring states the contract for every call",+ _contract.startswith(" is checked against the same shape, the elimination rounds included")+ and "answers" in _contract,+ "the docstring says: " + (_contract or "(no such paragraph)")[:160])++# ------------------------------------------------- narrowing drops a stale option_order# Nandha on #958: narrowing `criteria` while forwarding a full-length `option_order`# "is a core contract question"; his preferred remedy drops the order with a warning,