Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(shortlist): hold every tournament call to the reply shape it indexes

Aashishfd6b726fe235
diff --git a/laya/shortlist.py b/laya/shortlist.py
index 4cf8a0e6..2aedb729 100644
--- a/laya/shortlist.py
+++ b/laya/shortlist.py
@@ -131,10 +131,6 @@ def predict_shortlist(
reduced[qid] = _drop_stale_order(qid, updated, len(labels), warned)
result = _call_predict(agent, state, reduced, **predict_kwargs)
- if not isinstance(result, dict):
- raise TypeError(
- "predict/system_one must return a dict, got %s" % type(result).__name__
- )
out = dict(result)
out["shortlist"] = meta
return out
@@ -178,6 +174,12 @@ def predict_tournament(
come from the final call, so a tournament choice's probabilities are over its
finalists only.
+ Every call's reply is checked against the same shape, the elimination rounds included: a
+ reply that is not a dict, one whose ``answers`` is not a dict, and a round whose answers
+ leave one of the numbered group questions out each raise that contract by name. The group
+ numbers are invented here, so an unanswered group would otherwise surface as a ``KeyError``
+ on a question id no request contains.
+
Extra keyword arguments are forwarded to every call (for example ``model=`` on a ``Router``).
"""
if not isinstance(questions, dict):
@@ -208,7 +210,8 @@ def predict_tournament(
if not groups:
break
round_questions = {str(i): cut(qid, labels) for i, (qid, labels) in enumerate(groups)}
- answers = _call_predict(agent, state, round_questions, **predict_kwargs)["answers"]
+ answers = _round_answers(_call_predict(agent, state, round_questions, **predict_kwargs),
+ len(groups))
winners: Dict[str, List[Any]] = {}
for i, (qid, _labels) in enumerate(groups):
winners.setdefault(qid, []).append(answers[str(i)]["choice"])
@@ -221,10 +224,6 @@ def predict_tournament(
if entry["rounds"]:
final[qid] = cut(qid, entry["labels"])
result = _call_predict(agent, state, final, **predict_kwargs)
- if not isinstance(result, dict):
- raise TypeError(
- "predict/system_one must return a dict, got %s" % type(result).__name__
- )
out = dict(result)
out["tournament"] = meta
return out
@@ -523,12 +522,43 @@ def _cosine(query: np.ndarray, docs: np.ndarray) -> np.ndarray:
def _call_predict(agent, state, questions, **predict_kwargs):
+ """One ``predict``/``system_one`` call, checked against the shape every caller here reads.
+
+ Both public functions return that reply to their caller, and ``predict_tournament`` indexes
+ into it between rounds, so the check belongs to the call rather than to a call site: an agent
+ that answers with a list or ``None`` fails here, naming the contract, instead of raising a
+ bare indexing error from inside the module.
+ """
fn = getattr(agent, "predict", None)
if fn is None:
fn = getattr(agent, "system_one", None)
if fn is None:
raise TypeError("agent must provide predict or system_one")
- return fn(state, questions, **predict_kwargs)
+ result = fn(state, questions, **predict_kwargs)
+ if not isinstance(result, dict):
+ raise TypeError(
+ "predict/system_one must return a dict, got %s" % type(result).__name__
+ )
+ return result
+
+
+def _round_answers(result, n_groups):
+ """The answers of one elimination round, keyed by the group numbers this module invented.
+
+ ``predict_tournament`` numbers its own group questions, so the keys it indexes here are keys
+ the caller never wrote. An agent that answers fewer questions than it was handed would fail on
+ a ``KeyError`` naming a number no request contains; the omitted ids are named instead.
+ """
+ answers = result.get("answers")
+ if not isinstance(answers, dict):
+ raise TypeError(
+ "predict/system_one must return an answers dict, got %s" % type(answers).__name__
+ )
+ missing = [str(i) for i in range(n_groups) if str(i) not in answers]
+ if missing:
+ raise ValueError("the round answered %d of %d group questions, omitting %s"
+ % (n_groups - len(missing), n_groups, ", ".join(missing)))
+ return answers
def _hidden_size(encoder) -> int:
diff --git a/tests/test_shortlist.py b/tests/test_shortlist.py
index 06915cd5..893884dd 100644
--- a/tests/test_shortlist.py
+++ b/tests/test_shortlist.py
@@ -971,6 +971,109 @@ check_raises("tournament/choice without criteria",
lambda: predict_tournament(Ranker([]), "s", {"q": {"type": "choice", "instructions": "pick"}}))
+# --------------------------------------------- every call's reply is held to the same contract
+# `predict_shortlist` and the tournament's final call both refused a non-dict reply by naming that
+# contract; the tournament's round call indexed the reply instead. The same agent fault therefore
+# raised Python's indexing error from inside the module, and a reply that skipped one of the
+# numbered group questions raised a `KeyError` on an id no request contains -- the groups are
+# numbered here, so the caller has nothing to match that number against.
+class Reply:
+ """Answers every call with ``shape``, so only the reply's shape varies between these stubs.
+
+ A callable ``shape`` is handed the questions, which is how a stub replies to the caller's own
+ request and to the invented group questions differently.
+ """
+
+ def __init__(self, shape):
+ self.shape = shape
+ self.seen = []
+
+ def predict(self, state, questions, **kwargs):
+ self.seen.append(sorted(questions))
+ return self.shape(questions) if callable(self.shape) else self.shape
+
+
+def _message(fn):
+ try:
+ fn()
+ except Exception as exc:
+ return "%s: %s" % (type(exc).__name__, exc)
+ return "no exception"
+
+
+def _outcome(fn):
+ """What a call returns, or the exception it raised.
+
+ A guard that fires where it should not has to fail the check that owns that call site, not
+ abort the suite before the checks below it run.
+ """
+ try:
+ return ("returned", fn())
+ except Exception as exc:
+ return ("raised", "%s: %s" % (type(exc).__name__, exc))
+
+
+# 20 labels at group_size 4 is two rounds -- five groups, then two -- and then the final call over
+# the finalists. `_round` hands one reply shape to every call of such a tournament, as a thunk.
+_TOUR_Q = {"q": {"type": "choice", "criteria": ["l%02d" % i for i in range(20)]}}
+_DICT_MSG = "TypeError: predict/system_one must return a dict, got list"
+
+
+def _round(shape):
+ return lambda: predict_tournament(Reply(shape), "s", _TOUR_Q, group_size=4)
+
+
+def _group_answers(questions):
+ return {"model": "fake", "answers": {qid: {"choice": next(iter(qdef["criteria"]))}
+ for qid, qdef in questions.items()}}
+
+
+check("tournament/round reply must be a dict, said by name", _message(_round(["not", "a", "dict"])),
+ _DICT_MSG)
+check("shortlist/reply must be a dict, said by name", _message(
+ lambda: predict_shortlist(Reply(["not", "a", "dict"]), "s", _TOUR_Q,
+ embed_fn=lambda texts: [[float(i)] for i in range(len(texts))], k=5)),
+ _DICT_MSG)
+check("tournament/round reply without an answers mapping is named too",
+ _message(_round({"model": "fake"})),
+ "TypeError: predict/system_one must return an answers dict, got NoneType")
+check("tournament/a round that leaves groups out names the omitted ones",
+ _message(_round(lambda q: {"model": "fake", "answers": {"0": {"choice": "l00"}}})),
+ "ValueError: the round answered 1 of 5 group questions, omitting 1, 2, 3, 4")
+# Each call site is guarded on its own: an agent can answer the caller's request and hand back a
+# list for the invented groups, or the other way round. Either way the round has already run, so a
+# guard only on the last call would not catch the first of these.
+check("tournament/a bad round reply is refused before the final call",
+ _message(_round(lambda q: ["not", "a", "dict"] if "q" not in q else _group_answers(q))),
+ _DICT_MSG)
+_bad_final = Reply(lambda q: ["not", "a", "dict"] if "q" in q else _group_answers(q))
+check("tournament/a bad final reply is refused after the rounds ran",
+ (_message(lambda: predict_tournament(_bad_final, "s", _TOUR_Q, group_size=4)), _bad_final.seen),
+ (_DICT_MSG, [["0", "1", "2", "3", "4"], ["0", "1"], ["q"]]))
+# And a reply that satisfies the contract reaches the same result it did before: the checks are on
+# the shape this module indexes, not on an answer's contents, which the final dict passes through.
+_clean = Reply(_group_answers)
+check("tournament/a well-shaped reply still plays its rounds",
+ _outcome(lambda: (predict_tournament(_clean, "s", _TOUR_Q, group_size=4)["tournament"]["q"]["rounds"],
+ _clean.seen)),
+ ("returned", (2, [["0", "1", "2", "3", "4"], ["0", "1"], ["q"]])))
+check("tournament/an answer-less final reply is still returned as given",
+ _outcome(lambda: predict_tournament(Reply({"model": "fake"}), "s",
+ {"q": {"type": "choice", "criteria": ["a", "b"]}})),
+ ("returned", {"model": "fake",
+ "tournament": {"q": {"labels": ["a", "b"], "n": 2, "rounds": 0}}}))
+# The contract belongs in the docstring, because a caller picking an agent for this function reads
+# it there -- and it has to say the rounds are covered, since that is the call site that used to be
+# unchecked.
+_doc = " ".join(predict_tournament.__doc__.split())
+_contract = (_doc.split("Every call's reply")[1].split("Extra keyword")[0]
+ if "Every call's reply" in _doc else "")
+check_true("tournament/docstring states the contract for every call",
+ _contract.startswith(" is checked against the same shape, the elimination rounds included")
+ and "answers" in _contract,
+ "the docstring says: " + (_contract or "(no such paragraph)")[:160])
+
+
# ------------------------------------------------- narrowing drops a stale option_order
# Nandha on #958: narrowing `criteria` while forwarding a full-length `option_order`
# "is a core contract question"; his preferred remedy drops the order with a warning,