Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
feat(agent): let a question choose the order its options are shown in
sucf7b4b3f9d0382
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.ymlindex d4f38072..f5686f93 100644--- a/.github/workflows/ci.yml+++ b/.github/workflows/ci.yml@@ -102,6 +102,7 @@ jobs:python tests/test_batch.pypython tests/test_predict_long.pypython tests/test_attention_dynamic_shapes.py+ python tests/test_option_order.pypython tests/test_hooks.pypython tests/test_hooks_api.pypython tests/test_structured.pydiff --git a/laya/agent.py b/laya/agent.pyindex 5ca3861c..17c5f8e4 100644--- a/laya/agent.py+++ b/laya/agent.py@@ -220,6 +220,17 @@ def _cuda_amp_dtype(checkpoint_default: Optional[str]) -> torch.dtype:return torch.bfloat16return amp_dtype(checkpoint_default)+def _option_count(qdef: Dict) -> int:+ """How many options a validated question definition renders to.++ Mirrors `render_options`: a choice has one option per criterion, a score one per level,+ and a noul is always the pair [false, true].+ """+ if qdef.get("type") == "noul":+ return 2+ crit = qdef.get("criteria")+ return len(crit) if isinstance(crit, (dict, list, tuple)) else 0+def _start_evidence():"""A recorder for `predict_long`: what the start-hook chain left for inference to run on.@@ -719,6 +730,18 @@ class Agent(HookRegistry):"reads; any other key was silently dropped and replaced with the defaults. If you ""want the answer worded differently, keep 'criteria' keyed 'true'/'false' and set ""'labels' instead." % (qid, sorted(keys)))+ if "option_order" in qdef:+ # Slot s shows option `order[s]`. Anything other than a permutation of the option+ # indices would either drop an option or show one twice, so reject it here rather+ # than let it reach the encoder.+ order = qdef["option_order"]+ n = _option_count(qdef)+ if (not isinstance(order, (list, tuple))+ or len(order) != n+ or sorted(int(i) for i in order if isinstance(i, int) and not isinstance(i, bool)) != list(range(n))):+ raise ValueError(+ "question %r: 'option_order' must be a permutation of range(%d) -- one slot per "+ "option, each option once -- got %r" % (qid, n, order))if "labels" in qdef:if t != "noul":raise ValueError("question %r: 'labels' is only supported for noul questions" % (qid,))@@ -747,6 +770,8 @@ class Agent(HookRegistry):q = {"t": t, "ins": ins, "crit": crit}if "labels" in qdef:q["labels"] = qdef["labels"]+ if "option_order" in qdef:+ q["option_order"] = [int(i) for i in qdef["option_order"]]return qdef _encode_state(self, state: Union[str, dict, list], ids: List[str], internal: Dict[str, Dict],@@ -775,6 +800,7 @@ class Agent(HookRegistry):for qid in ids:q = internal[qid]seq, markers, stats = build_sequence(self.tok, state, q, max_len, head_max_len,+ option_order=q.get("option_order"),truncate_left=truncate_left, state_ids=state_ids,return_stats=True)if len(markers) != len(render_options(q)):@@ -883,6 +909,16 @@ class Agent(HookRegistry):p = np.exp(z - z.max())p = p / p.sum()+ # Slot s scored option `order[s]`, so the row comes back in slot order. Everything+ # below indexes by the caller's own option order -- `zip(keys, p)` for a choice,+ # `arange(k) * p` for a score level, `p[1]` for noul-true -- so put it back first.+ # Without this the probabilities would be attached to the wrong labels.+ order = q.get("option_order")+ if order is not None and len(order) == k:+ canonical = np.empty_like(p)+ canonical[np.asarray(order, dtype=int)] = p+ p = canonical+# `confidence` means one thing for `noul` (max(p)) and another for `choice` and# `score` (normalized entropy), and only the first is the quantity temperature# scaling fits and ECE measures. Rather than change one underneath existingdiff --git a/tests/test_option_order.py b/tests/test_option_order.pynew file mode 100644index 00000000..223f6142--- /dev/null+++ b/tests/test_option_order.py@@ -0,0 +1,166 @@+"""`option_order` must reach the encoder and be undone on the way back.++`build_sequence` has always taken an `option_order` -- slot `s` shows option `order[s]` -- and+training used it to shuffle. Nothing in the inference path ever passed it, so a question dict+carrying one was silently ignored: `_check_question` does not reject unknown keys and+`_to_internal` dropped it, so every call showed the canonical order.++That matters because where an option sits changes the answer. `research/results/+presentation_checks_shipped.json` measures the English checkpoint on five options whose text is+*identical*: the per-slot logits centre at [+1.68, +1.47, +0.11, -1.35, -1.91], a 3.59 spread+decided by position alone. `research/eval/metamorphic.py` and the `option_order_flip_rate` in+`research/scripts/bench_apps.py` exist for the same reason. Until now a caller had no way to+present the options any differently and see what changed.++The half that is easy to get wrong is the way back. The logit row comes out in *slot* order,+while everything downstream indexes by the caller's option order -- `zip(keys, p)` for a+choice, `arange(k) * p` for a score, `p[1]` for noul-true. Without the inverse permutation the+probabilities come back attached to the wrong labels, which is silent and worse than ignoring+the argument. These tests pin both halves.++No weights are loaded: the encoder is driven with a stub tokenizer and the decoder through+`Agent.__new__`, as in tests/test_batch.py.++Run: python tests/test_option_order.py+"""+import os+import sys++import numpy as np++sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))++from laya.agent import Agent, _option_count # noqa: E402+from laya.common import build_sequence # noqa: E402++PASS, FAIL = [], []+++def check(name, got, want):+ if got == want:+ PASS.append(name)+ else:+ FAIL.append("%s: got %r, want %r" % (name, got, want))+++def check_true(name, cond, detail=""):+ if cond:+ PASS.append(name)+ else:+ FAIL.append("%s %s" % (name, detail))+++def check_raises(name, exc, fn):+ try:+ fn()+ except exc:+ PASS.append(name)+ return+ except Exception as e: # noqa: BLE001 - the point is the wrong type+ FAIL.append("%s: raised %s" % (name, type(e).__name__))+ return+ FAIL.append("%s: did not raise" % name)+++class StubTok:+ mask_token = "[MASK]"+ mask_token_id, cls_token_id, sep_token_id, pad_token_id = 3, 1, 2, 0++ def __init__(self):+ self._vocab = {}++ def __call__(self, text, add_special_tokens=True, truncation=False, max_length=None):+ ids = [self._vocab.setdefault(w, len(self._vocab) + 10) for w in str(text).split()]+ return {"input_ids": ids[:max_length] if (truncation and max_length) else ids}+++# ------------------------------------------------------------------ option count+check("count/choice dict", _option_count({"type": "choice", "criteria": {"a": None, "b": None}}), 2)+check("count/choice list", _option_count({"type": "choice", "criteria": ["a", "b", "c"]}), 3)+check("count/score levels", _option_count({"type": "score", "criteria": ["lo", "mid", "hi"]}), 3)+check("count/noul is always two", _option_count({"type": "noul"}), 2)++# ------------------------------------------------------------------ validation+def _q(**extra):+ q = {"type": "choice", "instructions": "which team",+ "criteria": {"refund": "money", "tech": "a bug", "sales": "pricing"}}+ q.update(extra)+ return q+++Agent._check_question("q", _q()) # no option_order is fine+Agent._check_question("q", _q(option_order=[2, 0, 1])) # a permutation is fine+for bad, why in (([0, 1], "too short"), ([0, 1, 2, 0], "too long"), ([0, 1, 1], "repeats"),+ ([0, 1, 3], "out of range"), ("abc", "not a list"), ([0, 1, True], "bool")):+ check_raises("validate/rejects %s" % why, ValueError,+ lambda b=bad: Agent._check_question("q", _q(option_order=b)))++# ------------------------------------------------------------------ carried into the internal form+check("internal/absent stays absent", "option_order" in Agent._to_internal(_q()), False)+check("internal/carried through", Agent._to_internal(_q(option_order=[2, 0, 1]))["option_order"], [2, 0, 1])++# ------------------------------------------------------------------ reaches the encoder+tok = StubTok()+qi = {"t": "choice", "ins": "which team",+ "crit": {"refund": "money back", "tech": "a bug", "sales": "pricing questions"}}+canonical, m_canon = build_sequence(tok, {"x": "hello"}, qi, 512, 192)+reversed_, m_rev = build_sequence(tok, {"x": "hello"}, qi, 512, 192, option_order=[2, 1, 0])+check_true("encode/a different order encodes differently", canonical != reversed_)+check("encode/same option count either way", (len(m_canon), len(m_rev)), (3, 3))+check_true("encode/identity order matches no order at all",+ build_sequence(tok, {"x": "hello"}, qi, 512, 192, option_order=[0, 1, 2])[0] == canonical)++# ------------------------------------------------------------------ undone on the way back+# One logit row, deliberately lopsided, decoded under several presentations. Whatever order the+# slots were in, the probabilities must come back keyed in the caller's own option order.+decoder = Agent.__new__(Agent)+decoder.temperature = {0: 1.0, 1: 1.0, 2: 1.0}+decoder.temperature_by_options = {}+decoder.lang_temperatures = {}++CHOICE = {"t": "choice", "crit": {"a": None, "b": None, "c": None}}+SLOT_P = [0.7, 0.2, 0.1] # slot 0 wins, by construction+logits = np.log([SLOT_P])+act = np.array([[0.25, 0.75]])+items = [{"markers": [0, 1, 2]}]++for order, expect_winner in (([0, 1, 2], "a"), ([2, 1, 0], "c"), ([1, 2, 0], "b")):+ q = dict(CHOICE, option_order=order)+ ans = decoder._decode_answers(logits, act, items, ["q"], {"q": q}, 0)["q"]+ check("decode/keys stay in the caller's order %s" % (order,), list(ans["probabilities"]), ["a", "b", "c"])+ # slot 0 held option `order[0]`, so that option must carry slot 0's mass+ check("decode/slot 0 mass lands on option %s" % expect_winner, ans["choice"], expect_winner)+ check("decode/mass follows the slot for %s" % (order,),+ round(ans["probabilities"][expect_winner], 4), round(SLOT_P[0], 4))+ check("decode/probabilities still sum to one %s" % (order,),+ round(sum(ans["probabilities"].values()), 4), 1.0)++no_order = decoder._decode_answers(logits, act, items, ["q"], {"q": dict(CHOICE)}, 0)["q"]+identity = decoder._decode_answers(logits, act, items, ["q"], {"q": dict(CHOICE, option_order=[0, 1, 2])}, 0)["q"]+check("decode/identity order changes nothing", identity["probabilities"], no_order["probabilities"])++# A score level is an ordinal index, so the expected value has to be computed after un-permuting.+SCORE = {"t": "score", "crit": ["low", "mid", "high"]}+plain = decoder._decode_answers(logits, act, items, ["s"], {"s": dict(SCORE)}, 0)["s"]+flipped = decoder._decode_answers(logits, act, items, ["s"], {"s": dict(SCORE, option_order=[2, 1, 0])}, 0)["s"]+check("decode/score keeps its level legend", list(flipped["probabilities"]), ["0", "1", "2"])+check("decode/score mass moves to the level that held slot 0",+ round(flipped["probabilities"]["2"], 4), round(SLOT_P[0], 4))+check_true("decode/score expected value follows the levels, not the slots",+ flipped["score"] > plain["score"], "%.4f vs %.4f" % (flipped["score"], plain["score"]))++# noul: `p[1]` is P(true) in option order, so a flipped presentation must not invert the answer.+NOUL = {"t": "noul", "crit": None}+n_items = [{"markers": [0, 1]}]+n_logits = np.log([[0.7, 0.3]])+straight = decoder._decode_answers(n_logits, act, n_items, ["n"], {"n": dict(NOUL)}, 0)["n"]+swapped = decoder._decode_answers(n_logits, act, n_items, ["n"], {"n": dict(NOUL, option_order=[1, 0])}, 0)["n"]+check("decode/noul canonical P(true)", round(straight["noul"], 4), 0.3)+check("decode/noul swapped slots report P(true), not P(slot 1)", round(swapped["noul"], 4), 0.7)++print("\n%d passed, %d failed" % (len(PASS), len(FAIL)))+for f in FAIL:+ print(" FAIL", f)+if not FAIL:+ print("all option-order tests passed")+sys.exit(1 if FAIL else 0)