Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
feat(onnx): honor option_order on the ONNX runtime, and document the key
sucf4dc85e82a704
diff --git a/README.md b/README.mdindex 4aadb0c4..5fd635e5 100644--- a/README.md+++ b/README.md@@ -893,6 +893,47 @@ whose values must be distinct non-empty strings. Mapping order does not matter,`noul` value is still P(true). Label sensitivity varies by checkpoint and state, so validate anyoverride on your own data rather than treating `A`/`B` as a universal fix.+### Option order++Every question type also takes an optional `option_order`: a permutation of the option indices+saying which option goes in which slot. Slot `s` shows option `option_order[s]`. Probabilities+always come back keyed in your own option order, whatever order the model saw them in, so the+key is presentation only — it never changes what a returned label means.++```python+question = {+ "type": "choice",+ "instructions": "Which team should handle this?",+ "criteria": {"billing": "...", "technical": "...", "sales": "..."},+ "option_order": [2, 0, 1], # slot 0 shows `sales`, slot 1 `billing`, slot 2 `technical`+}+```++It exists because where an option sits changes the answer. `research/eval/presentation_checks.py`+measures the English checkpoint on five options whose text is *identical*: the per-slot logits+centre at `[+1.68, +1.47, +0.11, -1.35, -1.91]`, a spread decided by position alone, and the+multilingual checkpoint leans the other way on `score` (see [#131](https://github.com/NandhaKishorM/laya/issues/131)).+The checkpoint-side fix is a position-balanced retrain; until then `option_order` lets a caller+average the effect out, by asking the same question under orders in which every option occupies+every slot equally often:++```python+k = len(question["criteria"])+totals = {label: 0.0 for label in question["criteria"]}+for r in range(k): # k cyclic rotations+ rotated = dict(question, option_order=[(i + r) % k for i in range(k)])+ answer = agent.predict(state, {"q": rotated})["answers"]["q"]+ for label, value in answer["probabilities"].items():+ totals[label] += value / k++decision = max(totals, key=totals.get)+```++That costs `k` rows in the same forward pass rather than `k` forward passes. On 62 banking+intents shortlisted to 20 over 49 states, it took the share of answers that change with+presentation order from 16.3% to 6.1%. Omitting `option_order` keeps the canonical order and the+behaviour every existing caller already has.+---## MCP Server (Optional)diff --git a/laya/agent.py b/laya/agent.pyindex 17c5f8e4..f3f21c46 100644--- a/laya/agent.py+++ b/laya/agent.py@@ -31,6 +31,7 @@ from .common import (render_options,serialize_state,temp_bucket,+ unpermute_probs,)from .confidence import check_min_confidence, flag_low_confidencefrom .hooks import (@@ -909,15 +910,8 @@ class Agent(HookRegistry):p = np.exp(z - z.max())p = p / p.sum()- # Slot s scored option `order[s]`, so the row comes back in slot order. Everything- # below indexes by the caller's own option order -- `zip(keys, p)` for a choice,- # `arange(k) * p` for a score level, `p[1]` for noul-true -- so put it back first.- # Without this the probabilities would be attached to the wrong labels.- order = q.get("option_order")- if order is not None and len(order) == k:- canonical = np.empty_like(p)- canonical[np.asarray(order, dtype=int)] = p- p = canonical+ # The row comes back in slot order; everything below indexes by option.+ p = unpermute_probs(p, q.get("option_order"))# `confidence` means one thing for `noul` (max(p)) and another for `choice` and# `score` (normalized entropy), and only the first is the quantity temperaturediff --git a/laya/common.py b/laya/common.pyindex e5339320..eb225bff 100644--- a/laya/common.py+++ b/laya/common.py@@ -275,6 +275,23 @@ class _DynamicMultiheadAttention(nn.MultiheadAttention):return self.out_proj(attn), None+def unpermute_probs(p: np.ndarray, option_order: Optional[List[int]]) -> np.ndarray:+ """Put a slot-ordered probability row back into the caller's option order.++ `build_sequence` puts option `option_order[s]` in slot `s`, so a model row comes back+ indexed by slot. Everything downstream indexes by option -- `zip(keys, p)` for a choice,+ `arange(k) * p` for a score level, `p[1]` for noul-true -- so the row has to be inverted+ first or the probabilities end up attached to the wrong options, which is silent.++ A missing or mismatched order returns `p` untouched, so the canonical path is unaffected.+ """+ if option_order is None or len(option_order) != len(p):+ return p+ canonical = np.empty_like(p)+ canonical[np.asarray(option_order, dtype=int)] = p+ return canonical++class DecisionModel(nn.Module):"""Bidirectional transformer encoder backbone + typed decision head."""diff --git a/laya/onnx_agent.py b/laya/onnx_agent.pyindex c31778a9..105fdf95 100644--- a/laya/onnx_agent.py+++ b/laya/onnx_agent.py@@ -23,6 +23,7 @@ from laya.common import (render_options,serialize_state,temp_bucket,+ unpermute_probs,TEMP_MIN,TEMP_MAX,clamp_temperature,@@ -228,6 +229,8 @@ class ONNXAgent(HookRegistry):q = {"t": t, "ins": ins, "crit": crit}if "labels" in qdef:q["labels"] = qdef["labels"]+ if "option_order" in qdef:+ q["option_order"] = [int(i) for i in qdef["option_order"]]return qdef system_one(self, state: Union[str, dict, list], questions: Dict[str, Dict[str, Any]],@@ -542,6 +545,7 @@ class ONNXAgent(HookRegistry):q = internal[qid]seq, markers, stats = build_sequence(self.tok, state, q, max_len, head_max_len,+ option_order=q.get("option_order"),truncate_left=truncate_left, state_ids=state_ids, return_stats=True,)n_opts = len(render_options(q))@@ -577,6 +581,9 @@ class ONNXAgent(HookRegistry):p = np.exp(z - z.max())p = p / p.sum()+ # The row comes back in slot order; everything below indexes by option.+ p = unpermute_probs(p, q.get("option_order"))+conf_score = round(confidence_from_probs(p, k), 4)# `answer_confidence` is the calibrated max(p) confidence, reported on every question# type so a caller can gate across types on one number -- matching the PyTorch Agent,diff --git a/tests/test_option_order.py b/tests/test_option_order.pyindex 223f6142..50180103 100644--- a/tests/test_option_order.py+++ b/tests/test_option_order.py@@ -18,8 +18,12 @@ choice, `arange(k) * p` for a score, `p[1]` for noul-true. Without the inverse pprobabilities come back attached to the wrong labels, which is silent and worse than ignoringthe argument. These tests pin both halves.-No weights are loaded: the encoder is driven with a stub tokenizer and the decoder through-`Agent.__new__`, as in tests/test_batch.py.+Both runtimes are covered. `ONNXAgent` validates with the same `_check_question`, so it accepts+`option_order` too and has to honour it identically -- otherwise the argument is accepted and+ignored on one of the two backends, which is the same silent failure this change removes.++No weights are loaded: the encoder is driven with a stub tokenizer, and both decoders through+`__new__` with a stub session, as in tests/test_batch.py.Run: python tests/test_option_order.py"""@@ -31,7 +35,8 @@ import numpy as npsys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))from laya.agent import Agent, _option_count # noqa: E402-from laya.common import build_sequence # noqa: E402+from laya.common import build_sequence, unpermute_probs # noqa: E402+from laya.onnx_agent import ONNXAgent # noqa: E402PASS, FAIL = [], []@@ -158,6 +163,70 @@ swapped = decoder._decode_answers(n_logits, act, n_items, ["n"], {"n": dict(NOULcheck("decode/noul canonical P(true)", round(straight["noul"], 4), 0.3)check("decode/noul swapped slots report P(true), not P(slot 1)", round(swapped["noul"], 4), 0.7)+# ------------------------------------------------------------------ the shared helper+check("helper/no order is a no-op", list(unpermute_probs(np.array([0.5, 0.3, 0.2]), None)), [0.5, 0.3, 0.2])+check("helper/identity is a no-op", list(unpermute_probs(np.array([0.5, 0.3, 0.2]), [0, 1, 2])), [0.5, 0.3, 0.2])+check("helper/slot s lands on option order[s]",+ list(unpermute_probs(np.array([0.5, 0.3, 0.2]), [2, 0, 1])), [0.3, 0.2, 0.5])+check("helper/a mismatched length is left alone",+ list(unpermute_probs(np.array([0.5, 0.5]), [0, 1, 2])), [0.5, 0.5])+check_true("helper/round-trips for every permutation of 4",+ all(list(unpermute_probs(np.array([0.4, 0.3, 0.2, 0.1]), list(o))) ==+ [dict(zip(o, [0.4, 0.3, 0.2, 0.1]))[i] for i in range(4)]+ for o in __import__("itertools").permutations(range(4))))++# ------------------------------------------------------------------ the ONNX backend agrees+# ONNXAgent validates with the same `_check_question`, so it accepts `option_order`; it has to+# act on it too. A stub session returns a fixed logit row, so the two backends can be compared+# on identical numbers without onnxruntime or a model file.+onnx = ONNXAgent.__new__(ONNXAgent)+onnx.tok = StubTok()+onnx.cfg = {"max_len": 512, "head_max_len": 192}+onnx.temperature = {0: 1.0, 1: 1.0, 2: 1.0}+onnx.temperature_by_options = {}+onnx.model_id = "stub"++ONNX_P = [0.7, 0.2, 0.1]+++class StubSession:+ def run(self, names, feeds):+ rows = feeds["input_ids"].shape[0]+ return [np.log(np.array([ONNX_P] * rows)), np.zeros((rows, 2))]+++onnx.session = StubSession()++onnx_q = {"type": "choice", "instructions": "which team",+ "criteria": {"a": "first", "b": "second", "c": "third"}}+check("onnx/carries option_order into the internal form",+ ONNXAgent._to_internal(dict(onnx_q, option_order=[2, 0, 1]))["option_order"], [2, 0, 1])+check("onnx/omits it when absent", "option_order" in ONNXAgent._to_internal(dict(onnx_q)), False)++for order, winner in (([0, 1, 2], "a"), ([2, 1, 0], "c"), ([1, 2, 0], "b")):+ ans = onnx._infer({"x": "hello"}, {"q": dict(onnx_q, option_order=order)})["answers"]["q"]+ check("onnx/keys stay in the caller's order %s" % (order,), list(ans["probabilities"]), ["a", "b", "c"])+ check("onnx/slot 0 mass lands on %s" % winner, ans["choice"], winner)+ check("onnx/mass follows the slot %s" % (order,),+ round(ans["probabilities"][winner], 4), round(ONNX_P[0], 4))++plain_onnx = onnx._infer({"x": "hello"}, {"q": dict(onnx_q)})["answers"]["q"]+ident_onnx = onnx._infer({"x": "hello"}, {"q": dict(onnx_q, option_order=[0, 1, 2])})["answers"]["q"]+check("onnx/identity order changes nothing", ident_onnx["probabilities"], plain_onnx["probabilities"])+check_raises("onnx/rejects a bad order the same way", ValueError,+ lambda: onnx._infer({"x": "hi"}, {"q": dict(onnx_q, option_order=[0, 1])}))++# The two backends must not disagree about what an order means.+torch_side = decoder._decode_answers(np.log([ONNX_P]), np.zeros((1, 2)),+ [{"markers": [0, 1, 2]}], ["q"],+ {"q": {"t": "choice", "crit": {"a": None, "b": None, "c": None},+ "option_order": [2, 0, 1]}}, 0)["q"]+onnx_side = onnx._infer({"x": "hello"}, {"q": dict(onnx_q, option_order=[2, 0, 1])})["answers"]["q"]+check("backends/torch and onnx agree on the permuted answer",+ (torch_side["choice"], torch_side["probabilities"]),+ (onnx_side["choice"], onnx_side["probabilities"]))++print("\n%d passed, %d failed" % (len(PASS), len(FAIL)))for f in FAIL:print(" FAIL", f)