Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

feat(onnx): honor option_order on the ONNX runtime, and document the key

sucf4dc85e82a704
diff --git a/README.md b/README.md
index 4aadb0c4..5fd635e5 100644
--- a/README.md
+++ b/README.md
@@ -893,6 +893,47 @@ whose values must be distinct non-empty strings. Mapping order does not matter,
`noul` value is still P(true). Label sensitivity varies by checkpoint and state, so validate any
override on your own data rather than treating `A`/`B` as a universal fix.
+### Option order
+
+Every question type also takes an optional `option_order`: a permutation of the option indices
+saying which option goes in which slot. Slot `s` shows option `option_order[s]`. Probabilities
+always come back keyed in your own option order, whatever order the model saw them in, so the
+key is presentation only — it never changes what a returned label means.
+
+```python
+question = {
+ "type": "choice",
+ "instructions": "Which team should handle this?",
+ "criteria": {"billing": "...", "technical": "...", "sales": "..."},
+ "option_order": [2, 0, 1], # slot 0 shows `sales`, slot 1 `billing`, slot 2 `technical`
+}
+```
+
+It exists because where an option sits changes the answer. `research/eval/presentation_checks.py`
+measures the English checkpoint on five options whose text is *identical*: the per-slot logits
+centre at `[+1.68, +1.47, +0.11, -1.35, -1.91]`, a spread decided by position alone, and the
+multilingual checkpoint leans the other way on `score` (see [#131](https://github.com/NandhaKishorM/laya/issues/131)).
+The checkpoint-side fix is a position-balanced retrain; until then `option_order` lets a caller
+average the effect out, by asking the same question under orders in which every option occupies
+every slot equally often:
+
+```python
+k = len(question["criteria"])
+totals = {label: 0.0 for label in question["criteria"]}
+for r in range(k): # k cyclic rotations
+ rotated = dict(question, option_order=[(i + r) % k for i in range(k)])
+ answer = agent.predict(state, {"q": rotated})["answers"]["q"]
+ for label, value in answer["probabilities"].items():
+ totals[label] += value / k
+
+decision = max(totals, key=totals.get)
+```
+
+That costs `k` rows in the same forward pass rather than `k` forward passes. On 62 banking
+intents shortlisted to 20 over 49 states, it took the share of answers that change with
+presentation order from 16.3% to 6.1%. Omitting `option_order` keeps the canonical order and the
+behaviour every existing caller already has.
+
---
## MCP Server (Optional)
diff --git a/laya/agent.py b/laya/agent.py
index 17c5f8e4..f3f21c46 100644
--- a/laya/agent.py
+++ b/laya/agent.py
@@ -31,6 +31,7 @@ from .common import (
render_options,
serialize_state,
temp_bucket,
+ unpermute_probs,
)
from .confidence import check_min_confidence, flag_low_confidence
from .hooks import (
@@ -909,15 +910,8 @@ class Agent(HookRegistry):
p = np.exp(z - z.max())
p = p / p.sum()
- # Slot s scored option `order[s]`, so the row comes back in slot order. Everything
- # below indexes by the caller's own option order -- `zip(keys, p)` for a choice,
- # `arange(k) * p` for a score level, `p[1]` for noul-true -- so put it back first.
- # Without this the probabilities would be attached to the wrong labels.
- order = q.get("option_order")
- if order is not None and len(order) == k:
- canonical = np.empty_like(p)
- canonical[np.asarray(order, dtype=int)] = p
- p = canonical
+ # The row comes back in slot order; everything below indexes by option.
+ p = unpermute_probs(p, q.get("option_order"))
# `confidence` means one thing for `noul` (max(p)) and another for `choice` and
# `score` (normalized entropy), and only the first is the quantity temperature
diff --git a/laya/common.py b/laya/common.py
index e5339320..eb225bff 100644
--- a/laya/common.py
+++ b/laya/common.py
@@ -275,6 +275,23 @@ class _DynamicMultiheadAttention(nn.MultiheadAttention):
return self.out_proj(attn), None
+def unpermute_probs(p: np.ndarray, option_order: Optional[List[int]]) -> np.ndarray:
+ """Put a slot-ordered probability row back into the caller's option order.
+
+ `build_sequence` puts option `option_order[s]` in slot `s`, so a model row comes back
+ indexed by slot. Everything downstream indexes by option -- `zip(keys, p)` for a choice,
+ `arange(k) * p` for a score level, `p[1]` for noul-true -- so the row has to be inverted
+ first or the probabilities end up attached to the wrong options, which is silent.
+
+ A missing or mismatched order returns `p` untouched, so the canonical path is unaffected.
+ """
+ if option_order is None or len(option_order) != len(p):
+ return p
+ canonical = np.empty_like(p)
+ canonical[np.asarray(option_order, dtype=int)] = p
+ return canonical
+
+
class DecisionModel(nn.Module):
"""Bidirectional transformer encoder backbone + typed decision head."""
diff --git a/laya/onnx_agent.py b/laya/onnx_agent.py
index c31778a9..105fdf95 100644
--- a/laya/onnx_agent.py
+++ b/laya/onnx_agent.py
@@ -23,6 +23,7 @@ from laya.common import (
render_options,
serialize_state,
temp_bucket,
+ unpermute_probs,
TEMP_MIN,
TEMP_MAX,
clamp_temperature,
@@ -228,6 +229,8 @@ class ONNXAgent(HookRegistry):
q = {"t": t, "ins": ins, "crit": crit}
if "labels" in qdef:
q["labels"] = qdef["labels"]
+ if "option_order" in qdef:
+ q["option_order"] = [int(i) for i in qdef["option_order"]]
return q
def system_one(self, state: Union[str, dict, list], questions: Dict[str, Dict[str, Any]],
@@ -542,6 +545,7 @@ class ONNXAgent(HookRegistry):
q = internal[qid]
seq, markers, stats = build_sequence(
self.tok, state, q, max_len, head_max_len,
+ option_order=q.get("option_order"),
truncate_left=truncate_left, state_ids=state_ids, return_stats=True,
)
n_opts = len(render_options(q))
@@ -577,6 +581,9 @@ class ONNXAgent(HookRegistry):
p = np.exp(z - z.max())
p = p / p.sum()
+ # The row comes back in slot order; everything below indexes by option.
+ p = unpermute_probs(p, q.get("option_order"))
+
conf_score = round(confidence_from_probs(p, k), 4)
# `answer_confidence` is the calibrated max(p) confidence, reported on every question
# type so a caller can gate across types on one number -- matching the PyTorch Agent,
diff --git a/tests/test_option_order.py b/tests/test_option_order.py
index 223f6142..50180103 100644
--- a/tests/test_option_order.py
+++ b/tests/test_option_order.py
@@ -18,8 +18,12 @@ choice, `arange(k) * p` for a score, `p[1]` for noul-true. Without the inverse p
probabilities come back attached to the wrong labels, which is silent and worse than ignoring
the argument. These tests pin both halves.
-No weights are loaded: the encoder is driven with a stub tokenizer and the decoder through
-`Agent.__new__`, as in tests/test_batch.py.
+Both runtimes are covered. `ONNXAgent` validates with the same `_check_question`, so it accepts
+`option_order` too and has to honour it identically -- otherwise the argument is accepted and
+ignored on one of the two backends, which is the same silent failure this change removes.
+
+No weights are loaded: the encoder is driven with a stub tokenizer, and both decoders through
+`__new__` with a stub session, as in tests/test_batch.py.
Run: python tests/test_option_order.py
"""
@@ -31,7 +35,8 @@ import numpy as np
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from laya.agent import Agent, _option_count # noqa: E402
-from laya.common import build_sequence # noqa: E402
+from laya.common import build_sequence, unpermute_probs # noqa: E402
+from laya.onnx_agent import ONNXAgent # noqa: E402
PASS, FAIL = [], []
@@ -158,6 +163,70 @@ swapped = decoder._decode_answers(n_logits, act, n_items, ["n"], {"n": dict(NOUL
check("decode/noul canonical P(true)", round(straight["noul"], 4), 0.3)
check("decode/noul swapped slots report P(true), not P(slot 1)", round(swapped["noul"], 4), 0.7)
+# ------------------------------------------------------------------ the shared helper
+check("helper/no order is a no-op", list(unpermute_probs(np.array([0.5, 0.3, 0.2]), None)), [0.5, 0.3, 0.2])
+check("helper/identity is a no-op", list(unpermute_probs(np.array([0.5, 0.3, 0.2]), [0, 1, 2])), [0.5, 0.3, 0.2])
+check("helper/slot s lands on option order[s]",
+ list(unpermute_probs(np.array([0.5, 0.3, 0.2]), [2, 0, 1])), [0.3, 0.2, 0.5])
+check("helper/a mismatched length is left alone",
+ list(unpermute_probs(np.array([0.5, 0.5]), [0, 1, 2])), [0.5, 0.5])
+check_true("helper/round-trips for every permutation of 4",
+ all(list(unpermute_probs(np.array([0.4, 0.3, 0.2, 0.1]), list(o))) ==
+ [dict(zip(o, [0.4, 0.3, 0.2, 0.1]))[i] for i in range(4)]
+ for o in __import__("itertools").permutations(range(4))))
+
+# ------------------------------------------------------------------ the ONNX backend agrees
+# ONNXAgent validates with the same `_check_question`, so it accepts `option_order`; it has to
+# act on it too. A stub session returns a fixed logit row, so the two backends can be compared
+# on identical numbers without onnxruntime or a model file.
+onnx = ONNXAgent.__new__(ONNXAgent)
+onnx.tok = StubTok()
+onnx.cfg = {"max_len": 512, "head_max_len": 192}
+onnx.temperature = {0: 1.0, 1: 1.0, 2: 1.0}
+onnx.temperature_by_options = {}
+onnx.model_id = "stub"
+
+ONNX_P = [0.7, 0.2, 0.1]
+
+
+class StubSession:
+ def run(self, names, feeds):
+ rows = feeds["input_ids"].shape[0]
+ return [np.log(np.array([ONNX_P] * rows)), np.zeros((rows, 2))]
+
+
+onnx.session = StubSession()
+
+onnx_q = {"type": "choice", "instructions": "which team",
+ "criteria": {"a": "first", "b": "second", "c": "third"}}
+check("onnx/carries option_order into the internal form",
+ ONNXAgent._to_internal(dict(onnx_q, option_order=[2, 0, 1]))["option_order"], [2, 0, 1])
+check("onnx/omits it when absent", "option_order" in ONNXAgent._to_internal(dict(onnx_q)), False)
+
+for order, winner in (([0, 1, 2], "a"), ([2, 1, 0], "c"), ([1, 2, 0], "b")):
+ ans = onnx._infer({"x": "hello"}, {"q": dict(onnx_q, option_order=order)})["answers"]["q"]
+ check("onnx/keys stay in the caller's order %s" % (order,), list(ans["probabilities"]), ["a", "b", "c"])
+ check("onnx/slot 0 mass lands on %s" % winner, ans["choice"], winner)
+ check("onnx/mass follows the slot %s" % (order,),
+ round(ans["probabilities"][winner], 4), round(ONNX_P[0], 4))
+
+plain_onnx = onnx._infer({"x": "hello"}, {"q": dict(onnx_q)})["answers"]["q"]
+ident_onnx = onnx._infer({"x": "hello"}, {"q": dict(onnx_q, option_order=[0, 1, 2])})["answers"]["q"]
+check("onnx/identity order changes nothing", ident_onnx["probabilities"], plain_onnx["probabilities"])
+check_raises("onnx/rejects a bad order the same way", ValueError,
+ lambda: onnx._infer({"x": "hi"}, {"q": dict(onnx_q, option_order=[0, 1])}))
+
+# The two backends must not disagree about what an order means.
+torch_side = decoder._decode_answers(np.log([ONNX_P]), np.zeros((1, 2)),
+ [{"markers": [0, 1, 2]}], ["q"],
+ {"q": {"t": "choice", "crit": {"a": None, "b": None, "c": None},
+ "option_order": [2, 0, 1]}}, 0)["q"]
+onnx_side = onnx._infer({"x": "hello"}, {"q": dict(onnx_q, option_order=[2, 0, 1])})["answers"]["q"]
+check("backends/torch and onnx agree on the permuted answer",
+ (torch_side["choice"], torch_side["probabilities"]),
+ (onnx_side["choice"], onnx_side["probabilities"]))
+
+
print("\n%d passed, %d failed" % (len(PASS), len(FAIL)))
for f in FAIL:
print(" FAIL", f)