Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

feat(agent): let a question choose the order its options are shown in

sucf7b4b3f9d0382
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index d4f38072..f5686f93 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -102,6 +102,7 @@ jobs:
python tests/test_batch.py
python tests/test_predict_long.py
python tests/test_attention_dynamic_shapes.py
+ python tests/test_option_order.py
python tests/test_hooks.py
python tests/test_hooks_api.py
python tests/test_structured.py
diff --git a/laya/agent.py b/laya/agent.py
index 5ca3861c..17c5f8e4 100644
--- a/laya/agent.py
+++ b/laya/agent.py
@@ -220,6 +220,17 @@ def _cuda_amp_dtype(checkpoint_default: Optional[str]) -> torch.dtype:
return torch.bfloat16
return amp_dtype(checkpoint_default)
+def _option_count(qdef: Dict) -> int:
+ """How many options a validated question definition renders to.
+
+ Mirrors `render_options`: a choice has one option per criterion, a score one per level,
+ and a noul is always the pair [false, true].
+ """
+ if qdef.get("type") == "noul":
+ return 2
+ crit = qdef.get("criteria")
+ return len(crit) if isinstance(crit, (dict, list, tuple)) else 0
+
def _start_evidence():
"""A recorder for `predict_long`: what the start-hook chain left for inference to run on.
@@ -719,6 +730,18 @@ class Agent(HookRegistry):
"reads; any other key was silently dropped and replaced with the defaults. If you "
"want the answer worded differently, keep 'criteria' keyed 'true'/'false' and set "
"'labels' instead." % (qid, sorted(keys)))
+ if "option_order" in qdef:
+ # Slot s shows option `order[s]`. Anything other than a permutation of the option
+ # indices would either drop an option or show one twice, so reject it here rather
+ # than let it reach the encoder.
+ order = qdef["option_order"]
+ n = _option_count(qdef)
+ if (not isinstance(order, (list, tuple))
+ or len(order) != n
+ or sorted(int(i) for i in order if isinstance(i, int) and not isinstance(i, bool)) != list(range(n))):
+ raise ValueError(
+ "question %r: 'option_order' must be a permutation of range(%d) -- one slot per "
+ "option, each option once -- got %r" % (qid, n, order))
if "labels" in qdef:
if t != "noul":
raise ValueError("question %r: 'labels' is only supported for noul questions" % (qid,))
@@ -747,6 +770,8 @@ class Agent(HookRegistry):
q = {"t": t, "ins": ins, "crit": crit}
if "labels" in qdef:
q["labels"] = qdef["labels"]
+ if "option_order" in qdef:
+ q["option_order"] = [int(i) for i in qdef["option_order"]]
return q
def _encode_state(self, state: Union[str, dict, list], ids: List[str], internal: Dict[str, Dict],
@@ -775,6 +800,7 @@ class Agent(HookRegistry):
for qid in ids:
q = internal[qid]
seq, markers, stats = build_sequence(self.tok, state, q, max_len, head_max_len,
+ option_order=q.get("option_order"),
truncate_left=truncate_left, state_ids=state_ids,
return_stats=True)
if len(markers) != len(render_options(q)):
@@ -883,6 +909,16 @@ class Agent(HookRegistry):
p = np.exp(z - z.max())
p = p / p.sum()
+ # Slot s scored option `order[s]`, so the row comes back in slot order. Everything
+ # below indexes by the caller's own option order -- `zip(keys, p)` for a choice,
+ # `arange(k) * p` for a score level, `p[1]` for noul-true -- so put it back first.
+ # Without this the probabilities would be attached to the wrong labels.
+ order = q.get("option_order")
+ if order is not None and len(order) == k:
+ canonical = np.empty_like(p)
+ canonical[np.asarray(order, dtype=int)] = p
+ p = canonical
+
# `confidence` means one thing for `noul` (max(p)) and another for `choice` and
# `score` (normalized entropy), and only the first is the quantity temperature
# scaling fits and ECE measures. Rather than change one underneath existing
diff --git a/tests/test_option_order.py b/tests/test_option_order.py
new file mode 100644
index 00000000..223f6142
--- /dev/null
+++ b/tests/test_option_order.py
@@ -0,0 +1,166 @@
+"""`option_order` must reach the encoder and be undone on the way back.
+
+`build_sequence` has always taken an `option_order` -- slot `s` shows option `order[s]` -- and
+training used it to shuffle. Nothing in the inference path ever passed it, so a question dict
+carrying one was silently ignored: `_check_question` does not reject unknown keys and
+`_to_internal` dropped it, so every call showed the canonical order.
+
+That matters because where an option sits changes the answer. `research/results/
+presentation_checks_shipped.json` measures the English checkpoint on five options whose text is
+*identical*: the per-slot logits centre at [+1.68, +1.47, +0.11, -1.35, -1.91], a 3.59 spread
+decided by position alone. `research/eval/metamorphic.py` and the `option_order_flip_rate` in
+`research/scripts/bench_apps.py` exist for the same reason. Until now a caller had no way to
+present the options any differently and see what changed.
+
+The half that is easy to get wrong is the way back. The logit row comes out in *slot* order,
+while everything downstream indexes by the caller's option order -- `zip(keys, p)` for a
+choice, `arange(k) * p` for a score, `p[1]` for noul-true. Without the inverse permutation the
+probabilities come back attached to the wrong labels, which is silent and worse than ignoring
+the argument. These tests pin both halves.
+
+No weights are loaded: the encoder is driven with a stub tokenizer and the decoder through
+`Agent.__new__`, as in tests/test_batch.py.
+
+Run: python tests/test_option_order.py
+"""
+import os
+import sys
+
+import numpy as np
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from laya.agent import Agent, _option_count # noqa: E402
+from laya.common import build_sequence # noqa: E402
+
+PASS, FAIL = [], []
+
+
+def check(name, got, want):
+ if got == want:
+ PASS.append(name)
+ else:
+ FAIL.append("%s: got %r, want %r" % (name, got, want))
+
+
+def check_true(name, cond, detail=""):
+ if cond:
+ PASS.append(name)
+ else:
+ FAIL.append("%s %s" % (name, detail))
+
+
+def check_raises(name, exc, fn):
+ try:
+ fn()
+ except exc:
+ PASS.append(name)
+ return
+ except Exception as e: # noqa: BLE001 - the point is the wrong type
+ FAIL.append("%s: raised %s" % (name, type(e).__name__))
+ return
+ FAIL.append("%s: did not raise" % name)
+
+
+class StubTok:
+ mask_token = "[MASK]"
+ mask_token_id, cls_token_id, sep_token_id, pad_token_id = 3, 1, 2, 0
+
+ def __init__(self):
+ self._vocab = {}
+
+ def __call__(self, text, add_special_tokens=True, truncation=False, max_length=None):
+ ids = [self._vocab.setdefault(w, len(self._vocab) + 10) for w in str(text).split()]
+ return {"input_ids": ids[:max_length] if (truncation and max_length) else ids}
+
+
+# ------------------------------------------------------------------ option count
+check("count/choice dict", _option_count({"type": "choice", "criteria": {"a": None, "b": None}}), 2)
+check("count/choice list", _option_count({"type": "choice", "criteria": ["a", "b", "c"]}), 3)
+check("count/score levels", _option_count({"type": "score", "criteria": ["lo", "mid", "hi"]}), 3)
+check("count/noul is always two", _option_count({"type": "noul"}), 2)
+
+# ------------------------------------------------------------------ validation
+def _q(**extra):
+ q = {"type": "choice", "instructions": "which team",
+ "criteria": {"refund": "money", "tech": "a bug", "sales": "pricing"}}
+ q.update(extra)
+ return q
+
+
+Agent._check_question("q", _q()) # no option_order is fine
+Agent._check_question("q", _q(option_order=[2, 0, 1])) # a permutation is fine
+for bad, why in (([0, 1], "too short"), ([0, 1, 2, 0], "too long"), ([0, 1, 1], "repeats"),
+ ([0, 1, 3], "out of range"), ("abc", "not a list"), ([0, 1, True], "bool")):
+ check_raises("validate/rejects %s" % why, ValueError,
+ lambda b=bad: Agent._check_question("q", _q(option_order=b)))
+
+# ------------------------------------------------------------------ carried into the internal form
+check("internal/absent stays absent", "option_order" in Agent._to_internal(_q()), False)
+check("internal/carried through", Agent._to_internal(_q(option_order=[2, 0, 1]))["option_order"], [2, 0, 1])
+
+# ------------------------------------------------------------------ reaches the encoder
+tok = StubTok()
+qi = {"t": "choice", "ins": "which team",
+ "crit": {"refund": "money back", "tech": "a bug", "sales": "pricing questions"}}
+canonical, m_canon = build_sequence(tok, {"x": "hello"}, qi, 512, 192)
+reversed_, m_rev = build_sequence(tok, {"x": "hello"}, qi, 512, 192, option_order=[2, 1, 0])
+check_true("encode/a different order encodes differently", canonical != reversed_)
+check("encode/same option count either way", (len(m_canon), len(m_rev)), (3, 3))
+check_true("encode/identity order matches no order at all",
+ build_sequence(tok, {"x": "hello"}, qi, 512, 192, option_order=[0, 1, 2])[0] == canonical)
+
+# ------------------------------------------------------------------ undone on the way back
+# One logit row, deliberately lopsided, decoded under several presentations. Whatever order the
+# slots were in, the probabilities must come back keyed in the caller's own option order.
+decoder = Agent.__new__(Agent)
+decoder.temperature = {0: 1.0, 1: 1.0, 2: 1.0}
+decoder.temperature_by_options = {}
+decoder.lang_temperatures = {}
+
+CHOICE = {"t": "choice", "crit": {"a": None, "b": None, "c": None}}
+SLOT_P = [0.7, 0.2, 0.1] # slot 0 wins, by construction
+logits = np.log([SLOT_P])
+act = np.array([[0.25, 0.75]])
+items = [{"markers": [0, 1, 2]}]
+
+for order, expect_winner in (([0, 1, 2], "a"), ([2, 1, 0], "c"), ([1, 2, 0], "b")):
+ q = dict(CHOICE, option_order=order)
+ ans = decoder._decode_answers(logits, act, items, ["q"], {"q": q}, 0)["q"]
+ check("decode/keys stay in the caller's order %s" % (order,), list(ans["probabilities"]), ["a", "b", "c"])
+ # slot 0 held option `order[0]`, so that option must carry slot 0's mass
+ check("decode/slot 0 mass lands on option %s" % expect_winner, ans["choice"], expect_winner)
+ check("decode/mass follows the slot for %s" % (order,),
+ round(ans["probabilities"][expect_winner], 4), round(SLOT_P[0], 4))
+ check("decode/probabilities still sum to one %s" % (order,),
+ round(sum(ans["probabilities"].values()), 4), 1.0)
+
+no_order = decoder._decode_answers(logits, act, items, ["q"], {"q": dict(CHOICE)}, 0)["q"]
+identity = decoder._decode_answers(logits, act, items, ["q"], {"q": dict(CHOICE, option_order=[0, 1, 2])}, 0)["q"]
+check("decode/identity order changes nothing", identity["probabilities"], no_order["probabilities"])
+
+# A score level is an ordinal index, so the expected value has to be computed after un-permuting.
+SCORE = {"t": "score", "crit": ["low", "mid", "high"]}
+plain = decoder._decode_answers(logits, act, items, ["s"], {"s": dict(SCORE)}, 0)["s"]
+flipped = decoder._decode_answers(logits, act, items, ["s"], {"s": dict(SCORE, option_order=[2, 1, 0])}, 0)["s"]
+check("decode/score keeps its level legend", list(flipped["probabilities"]), ["0", "1", "2"])
+check("decode/score mass moves to the level that held slot 0",
+ round(flipped["probabilities"]["2"], 4), round(SLOT_P[0], 4))
+check_true("decode/score expected value follows the levels, not the slots",
+ flipped["score"] > plain["score"], "%.4f vs %.4f" % (flipped["score"], plain["score"]))
+
+# noul: `p[1]` is P(true) in option order, so a flipped presentation must not invert the answer.
+NOUL = {"t": "noul", "crit": None}
+n_items = [{"markers": [0, 1]}]
+n_logits = np.log([[0.7, 0.3]])
+straight = decoder._decode_answers(n_logits, act, n_items, ["n"], {"n": dict(NOUL)}, 0)["n"]
+swapped = decoder._decode_answers(n_logits, act, n_items, ["n"], {"n": dict(NOUL, option_order=[1, 0])}, 0)["n"]
+check("decode/noul canonical P(true)", round(straight["noul"], 4), 0.3)
+check("decode/noul swapped slots report P(true), not P(slot 1)", round(swapped["noul"], 4), 0.7)
+
+print("\n%d passed, %d failed" % (len(PASS), len(FAIL)))
+for f in FAIL:
+ print(" FAIL", f)
+if not FAIL:
+ print("all option-order tests passed")
+sys.exit(1 if FAIL else 0)