Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
fix(agent): a choice criteria list that cannot give every option its own answer key
Aashishcc622bf6e68a
diff --git a/laya/agent.py b/laya/agent.pyindex 1b58de6f..fe570cb9 100644--- a/laya/agent.py+++ b/laya/agent.py@@ -588,6 +588,32 @@ class Agent(HookRegistry):"question %r: choice label %d is a %s; a label is rendered as option text ""and used as the answer key, so it must be a scalar (a string, number or ""None), got %r" % (qid, i, type(label).__name__, label))+ # `_to_internal` normalises the list form to `{label: None}`, so those labels become+ # the answer keys. Two entries that land on one key made the model score fewer options+ # than the caller wrote, and the response carry fewer probabilities than their list,+ # without a word -- the same silent-shape class as the two checks above. Python+ # treats values as one key whenever they compare equal, so `[1, 1.0]` and `[True, 1]`+ # collapse as well as an exact repeat. An unhashable label raised+ # `TypeError: cannot use 'tuple' as a dict key` from `_to_internal`, three frames down,+ # which names neither the question nor the label -- and which `serve` cannot classify+ # as a caller error, so over HTTP it became a 500 "inference failed" instead of a 422.+ if isinstance(crit, list):+ keys: Dict[Any, int] = {}+ for i, label in enumerate(crit):+ try:+ first = keys[label]+ except TypeError as exc:+ raise ValueError(+ "question %r: choice label %d (%r) cannot be an answer key because it is "+ "unhashable; a label is rendered as option text and used as the answer "+ "key" % (qid, i, label)) from exc+ except KeyError:+ keys[label] = i+ else:+ raise ValueError(+ "question %r: choice label %d (%r) repeats label %d; the labels are the "+ "answer keys, so every option needs its own (1, 1.0 and True are one "+ "key)" % (qid, i, label, first))elif t == "score":if not isinstance(crit, list):raise ValueError("question %r: a score question takes 'criteria' as a list of level "diff --git a/tests/test_criteria.py b/tests/test_criteria.pyindex a5dacea5..44f25c8a 100644--- a/tests/test_criteria.py+++ b/tests/test_criteria.py@@ -322,6 +322,19 @@ for label, qdef in [("unknown type", {"type": "bool", "instructions": "Is it spam?"}),("missing type", {"instructions": "Is it spam?"}),("no instructions", {"type": "noul"}),+ # A criteria list is normalised to `{label: None}`, so its labels are the answer keys. Two+ # entries that land on one key scored fewer options than the caller wrote and returned fewer+ # probabilities than their list, without a word. Python collapses keys that compare equal, so+ # `[1, 1.0]` and `[True, 1]` collapse like an exact repeat, and an unhashable label raised+ # `TypeError: cannot use 'tuple' as a dict key` from `_to_internal`, three frames down.+ ("choice with a duplicate label", {"type": "choice", "instructions": "Which team?",+ "criteria": ["billing", "billing", "tech"]}),+ ("choice with 1 and 1.0 labels", {"type": "choice", "instructions": "Which team?",+ "criteria": [1, 1.0]}),+ ("choice with True and 1 labels", {"type": "choice", "instructions": "Which team?",+ "criteria": [True, 1]}),+ ("choice with an unhashable label", {"type": "choice", "instructions": "Which team?",+ "criteria": [("billing", ["tech"]), "sales"]}),]:try:agent.system_one(STATE, {"q": qdef})@@ -339,6 +352,24 @@ try:except ValueError as e:check_true("rejected/score null level names the level", "level 1" in str(e), str(e))+# a colliding label names the repeat and the label it repeats, so a twenty-option question can be+# fixed without guessing which pair collided+try:+ agent.system_one(STATE, {"q": {"type": "choice", "instructions": "Which team?",+ "criteria": ["billing", "tech", "billing"]}})+ FAIL.append("rejected/duplicate choice label names both: no error raised")+except ValueError as e:+ check_true("rejected/duplicate choice label names both",+ "label 2" in str(e) and "label 0" in str(e) and "billing" in str(e), str(e))++# the option count the caller wrote is preserved, so the guard must not reject distinct labels+try:+ out = agent.system_one(STATE, {"q": {"type": "choice", "instructions": "Which team?",+ "criteria": ["billing", "tech", "sales"]}})+ check("rejected/distinct labels still answer", len(out["answers"]["q"]["probabilities"]), 3)+except Exception as e: # noqa: BLE001+ FAIL.append("rejected/distinct labels still answer: %s" % e)+# the same questions through the public entry point, not only the method under itrouter = Router()router.attach("english", agent)diff --git a/tests/test_serve.py b/tests/test_serve.pyindex ae52aae2..17846c5a 100644--- a/tests/test_serve.py+++ b/tests/test_serve.py@@ -392,6 +392,23 @@ def test_validation_errors_are_not_logged_as_failures(monkeypatch, caplog):assert not [r for r in caplog.records if r.levelno >= logging.ERROR], caplog.records+class ValidatingRouter:+ """The real guard, without a checkpoint: what `Agent.system_one` runs before encoding.++ The app does not validate `criteria` itself -- the agent does -- so the stub calls the+ same guard `system_one` calls, and any `ValueError` it raises is what `serve` has to map+ to 422. `predict` still fails loudly if the guard lets something through.+ """++ loaded = ["english"]++ def predict(self, state, questions, model=None):+ from laya.agent import Agent+ for qid, qdef in questions.items():+ Agent._check_question(qid, qdef)+ raise AssertionError("validation should have rejected this before predict()")++def test_a_nested_choice_label_is_a_caller_error_not_a_server_fault(monkeypatch):"""A `criteria` list containing a list/dict label is the caller's mistake, so it must be 422.@@ -400,22 +417,6 @@ def test_a_nested_choice_label_is_a_caller_error_not_a_server_fault(monkeypatch)`ValueError` to 422, so the caller got a 500 "inference failed" with the reason discarded.`ValueError` is what carries the message to the client, so the guard has to raise that type."""- class ValidatingRouter:- """The real guard, without a checkpoint: what `Agent.system_one` runs before encoding.-- The app does not validate `criteria` itself -- the agent does -- so the stub calls the- same guard `system_one` calls, and any `ValueError` it raises is what `serve` has to map- to 422. `predict` still fails loudly if the guard lets something through.- """-- loaded = ["english"]-- def predict(self, state, questions, model=None):- from laya.agent import Agent- for qid, qdef in questions.items():- Agent._check_question(qid, qdef)- raise AssertionError("validation should have rejected this before predict()")-monkeypatch.delenv("LAYA_API_KEY", raising=False)client = TestClient(create_app(router=ValidatingRouter()), raise_server_exceptions=False)@@ -428,6 +429,32 @@ def test_a_nested_choice_label_is_a_caller_error_not_a_server_fault(monkeypatch)assert "choice label 0" in response.text, response.text+def test_a_colliding_choice_label_is_a_caller_error_not_a_server_fault(monkeypatch):+ """A `criteria` list that cannot produce one answer key per option must be 422, not 500.++ `_to_internal` normalises the list form to `{label: None}`, so two entries that land on one+ key scored fewer options than the caller wrote. `serve` maps `ValueError` to 422 and anything+ else to a 500 "inference failed", so the guard has to raise `ValueError` and say which labels+ collided. (An unhashable label is the same class of mistake, but JSON has no tuple: a list or+ dict label arrives as one of those and the guard above already names it, which+ `test_a_nested_choice_label_is_a_caller_error_not_a_server_fault` covers.)+ """+ monkeypatch.delenv("LAYA_API_KEY", raising=False)+ client = TestClient(create_app(router=ValidatingRouter()), raise_server_exceptions=False)++ for criteria, expect in (+ (["billing", "billing", "tech"], "repeats label 0"),+ ([1, 1.0], "repeats label 0"), # one dict key, two entries+ ([True, 1], "repeats label 0"), # `True == 1` is one dict key too+ ):+ body = dict(REQ)+ body["questions"] = {"dept": {"type": "choice", "instructions": "Which team?",+ "criteria": criteria}}+ response = client.post("/v1/systemone", json=body)+ assert response.status_code == 422, (criteria, response.status_code, response.text)+ assert expect in response.text, (criteria, response.text)++def test_inference_timing_headers():"""POST /v1/systemone returns Server-Timing and X-Inference-Time-Ms headers."""router = FakeRouter()