Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(agent): a choice criteria list that cannot give every option its own answer key

Aashishcc622bf6e68a
diff --git a/laya/agent.py b/laya/agent.py
index 1b58de6f..fe570cb9 100644
--- a/laya/agent.py
+++ b/laya/agent.py
@@ -588,6 +588,32 @@ class Agent(HookRegistry):
"question %r: choice label %d is a %s; a label is rendered as option text "
"and used as the answer key, so it must be a scalar (a string, number or "
"None), got %r" % (qid, i, type(label).__name__, label))
+ # `_to_internal` normalises the list form to `{label: None}`, so those labels become
+ # the answer keys. Two entries that land on one key made the model score fewer options
+ # than the caller wrote, and the response carry fewer probabilities than their list,
+ # without a word -- the same silent-shape class as the two checks above. Python
+ # treats values as one key whenever they compare equal, so `[1, 1.0]` and `[True, 1]`
+ # collapse as well as an exact repeat. An unhashable label raised
+ # `TypeError: cannot use 'tuple' as a dict key` from `_to_internal`, three frames down,
+ # which names neither the question nor the label -- and which `serve` cannot classify
+ # as a caller error, so over HTTP it became a 500 "inference failed" instead of a 422.
+ if isinstance(crit, list):
+ keys: Dict[Any, int] = {}
+ for i, label in enumerate(crit):
+ try:
+ first = keys[label]
+ except TypeError as exc:
+ raise ValueError(
+ "question %r: choice label %d (%r) cannot be an answer key because it is "
+ "unhashable; a label is rendered as option text and used as the answer "
+ "key" % (qid, i, label)) from exc
+ except KeyError:
+ keys[label] = i
+ else:
+ raise ValueError(
+ "question %r: choice label %d (%r) repeats label %d; the labels are the "
+ "answer keys, so every option needs its own (1, 1.0 and True are one "
+ "key)" % (qid, i, label, first))
elif t == "score":
if not isinstance(crit, list):
raise ValueError("question %r: a score question takes 'criteria' as a list of level "
diff --git a/tests/test_criteria.py b/tests/test_criteria.py
index a5dacea5..44f25c8a 100644
--- a/tests/test_criteria.py
+++ b/tests/test_criteria.py
@@ -322,6 +322,19 @@ for label, qdef in [
("unknown type", {"type": "bool", "instructions": "Is it spam?"}),
("missing type", {"instructions": "Is it spam?"}),
("no instructions", {"type": "noul"}),
+ # A criteria list is normalised to `{label: None}`, so its labels are the answer keys. Two
+ # entries that land on one key scored fewer options than the caller wrote and returned fewer
+ # probabilities than their list, without a word. Python collapses keys that compare equal, so
+ # `[1, 1.0]` and `[True, 1]` collapse like an exact repeat, and an unhashable label raised
+ # `TypeError: cannot use 'tuple' as a dict key` from `_to_internal`, three frames down.
+ ("choice with a duplicate label", {"type": "choice", "instructions": "Which team?",
+ "criteria": ["billing", "billing", "tech"]}),
+ ("choice with 1 and 1.0 labels", {"type": "choice", "instructions": "Which team?",
+ "criteria": [1, 1.0]}),
+ ("choice with True and 1 labels", {"type": "choice", "instructions": "Which team?",
+ "criteria": [True, 1]}),
+ ("choice with an unhashable label", {"type": "choice", "instructions": "Which team?",
+ "criteria": [("billing", ["tech"]), "sales"]}),
]:
try:
agent.system_one(STATE, {"q": qdef})
@@ -339,6 +352,24 @@ try:
except ValueError as e:
check_true("rejected/score null level names the level", "level 1" in str(e), str(e))
+# a colliding label names the repeat and the label it repeats, so a twenty-option question can be
+# fixed without guessing which pair collided
+try:
+ agent.system_one(STATE, {"q": {"type": "choice", "instructions": "Which team?",
+ "criteria": ["billing", "tech", "billing"]}})
+ FAIL.append("rejected/duplicate choice label names both: no error raised")
+except ValueError as e:
+ check_true("rejected/duplicate choice label names both",
+ "label 2" in str(e) and "label 0" in str(e) and "billing" in str(e), str(e))
+
+# the option count the caller wrote is preserved, so the guard must not reject distinct labels
+try:
+ out = agent.system_one(STATE, {"q": {"type": "choice", "instructions": "Which team?",
+ "criteria": ["billing", "tech", "sales"]}})
+ check("rejected/distinct labels still answer", len(out["answers"]["q"]["probabilities"]), 3)
+except Exception as e: # noqa: BLE001
+ FAIL.append("rejected/distinct labels still answer: %s" % e)
+
# the same questions through the public entry point, not only the method under it
router = Router()
router.attach("english", agent)
diff --git a/tests/test_serve.py b/tests/test_serve.py
index ae52aae2..17846c5a 100644
--- a/tests/test_serve.py
+++ b/tests/test_serve.py
@@ -392,6 +392,23 @@ def test_validation_errors_are_not_logged_as_failures(monkeypatch, caplog):
assert not [r for r in caplog.records if r.levelno >= logging.ERROR], caplog.records
+class ValidatingRouter:
+ """The real guard, without a checkpoint: what `Agent.system_one` runs before encoding.
+
+ The app does not validate `criteria` itself -- the agent does -- so the stub calls the
+ same guard `system_one` calls, and any `ValueError` it raises is what `serve` has to map
+ to 422. `predict` still fails loudly if the guard lets something through.
+ """
+
+ loaded = ["english"]
+
+ def predict(self, state, questions, model=None):
+ from laya.agent import Agent
+ for qid, qdef in questions.items():
+ Agent._check_question(qid, qdef)
+ raise AssertionError("validation should have rejected this before predict()")
+
+
def test_a_nested_choice_label_is_a_caller_error_not_a_server_fault(monkeypatch):
"""A `criteria` list containing a list/dict label is the caller's mistake, so it must be 422.
@@ -400,22 +417,6 @@ def test_a_nested_choice_label_is_a_caller_error_not_a_server_fault(monkeypatch)
`ValueError` to 422, so the caller got a 500 "inference failed" with the reason discarded.
`ValueError` is what carries the message to the client, so the guard has to raise that type.
"""
- class ValidatingRouter:
- """The real guard, without a checkpoint: what `Agent.system_one` runs before encoding.
-
- The app does not validate `criteria` itself -- the agent does -- so the stub calls the
- same guard `system_one` calls, and any `ValueError` it raises is what `serve` has to map
- to 422. `predict` still fails loudly if the guard lets something through.
- """
-
- loaded = ["english"]
-
- def predict(self, state, questions, model=None):
- from laya.agent import Agent
- for qid, qdef in questions.items():
- Agent._check_question(qid, qdef)
- raise AssertionError("validation should have rejected this before predict()")
-
monkeypatch.delenv("LAYA_API_KEY", raising=False)
client = TestClient(create_app(router=ValidatingRouter()), raise_server_exceptions=False)
@@ -428,6 +429,32 @@ def test_a_nested_choice_label_is_a_caller_error_not_a_server_fault(monkeypatch)
assert "choice label 0" in response.text, response.text
+def test_a_colliding_choice_label_is_a_caller_error_not_a_server_fault(monkeypatch):
+ """A `criteria` list that cannot produce one answer key per option must be 422, not 500.
+
+ `_to_internal` normalises the list form to `{label: None}`, so two entries that land on one
+ key scored fewer options than the caller wrote. `serve` maps `ValueError` to 422 and anything
+ else to a 500 "inference failed", so the guard has to raise `ValueError` and say which labels
+ collided. (An unhashable label is the same class of mistake, but JSON has no tuple: a list or
+ dict label arrives as one of those and the guard above already names it, which
+ `test_a_nested_choice_label_is_a_caller_error_not_a_server_fault` covers.)
+ """
+ monkeypatch.delenv("LAYA_API_KEY", raising=False)
+ client = TestClient(create_app(router=ValidatingRouter()), raise_server_exceptions=False)
+
+ for criteria, expect in (
+ (["billing", "billing", "tech"], "repeats label 0"),
+ ([1, 1.0], "repeats label 0"), # one dict key, two entries
+ ([True, 1], "repeats label 0"), # `True == 1` is one dict key too
+ ):
+ body = dict(REQ)
+ body["questions"] = {"dept": {"type": "choice", "instructions": "Which team?",
+ "criteria": criteria}}
+ response = client.post("/v1/systemone", json=body)
+ assert response.status_code == 422, (criteria, response.status_code, response.text)
+ assert expect in response.text, (criteria, response.text)
+
+
def test_inference_timing_headers():
"""POST /v1/systemone returns Server-Timing and X-Inference-Time-Ms headers."""
router = FakeRouter()