Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(agent): name the question when its definition cannot be answered (#183)

devloper961-makerfcb2cd9c277b
diff --git a/laya/agent.py b/laya/agent.py
index 39553a74..5017904b 100644
--- a/laya/agent.py
+++ b/laya/agent.py
@@ -258,6 +258,41 @@ class Agent:
" See https://pytorch.org/get-started/locally/\n"
% (fell_back_from, fell_back_why), flush=True)
+ @staticmethod
+ def _check_question(qid: str, qdef: Any) -> None:
+ """Reject a question that cannot be answered, naming it and what to fix.
+
+ `render_options` reads `criteria` in the shape the question's type expects and the decision
+ head needs at least one option, so a malformed definition used to surface from three frames
+ down as something that names neither the question nor the problem: `AttributeError:
+ 'NoneType' object has no attribute 'items'`, `KeyError: 'bool'`, or a `selected index k out
+ of range` raised inside the model for a question that ended up with no options at all.
+ """
+ if not isinstance(qdef, dict):
+ raise ValueError("question %r: definition must be a dict, got %s"
+ % (qid, type(qdef).__name__))
+ t = qdef.get("type")
+ if t not in QTYPES:
+ raise ValueError("question %r: unknown type %r; use one of %s" % (qid, t, sorted(QTYPES)))
+ if "instructions" not in qdef:
+ raise ValueError("question %r: no 'instructions'; add the text the model should answer" % (qid,))
+ crit = qdef.get("criteria")
+ if t == "choice":
+ if not isinstance(crit, (dict, list)):
+ raise ValueError("question %r: a choice question takes 'criteria' as a dict of "
+ "label -> description, or a list of labels" % (qid,))
+ if not crit:
+ raise ValueError("question %r: a choice question needs at least one criterion" % (qid,))
+ elif t == "score":
+ if not isinstance(crit, list):
+ raise ValueError("question %r: a score question takes 'criteria' as a list of level "
+ "descriptions, index 0 first" % (qid,))
+ if not crit:
+ raise ValueError("question %r: a score question needs at least one level" % (qid,))
+ elif crit is not None and not isinstance(crit, dict):
+ raise ValueError("question %r: a noul question takes 'criteria' as a dict with optional "
+ "'true'/'false' descriptions, or omits it" % (qid,))
+
@staticmethod
def _to_internal(qdef: Dict) -> Dict:
t = qdef["type"]
@@ -289,6 +324,7 @@ class Agent:
head_max_len = self.cfg.get("head_max_len", 192)
for qid in ids:
+ self._check_question(qid, questions[qid])
q = self._to_internal(questions[qid])
seq, markers = build_sequence(self.tok, state, q, max_len, head_max_len)
if len(markers) != len(render_options(q)):
diff --git a/tests/test_criteria.py b/tests/test_criteria.py
index 42eb6407..1dd49bb1 100644
--- a/tests/test_criteria.py
+++ b/tests/test_criteria.py
@@ -3,6 +3,11 @@
Regression tests for the bug reported in PR #2 (trocker): a `noul` question whose criteria
values were dicts raised `TypeError: can only concatenate str (not "dict") to str`, and
`choice`/`score` stringified dicts as Python reprs instead of JSON.
+
+A question whose *shape* is wrong is the other half of the same promise and lives at the bottom of
+this file: `Agent.system_one` rejects it by name before anything is rendered or tokenized, instead
+of raising `AttributeError: 'NoneType' object has no attribute 'items'` from `render_options` or a
+`selected index k out of range` from inside the decision head (#182).
"""
import json
import os
@@ -116,6 +121,130 @@ check_true("fallback/no bare cuda probe for the warning",
"torch.cuda.is_available() or getattr(torch.version" not in _src)
+# --------------------------------------------------------------- malformed question shapes (#182)
+# A question that cannot be answered used to fail three frames down, as an exception that named
+# neither the question nor the fix: `AttributeError: 'NoneType' object has no attribute 'items'`
+# from `render_options` for a `choice` without criteria, `KeyError: 'bool'` from the type table,
+# and -- for a question that ended up with no options at all -- a `selected index k out of range`
+# raised inside `DecisionModel.forward`, which reads like a bug in laya rather than in the caller's
+# definition. The inference path below is real: a tiny from-config encoder, no checkpoint
+# downloaded (tests/test_local_e2e.py covers the real weights).
+import torch # noqa: E402
+from transformers import AutoConfig, AutoModel # noqa: E402
+
+from laya.agent import Agent # noqa: E402
+from laya.common import DecisionModel # noqa: E402
+from laya.router import Router # noqa: E402
+
+
+class _FakeTok:
+ """The tokenizer surface `build_sequence` uses, with predictable ids."""
+ cls_token_id, sep_token_id, mask_token_id, pad_token_id = 0, 1, 4, 2
+ mask_token = "[MASK]"
+
+ def __call__(self, text, add_special_tokens=False):
+ return {"input_ids": [10 + (len(w) % 90) for w in text.split() if w]}
+
+
+def _tiny_agent():
+ """An `Agent` with a tiny random encoder: the inference path, without the download."""
+ agent = object.__new__(Agent)
+ cfg = AutoConfig.for_model("bert", hidden_size=16, num_hidden_layers=1, num_attention_heads=1,
+ intermediate_size=32, vocab_size=64)
+ agent.cfg = {"max_len": 256, "head_max_len": 96, "encoder": "tiny"}
+ agent.tok = _FakeTok()
+ agent.model = DecisionModel(AutoModel.from_config(cfg), head_layers=1, n_act=2).eval()
+ agent.device = torch.device("cpu")
+ agent.dtype = torch.float32
+ agent.temperature = [1.0, 1.0, 1.0]
+ agent.temperature_by_options = {}
+ return agent
+
+
+STATE = {"body": "I was charged twice for invoice 4411 and want a refund"}
+agent = _tiny_agent()
+for label, qdef in [
+ ("choice without criteria", {"type": "choice", "instructions": "Which team?"}),
+ ("choice with criteria None", {"type": "choice", "instructions": "Which team?", "criteria": None}),
+ ("choice with empty criteria", {"type": "choice", "instructions": "Which team?", "criteria": {}}),
+ ("choice with a tuple of labels", {"type": "choice", "instructions": "Which team?",
+ "criteria": ("billing", "tech")}),
+ ("score without criteria", {"type": "score", "instructions": "How urgent?"}),
+ ("score with an empty list", {"type": "score", "instructions": "How urgent?", "criteria": []}),
+ ("score with a dict of levels", {"type": "score", "instructions": "How urgent?",
+ "criteria": {"low": "no pressure", "high": "blocking"}}),
+ ("noul with list criteria", {"type": "noul", "instructions": "Is it spam?", "criteria": ["a", "b"]}),
+ ("noul with string criteria", {"type": "noul", "instructions": "Is it spam?", "criteria": "spam?"}),
+ ("unknown type", {"type": "bool", "instructions": "Is it spam?"}),
+ ("missing type", {"instructions": "Is it spam?"}),
+ ("no instructions", {"type": "noul"}),
+]:
+ try:
+ agent.system_one(STATE, {"q": qdef})
+ FAIL.append("rejected/%s: no error raised" % label)
+ except ValueError as e:
+ # the message must name the question: a caller with twenty of them needs to know which
+ check_true("rejected/%s names the question" % label, "'q'" in str(e), str(e))
+ check_true("rejected/%s says what to fix" % label, len(str(e)) > 40, str(e))
+ except Exception as e:
+ FAIL.append("rejected/%s: %s instead of ValueError: %s" % (label, type(e).__name__, e))
+
+# the same questions through the public entry point, not only the method under it
+router = Router()
+router.attach("english", agent)
+for label, qdef in (("choice without criteria", {"type": "choice", "instructions": "x"}),):
+ try:
+ router.predict(STATE, {"q": qdef}, model="english")
+ FAIL.append("rejected/router %s: no error raised" % label)
+ except ValueError as e:
+ check_true("rejected/router %s names the question" % label, "'q'" in str(e), str(e))
+ except Exception as e:
+ FAIL.append("rejected/router %s: %s instead of ValueError: %s" % (label, type(e).__name__, e))
+
+# every question id is validated, not only the first one put in the dict
+try:
+ agent.system_one(STATE, {"ok": {"type": "noul", "instructions": "Is it urgent?"},
+ "broken": {"type": "choice", "instructions": "Which team?"}})
+ FAIL.append("rejected/second question: no error raised")
+except ValueError as e:
+ check_true("rejected/second question names it", "'broken'" in str(e), str(e))
+except Exception as e:
+ FAIL.append("rejected/second question: %s instead of ValueError: %s" % (type(e).__name__, e))
+
+# ...and the shapes that are valid still answer, so this is not validation-only coverage
+GOOD = {
+ "choice": {"type": "choice", "instructions": "Which team?",
+ "criteria": {"billing": "invoices and refunds", "tech": "bugs"}},
+ "choice as a list": {"type": "choice", "instructions": "Which team?",
+ "criteria": ["billing", "tech"]},
+ "score": {"type": "score", "instructions": "How urgent?",
+ "criteria": ["no pressure", "soon", "blocking"]},
+ "noul": {"type": "noul", "instructions": "Does the sender want a reply?"},
+ "noul with criteria": {"type": "noul", "instructions": "Is it phishing?",
+ "criteria": {"true": "phishing", "false": "legitimate"}},
+ "non-string instructions": {"type": "noul", "instructions": {"asks": "for a refund"}},
+}
+out = agent.system_one(STATE, GOOD)
+check("good/one answer per question", sorted(out["answers"]), sorted(GOOD))
+check_true("good/choice label", out["answers"]["choice"]["choice"] in ("billing", "tech"),
+ str(out["answers"]["choice"]))
+check_true("good/choice from a list",
+ out["answers"]["choice as a list"]["choice"] in ("billing", "tech"),
+ str(out["answers"]["choice as a list"]))
+check("good/choice probabilities sum", round(sum(out["answers"]["choice"]["probabilities"].values()), 3), 1.0)
+check_true("good/score is in range", 0.0 <= out["answers"]["score"]["score"] <= 2.0,
+ str(out["answers"]["score"]))
+check_true("good/score legend", out["answers"]["score"]["legend"],
+ {"0": "no pressure", "1": "soon", "2": "blocking"})
+check_true("good/noul is a probability", 0.0 <= out["answers"]["noul"]["noul"] <= 1.0,
+ str(out["answers"]["noul"]))
+check_true("good/noul with criteria is a probability",
+ 0.0 <= out["answers"]["noul with criteria"]["noul"] <= 1.0,
+ str(out["answers"]["noul with criteria"]))
+check("good/usage has no output tokens", out["usage"]["output_tokens"], 0)
+check_true("good/usage counted input tokens", out["usage"]["input_tokens"] > 0, str(out["usage"]))
+
+
print("\n%d passed, %d failed" % (len(PASS), len(FAIL)))
for f in FAIL:
print(" FAIL " + f)