Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
fix(agent): a noul question ignores criteria it cannot read (#249)
PerryLink4bae1b32c0c0
diff --git a/README.md b/README.mdindex 0949639f..060035c1 100644--- a/README.md+++ b/README.md@@ -754,6 +754,13 @@ result["shortlist"]["intent"]["labels"] # the top 20 labels sent to the model{"type": "choice", "instructions": "Is this review positive?","criteria": {"A": "yes, the review is positive", "B": "no, the review is negative"}}```++ `criteria` on a `noul` must be keyed `true`/`false` — those two keys *are* the option text the+ model reads, so any other key is rejected instead of being quietly replaced with the defaults.+ Before that check, `criteria: {"yes": ..., "no": ...}` was accepted, dropped, and answered+ against `false:` / `true:` anyway, which cost 2 of 3 clearly positive reviews on the English+ checkpoint ([#156](https://github.com/NandhaKishorM/laya/issues/156)). Use `labels` as above to+ change the wording without touching the option text.* **`laya-multilingual` has a position bias on `score` questions** (#131): it rarely picks the first-listed level, in any language. For English score questions, route to `model="english"`, and for other languages validate score outputs on your own data before relying on them.* **`action.act_probability` carries no usable signal yet** (#185). It reads 1.0 for almost every input, and its raw logits run against correctness (AUROC 0.30 on 396 labelled decisions). Gate on `confidence` instead, which reaches an AUROC of 0.77 on the same items.* `laya` collapses outside English; `laya-multilingual` is weaker on English. Route, or pickdiff --git a/laya/agent.py b/laya/agent.pyindex 8e1d94ae..36abee8c 100644--- a/laya/agent.py+++ b/laya/agent.py@@ -419,6 +419,21 @@ class Agent:elif crit is not None and not isinstance(crit, dict):raise ValueError("question %r: a noul question takes 'criteria' as a dict with optional ""'true'/'false' descriptions, or omits it" % (qid,))+ elif isinstance(crit, dict):+ # `render_options` reads these two descriptions out by name -- `crit.get("false")` and+ # `crit.get("true")` -- so a dict keyed any other way is not a noul description at all.+ # It used to be substituted with the default pair without a word, so a caller saw their+ # descriptions accepted and never reach the model (#156). `labels` just below has+ # rejected the same mistake since #163; this is the same rule on the other parameter,+ # and a noul is a boolean question either way, so those are the only two keys it can have.+ keys = {str(k).lower() for k in crit}+ if not keys <= {"true", "false"}:+ raise ValueError(+ "question %r: a noul question takes 'criteria' keyed only 'true'/'false' (either "+ "or both, and omitted is fine), got %s. Those keys are the option texts the model "+ "reads; any other key was silently dropped and replaced with the defaults. If you "+ "want the answer worded differently, keep 'criteria' keyed 'true'/'false' and set "+ "'labels' instead." % (qid, sorted(keys)))if "labels" in qdef:if t != "noul":raise ValueError("question %r: 'labels' is only supported for noul questions" % (qid,))diff --git a/tests/test_criteria.py b/tests/test_criteria.pyindex 791da443..dea5703f 100644--- a/tests/test_criteria.py+++ b/tests/test_criteria.py@@ -281,6 +281,19 @@ for label, qdef in ["labels": {"true": "A"}}),("noul with duplicate labels", {"type": "noul", "instructions": "Is it spam?","labels": {"false": "A", "true": "A"}}),+ # `render_options` reads the two noul descriptions by name, so any other key used to be+ # dropped and replaced with the defaults without a word (#156). These are the shapes a+ # caller reaches for when they want to word the two options themselves.+ ("noul with yes/no criteria", {"type": "noul", "instructions": "Is it spam?",+ "criteria": {"yes": "it is spam", "no": "it is not"}}),+ ("noul with neutral keys", {"type": "noul", "instructions": "Is it spam?",+ "criteria": {"spam": "it is spam", "ham": "it is not"}}),+ ("noul with alpha/beta criteria", {"type": "noul", "instructions": "Is it spam?",+ "criteria": {"alpha": "yes", "beta": "no"}}),+ ("noul with a typo'd key", {"type": "noul", "instructions": "Is it spam?",+ "criteria": {"ture": "yes", "false": "no"}}),+ ("noul with an extra key", {"type": "noul", "instructions": "Is it spam?",+ "criteria": {"true": "y", "false": "n", "maybe": "?"}}),("unknown type", {"type": "bool", "instructions": "Is it spam?"}),("missing type", {"instructions": "Is it spam?"}),("no instructions", {"type": "noul"}),@@ -354,6 +367,67 @@ check_true("good/noul with labels is a probability",0.0 <= out["answers"]["noul with labels"]["noul"] <= 1.0,str(out["answers"]["noul with labels"]))check("good/usage has no output tokens", out["usage"]["output_tokens"], 0)+++# ------------------------------------------------- noul criteria keys that must keep working+# The guard above rejects a key it cannot use. These are the spellings it must still accept,+# and the check is on the rendered text rather than on "no exception", because the whole point+# is that the caller's descriptions reach the model. Before the guard, the yes/no row below+# would have rendered the defaults instead and the caller had no way to tell (#156).+_DEFAULT_FALSE_TEXT = "false: no, the statement does not hold"+_DEFAULT_TRUE_TEXT = "true: yes, the statement holds"++for label, crit, want in [+ ("true/false use the caller's text",+ {"true": "the review is positive", "false": "the review is negative"},+ ["false: the review is negative", "true: the review is positive"]),+ ("uppercase keys work, via the .lower() in _to_internal",+ {"TRUE": "the review is positive", "FALSE": "the review is negative"},+ ["false: the review is negative", "true: the review is positive"]),+ ("Python bool keys work, which is how JSON true/false arrive",+ {True: "the review is positive", False: "the review is negative"},+ ["false: the review is negative", "true: the review is positive"]),+ ("one key is enough",+ {"true": "the review is positive"},+ [_DEFAULT_FALSE_TEXT, "true: the review is positive"]),+ ("an empty dict falls back to both defaults", {},+ [_DEFAULT_FALSE_TEXT, _DEFAULT_TRUE_TEXT]),+ ("omitting criteria falls back to both defaults", None,+ [_DEFAULT_FALSE_TEXT, _DEFAULT_TRUE_TEXT]),+ ("a description equal to the default wording still counts as given",+ {"true": "yes, the statement holds", "false": "no, the statement does not hold"},+ [_DEFAULT_FALSE_TEXT, _DEFAULT_TRUE_TEXT]),+]:+ qdef = {"type": "noul", "instructions": "Is the review positive?"}+ if crit is not None:+ qdef["criteria"] = crit+ try:+ check("noul keys/%s" % label, render_options(Agent._to_internal(qdef)), want)+ except Exception as exc: # noqa: BLE001+ FAIL.append("noul keys/%s raised %s: %s" % (label, type(exc).__name__, exc))++# `labels` is the supported way to word the answer without touching the option text, so the+# message the guard raises points at it. It replaces the `false:`/`true:` prefixes the model+# reads; the criteria text after them is unchanged, and the result stays P(true).+_mixed = Agent._to_internal({"type": "noul", "instructions": "Is the review positive?",+ "criteria": {"true": "the review is positive",+ "false": "the review is negative"},+ "labels": {"true": "positive", "false": "negative"}})+check("noul keys/criteria text survives alongside labels",+ render_options(_mixed),+ ["negative: the review is negative", "positive: the review is positive"])+check("noul keys/labels are carried through", _mixed["labels"],+ {"true": "positive", "false": "negative"})+check("noul keys/labels keep the false/true slot order",+ render_options(_mixed)[0].startswith("negative:"),+ True)+# ...and the criteria descriptions are all `labels` changes -- the same pair without labels+# is the same text behind the default prefixes.+check("noul keys/labels change only the prefix",+ [o.split(": ", 1)[1] for o in render_options(_mixed)],+ [o.split(": ", 1)[1] for o in render_options(Agent._to_internal(+ {"type": "noul", "instructions": "Is the review positive?",+ "criteria": {"true": "the review is positive", "false": "the review is negative"}}))])check_true("good/usage counted input tokens", out["usage"]["input_tokens"] > 0, str(out["usage"]))# --------------------------------------------------------------- build_sequence left truncation