Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(agent): a noul question ignores criteria it cannot read (#249)

PerryLink4bae1b32c0c0
diff --git a/README.md b/README.md
index 0949639f..060035c1 100644
--- a/README.md
+++ b/README.md
@@ -754,6 +754,13 @@ result["shortlist"]["intent"]["labels"] # the top 20 labels sent to the model
{"type": "choice", "instructions": "Is this review positive?",
"criteria": {"A": "yes, the review is positive", "B": "no, the review is negative"}}
```
+
+ `criteria` on a `noul` must be keyed `true`/`false` — those two keys *are* the option text the
+ model reads, so any other key is rejected instead of being quietly replaced with the defaults.
+ Before that check, `criteria: {"yes": ..., "no": ...}` was accepted, dropped, and answered
+ against `false:` / `true:` anyway, which cost 2 of 3 clearly positive reviews on the English
+ checkpoint ([#156](https://github.com/NandhaKishorM/laya/issues/156)). Use `labels` as above to
+ change the wording without touching the option text.
* **`laya-multilingual` has a position bias on `score` questions** (#131): it rarely picks the first-listed level, in any language. For English score questions, route to `model="english"`, and for other languages validate score outputs on your own data before relying on them.
* **`action.act_probability` carries no usable signal yet** (#185). It reads 1.0 for almost every input, and its raw logits run against correctness (AUROC 0.30 on 396 labelled decisions). Gate on `confidence` instead, which reaches an AUROC of 0.77 on the same items.
* `laya` collapses outside English; `laya-multilingual` is weaker on English. Route, or pick
diff --git a/laya/agent.py b/laya/agent.py
index 8e1d94ae..36abee8c 100644
--- a/laya/agent.py
+++ b/laya/agent.py
@@ -419,6 +419,21 @@ class Agent:
elif crit is not None and not isinstance(crit, dict):
raise ValueError("question %r: a noul question takes 'criteria' as a dict with optional "
"'true'/'false' descriptions, or omits it" % (qid,))
+ elif isinstance(crit, dict):
+ # `render_options` reads these two descriptions out by name -- `crit.get("false")` and
+ # `crit.get("true")` -- so a dict keyed any other way is not a noul description at all.
+ # It used to be substituted with the default pair without a word, so a caller saw their
+ # descriptions accepted and never reach the model (#156). `labels` just below has
+ # rejected the same mistake since #163; this is the same rule on the other parameter,
+ # and a noul is a boolean question either way, so those are the only two keys it can have.
+ keys = {str(k).lower() for k in crit}
+ if not keys <= {"true", "false"}:
+ raise ValueError(
+ "question %r: a noul question takes 'criteria' keyed only 'true'/'false' (either "
+ "or both, and omitted is fine), got %s. Those keys are the option texts the model "
+ "reads; any other key was silently dropped and replaced with the defaults. If you "
+ "want the answer worded differently, keep 'criteria' keyed 'true'/'false' and set "
+ "'labels' instead." % (qid, sorted(keys)))
if "labels" in qdef:
if t != "noul":
raise ValueError("question %r: 'labels' is only supported for noul questions" % (qid,))
diff --git a/tests/test_criteria.py b/tests/test_criteria.py
index 791da443..dea5703f 100644
--- a/tests/test_criteria.py
+++ b/tests/test_criteria.py
@@ -281,6 +281,19 @@ for label, qdef in [
"labels": {"true": "A"}}),
("noul with duplicate labels", {"type": "noul", "instructions": "Is it spam?",
"labels": {"false": "A", "true": "A"}}),
+ # `render_options` reads the two noul descriptions by name, so any other key used to be
+ # dropped and replaced with the defaults without a word (#156). These are the shapes a
+ # caller reaches for when they want to word the two options themselves.
+ ("noul with yes/no criteria", {"type": "noul", "instructions": "Is it spam?",
+ "criteria": {"yes": "it is spam", "no": "it is not"}}),
+ ("noul with neutral keys", {"type": "noul", "instructions": "Is it spam?",
+ "criteria": {"spam": "it is spam", "ham": "it is not"}}),
+ ("noul with alpha/beta criteria", {"type": "noul", "instructions": "Is it spam?",
+ "criteria": {"alpha": "yes", "beta": "no"}}),
+ ("noul with a typo'd key", {"type": "noul", "instructions": "Is it spam?",
+ "criteria": {"ture": "yes", "false": "no"}}),
+ ("noul with an extra key", {"type": "noul", "instructions": "Is it spam?",
+ "criteria": {"true": "y", "false": "n", "maybe": "?"}}),
("unknown type", {"type": "bool", "instructions": "Is it spam?"}),
("missing type", {"instructions": "Is it spam?"}),
("no instructions", {"type": "noul"}),
@@ -354,6 +367,67 @@ check_true("good/noul with labels is a probability",
0.0 <= out["answers"]["noul with labels"]["noul"] <= 1.0,
str(out["answers"]["noul with labels"]))
check("good/usage has no output tokens", out["usage"]["output_tokens"], 0)
+
+
+# ------------------------------------------------- noul criteria keys that must keep working
+# The guard above rejects a key it cannot use. These are the spellings it must still accept,
+# and the check is on the rendered text rather than on "no exception", because the whole point
+# is that the caller's descriptions reach the model. Before the guard, the yes/no row below
+# would have rendered the defaults instead and the caller had no way to tell (#156).
+_DEFAULT_FALSE_TEXT = "false: no, the statement does not hold"
+_DEFAULT_TRUE_TEXT = "true: yes, the statement holds"
+
+for label, crit, want in [
+ ("true/false use the caller's text",
+ {"true": "the review is positive", "false": "the review is negative"},
+ ["false: the review is negative", "true: the review is positive"]),
+ ("uppercase keys work, via the .lower() in _to_internal",
+ {"TRUE": "the review is positive", "FALSE": "the review is negative"},
+ ["false: the review is negative", "true: the review is positive"]),
+ ("Python bool keys work, which is how JSON true/false arrive",
+ {True: "the review is positive", False: "the review is negative"},
+ ["false: the review is negative", "true: the review is positive"]),
+ ("one key is enough",
+ {"true": "the review is positive"},
+ [_DEFAULT_FALSE_TEXT, "true: the review is positive"]),
+ ("an empty dict falls back to both defaults", {},
+ [_DEFAULT_FALSE_TEXT, _DEFAULT_TRUE_TEXT]),
+ ("omitting criteria falls back to both defaults", None,
+ [_DEFAULT_FALSE_TEXT, _DEFAULT_TRUE_TEXT]),
+ ("a description equal to the default wording still counts as given",
+ {"true": "yes, the statement holds", "false": "no, the statement does not hold"},
+ [_DEFAULT_FALSE_TEXT, _DEFAULT_TRUE_TEXT]),
+]:
+ qdef = {"type": "noul", "instructions": "Is the review positive?"}
+ if crit is not None:
+ qdef["criteria"] = crit
+ try:
+ check("noul keys/%s" % label, render_options(Agent._to_internal(qdef)), want)
+ except Exception as exc: # noqa: BLE001
+ FAIL.append("noul keys/%s raised %s: %s" % (label, type(exc).__name__, exc))
+
+# `labels` is the supported way to word the answer without touching the option text, so the
+# message the guard raises points at it. It replaces the `false:`/`true:` prefixes the model
+# reads; the criteria text after them is unchanged, and the result stays P(true).
+_mixed = Agent._to_internal({"type": "noul", "instructions": "Is the review positive?",
+ "criteria": {"true": "the review is positive",
+ "false": "the review is negative"},
+ "labels": {"true": "positive", "false": "negative"}})
+check("noul keys/criteria text survives alongside labels",
+ render_options(_mixed),
+ ["negative: the review is negative", "positive: the review is positive"])
+check("noul keys/labels are carried through", _mixed["labels"],
+ {"true": "positive", "false": "negative"})
+check("noul keys/labels keep the false/true slot order",
+ render_options(_mixed)[0].startswith("negative:"),
+ True)
+# ...and the criteria descriptions are all `labels` changes -- the same pair without labels
+# is the same text behind the default prefixes.
+check("noul keys/labels change only the prefix",
+ [o.split(": ", 1)[1] for o in render_options(_mixed)],
+ [o.split(": ", 1)[1] for o in render_options(Agent._to_internal(
+ {"type": "noul", "instructions": "Is the review positive?",
+ "criteria": {"true": "the review is positive", "false": "the review is negative"}}))])
check_true("good/usage counted input tokens", out["usage"]["input_tokens"] > 0, str(out["usage"]))
# --------------------------------------------------------------- build_sequence left truncation