Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(agent): keep non-ASCII characters in non-string instructions (#228)

PerryLink8dacc3996b02
diff --git a/laya/agent.py b/laya/agent.py
index 70163bf9..e006ffef 100644
--- a/laya/agent.py
+++ b/laya/agent.py
@@ -312,7 +312,12 @@ class Agent:
crit = {str(k).lower(): v for k, v in crit.items()}
ins = qdef["instructions"]
if not isinstance(ins, str):
- ins = json.dumps(ins)
+ # `ensure_ascii=False`, matching `serialize_state` and `render_criterion` in
+ # common.py and the instructions path in shortlist.py. The default escaped
+ # non-ASCII to literal `\uXXXX`, which the tokenizer then read as escape text:
+ # on the English checkpoint one German question answered noul=0.1652 as a dict
+ # and noul=0.2650 as the identical plain string.
+ ins = json.dumps(ins, ensure_ascii=False)
return {"t": t, "ins": ins, "crit": crit}
@torch.no_grad()
diff --git a/tests/test_criteria.py b/tests/test_criteria.py
index 17236245..d9418fdf 100644
--- a/tests/test_criteria.py
+++ b/tests/test_criteria.py
@@ -146,6 +146,31 @@ from laya.common import DecisionModel # noqa: E402
from laya.router import Router # noqa: E402
+# `_to_internal` serialises non-string `instructions` with json.dumps. The default
+# `ensure_ascii=True` escaped non-ASCII to literal `\uXXXX`, which the tokenizer then
+# read as escape text: on the English checkpoint one German question answered noul=0.1652
+# as a dict and noul=0.2650 as the identical plain string. Every other text path keeps
+# its characters -- see the `criterion/non-ascii kept` case above.
+_internal = Agent._to_internal(
+ {"type": "noul", "instructions": {"frage": "Bittet um eine R\u00fcckerstattung?"},
+ "criteria": None})
+check("instructions/non-ascii kept as a dict",
+ _internal["ins"], '{"frage": "Bittet um eine R\u00fcckerstattung?"}')
+check_true("instructions/no escape sequences in the prompt",
+ "\\u" not in _internal["ins"], repr(_internal["ins"]))
+check("instructions/ascii is unchanged",
+ Agent._to_internal({"type": "noul", "instructions": {"asks": "for a refund"},
+ "criteria": None})["ins"],
+ '{"asks": "for a refund"}')
+check("instructions/plain string is untouched",
+ Agent._to_internal({"type": "noul", "instructions": "Bittet der Kunde um eine "
+ "R\u00fcckerstattung?", "criteria": None})["ins"],
+ "Bittet der Kunde um eine R\u00fcckerstattung?")
+check("instructions/non-string still renders as json",
+ Agent._to_internal({"type": "noul", "instructions": ["a", "b"],
+ "criteria": None})["ins"], '["a", "b"]')
+
+
class _FakeTok:
"""The tokenizer surface `build_sequence` uses, with predictable ids."""
cls_token_id, sep_token_id, mask_token_id, pad_token_id = 0, 1, 4, 2