Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
"""Batch inference: predict_batch packs many states into shared forward passes.
These tests need no model weights. They cover:
1. Contract — predict_batch exists and system_one is defined in terms of it, so the single-state
and batched paths can never numerically drift apart.
2. Orchestration — empty input, bad input, per-state row mapping, and batch_size chunking, all
exercised against a fake whose forward is stubbed. The numerical equivalence of the real
model path (predict_batch(states) == [system_one(s) for s in states]) is checked with weights
in tests/test_local_e2e.py.
3. Decoding — calibrated answer values and confidence, including mixed question types.
"""
import inspect
import os
import re
import sys
from contextlib import contextmanager
from types import SimpleNamespace
from unittest.mock import patch
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
import numpy as np # noqa: E402
import torch # noqa: E402
from laya import agent as _agent # noqa: E402
from laya.agent import Agent # noqa: E402
PASS, FAIL = [], []
def check(name, got, want):
if got == want:
PASS.append(name)
else:
FAIL.append("%s:\n got %r\n want %r" % (name, got, want))
def check_true(name, cond, detail=""):
if cond:
PASS.append(name)
else:
FAIL.append("%s %s" % (name, detail))
def check_raises(name, exc, fn):
try:
fn()
except exc:
PASS.append(name)
except Exception as e: # noqa: BLE001
FAIL.append("%s: raised %r, expected %s" % (name, e, exc.__name__))
else:
FAIL.append("%s: did not raise %s" % (name, exc.__name__))
# --------------------------------------------------------------- contract (source inspection)
check_true("contract/predict_batch is defined", callable(getattr(Agent, "predict_batch", None)))
_so = inspect.getsource(Agent.system_one)
check_true("contract/system_one delegates to predict_batch", "self.predict_batch(" in _so)
check_true("contract/predict alias preserved", Agent.predict is Agent.system_one)
_sig = inspect.signature(Agent.predict_batch)
check_true("contract/predict_batch has batch_size kwarg", "batch_size" in _sig.parameters)
# --------------------------------------------------------------- orchestration (weight-free fake)
# We drive the real predict_batch method but stub the three helpers it composes, so no encoder,
# tokenizer vocabulary, or weights are needed. collate_items runs for real on the fake items.
NQ = 2
QUESTIONS = {"a": {"type": "noul", "instructions": "?"}, "b": {"type": "noul", "instructions": "?"}}
# `_encode_state` items carry the state truncation counts that `predict_batch` reports in `usage` (#174)
NO_STATE_STATS = {"state_tokens": 0, "state_tokens_used": 0, "state_tokens_dropped": 0, "truncated": False}
def make_fake():
fake = _agent.Agent.__new__(_agent.Agent)
fake.tok = type("Tok", (), {"pad_token_id": 0})()
fake._to_internal = staticmethod(Agent._to_internal).__func__ # reuse the real normalizer
fake._forward_calls = []
def _encode_state(state, ids, internal):
# one 3-token, 2-marker item per question; content is irrelevant to the mapping test
return [{"ids": [1, 2, 3], "markers": [0, 1], "qtype": 2, "state_stats": NO_STATE_STATS} for _ in ids]
def _forward(b):
n = b["input_ids"].shape[0]
fake._forward_calls.append(n)
# deterministic logits so decode is stable; act pre-softmaxed as _forward would return it
return np.zeros((n, 2), dtype=np.float32), np.full((n, 2), 0.5, dtype=np.float32)
def _decode_answers(logits, act, items, ids, internal, offset):
# record the row offset this state was decoded from, to verify per-state alignment
return {"_offset": offset}
fake._encode_state = _encode_state
fake._forward = _forward
fake._decode_answers = _decode_answers
return fake
# --------------------------------------------------------------- length grouping and restoration
def make_length_fake():
fake = make_fake()
fake.encoded = []
fake.batch_rows = []
fake.slots = 0
fake.encoded_at_forward = []
def encode(state, ids, internal, **overrides):
fake.encoded.append(state)
limit = overrides.get("max_len", 100)
# Question rows have different lengths; sorting must use the longest.
return [{"ids": [state["id"] + 1] * min(limit, size), "markers": [0, 1], "qtype": 2,
"state_stats": NO_STATE_STATS}
for size in (3, state["length"])]
def forward(batch):
fake.slots += batch["input_ids"].numel()
fake.batch_rows.append(batch["input_ids"][:, 0].tolist())
fake.encoded_at_forward.append(len(fake.encoded))
return batch["input_ids"][:, :1].numpy(), None
def decode(logits, act, items, ids, internal, offset):
return {qid: int(logits[offset + i, 0]) - 1 for i, qid in enumerate(ids)}
fake._encode_state = encode
fake._forward = forward
fake._decode_answers = decode
return fake
states = [{"id": i, "length": [40, 5, 25, 8][i % 4]} for i in range(37)]
plain, grouped = make_length_fake(), make_length_fake()
expected = plain.predict_batch(states, QUESTIONS, batch_size=2)
events = []
actual = grouped.predict_batch(states, QUESTIONS, batch_size=2, sort_by_length=True,
on_predict_end=lambda ctx: events.append((ctx.results, ctx.usage)))
check("length/answers and usage retain input order across windows", actual, expected)
check("length/each state encoded once in input order", grouped.encoded, states)
check_true("length/grouping reduces padded slots", grouped.slots < plain.slots)
check("length/same number of forward passes", len(grouped.batch_rows), len(plain.batch_rows))
check_true("length/batch size caps all forward passes", all(len(rows) <= 4 for rows in grouped.batch_rows))
check("length/partial final batch", len(grouped.batch_rows[-1]), 2)
check("length/first forward buffers at most eight batches", grouped.encoded_at_forward[0], 16)
check("length/end hook sees restored results", events[0][0], expected)
check("length/end hook usage excludes padding", events[0][1]["input_tokens"],
sum(3 + st["length"] for st in states))
check("length/does not mutate caller states", [st["id"] for st in states], list(range(37)))
# A hook may replace both inputs and the length budget before grouping occurs.
def rewrite_length_inputs(ctx):
ctx.states = states
ctx.max_len = 10
rewritten = make_length_fake().predict_batch(["unused"], QUESTIONS, batch_size=2,
sort_by_length=True, on_predict_start=rewrite_length_inputs)
reference = make_length_fake().predict_batch(states, QUESTIONS, batch_size=2, max_len=10)
check("length/hook rewrites and truncation budgets precede sorting", rewritten, reference)
for batch_size in (None, 0, -1, 1, 100):
plain, grouped = make_length_fake(), make_length_fake()
check("length/no-op results for batch_size=%r" % batch_size,
grouped.predict_batch(states, QUESTIONS, batch_size=batch_size, sort_by_length=True),
plain.predict_batch(states, QUESTIONS, batch_size=batch_size))
check("length/no-op grouping for batch_size=%r" % batch_size, grouped.batch_rows, plain.batch_rows)
f = make_length_fake()
check("length/empty inputs", f.predict_batch([], QUESTIONS, sort_by_length=True), [])
check("length/empty questions", f.predict_batch(states[:1], {}, sort_by_length=True)[0]["answers"], {})
check("length/empty cases do not encode", f.encoded, [])
# The cache pattern carries one entry per state (see docs/hooks/api.md); an empty skip
# would return zero rows for this 37-state call, which `skip()` refuses as the contract
# violation it is. What this pins is that a skip short-circuits before sorting/encoding.
f.predict_batch(states, QUESTIONS, sort_by_length=True,
on_predict_start=lambda ctx: ctx.skip(
[{"model": "cached", "answers": {}, "usage": {}} for _ in ctx.states]))
check("length/cache skip does not encode", f.encoded, [])
plain, grouped = make_length_fake(), make_length_fake()
equal_lengths = [{"id": i, "length": 10} for i in range(20)]
plain.predict_batch(equal_lengths, QUESTIONS, batch_size=3)
grouped.predict_batch(equal_lengths, QUESTIONS, batch_size=3, sort_by_length=True)
check("length/equal length ties preserve row order", grouped.batch_rows, plain.batch_rows)
errors = []
class RecordFailure:
def on_error(self, ctx):
errors.append(ctx.error)
def fail_forward(batch):
raise RuntimeError("forward failed")
f = make_length_fake()
f._forward = fail_forward
check_raises("length/forward error propagates", RuntimeError,
lambda: f.predict_batch(states, QUESTIONS, batch_size=2,
sort_by_length=True, hooks=[RecordFailure()]))
check("length/error hook receives failure once", [str(e) for e in errors], ["forward failed"])
f = make_fake()
check("orchestration/empty states -> empty list", f.predict_batch([], QUESTIONS), [])
# What the end hook of that same call sees. `predict_batch`'s finally block guards the usage
# aggregation with `ctx.results is not None`, not with truthiness, and `[]` is the one value where
# the two disagree: truthiness would leave `ctx.usage` as None on an empty batch, which is exactly
# the value a call that raised before any result leaves behind. docs/hooks/lifecycle.md's
# "Empty inputs" table states the difference to readers who write audit hooks against `ctx.usage`,
# so it is pinned here against the real method.
end_seen = []
class WatchEnd:
def on_predict_end(self, ctx):
end_seen.append((ctx.results, ctx.usage))
f = make_fake()
f.predict_batch([], QUESTIONS, hooks=[WatchEnd()])
check("orchestration/empty batch: the end hook still runs, once", len(end_seen), 1)
check("orchestration/empty batch: results [] but usage summed, not None", end_seen[0],
([], {"input_tokens": 0, "output_tokens": 0}))
end_seen.clear()
f = make_fake()
f.predict_batch(["s0", "s1"], {}, hooks=[WatchEnd()])
check("orchestration/no questions: usage is the sum of the empty payloads", end_seen[0][1],
{"input_tokens": 0, "output_tokens": 0})
end_seen.clear()
f = make_fake()
f._forward = fail_forward
check_raises("orchestration/forward error propagates on one state", RuntimeError,
lambda: f.predict_batch(["s0"], QUESTIONS, hooks=[WatchEnd()]))
check("orchestration/a failed call is the only one that leaves usage None", end_seen,
[(None, None)])
# The same guard read from both sides. lifecycle.md transcribes this `finally` block into its
# diagram, so the transcription has to track the code -- a guard that changes shape in one of the
# two is a page that now lies about when `ctx.usage` is None. Source inspection is already how this
# file's contract section pins `predict_batch` (it reads `Agent.predict is Agent.system_one` off the
# class); the doc side is a literal pull out of the fenced diagram. Box-drawing characters and the
# diagram's own arrows are why both sides go through one regex instead of a line compare.
_REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
def _finally_guard(text):
"""The condition the first `if` after `finally:` tests, as written."""
tail = text.split("finally:", 1)
if len(tail) < 2:
return None
found = re.search(r"if (ctx\.results[^:]*)", tail[1])
return found.group(1).strip() if found else None
_code_text = inspect.getsource(Agent.predict_batch)
with open(os.path.join(_REPO, "docs", "hooks", "lifecycle.md"), encoding="utf-8") as _life:
_doc_text = _life.read()
check("guard/predict_batch has exactly one finally block to transcribe",
_code_text.count("finally:"), 1)
check_true("guard/lifecycle.md's diagram has a finally block", "finally:" in _doc_text)
# The diagram is the text between the section heading and its closing fence.
_diagram = _doc_text.split("## Agent.predict_batch", 1)[1].split("```", 2)[1]
_GUARD_CODE = _finally_guard(_code_text)
_GUARD_DOC = _finally_guard(_diagram)
check("guard/the diagram tests what the code tests", _GUARD_DOC, _GUARD_CODE)
check_true("guard/and that guard is the identity test, not truthiness",
_GUARD_CODE == "ctx.results is not None", repr(_GUARD_CODE))
def _stmts_after_guard(text):
"""The call names the two statements that follow that guard, in source order.
Scoped after `finally:` for the same reason `_finally_guard` is: the body has an earlier
`if ctx.results is None:` on the cache-hit branch, and transcribing that one would be a
different claim.
"""
tail = text.split("finally:", 1)
if len(tail) < 2:
return []
lines = tail[1].splitlines()
for n, ln in enumerate(lines):
if "if ctx.results" in ln:
return re.findall(r"\b([a-z_]+)\(", "\n".join(lines[n + 1:n + 3]))
return []
_code_stmts = _stmts_after_guard(_code_text)
check("guard/the guard runs two statements worth transcribing", _code_stmts,
["aggregate_usage", "apply_confidence_gate"])
check("guard/and the diagram transcribes both of them",
_stmts_after_guard(_diagram), _code_stmts)
f = make_fake()
res = f.predict_batch(["s0", "s1", "s2"], QUESTIONS)
check("orchestration/one result per state", len(res), 3)
check("orchestration/state 0 decoded from row 0", res[0]["answers"]["_offset"], 0)
check("orchestration/state 1 decoded from row NQ", res[1]["answers"]["_offset"], NQ)
check("orchestration/state 2 decoded from row 2*NQ", res[2]["answers"]["_offset"], 2 * NQ)
check_true("orchestration/result shape matches system_one",
all(set(r) == {"model", "answers", "usage"} for r in res))
check("orchestration/single forward pass by default", f._forward_calls, [6]) # 3 states x 2 questions
f = make_fake()
f.predict_batch(["s0", "s1", "s2"], QUESTIONS, batch_size=1)
check("orchestration/batch_size=1 chunks into three passes", f._forward_calls, [2, 2, 2])
f = make_fake()
f.predict_batch(["s0", "s1", "s2", "s3", "s4"], QUESTIONS, batch_size=2)
check("orchestration/batch_size=2 chunks 5 states as 2+2+1", f._forward_calls, [4, 4, 2])
f = make_fake()
check_raises("orchestration/bare string rejected", TypeError, lambda: f.predict_batch("just a string", QUESTIONS))
f = make_fake()
check_raises("orchestration/bare dict rejected", TypeError, lambda: f.predict_batch({"body": "x"}, QUESTIONS))
# --------------------------------------------------------------- real answer decoding (no weights)
# Exercise the real decoder with a row offset and distinct per-type/bucket temperatures.
# Noul uses top probability for confidence; its unused entropy calculation can be skipped.
decoder = Agent.__new__(Agent)
decoder.temperature = {0: 1.0, 1: 2.0, 2: 3.0}
decoder.temperature_by_options = {"choice:2": 0.5}
decode_ids = ["pick", "level", "flag"]
decode_internal = {
"pick": {"t": "choice", "crit": {"left": "left", "right": "right"}},
"level": {"t": "score", "crit": ["low", "high"]},
"flag": {"t": "noul", "crit": None},
}
decode_items = [{"markers": [0, 1]} for _ in decode_ids]
decode_logits = np.array([
[99.0, -99.0], # preceding state: must not be decoded
np.log([0.25, 0.75]) * 0.5,
np.log([0.25, 0.75]) * 2.0,
np.log([0.8, 0.2]) * 3.0,
])
decode_act = np.array([[0.0, 1.0], [0.125, 0.875], [0.25, 0.75], [0.75, 0.25]])
with patch.object(_agent, "confidence_from_probs", wraps=_agent.confidence_from_probs) as entropy:
decoded = decoder._decode_answers(
decode_logits, decode_act, decode_items, decode_ids, decode_internal, 1
)
check("decode/mixed answers retain calibrated values", decoded, {
"pick": {"type": "choice", "choice": "right",
"probabilities": {"left": 0.25, "right": 0.75}, "confidence": 0.1887,
"answer_confidence": 0.75,
"action": {"act_probability": 0.125}},
"level": {"type": "score", "score": 0.75, "legend": {"0": "low", "1": "high"},
"probabilities": {"0": 0.25, "1": 0.75}, "confidence": 0.1887,
"answer_confidence": 0.75,
"action": {"act_probability": 0.25}},
"flag": {"type": "noul", "noul": 0.2, "confidence": 0.8,
# max(p) over two options is max(p_true, 1 - p_true): the same number
"answer_confidence": 0.8,
"action": {"act_probability": 0.75}},
})
check("decode/entropy only computed for choice and score", entropy.call_count, 2)
entropy.reset_mock()
for true_probability in (0.1, 0.5, 0.9):
logits = np.log([[1.0 - true_probability, true_probability]]) * 3.0
result = decoder._decode_answers(
logits, decode_act[-1:], decode_items[-1:], ["flag"], decode_internal, 0
)
check("decode/noul confidence at p=%s" % true_probability, result["flag"], {
"type": "noul", "noul": true_probability,
"confidence": max(true_probability, 1.0 - true_probability),
"answer_confidence": max(true_probability, 1.0 - true_probability),
"action": {"act_probability": 0.75},
})
check("decode/noul-only answers skip entropy", entropy.call_count, 0)
# --------------------------------------------------------------- min_confidence abstention (#361)
# The flag is set before `on_predict_end` runs, so an audit hook sees exactly what the caller gets.
def make_confidence_fake():
fake = make_fake()
def decode(logits, act, items, ids, internal, offset):
return {qid: {"type": "choice", "choice": qid,
"answer_confidence": 0.90 if qid == "a" else 0.40}
for qid in ids}
fake._decode_answers = decode
return fake
seen = []
gated = make_confidence_fake().predict_batch(
["s0", "s1"], QUESTIONS, min_confidence=0.5,
on_predict_end=lambda ctx: seen.append(
[{q: a.get("low_confidence", False) for q, a in r["answers"].items()} for r in ctx.results]))
check("min_confidence/flags only answers below the threshold",
[[r["answers"][q].get("low_confidence", False) for q in ("a", "b")] for r in gated],
[[False, True], [False, True]])
check("min_confidence/end hook sees the same flags", seen, [[{"a": False, "b": True}] * 2])
check("min_confidence/raw answer and confidence intact",
(gated[0]["answers"]["a"]["answer_confidence"], "low_confidence" in gated[0]["answers"]["a"]),
(0.90, False))
check("min_confidence/default leaves results untouched",
"low_confidence" in make_confidence_fake().predict_batch(["s0"], QUESTIONS)[0]["answers"]["b"], False)
check_raises("min_confidence/out-of-range rejected", ValueError,
lambda: make_confidence_fake().predict_batch(["s0"], QUESTIONS, min_confidence=1.5))
check_raises("min_confidence/bool rejected", ValueError,
lambda: make_confidence_fake().predict_batch(["s0"], QUESTIONS, min_confidence=True))
# A legend maps a level index to the TEXT of that level. Its keys are already strings, and
# `structured` stringifies every level it builds, so a numeric scale written directly used to be
# the one path that echoed the caller's own type back: `{"0": 1, "1": 2}` instead of
# `{"0": "1", "1": "2"}`. That made the response's JSON types depend on the input's types, so a
# client that reads a level as a string had to handle a number as well.
#
# `render_criterion` is what renders the text, so a structured level comes back as the JSON the
# model was shown rather than a Python repr, and bool/None follow JSON spelling (`true`, `null`).
for label, crit, want in (
("int levels", [1, 2, 3], {"0": "1", "1": "2", "2": "3"}),
("float levels", [1.5, 2.5], {"0": "1.5", "1": "2.5"}),
("bool levels", [True, False], {"0": "true", "1": "false"}),
("string levels", ["low", "high"], {"0": "low", "1": "high"}),
("dict level", [{"a": 1}], {"0": '{"a": 1}'}),
("list level", [[1, 2]], {"0": "[1, 2]"}),
):
internal = {"level": {"t": "score", "crit": crit}}
k = len(crit)
logits = np.tile(np.log([1.0 / k] * k), (k, 1))
act = np.array([[0.25, 0.75]] * k)
result = decoder._decode_answers(
logits, act, [{"markers": [0, 1]}] * k, ["level"] * k, internal, 0
)
check("decode/score legend values are str (%s)" % label, result["level"]["legend"], want)
check_true("decode/score legend values have no non-str (%s)" % label,
all(isinstance(v, str) for v in result["level"]["legend"].values()),
result["level"]["legend"])
# --------------------------------------------------------------- CUDA batch autocast lifetime
# Fake CUDA device and autocast so this orchestration test also runs in CPU-only CI.
def run_amp_scope_case(batch_size=2, device="cuda", amp=True, fast=None, compiled=False, fail=False):
fake = make_fake()
fake.device = torch.device(device)
fake.dtype = torch.bfloat16
fake.amp_enabled = amp
fake._fast = fast
fake._compiled = compiled
events = []
@contextmanager
def record_autocast(**kwargs):
events.append(("enter", kwargs))
try:
yield
finally:
events.append(("exit", kwargs))
original_forward = fake._forward
def forward(batch):
events.append(("forward", _agent._BATCH_AUTOCAST_CACHE.get()))
if fail:
raise ValueError("forward failed")
return original_forward(batch)
fake._forward = forward
with patch.object(torch, "autocast", side_effect=record_autocast):
try:
fake.predict_batch(["s0", "s1", "s2"], QUESTIONS, batch_size=batch_size,
on_predict_end=lambda ctx: events.append(("end", None)),
hooks=SimpleNamespace(on_error=lambda ctx: events.append(("error", None))))
except ValueError:
if not fail:
raise
return events
scoped = run_amp_scope_case()
check("autocast/multiple forwards use one enclosing scope",
[name for name, _ in scoped], ["enter", "forward", "forward", "exit", "end"])
check("autocast/outer scope disables autocast between forwards",
scoped[0][1], {"device_type": "cuda", "dtype": torch.bfloat16, "enabled": False})
check("autocast/batch marker only active during forwards",
[value for name, value in scoped if name == "forward"], [True, True])
check("autocast/batch marker reset before end hook", _agent._BATCH_AUTOCAST_CACHE.get(), False)
check("autocast/single forward skips extra scope",
[name for name, _ in run_amp_scope_case(batch_size=None)], ["forward", "end"])
for label, settings in (("CPU", {"device": "cpu"}), ("disabled AMP", {"amp": False}),
("fast path", {"fast": object()}), ("compiled path", {"compiled": True})):
check("autocast/%s skips extra scope" % label,
[name for name, _ in run_amp_scope_case(**settings)], ["forward", "forward", "end"])
with patch.object(torch, "is_autocast_enabled", return_value=True):
check("autocast/caller-owned scope needs no extra scope",
[name for name, _ in run_amp_scope_case()], ["forward", "forward", "end"])
check("autocast/error closes scope before error and end hooks",
[name for name, _ in run_amp_scope_case(fail=True)], ["enter", "forward", "exit", "error", "end"])
# --------------------------------------------------------------------------- report
print("\n%d passed, %d failed" % (len(PASS), len(FAIL)))
for f_ in FAIL:
print(" FAIL " + f_)
sys.exit(1 if FAIL else 0)