Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
perf(agent): reuse the tokenized state across questions; autocast on MPS (#109)
Emiliano G.O.34d27dfa8eb0
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.ymlindex 0674f428..00ec5f27 100644--- a/.github/workflows/ci.yml+++ b/.github/workflows/ci.yml@@ -69,6 +69,7 @@ jobs:python tests/test_doc_tables.pypython tests/test_tokenizer_cache.pypython tests/test_lazy_import.py+ python tests/test_runtime_fixes.pypython tests/test_lang_guess.pypython tests/test_calibration_persistence.pypython tests/test_context_manager.pydiff --git a/laya/agent.py b/laya/agent.pyindex 4182f3ed..5af79da9 100644--- a/laya/agent.py+++ b/laya/agent.py@@ -23,6 +23,7 @@ from .common import (confidence_from_probs,_resolve_noul_labels,render_options,+ serialize_state,temp_bucket,)from .hooks import HookRegistry, PredictContext, aggregate_usage, compose_hooks, dispatch, normalise_hooks@@ -166,17 +167,28 @@ def _load_tokenizer(tok_dir: str, cfg: Dict) -> Any:return tokenizer-def _amp_context(device, dtype):+def _amp_context(device, dtype, enabled: bool):"""Autocast context for the forward pass, or a no-op when mixed precision is not in use.- Autocast is a CUDA-only win here. Entering `torch.autocast` on a device torch has no- autocast backend for raises even with `enabled=False` ('User specified an unsupported- autocast device_type mps'), which broke every `predict()` call on the MPS GPU that torch- selects automatically on Apple/AMD machines. Only wrap the forward pass when we use it.+ Entering `torch.autocast` on a device torch has no autocast backend for raises even with+ `enabled=False` ("User specified an unsupported autocast device_type mps"), which broke+ every `predict()` on Apple Silicon on some torch builds. Only enter it when we use it."""- if device.type == "cuda":- return torch.autocast(device_type="cuda", dtype=dtype)- return nullcontext()+ if not enabled:+ return nullcontext()+ return torch.autocast(device_type=device.type, dtype=dtype)+++MPS_AMP_MIN_ROWS_DEFAULT = 5+++def _mps_amp_min_rows() -> int:+ """Rows at which MPS fp16 autocast starts to pay off. Override with LAYA_MPS_AMP_MIN_ROWS."""+ raw = os.environ.get("LAYA_MPS_AMP_MIN_ROWS", "")+ try:+ return max(1, int(raw))+ except (TypeError, ValueError):+ return MPS_AMP_MIN_ROWS_DEFAULTclass Agent(HookRegistry):@@ -188,6 +200,10 @@ class Agent(HookRegistry):hooks_concurrent = True_hooks_lock = Nonemodel_id = None+ # Autocast is chosen per device in __init__; this default covers instances built+ # without it (for example a hand-constructed runtime in tests).+ amp_enabled = False+ mps_amp_min_rows = MPS_AMP_MIN_ROWS_DEFAULTdef __init__(self,@@ -351,13 +367,30 @@ class Agent(HookRegistry):"using %s. Treat confidence from the affected entries as uncalibrated."% (TEMP_MIN, TEMP_MAX, ", ".join(rejected)),RuntimeWarning, stacklevel=2)- self.dtype = amp_dtype(self.cfg.get("amp_dtype", "fp16"))+ # Autocast policy. CUDA and MPS both support fp16/bf16 autocast and the shipped+ # checkpoints are trained in reduced precision. CPU bf16 is only a win on hardware with+ # native BF16, so it stays opt-in via LAYA_CPU_AMP=bf16. MPS fp16 is slower than fp32 on+ # a single small row (autocast overhead dominates) and only wins once the batch has+ # several rows, so it is gated per call by `mps_amp_min_rows` (default 5, override with+ # LAYA_MPS_AMP_MIN_ROWS) rather than enabled unconditionally.+ self.dtype = torch.float32+ self.amp_enabled = False+ self.mps_amp_min_rows = _mps_amp_min_rows()+ if self.device.type == "cuda":+ self.amp_enabled = True+ if torch.cuda.get_device_capability(self.device)[0] < 8:+ self.dtype = torch.float16+ else:+ self.dtype = amp_dtype(self.cfg.get("amp_dtype", "fp16"))+ elif self.device.type == "mps":+ self.amp_enabled = True+ self.dtype = torch.float16+ elif self.device.type == "cpu":+ if os.environ.get("LAYA_CPU_AMP", "").lower() in ("bf16", "bfloat16"):+ self.amp_enabled = True+ self.dtype = torch.bfloat16self._fast = None- if self.device.type == "cuda" and torch.cuda.get_device_capability(self.device)[0] < 8:- self.dtype = torch.float16- elif self.device.type in ("cpu", "mps", "xpu"):- self.dtype = torch.float32# 2. Place on device with graceful fallback to CPU on memory errorfell_back_from = fell_back_why = None@@ -370,6 +403,7 @@ class Agent(HookRegistry):fell_back_from, fell_back_why = self.device, eself.device = torch.device("cpu")self.dtype = torch.float32+ self.amp_enabled = Falseself.model.to(self.device).eval()else:raise e@@ -516,20 +550,45 @@ class Agent(HookRegistry):# right-truncation (st[:room]) would silently drop the newest turn. Truncate# from the left for lists so the most recent intent is preserved.truncate_left = isinstance(state, list)+ # Tokenize the shared state once. The ids are identical for every question, so+ # re-serializing and re-tokenizing it inside build_sequence per question was pure+ # duplicated work. Tokenize in full and let build_sequence slice per question, so+ # left-truncation for conversation lists keeps its meaning.+ state_ids = self.tok(+ serialize_state(state).replace(self.tok.mask_token, " "),+ add_special_tokens=False,+ )["input_ids"]items = []for qid in ids:q = internal[qid]seq, markers = build_sequence(self.tok, state, q, max_len, head_max_len,- truncate_left=truncate_left)+ truncate_left=truncate_left, state_ids=state_ids)if len(markers) != len(render_options(q)):raise ValueError("question %r options exceed head_max_len=%d" % (qid, head_max_len))items.append({"ids": seq, "markers": markers, "qtype": QTYPES[q["t"]]})return items- def _forward(self, b: Dict):- """Run the model on a collated batch, with the GPU->CPU OOM fallback, and return numpy outputs."""+ def _amp_enabled_for(self, rows: int) -> bool:+ """Whether to autocast a forward with `rows` question rows.++ MPS fp16 loses to fp32 on a single small row and wins once the batch grows, so it is+ only enabled at or above `mps_amp_min_rows`. Other devices are unaffected.+ """+ if not self.amp_enabled:+ return False+ if self.device.type == "mps" and rows < self.mps_amp_min_rows:+ return False+ return True++ def _infer(self, b: Dict):+ """Run the forward pass under autocast, degrading gracefully on OOM or unsupported autocast."""+ use_amp = self._amp_enabled_for(b["input_ids"].shape[0])+def run():- with _amp_context(self.device, self.dtype):+ # Recomputed inside run() so a fallback that disables amp (or moves to CPU) takes+ # effect on the retry. A disabled gate never enters torch.autocast at all.+ enabled = self._amp_enabled_for(b["input_ids"].shape[0])+ with _amp_context(self.device, self.dtype, enabled):return self.model(b["input_ids"].to(self.device),b["attention_mask"].to(self.device),@@ -539,17 +598,27 @@ class Agent(HookRegistry):)try:- logits, act = run()+ return run()except (RuntimeError, torch.cuda.OutOfMemoryError) as e:- if self.device.type != "cpu" and ("memory" in str(e).lower() or "cuda" in str(e).lower()):+ low = str(e).lower()+ if self.device.type != "cpu" and ("memory" in low or "cuda" in low):print("Warning: GPU memory exceeded during inference. Falling back to CPU...")self.device = torch.device("cpu")self.dtype = torch.float32+ self.amp_enabled = Falseself.model.to(self.device)- logits, act = run()- else:- raise e+ return run()+ if use_amp and self.device.type in ("mps", "cpu"):+ # Not every MPS/CPU build implements autocast for every op. Drop to full+ # precision once rather than failing the request.+ self.amp_enabled = False+ self.dtype = torch.float32+ return run()+ raise+ def _forward(self, b: Dict):+ """Run the model on a collated batch, with the GPU->CPU OOM fallback, and return numpy outputs."""+ logits, act = self._infer(b)return logits.float().cpu().numpy(), torch.softmax(act.float(), -1).cpu().numpy()def _decode_answers(self, logits, act, items: List[Dict], ids: List[str],diff --git a/laya/common.py b/laya/common.pyindex 6d0859bb..45aa99e5 100644--- a/laya/common.py+++ b/laya/common.py@@ -75,8 +75,13 @@ def build_sequence(head_max_len: int = 192,option_order: Optional[List[int]] = None,truncate_left: bool = False,+ state_ids: Optional[List[int]] = None,):- """Format: [CLS] <type> instructions [SEP] [MASK] opt0 [MASK] opt1 ... [SEP] state [SEP]."""+ """Format: [CLS] <type> instructions [SEP] [MASK] opt0 [MASK] opt1 ... [SEP] state [SEP].++ `state_ids` lets a caller tokenize the shared state once and reuse it across every question,+ instead of re-serializing and re-tokenizing the same document per question.+ """mask_tok = tok.mask_tokenopts = render_options(q)order = option_order if option_order is not None else list(range(len(opts)))@@ -84,10 +89,16 @@ def build_sequence(head_ids = tok("%s question: %s" % (q["t"], ins), add_special_tokens=False)["input_ids"]opt_ids = []for i in order:- opt_ids.append(- [tok.mask_token_id]- + tok(" " + opts[i].replace(mask_tok, " "), add_special_tokens=False)["input_ids"][:48]- )+ # Cap at the tokenizer, not after the fact: `[:48]` still makes the tokenizer process the+ # whole (possibly long) description. truncation=True, max_length=48 keeps the first 48+ # tokens, which is exactly what the previous slice produced.+ opt_tokens = tok(+ " " + opts[i].replace(mask_tok, " "),+ add_special_tokens=False,+ truncation=True,+ max_length=48,+ )["input_ids"]+ opt_ids.append([tok.mask_token_id] + opt_tokens)opt_budget = head_max_len - sum(len(o) for o in opt_ids)if opt_budget < 16:per = max(4, (head_max_len - 16) // max(1, len(opt_ids)))@@ -101,9 +112,10 @@ def build_sequence(ids.extend(o)ids.append(tok.sep_token_id)room = max(0, max_len - len(ids) - 1)- st = tok(serialize_state(state).replace(mask_tok, " "), add_special_tokens=False)["input_ids"]- # not st[-room:]: with no room left, st[-0:] is the whole state rather than none of it- st = st[max(0, len(st) - room):] if truncate_left else st[:room]+ if state_ids is None:+ state_ids = tok(serialize_state(state).replace(mask_tok, " "), add_special_tokens=False)["input_ids"]+ # not state_ids[-room:]: with no room left, state_ids[-0:] is the whole state rather than none of it+ st = state_ids[max(0, len(state_ids) - room):] if truncate_left else state_ids[:room]ids = ids + st + [tok.sep_token_id]return ids[:max_len], [m for m in markers if m < max_len]diff --git a/tests/test_criteria.py b/tests/test_criteria.pyindex dea5703f..738ea87f 100644--- a/tests/test_criteria.py+++ b/tests/test_criteria.py@@ -238,8 +238,11 @@ class _FakeTok:cls_token_id, sep_token_id, mask_token_id, pad_token_id = 0, 1, 4, 2mask_token = "[MASK]"- def __call__(self, text, add_special_tokens=False):- return {"input_ids": [10 + (len(w) % 90) for w in text.split() if w]}+ def __call__(self, text, add_special_tokens=False, truncation=False, max_length=None):+ ids = [10 + (len(w) % 90) for w in text.split() if w]+ if truncation and max_length:+ ids = ids[:max_length]+ return {"input_ids": ids}def _tiny_agent():@@ -442,8 +445,11 @@ class _SeqTok:def __init__(self):self.vocab = {}- def __call__(self, text, add_special_tokens=False):- return {"input_ids": [self.vocab.setdefault(w, 100 + len(self.vocab)) for w in text.split()]}+ def __call__(self, text, add_special_tokens=False, truncation=False, max_length=None):+ ids = [self.vocab.setdefault(w, 100 + len(self.vocab)) for w in text.split()]+ if truncation and max_length:+ ids = ids[:max_length]+ return {"input_ids": ids}_tok, _q = _SeqTok(), {"t": "noul", "ins": "Is it urgent?", "crit": None}diff --git a/tests/test_portability.py b/tests/test_portability.pyindex fffe24b3..878c9a64 100644--- a/tests/test_portability.py+++ b/tests/test_portability.py@@ -100,29 +100,35 @@ def _autocast_that_rejects_non_cuda(entered):return fake+# A disabled context must never enter torch.autocast: on a torch build with no MPS autocast+# backend, entering it raises. The gate (Agent._amp_enabled_for) decides `enabled`.for name in ("cpu", "mps", "xpu"):entered = []with mock.patch.object(_agent.torch, "autocast", _autocast_that_rejects_non_cuda(entered)):try:- ctx = _agent._amp_context(SimpleNamespace(type=name), torch.float32)+ ctx = _agent._amp_context(SimpleNamespace(type=name), torch.float32, False)with ctx:pass- check_true("amp/%s never enters autocast" % name, entered == [], "entered %s" % entered)+ check_true("amp/%s never enters autocast when disabled" % name, entered == [],+ "entered %s" % entered)except RuntimeError as e:FAIL.append("amp/%s raised %s" % (name, e))+check_true("amp/disabled is a no-op",+ isinstance(_agent._amp_context(SimpleNamespace(type="mps"), torch.float16, False), nullcontext))+# CUDA still gets mixed precision, with the dtype the model was configured forentered = []with mock.patch.object(_agent.torch, "autocast", _autocast_that_rejects_non_cuda(entered)):- with _agent._amp_context(SimpleNamespace(type="cuda"), torch.float16):+ with _agent._amp_context(SimpleNamespace(type="cuda"), torch.float16, True):passcheck("amp/cuda uses autocast", entered, ["cuda"])# the old unconditional call is what broke MPS -- make sure it cannot come backimport inspect # noqa: E402-_src = inspect.getsource(_agent.Agent._forward)-check_true("amp/forward pass goes through _amp_context", "with _amp_context(self.device, self.dtype):" in _src)+_src = inspect.getsource(_agent.Agent._infer)+check_true("amp/infer goes through _amp_context", "with _amp_context(self.device, self.dtype, enabled)" in _src)check_true("amp/no unconditional autocast in the forward pass","torch.autocast(device_type=self.device.type" not in _src)@@ -135,8 +141,11 @@ class _FakeTok:cls_token_id, sep_token_id, mask_token_id, pad_token_id = 1, 2, 3, 0mask_token = "[M]"- def __call__(self, text, add_special_tokens=False):- return {"input_ids": [10 + (ord(c) % 40) for c in text]}+ def __call__(self, text, add_special_tokens=False, truncation=False, max_length=None):+ ids = [10 + (ord(c) % 40) for c in text]+ if truncation and max_length:+ ids = ids[:max_length]+ return {"input_ids": ids}class _FailsOnce(torch.nn.Module):diff --git a/tests/test_runtime_fixes.py b/tests/test_runtime_fixes.pynew file mode 100644index 00000000..5875ee86--- /dev/null+++ b/tests/test_runtime_fixes.py@@ -0,0 +1,240 @@+"""Regression tests for the state-reuse and autocast changes. No weights are downloaded.++Covers:+ * the state is serialized/tokenized once per call, not once per question+ * `build_sequence(..., state_ids=...)` is equivalent to the tokenizing path+ * the autocast path works and degrades to full precision instead of failing+"""+import os+import sys+from contextlib import nullcontext+from types import SimpleNamespace++import torch+import torch.nn as nn++sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))++from laya.agent import MPS_AMP_MIN_ROWS_DEFAULT, Agent, _amp_context, _mps_amp_min_rows # noqa: E402+from laya.common import DecisionModel, build_sequence, serialize_state # noqa: E402++PASS, FAIL = [], []+++def check(name, got, want):+ if got == want:+ PASS.append(name)+ else:+ FAIL.append("%s: got %r, want %r" % (name, got, want))+++# ------------------------------------------------------------------ fake tokenizer+class FakeTok:+ """Just enough of a tokenizer for `build_sequence` / `system_one`."""++ mask_token = "[MASK]"+ mask_token_id = 1+ cls_token_id = 2+ sep_token_id = 3+ pad_token_id = 0++ def __init__(self):+ self.calls = []++ def __call__(self, text, add_special_tokens=False, truncation=False, max_length=None):+ self.calls.append(text)+ n = max(1, len(text) // 4)+ if truncation and max_length:+ n = min(n, max_length)+ return {"input_ids": [5] * n}+++class FakeModel:+ def __call__(self, input_ids, attention_mask, marker_pos, marker_mask, qtype):+ logits = torch.where(+ marker_mask,+ torch.ones_like(marker_mask, dtype=torch.float32),+ torch.full_like(marker_mask, -1e4, dtype=torch.float32),+ )+ return logits, torch.zeros((input_ids.shape[0], 2))+++def _bare_agent(model, dtype=torch.float32, amp=False, tok=None):+ a = object.__new__(Agent)+ a.device = torch.device("cpu")+ a.dtype = dtype+ a.amp_enabled = amp+ a.cfg = {"max_len": 64, "head_max_len": 32}+ a.temperature = [1.0, 1.0, 1.0]+ a.temperature_by_options = {}+ a.tok = tok or FakeTok()+ a.model = model+ return a+++# ------------------------------------------------------------------ state reused once+STATE = {"subject": "Duplicate charge", "body": "x" * 400}+QUESTION = {"t": "choice", "ins": "pick", "crit": {"a": "x", "b": "y"}}++tok_ref = FakeTok()+seq_ref, markers_ref = build_sequence(tok_ref, STATE, QUESTION, 64, 32)++tok_shared = FakeTok()+state_ids = tok_shared(serialize_state(STATE), add_special_tokens=False)["input_ids"]+seq_shared, markers_shared = build_sequence(tok_shared, STATE, QUESTION, 64, 32, state_ids=state_ids)+check("build_sequence/state_ids identical ids", seq_shared, seq_ref)+check("build_sequence/state_ids identical markers", markers_shared, markers_ref)+check("build_sequence/state_ids does not re-tokenize state", tok_shared.calls.count(serialize_state(STATE)), 1)++agent = _bare_agent(FakeModel())+out = agent.system_one("the customer was charged twice", {+ "department": {"type": "choice", "instructions": "which?", "criteria": {"billing": "x", "technical": "y"}},+ "urgent": {"type": "noul", "instructions": "is it urgent?"},+})+state_text = serialize_state("the customer was charged twice").replace(agent.tok.mask_token, " ")+check("system_one/state tokenized once for two questions", agent.tok.calls.count(state_text), 1)+check("system_one/answers present", sorted(out["answers"]), ["department", "urgent"])+++class RecordingTok(FakeTok):+ def __init__(self):+ super().__init__()+ self.caps = []++ def __call__(self, text, add_special_tokens=False, truncation=False, max_length=None):+ self.caps.append((truncation, max_length))+ return super().__call__(text, add_special_tokens=add_special_tokens,+ truncation=truncation, max_length=max_length)+++long_q = {"t": "choice", "ins": "pick", "crit": {"a": "x" * 400, "b": "y" * 400}}+rtok = RecordingTok()+long_seq, long_markers = build_sequence(rtok, "state", long_q, 300, 200)+# options are capped at the tokenizer (truncation=True, max_length=48), not sliced afterwards+check("build_sequence/options capped at the tokenizer",+ len([c for c in rtok.caps if c == (True, 48)]), len(long_markers))+check("build_sequence/option segments stay <= 49 tokens",+ all(long_markers[i + 1] - long_markers[i] <= 49 for i in range(len(long_markers) - 1)), True)+++# ------------------------------------------------------------------ autocast+class DummyEnc(nn.Module):+ def __init__(self, d=16):+ super().__init__()+ self.config = SimpleNamespace(hidden_size=d, num_attention_heads=1)+ self.emb = nn.Embedding(16, d)++ def forward(self, input_ids, attention_mask):+ return SimpleNamespace(last_hidden_state=self.emb(input_ids))+++model = DecisionModel(DummyEnc(), head_layers=1, n_act=2)+iid = torch.tensor([[1, 2, 3, 4, 5]])+am = torch.ones_like(iid)+mp = torch.tensor([[1, 3]])+mm = torch.ones_like(mp, dtype=torch.bool)+qt = torch.tensor([0])+with torch.no_grad():+ logits32, act32 = model(iid, am, mp, mm, qt)+ with torch.autocast(device_type="cpu", dtype=torch.bfloat16, enabled=True):+ logits16, act16 = model(iid, am, mp, mm, qt)+check("autocast/shapes match", tuple(logits16.shape), tuple(logits32.shape))+check("autocast/finite", bool(torch.isfinite(logits16).all() and torch.isfinite(act16).all()), True)+check("autocast/close to fp32", float((logits32 - logits16).abs().max()) < 0.5, True)++calls = {"n": 0}+++class Flaky:+ def __call__(self, *args):+ calls["n"] += 1+ if calls["n"] == 1:+ raise RuntimeError("autocast not supported on this build")+ return torch.zeros((1, 2)), torch.zeros((1, 2))+++batch = {+ "input_ids": torch.tensor([[1]]),+ "attention_mask": torch.ones((1, 1), dtype=torch.long),+ "marker_pos": torch.zeros((1, 1), dtype=torch.long),+ "marker_mask": torch.ones((1, 1), dtype=torch.bool),+ "qtype": torch.tensor([0]),+}+flaky = _bare_agent(Flaky(), dtype=torch.bfloat16, amp=True)+flaky._infer(batch)+check("infer/falls back and disables amp", flaky.amp_enabled, False)+check("infer/retried once", calls["n"], 2)+++class Boom:+ def __call__(self, *args):+ raise RuntimeError("genuine failure")+++boom = _bare_agent(Boom())+raised = False+try:+ boom._infer(batch)+except RuntimeError:+ raised = True+check("infer/non-autocast error propagates", raised, True)+++# ------------------------------------------------------------------ MPS autocast gating+# MPS fp16 is slower than fp32 on one small row and only wins once the batch grows, so it is+# gated by row count. These check the gate without an MPS device.+def _mps_agent(amp=True, min_rows=None):+ a = _bare_agent(FakeModel(), dtype=torch.float16, amp=amp)+ a.device = torch.device("mps")+ if min_rows is not None:+ a.mps_amp_min_rows = min_rows+ return a+++a = _mps_agent()+check("mps-gate/one row stays fp32", a._amp_enabled_for(1), False)+check("mps-gate/below threshold stays fp32", a._amp_enabled_for(a.mps_amp_min_rows - 1), False)+check("mps-gate/at threshold enables fp16", a._amp_enabled_for(a.mps_amp_min_rows), True)+check("mps-gate/above threshold enables fp16", a._amp_enabled_for(a.mps_amp_min_rows + 3), True)++check("mps-gate/threshold override", _mps_agent(min_rows=2)._amp_enabled_for(2), True)+check("mps-gate/amp disabled stays off", _mps_agent(amp=False)._amp_enabled_for(100), False)++cpu = _bare_agent(FakeModel(), dtype=torch.bfloat16, amp=True)+check("cpu-gate/not gated by rows", cpu._amp_enabled_for(1), True)++os.environ["LAYA_MPS_AMP_MIN_ROWS"] = "2"+check("mps-gate/env override", _mps_amp_min_rows(), 2)+os.environ["LAYA_MPS_AMP_MIN_ROWS"] = "nonsense"+check("mps-gate/env invalid falls back", _mps_amp_min_rows(), MPS_AMP_MIN_ROWS_DEFAULT)+del os.environ["LAYA_MPS_AMP_MIN_ROWS"]+check("mps-gate/env default", _mps_amp_min_rows(), MPS_AMP_MIN_ROWS_DEFAULT)+++# ------------------------------------------------------------------ amp context shape+# A disabled gate must never construct torch.autocast: on torch builds without an MPS+# autocast backend, entering it raises even with enabled=False, which broke MPS predict().+mps = torch.device("mps")+check("amp-context/disabled mps is a no-op", isinstance(_amp_context(mps, torch.float16, False), nullcontext), True)+with _amp_context(mps, torch.float16, False): # must not raise+ pass+check("amp-context/disabled mps enters cleanly", True, True)++check("amp-context/disabled cpu is a no-op",+ isinstance(_amp_context(torch.device("cpu"), torch.float32, False), nullcontext), True)+check("amp-context/enabled cpu is autocast",+ not isinstance(_amp_context(torch.device("cpu"), torch.bfloat16, True), nullcontext), True)++# the disabled path through _infer completes on a plain CPU agent+cpu_disabled = _bare_agent(FakeModel()) # amp=False+cpu_disabled._infer(batch)+check("amp-context/_infer disabled completes", True, True)+++# ------------------------------------------------------------------ report+print("\n%d passed, %d failed" % (len(PASS), len(FAIL)))+for f in FAIL:+ print(" FAIL", f)+if not FAIL:+ print("all runtime-fix tests passed")+sys.exit(1 if FAIL else 0)diff --git a/tests/test_training.py b/tests/test_training.pyindex 796e5259..ab1394c5 100644--- a/tests/test_training.py+++ b/tests/test_training.py@@ -264,8 +264,11 @@ class _Tok:cls_token_id, sep_token_id, mask_token_id, pad_token_id = 1, 2, 3, 0mask_token = "[M]"- def __call__(self, text, add_special_tokens=False):- return {"input_ids": [10 + (ord(c) % 50) for c in text]}+ def __call__(self, text, add_special_tokens=False, truncation=False, max_length=None):+ ids = [10 + (ord(c) % 50) for c in text]+ if truncation and max_length:+ ids = ids[:max_length]+ return {"input_ids": ids}tok = _Tok()diff --git a/tests/test_truncation_direction.py b/tests/test_truncation_direction.pyindex 3cb1722d..c4de4078 100644--- a/tests/test_truncation_direction.py+++ b/tests/test_truncation_direction.py@@ -20,8 +20,11 @@ class _FakeTok:cls_token_id, sep_token_id, mask_token_id, pad_token_id = 0, 1, 4, 2mask_token = "[MASK]"- def __call__(self, text, add_special_tokens=False):- return {"input_ids": [10 + (len(w) % 90) for w in text.split() if w]}+ def __call__(self, text, add_special_tokens=False, truncation=False, max_length=None):+ ids = [10 + (len(w) % 90) for w in text.split() if w]+ if truncation and max_length:+ ids = ids[:max_length]+ return {"input_ids": ids}def _tiny_agent():@@ -56,12 +59,13 @@ def test_agent_list_vs_string_truncation_direction():list_capture = {}string_capture = {}- def fake_build(tok, state, q, max_len, head_max_len, truncate_left=False):+ def fake_build(tok, state, q, max_len, head_max_len, truncate_left=False, state_ids=None):if isinstance(state, list):list_capture["truncate_left"] = truncate_leftelse:string_capture["truncate_left"] = truncate_left- return build_sequence(tok, state, q, max_len, head_max_len, truncate_left=truncate_left)+ return build_sequence(tok, state, q, max_len, head_max_len,+ truncate_left=truncate_left, state_ids=state_ids)with patch("laya.agent.build_sequence", side_effect=fake_build):agent.system_one([{"role": "user", "content": "hi"}, {"role": "user", "content": "newest"}], questions)@@ -81,10 +85,11 @@ def test_agent_system_one_passes_truncate_left_for_list():captured = {}- def fake_build(tok, state, q, max_len, head_max_len, truncate_left=False):+ def fake_build(tok, state, q, max_len, head_max_len, truncate_left=False, state_ids=None):captured["truncate_left"] = truncate_leftcaptured["state"] = state- return build_sequence(tok, state, q, max_len, head_max_len, truncate_left=truncate_left)+ return build_sequence(tok, state, q, max_len, head_max_len,+ truncate_left=truncate_left, state_ids=state_ids)with patch("laya.agent.build_sequence", side_effect=fake_build):agent.system_one(conversation, questions)@@ -102,9 +107,10 @@ def test_agent_system_one_string_state_default_truncation():captured = {}- def fake_build(tok, state, q, max_len, head_max_len, truncate_left=False):+ def fake_build(tok, state, q, max_len, head_max_len, truncate_left=False, state_ids=None):captured["truncate_left"] = truncate_left- return build_sequence(tok, state, q, max_len, head_max_len, truncate_left=truncate_left)+ return build_sequence(tok, state, q, max_len, head_max_len,+ truncate_left=truncate_left, state_ids=state_ids)with patch("laya.agent.build_sequence", side_effect=fake_build):agent.system_one("just a string state", questions)