Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

perf(agent): reuse the tokenized state across questions; autocast on MPS (#109)

Emiliano G.O.34d27dfa8eb0
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 0674f428..00ec5f27 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -69,6 +69,7 @@ jobs:
python tests/test_doc_tables.py
python tests/test_tokenizer_cache.py
python tests/test_lazy_import.py
+ python tests/test_runtime_fixes.py
python tests/test_lang_guess.py
python tests/test_calibration_persistence.py
python tests/test_context_manager.py
diff --git a/laya/agent.py b/laya/agent.py
index 4182f3ed..5af79da9 100644
--- a/laya/agent.py
+++ b/laya/agent.py
@@ -23,6 +23,7 @@ from .common import (
confidence_from_probs,
_resolve_noul_labels,
render_options,
+ serialize_state,
temp_bucket,
)
from .hooks import HookRegistry, PredictContext, aggregate_usage, compose_hooks, dispatch, normalise_hooks
@@ -166,17 +167,28 @@ def _load_tokenizer(tok_dir: str, cfg: Dict) -> Any:
return tokenizer
-def _amp_context(device, dtype):
+def _amp_context(device, dtype, enabled: bool):
"""Autocast context for the forward pass, or a no-op when mixed precision is not in use.
- Autocast is a CUDA-only win here. Entering `torch.autocast` on a device torch has no
- autocast backend for raises even with `enabled=False` ('User specified an unsupported
- autocast device_type mps'), which broke every `predict()` call on the MPS GPU that torch
- selects automatically on Apple/AMD machines. Only wrap the forward pass when we use it.
+ Entering `torch.autocast` on a device torch has no autocast backend for raises even with
+ `enabled=False` ("User specified an unsupported autocast device_type mps"), which broke
+ every `predict()` on Apple Silicon on some torch builds. Only enter it when we use it.
"""
- if device.type == "cuda":
- return torch.autocast(device_type="cuda", dtype=dtype)
- return nullcontext()
+ if not enabled:
+ return nullcontext()
+ return torch.autocast(device_type=device.type, dtype=dtype)
+
+
+MPS_AMP_MIN_ROWS_DEFAULT = 5
+
+
+def _mps_amp_min_rows() -> int:
+ """Rows at which MPS fp16 autocast starts to pay off. Override with LAYA_MPS_AMP_MIN_ROWS."""
+ raw = os.environ.get("LAYA_MPS_AMP_MIN_ROWS", "")
+ try:
+ return max(1, int(raw))
+ except (TypeError, ValueError):
+ return MPS_AMP_MIN_ROWS_DEFAULT
class Agent(HookRegistry):
@@ -188,6 +200,10 @@ class Agent(HookRegistry):
hooks_concurrent = True
_hooks_lock = None
model_id = None
+ # Autocast is chosen per device in __init__; this default covers instances built
+ # without it (for example a hand-constructed runtime in tests).
+ amp_enabled = False
+ mps_amp_min_rows = MPS_AMP_MIN_ROWS_DEFAULT
def __init__(
self,
@@ -351,13 +367,30 @@ class Agent(HookRegistry):
"using %s. Treat confidence from the affected entries as uncalibrated."
% (TEMP_MIN, TEMP_MAX, ", ".join(rejected)),
RuntimeWarning, stacklevel=2)
- self.dtype = amp_dtype(self.cfg.get("amp_dtype", "fp16"))
+ # Autocast policy. CUDA and MPS both support fp16/bf16 autocast and the shipped
+ # checkpoints are trained in reduced precision. CPU bf16 is only a win on hardware with
+ # native BF16, so it stays opt-in via LAYA_CPU_AMP=bf16. MPS fp16 is slower than fp32 on
+ # a single small row (autocast overhead dominates) and only wins once the batch has
+ # several rows, so it is gated per call by `mps_amp_min_rows` (default 5, override with
+ # LAYA_MPS_AMP_MIN_ROWS) rather than enabled unconditionally.
+ self.dtype = torch.float32
+ self.amp_enabled = False
+ self.mps_amp_min_rows = _mps_amp_min_rows()
+ if self.device.type == "cuda":
+ self.amp_enabled = True
+ if torch.cuda.get_device_capability(self.device)[0] < 8:
+ self.dtype = torch.float16
+ else:
+ self.dtype = amp_dtype(self.cfg.get("amp_dtype", "fp16"))
+ elif self.device.type == "mps":
+ self.amp_enabled = True
+ self.dtype = torch.float16
+ elif self.device.type == "cpu":
+ if os.environ.get("LAYA_CPU_AMP", "").lower() in ("bf16", "bfloat16"):
+ self.amp_enabled = True
+ self.dtype = torch.bfloat16
self._fast = None
- if self.device.type == "cuda" and torch.cuda.get_device_capability(self.device)[0] < 8:
- self.dtype = torch.float16
- elif self.device.type in ("cpu", "mps", "xpu"):
- self.dtype = torch.float32
# 2. Place on device with graceful fallback to CPU on memory error
fell_back_from = fell_back_why = None
@@ -370,6 +403,7 @@ class Agent(HookRegistry):
fell_back_from, fell_back_why = self.device, e
self.device = torch.device("cpu")
self.dtype = torch.float32
+ self.amp_enabled = False
self.model.to(self.device).eval()
else:
raise e
@@ -516,20 +550,45 @@ class Agent(HookRegistry):
# right-truncation (st[:room]) would silently drop the newest turn. Truncate
# from the left for lists so the most recent intent is preserved.
truncate_left = isinstance(state, list)
+ # Tokenize the shared state once. The ids are identical for every question, so
+ # re-serializing and re-tokenizing it inside build_sequence per question was pure
+ # duplicated work. Tokenize in full and let build_sequence slice per question, so
+ # left-truncation for conversation lists keeps its meaning.
+ state_ids = self.tok(
+ serialize_state(state).replace(self.tok.mask_token, " "),
+ add_special_tokens=False,
+ )["input_ids"]
items = []
for qid in ids:
q = internal[qid]
seq, markers = build_sequence(self.tok, state, q, max_len, head_max_len,
- truncate_left=truncate_left)
+ truncate_left=truncate_left, state_ids=state_ids)
if len(markers) != len(render_options(q)):
raise ValueError("question %r options exceed head_max_len=%d" % (qid, head_max_len))
items.append({"ids": seq, "markers": markers, "qtype": QTYPES[q["t"]]})
return items
- def _forward(self, b: Dict):
- """Run the model on a collated batch, with the GPU->CPU OOM fallback, and return numpy outputs."""
+ def _amp_enabled_for(self, rows: int) -> bool:
+ """Whether to autocast a forward with `rows` question rows.
+
+ MPS fp16 loses to fp32 on a single small row and wins once the batch grows, so it is
+ only enabled at or above `mps_amp_min_rows`. Other devices are unaffected.
+ """
+ if not self.amp_enabled:
+ return False
+ if self.device.type == "mps" and rows < self.mps_amp_min_rows:
+ return False
+ return True
+
+ def _infer(self, b: Dict):
+ """Run the forward pass under autocast, degrading gracefully on OOM or unsupported autocast."""
+ use_amp = self._amp_enabled_for(b["input_ids"].shape[0])
+
def run():
- with _amp_context(self.device, self.dtype):
+ # Recomputed inside run() so a fallback that disables amp (or moves to CPU) takes
+ # effect on the retry. A disabled gate never enters torch.autocast at all.
+ enabled = self._amp_enabled_for(b["input_ids"].shape[0])
+ with _amp_context(self.device, self.dtype, enabled):
return self.model(
b["input_ids"].to(self.device),
b["attention_mask"].to(self.device),
@@ -539,17 +598,27 @@ class Agent(HookRegistry):
)
try:
- logits, act = run()
+ return run()
except (RuntimeError, torch.cuda.OutOfMemoryError) as e:
- if self.device.type != "cpu" and ("memory" in str(e).lower() or "cuda" in str(e).lower()):
+ low = str(e).lower()
+ if self.device.type != "cpu" and ("memory" in low or "cuda" in low):
print("Warning: GPU memory exceeded during inference. Falling back to CPU...")
self.device = torch.device("cpu")
self.dtype = torch.float32
+ self.amp_enabled = False
self.model.to(self.device)
- logits, act = run()
- else:
- raise e
+ return run()
+ if use_amp and self.device.type in ("mps", "cpu"):
+ # Not every MPS/CPU build implements autocast for every op. Drop to full
+ # precision once rather than failing the request.
+ self.amp_enabled = False
+ self.dtype = torch.float32
+ return run()
+ raise
+ def _forward(self, b: Dict):
+ """Run the model on a collated batch, with the GPU->CPU OOM fallback, and return numpy outputs."""
+ logits, act = self._infer(b)
return logits.float().cpu().numpy(), torch.softmax(act.float(), -1).cpu().numpy()
def _decode_answers(self, logits, act, items: List[Dict], ids: List[str],
diff --git a/laya/common.py b/laya/common.py
index 6d0859bb..45aa99e5 100644
--- a/laya/common.py
+++ b/laya/common.py
@@ -75,8 +75,13 @@ def build_sequence(
head_max_len: int = 192,
option_order: Optional[List[int]] = None,
truncate_left: bool = False,
+ state_ids: Optional[List[int]] = None,
):
- """Format: [CLS] <type> instructions [SEP] [MASK] opt0 [MASK] opt1 ... [SEP] state [SEP]."""
+ """Format: [CLS] <type> instructions [SEP] [MASK] opt0 [MASK] opt1 ... [SEP] state [SEP].
+
+ `state_ids` lets a caller tokenize the shared state once and reuse it across every question,
+ instead of re-serializing and re-tokenizing the same document per question.
+ """
mask_tok = tok.mask_token
opts = render_options(q)
order = option_order if option_order is not None else list(range(len(opts)))
@@ -84,10 +89,16 @@ def build_sequence(
head_ids = tok("%s question: %s" % (q["t"], ins), add_special_tokens=False)["input_ids"]
opt_ids = []
for i in order:
- opt_ids.append(
- [tok.mask_token_id]
- + tok(" " + opts[i].replace(mask_tok, " "), add_special_tokens=False)["input_ids"][:48]
- )
+ # Cap at the tokenizer, not after the fact: `[:48]` still makes the tokenizer process the
+ # whole (possibly long) description. truncation=True, max_length=48 keeps the first 48
+ # tokens, which is exactly what the previous slice produced.
+ opt_tokens = tok(
+ " " + opts[i].replace(mask_tok, " "),
+ add_special_tokens=False,
+ truncation=True,
+ max_length=48,
+ )["input_ids"]
+ opt_ids.append([tok.mask_token_id] + opt_tokens)
opt_budget = head_max_len - sum(len(o) for o in opt_ids)
if opt_budget < 16:
per = max(4, (head_max_len - 16) // max(1, len(opt_ids)))
@@ -101,9 +112,10 @@ def build_sequence(
ids.extend(o)
ids.append(tok.sep_token_id)
room = max(0, max_len - len(ids) - 1)
- st = tok(serialize_state(state).replace(mask_tok, " "), add_special_tokens=False)["input_ids"]
- # not st[-room:]: with no room left, st[-0:] is the whole state rather than none of it
- st = st[max(0, len(st) - room):] if truncate_left else st[:room]
+ if state_ids is None:
+ state_ids = tok(serialize_state(state).replace(mask_tok, " "), add_special_tokens=False)["input_ids"]
+ # not state_ids[-room:]: with no room left, state_ids[-0:] is the whole state rather than none of it
+ st = state_ids[max(0, len(state_ids) - room):] if truncate_left else state_ids[:room]
ids = ids + st + [tok.sep_token_id]
return ids[:max_len], [m for m in markers if m < max_len]
diff --git a/tests/test_criteria.py b/tests/test_criteria.py
index dea5703f..738ea87f 100644
--- a/tests/test_criteria.py
+++ b/tests/test_criteria.py
@@ -238,8 +238,11 @@ class _FakeTok:
cls_token_id, sep_token_id, mask_token_id, pad_token_id = 0, 1, 4, 2
mask_token = "[MASK]"
- def __call__(self, text, add_special_tokens=False):
- return {"input_ids": [10 + (len(w) % 90) for w in text.split() if w]}
+ def __call__(self, text, add_special_tokens=False, truncation=False, max_length=None):
+ ids = [10 + (len(w) % 90) for w in text.split() if w]
+ if truncation and max_length:
+ ids = ids[:max_length]
+ return {"input_ids": ids}
def _tiny_agent():
@@ -442,8 +445,11 @@ class _SeqTok:
def __init__(self):
self.vocab = {}
- def __call__(self, text, add_special_tokens=False):
- return {"input_ids": [self.vocab.setdefault(w, 100 + len(self.vocab)) for w in text.split()]}
+ def __call__(self, text, add_special_tokens=False, truncation=False, max_length=None):
+ ids = [self.vocab.setdefault(w, 100 + len(self.vocab)) for w in text.split()]
+ if truncation and max_length:
+ ids = ids[:max_length]
+ return {"input_ids": ids}
_tok, _q = _SeqTok(), {"t": "noul", "ins": "Is it urgent?", "crit": None}
diff --git a/tests/test_portability.py b/tests/test_portability.py
index fffe24b3..878c9a64 100644
--- a/tests/test_portability.py
+++ b/tests/test_portability.py
@@ -100,29 +100,35 @@ def _autocast_that_rejects_non_cuda(entered):
return fake
+# A disabled context must never enter torch.autocast: on a torch build with no MPS autocast
+# backend, entering it raises. The gate (Agent._amp_enabled_for) decides `enabled`.
for name in ("cpu", "mps", "xpu"):
entered = []
with mock.patch.object(_agent.torch, "autocast", _autocast_that_rejects_non_cuda(entered)):
try:
- ctx = _agent._amp_context(SimpleNamespace(type=name), torch.float32)
+ ctx = _agent._amp_context(SimpleNamespace(type=name), torch.float32, False)
with ctx:
pass
- check_true("amp/%s never enters autocast" % name, entered == [], "entered %s" % entered)
+ check_true("amp/%s never enters autocast when disabled" % name, entered == [],
+ "entered %s" % entered)
except RuntimeError as e:
FAIL.append("amp/%s raised %s" % (name, e))
+check_true("amp/disabled is a no-op",
+ isinstance(_agent._amp_context(SimpleNamespace(type="mps"), torch.float16, False), nullcontext))
+
# CUDA still gets mixed precision, with the dtype the model was configured for
entered = []
with mock.patch.object(_agent.torch, "autocast", _autocast_that_rejects_non_cuda(entered)):
- with _agent._amp_context(SimpleNamespace(type="cuda"), torch.float16):
+ with _agent._amp_context(SimpleNamespace(type="cuda"), torch.float16, True):
pass
check("amp/cuda uses autocast", entered, ["cuda"])
# the old unconditional call is what broke MPS -- make sure it cannot come back
import inspect # noqa: E402
-_src = inspect.getsource(_agent.Agent._forward)
-check_true("amp/forward pass goes through _amp_context", "with _amp_context(self.device, self.dtype):" in _src)
+_src = inspect.getsource(_agent.Agent._infer)
+check_true("amp/infer goes through _amp_context", "with _amp_context(self.device, self.dtype, enabled)" in _src)
check_true("amp/no unconditional autocast in the forward pass",
"torch.autocast(device_type=self.device.type" not in _src)
@@ -135,8 +141,11 @@ class _FakeTok:
cls_token_id, sep_token_id, mask_token_id, pad_token_id = 1, 2, 3, 0
mask_token = "[M]"
- def __call__(self, text, add_special_tokens=False):
- return {"input_ids": [10 + (ord(c) % 40) for c in text]}
+ def __call__(self, text, add_special_tokens=False, truncation=False, max_length=None):
+ ids = [10 + (ord(c) % 40) for c in text]
+ if truncation and max_length:
+ ids = ids[:max_length]
+ return {"input_ids": ids}
class _FailsOnce(torch.nn.Module):
diff --git a/tests/test_runtime_fixes.py b/tests/test_runtime_fixes.py
new file mode 100644
index 00000000..5875ee86
--- /dev/null
+++ b/tests/test_runtime_fixes.py
@@ -0,0 +1,240 @@
+"""Regression tests for the state-reuse and autocast changes. No weights are downloaded.
+
+Covers:
+ * the state is serialized/tokenized once per call, not once per question
+ * `build_sequence(..., state_ids=...)` is equivalent to the tokenizing path
+ * the autocast path works and degrades to full precision instead of failing
+"""
+import os
+import sys
+from contextlib import nullcontext
+from types import SimpleNamespace
+
+import torch
+import torch.nn as nn
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from laya.agent import MPS_AMP_MIN_ROWS_DEFAULT, Agent, _amp_context, _mps_amp_min_rows # noqa: E402
+from laya.common import DecisionModel, build_sequence, serialize_state # noqa: E402
+
+PASS, FAIL = [], []
+
+
+def check(name, got, want):
+ if got == want:
+ PASS.append(name)
+ else:
+ FAIL.append("%s: got %r, want %r" % (name, got, want))
+
+
+# ------------------------------------------------------------------ fake tokenizer
+class FakeTok:
+ """Just enough of a tokenizer for `build_sequence` / `system_one`."""
+
+ mask_token = "[MASK]"
+ mask_token_id = 1
+ cls_token_id = 2
+ sep_token_id = 3
+ pad_token_id = 0
+
+ def __init__(self):
+ self.calls = []
+
+ def __call__(self, text, add_special_tokens=False, truncation=False, max_length=None):
+ self.calls.append(text)
+ n = max(1, len(text) // 4)
+ if truncation and max_length:
+ n = min(n, max_length)
+ return {"input_ids": [5] * n}
+
+
+class FakeModel:
+ def __call__(self, input_ids, attention_mask, marker_pos, marker_mask, qtype):
+ logits = torch.where(
+ marker_mask,
+ torch.ones_like(marker_mask, dtype=torch.float32),
+ torch.full_like(marker_mask, -1e4, dtype=torch.float32),
+ )
+ return logits, torch.zeros((input_ids.shape[0], 2))
+
+
+def _bare_agent(model, dtype=torch.float32, amp=False, tok=None):
+ a = object.__new__(Agent)
+ a.device = torch.device("cpu")
+ a.dtype = dtype
+ a.amp_enabled = amp
+ a.cfg = {"max_len": 64, "head_max_len": 32}
+ a.temperature = [1.0, 1.0, 1.0]
+ a.temperature_by_options = {}
+ a.tok = tok or FakeTok()
+ a.model = model
+ return a
+
+
+# ------------------------------------------------------------------ state reused once
+STATE = {"subject": "Duplicate charge", "body": "x" * 400}
+QUESTION = {"t": "choice", "ins": "pick", "crit": {"a": "x", "b": "y"}}
+
+tok_ref = FakeTok()
+seq_ref, markers_ref = build_sequence(tok_ref, STATE, QUESTION, 64, 32)
+
+tok_shared = FakeTok()
+state_ids = tok_shared(serialize_state(STATE), add_special_tokens=False)["input_ids"]
+seq_shared, markers_shared = build_sequence(tok_shared, STATE, QUESTION, 64, 32, state_ids=state_ids)
+check("build_sequence/state_ids identical ids", seq_shared, seq_ref)
+check("build_sequence/state_ids identical markers", markers_shared, markers_ref)
+check("build_sequence/state_ids does not re-tokenize state", tok_shared.calls.count(serialize_state(STATE)), 1)
+
+agent = _bare_agent(FakeModel())
+out = agent.system_one("the customer was charged twice", {
+ "department": {"type": "choice", "instructions": "which?", "criteria": {"billing": "x", "technical": "y"}},
+ "urgent": {"type": "noul", "instructions": "is it urgent?"},
+})
+state_text = serialize_state("the customer was charged twice").replace(agent.tok.mask_token, " ")
+check("system_one/state tokenized once for two questions", agent.tok.calls.count(state_text), 1)
+check("system_one/answers present", sorted(out["answers"]), ["department", "urgent"])
+
+
+class RecordingTok(FakeTok):
+ def __init__(self):
+ super().__init__()
+ self.caps = []
+
+ def __call__(self, text, add_special_tokens=False, truncation=False, max_length=None):
+ self.caps.append((truncation, max_length))
+ return super().__call__(text, add_special_tokens=add_special_tokens,
+ truncation=truncation, max_length=max_length)
+
+
+long_q = {"t": "choice", "ins": "pick", "crit": {"a": "x" * 400, "b": "y" * 400}}
+rtok = RecordingTok()
+long_seq, long_markers = build_sequence(rtok, "state", long_q, 300, 200)
+# options are capped at the tokenizer (truncation=True, max_length=48), not sliced afterwards
+check("build_sequence/options capped at the tokenizer",
+ len([c for c in rtok.caps if c == (True, 48)]), len(long_markers))
+check("build_sequence/option segments stay <= 49 tokens",
+ all(long_markers[i + 1] - long_markers[i] <= 49 for i in range(len(long_markers) - 1)), True)
+
+
+# ------------------------------------------------------------------ autocast
+class DummyEnc(nn.Module):
+ def __init__(self, d=16):
+ super().__init__()
+ self.config = SimpleNamespace(hidden_size=d, num_attention_heads=1)
+ self.emb = nn.Embedding(16, d)
+
+ def forward(self, input_ids, attention_mask):
+ return SimpleNamespace(last_hidden_state=self.emb(input_ids))
+
+
+model = DecisionModel(DummyEnc(), head_layers=1, n_act=2)
+iid = torch.tensor([[1, 2, 3, 4, 5]])
+am = torch.ones_like(iid)
+mp = torch.tensor([[1, 3]])
+mm = torch.ones_like(mp, dtype=torch.bool)
+qt = torch.tensor([0])
+with torch.no_grad():
+ logits32, act32 = model(iid, am, mp, mm, qt)
+ with torch.autocast(device_type="cpu", dtype=torch.bfloat16, enabled=True):
+ logits16, act16 = model(iid, am, mp, mm, qt)
+check("autocast/shapes match", tuple(logits16.shape), tuple(logits32.shape))
+check("autocast/finite", bool(torch.isfinite(logits16).all() and torch.isfinite(act16).all()), True)
+check("autocast/close to fp32", float((logits32 - logits16).abs().max()) < 0.5, True)
+
+calls = {"n": 0}
+
+
+class Flaky:
+ def __call__(self, *args):
+ calls["n"] += 1
+ if calls["n"] == 1:
+ raise RuntimeError("autocast not supported on this build")
+ return torch.zeros((1, 2)), torch.zeros((1, 2))
+
+
+batch = {
+ "input_ids": torch.tensor([[1]]),
+ "attention_mask": torch.ones((1, 1), dtype=torch.long),
+ "marker_pos": torch.zeros((1, 1), dtype=torch.long),
+ "marker_mask": torch.ones((1, 1), dtype=torch.bool),
+ "qtype": torch.tensor([0]),
+}
+flaky = _bare_agent(Flaky(), dtype=torch.bfloat16, amp=True)
+flaky._infer(batch)
+check("infer/falls back and disables amp", flaky.amp_enabled, False)
+check("infer/retried once", calls["n"], 2)
+
+
+class Boom:
+ def __call__(self, *args):
+ raise RuntimeError("genuine failure")
+
+
+boom = _bare_agent(Boom())
+raised = False
+try:
+ boom._infer(batch)
+except RuntimeError:
+ raised = True
+check("infer/non-autocast error propagates", raised, True)
+
+
+# ------------------------------------------------------------------ MPS autocast gating
+# MPS fp16 is slower than fp32 on one small row and only wins once the batch grows, so it is
+# gated by row count. These check the gate without an MPS device.
+def _mps_agent(amp=True, min_rows=None):
+ a = _bare_agent(FakeModel(), dtype=torch.float16, amp=amp)
+ a.device = torch.device("mps")
+ if min_rows is not None:
+ a.mps_amp_min_rows = min_rows
+ return a
+
+
+a = _mps_agent()
+check("mps-gate/one row stays fp32", a._amp_enabled_for(1), False)
+check("mps-gate/below threshold stays fp32", a._amp_enabled_for(a.mps_amp_min_rows - 1), False)
+check("mps-gate/at threshold enables fp16", a._amp_enabled_for(a.mps_amp_min_rows), True)
+check("mps-gate/above threshold enables fp16", a._amp_enabled_for(a.mps_amp_min_rows + 3), True)
+
+check("mps-gate/threshold override", _mps_agent(min_rows=2)._amp_enabled_for(2), True)
+check("mps-gate/amp disabled stays off", _mps_agent(amp=False)._amp_enabled_for(100), False)
+
+cpu = _bare_agent(FakeModel(), dtype=torch.bfloat16, amp=True)
+check("cpu-gate/not gated by rows", cpu._amp_enabled_for(1), True)
+
+os.environ["LAYA_MPS_AMP_MIN_ROWS"] = "2"
+check("mps-gate/env override", _mps_amp_min_rows(), 2)
+os.environ["LAYA_MPS_AMP_MIN_ROWS"] = "nonsense"
+check("mps-gate/env invalid falls back", _mps_amp_min_rows(), MPS_AMP_MIN_ROWS_DEFAULT)
+del os.environ["LAYA_MPS_AMP_MIN_ROWS"]
+check("mps-gate/env default", _mps_amp_min_rows(), MPS_AMP_MIN_ROWS_DEFAULT)
+
+
+# ------------------------------------------------------------------ amp context shape
+# A disabled gate must never construct torch.autocast: on torch builds without an MPS
+# autocast backend, entering it raises even with enabled=False, which broke MPS predict().
+mps = torch.device("mps")
+check("amp-context/disabled mps is a no-op", isinstance(_amp_context(mps, torch.float16, False), nullcontext), True)
+with _amp_context(mps, torch.float16, False): # must not raise
+ pass
+check("amp-context/disabled mps enters cleanly", True, True)
+
+check("amp-context/disabled cpu is a no-op",
+ isinstance(_amp_context(torch.device("cpu"), torch.float32, False), nullcontext), True)
+check("amp-context/enabled cpu is autocast",
+ not isinstance(_amp_context(torch.device("cpu"), torch.bfloat16, True), nullcontext), True)
+
+# the disabled path through _infer completes on a plain CPU agent
+cpu_disabled = _bare_agent(FakeModel()) # amp=False
+cpu_disabled._infer(batch)
+check("amp-context/_infer disabled completes", True, True)
+
+
+# ------------------------------------------------------------------ report
+print("\n%d passed, %d failed" % (len(PASS), len(FAIL)))
+for f in FAIL:
+ print(" FAIL", f)
+if not FAIL:
+ print("all runtime-fix tests passed")
+sys.exit(1 if FAIL else 0)
diff --git a/tests/test_training.py b/tests/test_training.py
index 796e5259..ab1394c5 100644
--- a/tests/test_training.py
+++ b/tests/test_training.py
@@ -264,8 +264,11 @@ class _Tok:
cls_token_id, sep_token_id, mask_token_id, pad_token_id = 1, 2, 3, 0
mask_token = "[M]"
- def __call__(self, text, add_special_tokens=False):
- return {"input_ids": [10 + (ord(c) % 50) for c in text]}
+ def __call__(self, text, add_special_tokens=False, truncation=False, max_length=None):
+ ids = [10 + (ord(c) % 50) for c in text]
+ if truncation and max_length:
+ ids = ids[:max_length]
+ return {"input_ids": ids}
tok = _Tok()
diff --git a/tests/test_truncation_direction.py b/tests/test_truncation_direction.py
index 3cb1722d..c4de4078 100644
--- a/tests/test_truncation_direction.py
+++ b/tests/test_truncation_direction.py
@@ -20,8 +20,11 @@ class _FakeTok:
cls_token_id, sep_token_id, mask_token_id, pad_token_id = 0, 1, 4, 2
mask_token = "[MASK]"
- def __call__(self, text, add_special_tokens=False):
- return {"input_ids": [10 + (len(w) % 90) for w in text.split() if w]}
+ def __call__(self, text, add_special_tokens=False, truncation=False, max_length=None):
+ ids = [10 + (len(w) % 90) for w in text.split() if w]
+ if truncation and max_length:
+ ids = ids[:max_length]
+ return {"input_ids": ids}
def _tiny_agent():
@@ -56,12 +59,13 @@ def test_agent_list_vs_string_truncation_direction():
list_capture = {}
string_capture = {}
- def fake_build(tok, state, q, max_len, head_max_len, truncate_left=False):
+ def fake_build(tok, state, q, max_len, head_max_len, truncate_left=False, state_ids=None):
if isinstance(state, list):
list_capture["truncate_left"] = truncate_left
else:
string_capture["truncate_left"] = truncate_left
- return build_sequence(tok, state, q, max_len, head_max_len, truncate_left=truncate_left)
+ return build_sequence(tok, state, q, max_len, head_max_len,
+ truncate_left=truncate_left, state_ids=state_ids)
with patch("laya.agent.build_sequence", side_effect=fake_build):
agent.system_one([{"role": "user", "content": "hi"}, {"role": "user", "content": "newest"}], questions)
@@ -81,10 +85,11 @@ def test_agent_system_one_passes_truncate_left_for_list():
captured = {}
- def fake_build(tok, state, q, max_len, head_max_len, truncate_left=False):
+ def fake_build(tok, state, q, max_len, head_max_len, truncate_left=False, state_ids=None):
captured["truncate_left"] = truncate_left
captured["state"] = state
- return build_sequence(tok, state, q, max_len, head_max_len, truncate_left=truncate_left)
+ return build_sequence(tok, state, q, max_len, head_max_len,
+ truncate_left=truncate_left, state_ids=state_ids)
with patch("laya.agent.build_sequence", side_effect=fake_build):
agent.system_one(conversation, questions)
@@ -102,9 +107,10 @@ def test_agent_system_one_string_state_default_truncation():
captured = {}
- def fake_build(tok, state, q, max_len, head_max_len, truncate_left=False):
+ def fake_build(tok, state, q, max_len, head_max_len, truncate_left=False, state_ids=None):
captured["truncate_left"] = truncate_left
- return build_sequence(tok, state, q, max_len, head_max_len, truncate_left=truncate_left)
+ return build_sequence(tok, state, q, max_len, head_max_len,
+ truncate_left=truncate_left, state_ids=state_ids)
with patch("laya.agent.build_sequence", side_effect=fake_build):
agent.system_one("just a string state", questions)