Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
fix(agent): scope the autocast fallback to the failed request
KingPointerb775efc500b4
diff --git a/laya/agent.py b/laya/agent.pyindex 5ca3861c..d113022a 100644--- a/laya/agent.py+++ b/laya/agent.py@@ -198,6 +198,9 @@ def _amp_context(device, dtype, enabled: bool):MPS_AMP_MIN_ROWS_DEFAULT = 5+# An unsupported autocast op fails the same way on every request. Retry that request in+# full precision, and only turn AMP off after this many failures in a row (#351).+_AMP_FAIL_LIMIT = 3def _mps_amp_min_rows() -> int:@@ -269,6 +272,7 @@ class Agent(HookRegistry):# without it (for example a hand-constructed runtime in tests).amp_enabled = Falsemps_amp_min_rows = MPS_AMP_MIN_ROWS_DEFAULT+ _amp_failures = 0# The stock forward is also used by lightweight runtimes built with __new__ in tests._fast = None# Scoped CPU-fallback observability: how often _infer's per-request OOM fallback fired@@ -805,10 +809,12 @@ class Agent(HookRegistry):% (self._fast.max_len, b["input_ids"].shape[1], self._fast.max_len))- def run():- # Recomputed inside run() so a fallback that disables amp (or moves to CPU) takes- # effect on the retry. A disabled gate never enters torch.autocast at all.- enabled = self._amp_enabled_for(b["input_ids"].shape[0])+ def run(enabled=None):+ # Recomputed inside run() so a fallback that moves to CPU takes effect on the+ # retry. A disabled gate never enters torch.autocast at all. `enabled=False`+ # retries one request in full precision without reading the agent's AMP flag.+ if enabled is None:+ enabled = self._amp_enabled_for(b["input_ids"].shape[0])dims = independent_dims() if self._compiled else nullcontext()with _amp_context(self.device, self.dtype, enabled), dims:return self.model(@@ -820,7 +826,7 @@ class Agent(HookRegistry):)try:- return run()+ out = run()except (RuntimeError, torch.cuda.OutOfMemoryError) as e:low = str(e).lower()# An OOM is `torch.cuda.OutOfMemoryError` or says so in its message. The bare@@ -854,12 +860,23 @@ class Agent(HookRegistry):finally:self._restore_runtime(held_device, held_dtype, held_amp, had_fast)if use_amp and self.device.type in ("mps", "cpu"):- # Not every MPS/CPU build implements autocast for every op. Drop to full- # precision once rather than failing the request.- self.amp_enabled = False- self.dtype = torch.float32- return run()+ # Not every MPS/CPU build implements autocast for every op. Retry this+ # request in full precision. One miss must not turn AMP off; a build that+ # lacks the op fails the same way every time, so after a short streak the+ # process drops to full precision instead of paying for two forwards (#351).+ try:+ return run(enabled=False)+ finally:+ self._amp_failures += 1+ if self._amp_failures >= _AMP_FAIL_LIMIT:+ print("Warning: autocast failed %d times in a row. Disabling mixed precision."+ % self._amp_failures)+ self.amp_enabled = False+ self.dtype = torch.float32raise+ else:+ self._amp_failures = 0+ return outdef _forward(self, b: Dict):"""Run the model on a collated batch, with the GPU->CPU OOM fallback, and return numpy outputs."""diff --git a/tests/test_portability.py b/tests/test_portability.pyindex e085e0e3..7a19a921 100644--- a/tests/test_portability.py+++ b/tests/test_portability.py@@ -257,6 +257,44 @@ with mock.patch.object(torch.Tensor, "to", _to_that_ignores_mps):except Exception as e: # noqa: BLE001FAIL.append("fallback/restore-failure not survived: %s: %s" % (type(e).__name__, e))+ # one unsupported autocast op retries that request and leaves AMP on. Three misses in a+ # row disable it, so a build without the op does not pay for two forwards forever (#351).+ # CPU so the MPS row gate does not hide the branch.+ class _AutocastMisses(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.dummy = torch.nn.Parameter(torch.zeros(1))+ self.calls = 0++ def forward(self, input_ids, attention_mask, marker_pos, marker_mask, qtype):+ self.calls += 1+ if self.calls in (1, 3, 5):+ raise RuntimeError("User specified an unsupported autocast device_type cpu")+ logits = torch.zeros((input_ids.shape[0], marker_mask.shape[1]))+ logits[:, 0] = 1.0+ return logits, torch.tensor([[1.0, 0.0]])++ agent = _bare_agent(_AutocastMisses())+ agent.device = torch.device("cpu")+ agent.amp_enabled = True+ agent.dtype = torch.float16+ try:+ result = agent.predict({"body": "some state"}, QUESTIONS)+ check("autocast/answers after one miss", result["answers"]["q"]["choice"], "a")+ check("autocast/one miss retries once", agent.model.calls, 2)+ check("autocast/one miss keeps amp", agent.amp_enabled, True)+ check("autocast/one miss keeps dtype", agent.dtype, torch.float16)+ agent.predict({"body": "another state"}, QUESTIONS)+ check("autocast/two misses keep amp", agent.amp_enabled, True)+ agent.predict({"body": "third state"}, QUESTIONS)+ check("autocast/third miss disables amp", agent.amp_enabled, False)+ check("autocast/third miss drops dtype", agent.dtype, torch.float32)+ check("autocast/three misses are six forwards", agent.model.calls, 6)+ agent.predict({"body": "later state"}, QUESTIONS)+ check("autocast/later request is one forward", agent.model.calls, 7)+ except Exception as e: # noqa: BLE001+ FAIL.append("autocast/streak was not survived: %s: %s" % (type(e).__name__, e))+print("\n%d passed, %d failed" % (len(PASS), len(FAIL)))for f in FAIL:diff --git a/tests/test_runtime_fixes.py b/tests/test_runtime_fixes.pyindex a89e0890..cf3a77d1 100644--- a/tests/test_runtime_fixes.py+++ b/tests/test_runtime_fixes.py@@ -146,9 +146,11 @@ calls = {"n": 0}class Flaky:+ """Fail the autocast attempt of the first three requests; the fp32 retry succeeds."""+def __call__(self, *args):calls["n"] += 1- if calls["n"] == 1:+ if calls["n"] in (1, 3, 5):raise RuntimeError("autocast not supported on this build")return torch.zeros((1, 2)), torch.zeros((1, 2))@@ -162,8 +164,36 @@ batch = {}flaky = _bare_agent(Flaky(), dtype=torch.bfloat16, amp=True)flaky._infer(batch)-check("infer/falls back and disables amp", flaky.amp_enabled, False)+check("infer/one miss keeps amp", flaky.amp_enabled, True)+check("infer/one miss keeps dtype", flaky.dtype, torch.bfloat16)check("infer/retried once", calls["n"], 2)+flaky._infer(batch)+check("infer/two misses keep amp", flaky.amp_enabled, True)+flaky._infer(batch)+check("infer/third miss disables amp", flaky.amp_enabled, False)+check("infer/third miss drops dtype", flaky.dtype, torch.float32)+check("infer/three misses retried", calls["n"], 6)+flaky._infer(batch)+check("infer/later request is one forward", calls["n"], 7)+++class OneMiss:+ def __init__(self):+ self.n = 0++ def __call__(self, *args):+ self.n += 1+ if self.n == 1:+ raise RuntimeError("autocast not supported on this build")+ return torch.zeros((1, 2)), torch.zeros((1, 2))+++streak = _bare_agent(OneMiss(), dtype=torch.bfloat16, amp=True)+streak._infer(batch)+check("infer/streak is one after a miss", streak._amp_failures, 1)+streak._infer(batch)+check("infer/a clean forward clears the streak", streak._amp_failures, 0)+check("infer/a clean forward keeps amp", streak.amp_enabled, True)class Boom: