Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(agent): scope the autocast fallback to the failed request

KingPointerb775efc500b4
diff --git a/laya/agent.py b/laya/agent.py
index 5ca3861c..d113022a 100644
--- a/laya/agent.py
+++ b/laya/agent.py
@@ -198,6 +198,9 @@ def _amp_context(device, dtype, enabled: bool):
MPS_AMP_MIN_ROWS_DEFAULT = 5
+# An unsupported autocast op fails the same way on every request. Retry that request in
+# full precision, and only turn AMP off after this many failures in a row (#351).
+_AMP_FAIL_LIMIT = 3
def _mps_amp_min_rows() -> int:
@@ -269,6 +272,7 @@ class Agent(HookRegistry):
# without it (for example a hand-constructed runtime in tests).
amp_enabled = False
mps_amp_min_rows = MPS_AMP_MIN_ROWS_DEFAULT
+ _amp_failures = 0
# The stock forward is also used by lightweight runtimes built with __new__ in tests.
_fast = None
# Scoped CPU-fallback observability: how often _infer's per-request OOM fallback fired
@@ -805,10 +809,12 @@ class Agent(HookRegistry):
% (self._fast.max_len, b["input_ids"].shape[1], self._fast.max_len)
)
- def run():
- # Recomputed inside run() so a fallback that disables amp (or moves to CPU) takes
- # effect on the retry. A disabled gate never enters torch.autocast at all.
- enabled = self._amp_enabled_for(b["input_ids"].shape[0])
+ def run(enabled=None):
+ # Recomputed inside run() so a fallback that moves to CPU takes effect on the
+ # retry. A disabled gate never enters torch.autocast at all. `enabled=False`
+ # retries one request in full precision without reading the agent's AMP flag.
+ if enabled is None:
+ enabled = self._amp_enabled_for(b["input_ids"].shape[0])
dims = independent_dims() if self._compiled else nullcontext()
with _amp_context(self.device, self.dtype, enabled), dims:
return self.model(
@@ -820,7 +826,7 @@ class Agent(HookRegistry):
)
try:
- return run()
+ out = run()
except (RuntimeError, torch.cuda.OutOfMemoryError) as e:
low = str(e).lower()
# An OOM is `torch.cuda.OutOfMemoryError` or says so in its message. The bare
@@ -854,12 +860,23 @@ class Agent(HookRegistry):
finally:
self._restore_runtime(held_device, held_dtype, held_amp, had_fast)
if use_amp and self.device.type in ("mps", "cpu"):
- # Not every MPS/CPU build implements autocast for every op. Drop to full
- # precision once rather than failing the request.
- self.amp_enabled = False
- self.dtype = torch.float32
- return run()
+ # Not every MPS/CPU build implements autocast for every op. Retry this
+ # request in full precision. One miss must not turn AMP off; a build that
+ # lacks the op fails the same way every time, so after a short streak the
+ # process drops to full precision instead of paying for two forwards (#351).
+ try:
+ return run(enabled=False)
+ finally:
+ self._amp_failures += 1
+ if self._amp_failures >= _AMP_FAIL_LIMIT:
+ print("Warning: autocast failed %d times in a row. Disabling mixed precision."
+ % self._amp_failures)
+ self.amp_enabled = False
+ self.dtype = torch.float32
raise
+ else:
+ self._amp_failures = 0
+ return out
def _forward(self, b: Dict):
"""Run the model on a collated batch, with the GPU->CPU OOM fallback, and return numpy outputs."""
diff --git a/tests/test_portability.py b/tests/test_portability.py
index e085e0e3..7a19a921 100644
--- a/tests/test_portability.py
+++ b/tests/test_portability.py
@@ -257,6 +257,44 @@ with mock.patch.object(torch.Tensor, "to", _to_that_ignores_mps):
except Exception as e: # noqa: BLE001
FAIL.append("fallback/restore-failure not survived: %s: %s" % (type(e).__name__, e))
+ # one unsupported autocast op retries that request and leaves AMP on. Three misses in a
+ # row disable it, so a build without the op does not pay for two forwards forever (#351).
+ # CPU so the MPS row gate does not hide the branch.
+ class _AutocastMisses(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.dummy = torch.nn.Parameter(torch.zeros(1))
+ self.calls = 0
+
+ def forward(self, input_ids, attention_mask, marker_pos, marker_mask, qtype):
+ self.calls += 1
+ if self.calls in (1, 3, 5):
+ raise RuntimeError("User specified an unsupported autocast device_type cpu")
+ logits = torch.zeros((input_ids.shape[0], marker_mask.shape[1]))
+ logits[:, 0] = 1.0
+ return logits, torch.tensor([[1.0, 0.0]])
+
+ agent = _bare_agent(_AutocastMisses())
+ agent.device = torch.device("cpu")
+ agent.amp_enabled = True
+ agent.dtype = torch.float16
+ try:
+ result = agent.predict({"body": "some state"}, QUESTIONS)
+ check("autocast/answers after one miss", result["answers"]["q"]["choice"], "a")
+ check("autocast/one miss retries once", agent.model.calls, 2)
+ check("autocast/one miss keeps amp", agent.amp_enabled, True)
+ check("autocast/one miss keeps dtype", agent.dtype, torch.float16)
+ agent.predict({"body": "another state"}, QUESTIONS)
+ check("autocast/two misses keep amp", agent.amp_enabled, True)
+ agent.predict({"body": "third state"}, QUESTIONS)
+ check("autocast/third miss disables amp", agent.amp_enabled, False)
+ check("autocast/third miss drops dtype", agent.dtype, torch.float32)
+ check("autocast/three misses are six forwards", agent.model.calls, 6)
+ agent.predict({"body": "later state"}, QUESTIONS)
+ check("autocast/later request is one forward", agent.model.calls, 7)
+ except Exception as e: # noqa: BLE001
+ FAIL.append("autocast/streak was not survived: %s: %s" % (type(e).__name__, e))
+
print("\n%d passed, %d failed" % (len(PASS), len(FAIL)))
for f in FAIL:
diff --git a/tests/test_runtime_fixes.py b/tests/test_runtime_fixes.py
index a89e0890..cf3a77d1 100644
--- a/tests/test_runtime_fixes.py
+++ b/tests/test_runtime_fixes.py
@@ -146,9 +146,11 @@ calls = {"n": 0}
class Flaky:
+ """Fail the autocast attempt of the first three requests; the fp32 retry succeeds."""
+
def __call__(self, *args):
calls["n"] += 1
- if calls["n"] == 1:
+ if calls["n"] in (1, 3, 5):
raise RuntimeError("autocast not supported on this build")
return torch.zeros((1, 2)), torch.zeros((1, 2))
@@ -162,8 +164,36 @@ batch = {
}
flaky = _bare_agent(Flaky(), dtype=torch.bfloat16, amp=True)
flaky._infer(batch)
-check("infer/falls back and disables amp", flaky.amp_enabled, False)
+check("infer/one miss keeps amp", flaky.amp_enabled, True)
+check("infer/one miss keeps dtype", flaky.dtype, torch.bfloat16)
check("infer/retried once", calls["n"], 2)
+flaky._infer(batch)
+check("infer/two misses keep amp", flaky.amp_enabled, True)
+flaky._infer(batch)
+check("infer/third miss disables amp", flaky.amp_enabled, False)
+check("infer/third miss drops dtype", flaky.dtype, torch.float32)
+check("infer/three misses retried", calls["n"], 6)
+flaky._infer(batch)
+check("infer/later request is one forward", calls["n"], 7)
+
+
+class OneMiss:
+ def __init__(self):
+ self.n = 0
+
+ def __call__(self, *args):
+ self.n += 1
+ if self.n == 1:
+ raise RuntimeError("autocast not supported on this build")
+ return torch.zeros((1, 2)), torch.zeros((1, 2))
+
+
+streak = _bare_agent(OneMiss(), dtype=torch.bfloat16, amp=True)
+streak._infer(batch)
+check("infer/streak is one after a miss", streak._amp_failures, 1)
+streak._infer(batch)
+check("infer/a clean forward clears the streak", streak._amp_failures, 0)
+check("infer/a clean forward keeps amp", streak.amp_enabled, True)
class Boom: