Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
fix(agent): scope the per-inference CUDA OOM fallback to the failed request
Linz1248a9559a370194
diff --git a/laya/agent.py b/laya/agent.pyindex b0fb78a7..479598a1 100644--- a/laya/agent.py+++ b/laya/agent.py@@ -135,6 +135,11 @@ def _verify_compatibility(model: torch.nn.Module, cfg: Dict, weights: Dict[str,_TOKENIZERS: Dict[tuple, Any] = {}_TOKENIZERS_LOCK = threading.Lock()+# The per-inference CPU fallback rewrites shared runtime state (device, dtype, amp) and moves the+# model while other threads may be running their own forward, so the demotion and the restore are+# serialised. A second request that hits OOM waits here and re-demotes only if it needs to.+_OOM_FALLBACK_LOCK = threading.Lock()+def _load_tokenizer(tok_dir: str, cfg: Dict) -> Any:"""Tokenizer for a checkpoint, parsed once per process.@@ -474,6 +479,23 @@ class Agent(HookRegistry):self.model.forward = self._stock_forwardself._fast = None+ def _restore_runtime(self, device, dtype, amp_enabled: bool, had_fast: bool) -> None:+ """Undo the scoped CPU fallback of `_infer`, best effort.++ A model that no longer fits `device` after the CPU retry stays demoted: raising out of a+ request that already succeeded would trade a silent slowdown for a crash, the worse+ failure of the two.+ """+ try:+ self.model.to(device)+ except (RuntimeError, torch.cuda.OutOfMemoryError) as e:+ print("Warning: could not move the model back to %s after the CPU retry (%s); "+ "staying on CPU." % (device, e))+ return+ self.device, self.dtype, self.amp_enabled = device, dtype, amp_enabled+ if had_fast:+ self.accelerate()+@staticmethoddef _check_question(qid: str, qdef: Any) -> None:"""Reject a question that cannot be answered, naming it and what to fix.@@ -624,19 +646,32 @@ class Agent(HookRegistry):return run()except (RuntimeError, torch.cuda.OutOfMemoryError) as e:low = str(e).lower()- if self.device.type != "cpu" and ("memory" in low or "cuda" in low):- print("Warning: GPU memory exceeded during inference. Falling back to CPU...")- # FastLaya keeps copied CUDA weights and replaces model.forward. Move- # the model first without that replacement, or the retry would still- # execute on the failed CUDA fast path.- self.deaccelerate()- self.device = torch.device("cpu")- self.dtype = torch.float32- self.amp_enabled = False- self.model.to(self.device)- if torch.cuda.is_available():- torch.cuda.empty_cache()- return run()+ # An OOM is `torch.cuda.OutOfMemoryError` or says so in its message. The bare+ # "cuda" substring used to be accepted too, so any CUDA-shaped RuntimeError (a+ # shape, assert or kernel error) silently and permanently demoted the agent.+ if self.device.type != "cpu" and (isinstance(e, torch.cuda.OutOfMemoryError) or "memory" in low):+ print("Warning: GPU memory exceeded during inference. Retrying this request on CPU...")+ # Scoped, not permanent: the demotion used to rewrite device/dtype/amp and move+ # the model for the life of the process, so one oversized request left every+ # later call ~10-15x slower on CPU. Demote under the lock, answer this request+ # on CPU, then put the runtime back the way it was.+ with _OOM_FALLBACK_LOCK:+ held_device, held_dtype, held_amp = self.device, self.dtype, self.amp_enabled+ had_fast = self._fast is not None+ # FastLaya keeps copied CUDA weights and replaces model.forward. Move+ # the model first without that replacement, or the retry would still+ # execute on the failed CUDA fast path.+ self.deaccelerate()+ self.device = torch.device("cpu")+ self.dtype = torch.float32+ self.amp_enabled = False+ self.model.to(self.device)+ if torch.cuda.is_available():+ torch.cuda.empty_cache()+ try:+ return run()+ finally:+ self._restore_runtime(held_device, held_dtype, held_amp, had_fast)if use_amp and self.device.type in ("mps", "cpu"):# Not every MPS/CPU build implements autocast for every op. Drop to full# precision once rather than failing the request.diff --git a/tests/test_portability.py b/tests/test_portability.pyindex 878c9a64..e318acb5 100644--- a/tests/test_portability.py+++ b/tests/test_portability.py@@ -200,8 +200,13 @@ with mock.patch.object(torch.Tensor, "to", _to_that_ignores_mps):result = agent.predict({"body": "some state"}, QUESTIONS)check("fallback/answers after a memory failure", result["answers"]["q"]["choice"], "a")check("fallback/forward pass retried once", agent.model.calls, 2)- check("fallback/device is cpu now", agent.device.type, "cpu")- check("fallback/dtype downgraded to fp32", agent.dtype, torch.float32)+ # the demotion is scoped to the failed request (#344): one oversized call must not+ # leave every later call on a ~10-15x slower CPU path for the life of the process+ check("fallback/device restored after the retry", agent.device.type, "mps")+ check("fallback/a later request is answered without a new demotion",+ agent.predict({"body": "another state"}, QUESTIONS)["answers"]["q"]["choice"], "a")+ check("fallback/later request needs no extra forward", agent.model.calls, 3)+ check("fallback/device still the original after a later call", agent.device.type, "mps")except Exception as e: # noqa: BLE001FAIL.append("fallback/memory failure was not survived: %s: %s" % (type(e).__name__, e))@@ -216,6 +221,18 @@ with mock.patch.object(torch.Tensor, "to", _to_that_ignores_mps):except Exception as e: # noqa: BLE001FAIL.append("fallback/non-memory error raised %s instead of RuntimeError" % type(e).__name__)+ # a RuntimeError that merely mentions cuda is not an OOM (#344): it used to demote the+ # agent to CPU permanently and return silently-CPU results+ agent = _bare_agent(_FailsOnce("CUDA error: device-side assert triggered"))+ try:+ agent.predict({"body": "some state"}, QUESTIONS)+ FAIL.append("fallback/cuda-worded non-memory error propagates (nothing raised)")+ except RuntimeError:+ PASS.append("fallback/cuda-worded non-memory error propagates")+ except Exception as e: # noqa: BLE001+ FAIL.append("fallback/cuda-worded non-memory error raised %s instead of RuntimeError"+ % type(e).__name__)+print("\n%d passed, %d failed" % (len(PASS), len(FAIL)))for f in FAIL: