Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
fix(agent): finish the demotion when the restore after a CPU retry fails
Linz12481b71af8d88c9
diff --git a/laya/agent.py b/laya/agent.pyindex 479598a1..d52b41b4 100644--- a/laya/agent.py+++ b/laya/agent.py@@ -489,6 +489,12 @@ class Agent(HookRegistry):try:self.model.to(device)except (RuntimeError, torch.cuda.OutOfMemoryError) as e:+ # The move can fail partway through, leaving parameters split between devices, so+ # finish the demotion before giving up: every later call must find one device,+ # not a mix of both.+ self.model.to(torch.device("cpu"))+ if torch.cuda.is_available():+ torch.cuda.empty_cache()print("Warning: could not move the model back to %s after the CPU retry (%s); ""staying on CPU." % (device, e))returndiff --git a/tests/test_portability.py b/tests/test_portability.pyindex e318acb5..e085e0e3 100644--- a/tests/test_portability.py+++ b/tests/test_portability.py@@ -165,6 +165,17 @@ class _FailsOnce(torch.nn.Module):return logits, torch.tensor([[1.0, 0.0]])+class _RestoreFails(_FailsOnce):+ """OOMs once like _FailsOnce, but also refuses to move back to the accelerator afterwards,+ so `_restore_runtime`'s `model.to(device)` fails and the agent must stay on CPU."""+ def to(self, *args, **kwargs):+ target = args[0] if args else kwargs.get("device")+ dt = getattr(target, "type", None) or (target if isinstance(target, str) else None)+ if dt == "mps":+ raise RuntimeError("CUDA out of memory: model no longer fits after the retry")+ return super().to(*args, **kwargs)++def _bare_agent(model):agent = _agent.Agent.__new__(_agent.Agent) # no weights: exercise system_one onlyagent.device = torch.device("mps") # any non-CPU device enters the fallback branch@@ -233,6 +244,19 @@ with mock.patch.object(torch.Tensor, "to", _to_that_ignores_mps):FAIL.append("fallback/cuda-worded non-memory error raised %s instead of RuntimeError"% type(e).__name__)+ # restore fails: the model no longer fits the accelerator after the CPU retry, so it stays+ # demoted rather than crashing a request that already succeeded+ agent = _bare_agent(_RestoreFails("CUDA out of memory. Tried to allocate 2.00 GiB"))+ try:+ result = agent.predict({"body": "some state"}, QUESTIONS)+ check("fallback/restore-failure still answers the request", result["answers"]["q"]["choice"], "a")+ check("fallback/restore-failure stays on cpu", agent.device.type, "cpu")+ check("fallback/restore-failure: later call runs on cpu without crashing",+ agent.predict({"body": "another state"}, QUESTIONS)["answers"]["q"]["choice"], "a")+ check("fallback/restore-failure: no extra forward beyond retry + later call", agent.model.calls, 3)+ except Exception as e: # noqa: BLE001+ FAIL.append("fallback/restore-failure not survived: %s: %s" % (type(e).__name__, e))+print("\n%d passed, %d failed" % (len(PASS), len(FAIL)))for f in FAIL: