Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

feat(serve): report per-agent CPU fallback counters in /health

umran6667c923eaf2fd6
diff --git a/laya/agent.py b/laya/agent.py
index 1b58de6f..705037aa 100644
--- a/laya/agent.py
+++ b/laya/agent.py
@@ -232,6 +232,11 @@ class Agent(HookRegistry):
mps_amp_min_rows = MPS_AMP_MIN_ROWS_DEFAULT
# The stock forward is also used by lightweight runtimes built with __new__ in tests.
_fast = None
+ # Scoped CPU-fallback observability: how often _infer's per-request OOM fallback fired
+ # and what the last failure was, so an operator sees a slow lane in /health instead of
+ # discovering it by accident. Class defaults cover instances built without __init__.
+ cpu_fallback_count = 0
+ last_fallback_reason = None
def __init__(
self,
@@ -727,6 +732,10 @@ class Agent(HookRegistry):
# later call ~10-15x slower on CPU. Demote under the lock, answer this request
# on CPU, then put the runtime back the way it was.
with _OOM_FALLBACK_LOCK:
+ # Recorded on entry, under the same lock as the demotion: the count and
+ # reason must be readable by /health without racing a concurrent fallback.
+ self.cpu_fallback_count += 1
+ self.last_fallback_reason = str(e)
held_device, held_dtype, held_amp = self.device, self.dtype, self.amp_enabled
had_fast = self._fast is not None
# FastLaya keeps copied CUDA weights and replaces model.forward. Move
diff --git a/laya/serve.py b/laya/serve.py
index 2cd30922..b05a9ab6 100644
--- a/laya/serve.py
+++ b/laya/serve.py
@@ -306,11 +306,21 @@ def create_app(router: Optional[Any] = None):
@app.get("/health")
def health() -> Dict[str, Any]:
+ # Per-agent CPU-fallback counters. `_agents` is read the way laya.mcp.device
+ # reads it -- the only side-effect-free handle on a resident agent -- and every
+ # access is getattr-guarded so injected routers predating the counters (or
+ # without a `_agents` mapping) stay health-compatible, like `revisions`.
+ agents = getattr(router, "_agents", None) or {}
return {
"status": "ok",
"loaded": router.loaded,
"revisions": getattr(router, "loaded_revisions", {}),
"device": os.environ.get("LAYA_DEVICE") or "auto",
+ "cpu_fallbacks": {
+ name: {"count": getattr(agents.get(name), "cpu_fallback_count", 0),
+ "last_reason": getattr(agents.get(name), "last_fallback_reason", None)}
+ for name in router.loaded
+ },
}
@app.post("/v1/systemone")
diff --git a/tests/test_runtime_fixes.py b/tests/test_runtime_fixes.py
index a769a2db..a89e0890 100644
--- a/tests/test_runtime_fixes.py
+++ b/tests/test_runtime_fixes.py
@@ -242,6 +242,59 @@ cpu_disabled._infer(batch)
check("amp-context/_infer disabled completes", True, True)
+# ------------------------------------------------------------------ OOM fallback observability (#351)
+class FakeCUDAInput:
+ """An input_ids that fails the way a real CUDA OOM does when moved off CPU.
+
+ Only `input_ids` needs this: it is the first tensor `_infer` moves, so the
+ failure fires before the other (real, CPU-safe) tensors are touched, and on
+ the CPU retry `.to('cpu')` passes it straight through.
+ """
+
+ shape = (1, 8)
+
+ def __init__(self):
+ self.moves = 0
+
+ def to(self, device):
+ self.moves += 1
+ if str(device) != "cpu":
+ raise RuntimeError("CUDA out of memory. Tried to allocate 2.00 GiB")
+ return self
+
+
+class ImmovableModel:
+ """Accepts model.to() without moving anything (there is no real tensor to move)."""
+
+ def to(self, device):
+ self.placed = str(device)
+ return self
+
+ def __call__(self, *args):
+ return torch.zeros((1, 2)), torch.zeros((1, 2))
+
+
+oom_batch = dict(batch, input_ids=FakeCUDAInput())
+oom = _bare_agent(ImmovableModel(), dtype=torch.float16)
+oom.device = torch.device("cuda") # the OOM branch only reads .type
+check("oom-fallback/count starts at 0", oom.cpu_fallback_count, 0)
+check("oom-fallback/reason starts None", oom.last_fallback_reason, None)
+
+out = oom._infer(oom_batch) # first forward raises OOM -> scoped CPU retry
+check("oom-fallback/retry answered", isinstance(out, tuple), True)
+check("oom-fallback/count recorded", oom.cpu_fallback_count, 1)
+check("oom-fallback/reason recorded",
+ "out of memory" in (oom.last_fallback_reason or ""), True)
+check("oom-fallback/scoped: device restored", oom.device.type, "cuda")
+
+oom._infer(oom_batch) # a second OOM accumulates
+check("oom-fallback/second OOM counts too", oom.cpu_fallback_count, 2)
+
+# a plain forward never touches the counters
+check("oom-fallback/plain CPU infer stays 0", cpu_disabled.cpu_fallback_count, 0)
+check("oom-fallback/plain CPU reason stays None", cpu_disabled.last_fallback_reason, None)
+
+
# ------------------------------------------------------------------ report
print("\n%d passed, %d failed" % (len(PASS), len(FAIL)))
for f in FAIL:
diff --git a/tests/test_serve.py b/tests/test_serve.py
index ae52aae2..bc94136c 100644
--- a/tests/test_serve.py
+++ b/tests/test_serve.py
@@ -209,6 +209,25 @@ def test_health_supports_router_without_loaded_revisions(monkeypatch):
assert r.status_code == 200
assert r.json()["status"] == "ok"
assert r.json()["revisions"] == {}
+ # Same for the #351 fallback counters: a router with no _agents and no agents
+ # with counters still reports a stable, all-zero shape.
+ assert r.json()["cpu_fallbacks"] == {"english": {"count": 0, "last_reason": None}}
+
+
+def test_health_reports_cpu_fallback_counters(monkeypatch):
+ """A resident agent that triggered the scoped CPU fallback shows it in /health."""
+ from types import SimpleNamespace
+
+ client, fake = _client(monkeypatch)
+ fake._agents = {"english": SimpleNamespace(
+ cpu_fallback_count=2,
+ last_fallback_reason="CUDA out of memory. Tried to allocate 1.00 GiB",
+ )}
+ r = client.get("/health")
+ assert r.status_code == 200
+ fb = r.json()["cpu_fallbacks"]
+ assert fb["english"]["count"] == 2, fb
+ assert "out of memory" in fb["english"]["last_reason"], fb
def test_helpers():