Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
feat(serve): report per-agent CPU fallback counters in /health
umran6667c923eaf2fd6
diff --git a/laya/agent.py b/laya/agent.pyindex 1b58de6f..705037aa 100644--- a/laya/agent.py+++ b/laya/agent.py@@ -232,6 +232,11 @@ class Agent(HookRegistry):mps_amp_min_rows = MPS_AMP_MIN_ROWS_DEFAULT# The stock forward is also used by lightweight runtimes built with __new__ in tests._fast = None+ # Scoped CPU-fallback observability: how often _infer's per-request OOM fallback fired+ # and what the last failure was, so an operator sees a slow lane in /health instead of+ # discovering it by accident. Class defaults cover instances built without __init__.+ cpu_fallback_count = 0+ last_fallback_reason = Nonedef __init__(self,@@ -727,6 +732,10 @@ class Agent(HookRegistry):# later call ~10-15x slower on CPU. Demote under the lock, answer this request# on CPU, then put the runtime back the way it was.with _OOM_FALLBACK_LOCK:+ # Recorded on entry, under the same lock as the demotion: the count and+ # reason must be readable by /health without racing a concurrent fallback.+ self.cpu_fallback_count += 1+ self.last_fallback_reason = str(e)held_device, held_dtype, held_amp = self.device, self.dtype, self.amp_enabledhad_fast = self._fast is not None# FastLaya keeps copied CUDA weights and replaces model.forward. Movediff --git a/laya/serve.py b/laya/serve.pyindex 2cd30922..b05a9ab6 100644--- a/laya/serve.py+++ b/laya/serve.py@@ -306,11 +306,21 @@ def create_app(router: Optional[Any] = None):@app.get("/health")def health() -> Dict[str, Any]:+ # Per-agent CPU-fallback counters. `_agents` is read the way laya.mcp.device+ # reads it -- the only side-effect-free handle on a resident agent -- and every+ # access is getattr-guarded so injected routers predating the counters (or+ # without a `_agents` mapping) stay health-compatible, like `revisions`.+ agents = getattr(router, "_agents", None) or {}return {"status": "ok","loaded": router.loaded,"revisions": getattr(router, "loaded_revisions", {}),"device": os.environ.get("LAYA_DEVICE") or "auto",+ "cpu_fallbacks": {+ name: {"count": getattr(agents.get(name), "cpu_fallback_count", 0),+ "last_reason": getattr(agents.get(name), "last_fallback_reason", None)}+ for name in router.loaded+ },}@app.post("/v1/systemone")diff --git a/tests/test_runtime_fixes.py b/tests/test_runtime_fixes.pyindex a769a2db..a89e0890 100644--- a/tests/test_runtime_fixes.py+++ b/tests/test_runtime_fixes.py@@ -242,6 +242,59 @@ cpu_disabled._infer(batch)check("amp-context/_infer disabled completes", True, True)+# ------------------------------------------------------------------ OOM fallback observability (#351)+class FakeCUDAInput:+ """An input_ids that fails the way a real CUDA OOM does when moved off CPU.++ Only `input_ids` needs this: it is the first tensor `_infer` moves, so the+ failure fires before the other (real, CPU-safe) tensors are touched, and on+ the CPU retry `.to('cpu')` passes it straight through.+ """++ shape = (1, 8)++ def __init__(self):+ self.moves = 0++ def to(self, device):+ self.moves += 1+ if str(device) != "cpu":+ raise RuntimeError("CUDA out of memory. Tried to allocate 2.00 GiB")+ return self+++class ImmovableModel:+ """Accepts model.to() without moving anything (there is no real tensor to move)."""++ def to(self, device):+ self.placed = str(device)+ return self++ def __call__(self, *args):+ return torch.zeros((1, 2)), torch.zeros((1, 2))+++oom_batch = dict(batch, input_ids=FakeCUDAInput())+oom = _bare_agent(ImmovableModel(), dtype=torch.float16)+oom.device = torch.device("cuda") # the OOM branch only reads .type+check("oom-fallback/count starts at 0", oom.cpu_fallback_count, 0)+check("oom-fallback/reason starts None", oom.last_fallback_reason, None)++out = oom._infer(oom_batch) # first forward raises OOM -> scoped CPU retry+check("oom-fallback/retry answered", isinstance(out, tuple), True)+check("oom-fallback/count recorded", oom.cpu_fallback_count, 1)+check("oom-fallback/reason recorded",+ "out of memory" in (oom.last_fallback_reason or ""), True)+check("oom-fallback/scoped: device restored", oom.device.type, "cuda")++oom._infer(oom_batch) # a second OOM accumulates+check("oom-fallback/second OOM counts too", oom.cpu_fallback_count, 2)++# a plain forward never touches the counters+check("oom-fallback/plain CPU infer stays 0", cpu_disabled.cpu_fallback_count, 0)+check("oom-fallback/plain CPU reason stays None", cpu_disabled.last_fallback_reason, None)++# ------------------------------------------------------------------ reportprint("\n%d passed, %d failed" % (len(PASS), len(FAIL)))for f in FAIL:diff --git a/tests/test_serve.py b/tests/test_serve.pyindex ae52aae2..bc94136c 100644--- a/tests/test_serve.py+++ b/tests/test_serve.py@@ -209,6 +209,25 @@ def test_health_supports_router_without_loaded_revisions(monkeypatch):assert r.status_code == 200assert r.json()["status"] == "ok"assert r.json()["revisions"] == {}+ # Same for the #351 fallback counters: a router with no _agents and no agents+ # with counters still reports a stable, all-zero shape.+ assert r.json()["cpu_fallbacks"] == {"english": {"count": 0, "last_reason": None}}+++def test_health_reports_cpu_fallback_counters(monkeypatch):+ """A resident agent that triggered the scoped CPU fallback shows it in /health."""+ from types import SimpleNamespace++ client, fake = _client(monkeypatch)+ fake._agents = {"english": SimpleNamespace(+ cpu_fallback_count=2,+ last_fallback_reason="CUDA out of memory. Tried to allocate 1.00 GiB",+ )}+ r = client.get("/health")+ assert r.status_code == 200+ fb = r.json()["cpu_fallbacks"]+ assert fb["english"]["count"] == 2, fb+ assert "out of memory" in fb["english"]["last_reason"], fbdef test_helpers():