Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(serve): report where inference actually runs, not the requested device

Aashish64e91f15406a
diff --git a/docs/docker.md b/docs/docker.md
index 27487236..785b2b15 100644
--- a/docs/docker.md
+++ b/docs/docker.md
@@ -212,6 +212,13 @@ The service has a healthcheck on `/health`. The server preloads before it starts
listening, so with `LAYA_PRELOAD=1` a healthy container has its checkpoints loaded.
`docker compose ... up -d --wait laya-serve` returns once it is healthy.
+`/health` reports `device` as the device a resident checkpoint actually computes
+on, which is not always what `LAYA_DEVICE` asked for: a checkpoint that wants a
+GPU it cannot get falls back to CPU silently and still answers correctly.
+`checkpoint_devices` names each loaded checkpoint, and `device_is_preference` is
+`true` only while nothing is resident, so a deployment that quietly lost its GPU
+says so instead of echoing its own configuration back.
+
### Server configuration
These apply to the `laya-serve` service only.
diff --git a/laya/serve.py b/laya/serve.py
index 2cd30922..3bd40391 100644
--- a/laya/serve.py
+++ b/laya/serve.py
@@ -30,6 +30,11 @@ env var meaning default
gets 503 (see below)
====================== ============================================ =========
+``LAYA_DEVICE`` is a preference, not a guarantee: an ``Agent`` that asks for a
+GPU it cannot get falls back to CPU silently and keeps answering correctly.
+``/health`` reports where a resident checkpoint actually computes, using the
+same helpers the MCP status tool uses (``laya.mcp.device``).
+
Imports of heavy dependencies (fastapi, uvicorn, torch via Router) are all
deferred into the functions that need them, so ``import laya.serve`` stays cheap
and touches no GPU -- which is what keeps the Nix ``pythonImportsCheck`` honest.
@@ -229,10 +234,14 @@ def _apply_thread_limit():
def build_router():
"""Build a Router from the environment, preloading unless told otherwise."""
+ from .mcp.device import env_device
from .router import Router
_apply_thread_limit()
- device = os.environ.get("LAYA_DEVICE") or None
+ # `env_device` rather than a raw read: it strips, the way `LAYA_MODELS` two lines
+ # below is stripped. A value copied out of a Dockerfile or a `.env` file carries a
+ # trailing newline, and torch refuses that as an invalid device string.
+ device = env_device()
models_env = os.environ.get("LAYA_MODELS", "").strip()
preload_names = [m.strip() for m in models_env.split(",") if m.strip()] or None
router = Router(device=device, auto_task_detection=_env_bool("LAYA_AUTO_TASK", False))
@@ -249,6 +258,8 @@ def create_app(router: Optional[Any] = None):
from fastapi import FastAPI, Header, HTTPException, Request
+ from .mcp.device import agent_device, resolve_device, router_agent
+
if router is None:
router = build_router()
api_key = os.environ.get("LAYA_API_KEY") or None
@@ -306,11 +317,24 @@ def create_app(router: Optional[Any] = None):
@app.get("/health")
def health() -> Dict[str, Any]:
+ # `device` is where a resident checkpoint really computes, not what was asked for:
+ # `Agent.device` reflects the silent GPU -> CPU fallback, so a container that asked
+ # for a GPU it did not get says so. With nothing resident it is the configured
+ # preference, and `device_is_preference` tells the reader which of the two it is
+ # looking at. Same convention, and the same three keys, as `laya_status` over MCP.
+ checkpoint_devices: Dict[str, str] = {}
+ for name in (router.loaded or []):
+ device = agent_device(router_agent(router, name))
+ if device:
+ checkpoint_devices[name] = device
+ actual = next(iter(checkpoint_devices.values()), None)
return {
"status": "ok",
"loaded": router.loaded,
"revisions": getattr(router, "loaded_revisions", {}),
- "device": os.environ.get("LAYA_DEVICE") or "auto",
+ "device": actual or resolve_device(),
+ "device_is_preference": actual is None,
+ "checkpoint_devices": checkpoint_devices,
}
@app.post("/v1/systemone")
diff --git a/tests/test_serve.py b/tests/test_serve.py
index ae52aae2..b905afd1 100644
--- a/tests/test_serve.py
+++ b/tests/test_serve.py
@@ -5,12 +5,18 @@ HTTP layer maps requests/responses and enforces auth as hs-jev expects.
"""
import json
import logging
+import os
+import subprocess
+import sys
+from types import SimpleNamespace
import pytest
fastapi = pytest.importorskip("fastapi")
from fastapi.testclient import TestClient # noqa: E402
+ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+
from laya.serve import ( # noqa: E402
MAX_BODY_BYTES,
_apply_thread_limit,
@@ -201,6 +207,119 @@ def test_body_read_preserves_parse_error_codes(monkeypatch):
assert r.status_code == 400, (payload, r.status_code)
+class DeviceRouter:
+ """A Router with resident checkpoints whose real devices are known.
+
+ `Agent.device` is a `torch.device`; `laya.mcp.device.agent_device` also accepts the
+ plain string, which keeps this stub free of torch and of checkpoint weights.
+ """
+
+ def __init__(self, **devices):
+ self._agents = {name: SimpleNamespace(device=device)
+ for name, device in devices.items()}
+ self.loaded = list(devices)
+
+ def predict(self, state, questions, model=None):
+ return {"model": "laya-rl-agent",
+ "answers": {"dept": {"type": "choice", "choice": "billing",
+ "probabilities": {"billing": 1.0}, "confidence": 1.0}},
+ "usage": {"input_tokens": 1, "output_tokens": 0},
+ "routing": {"model": (self.loaded or ["english"])[0]}}
+
+
+def test_health_reports_where_inference_actually_runs(monkeypatch):
+ """`LAYA_DEVICE` is a request, not a fact: the Agent falls back to CPU silently."""
+ monkeypatch.setenv("LAYA_DEVICE", "cuda")
+ fake = DeviceRouter(english="cpu") # asked for cuda, ended up on cpu
+ body = TestClient(create_app(router=fake)).get("/health").json()
+ assert body["device"] == "cpu", body
+ assert body["device_is_preference"] is False, body
+ assert body["checkpoint_devices"] == {"english": "cpu"}, body
+
+
+def test_health_names_each_checkpoint_device(monkeypatch):
+ monkeypatch.setenv("LAYA_DEVICE", "cuda")
+ fake = DeviceRouter(english="cpu", multilingual="cuda")
+ body = TestClient(create_app(router=fake)).get("/health").json()
+ assert body["checkpoint_devices"] == {"english": "cpu", "multilingual": "cuda"}, body
+ # The top-level answer is the first resident one, exactly as `laya_status` reports it.
+ assert body["device"] == body["checkpoint_devices"][body["loaded"][0]], body
+
+
+def test_health_without_a_resident_checkpoint_flags_the_preference(monkeypatch):
+ monkeypatch.setenv("LAYA_DEVICE", "cuda")
+ body = TestClient(create_app(router=FakeRouter())).get("/health").json()
+ assert body["device_is_preference"] is True, body
+ assert body["checkpoint_devices"] == {}, body
+ assert body["device"] == "cuda", body # what was asked for, labelled as such
+
+
+def test_health_agrees_with_the_mcp_status_tool(monkeypatch):
+ """One fact about the device, reported the same way by both surfaces."""
+ pytest.importorskip("mcp")
+ from laya.mcp.tools import laya_status
+
+ monkeypatch.setenv("LAYA_DEVICE", "cuda")
+ fake = DeviceRouter(english="cpu")
+ body = TestClient(create_app(router=fake)).get("/health").json()
+ status = laya_status(router=fake, loaded=list(fake.loaded))
+ assert body["device"] == status["device"], (body["device"], status["device"])
+ assert body["checkpoint_devices"] == status["checkpoint_devices"], body
+ assert body["device_is_preference"] == status["device_is_preference"], body
+
+
+def test_health_needs_neither_the_mcp_extra_nor_torch():
+ """`laya.mcp.device` is documented as importable without `mcp`; prove the server agrees."""
+ probe = r'''
+import sys
+class Blocker:
+ def find_spec(self, name, path=None, target=None):
+ if name == "mcp" or name.startswith("mcp."):
+ raise ImportError("mcp blocked")
+sys.meta_path.insert(0, Blocker())
+sys.path.insert(0, %r)
+from laya.mcp.device import agent_device, env_device, resolve_device, router_agent
+assert agent_device(type("A", (), {"device": "cpu"})()) == "cpu"
+assert env_device.__module__ == "laya.mcp.device"
+import laya.serve
+assert "mcp" not in sys.modules, "laya.mcp.device reached the mcp distribution"
+from fastapi.testclient import TestClient
+client = TestClient(laya.serve.create_app(router=type("R", (), {"loaded": ["english"],
+ "_agents": {"english": type("A", (), {"device": "cpu"})()}})()))
+body = client.get("/health").json()
+assert body["device"] == "cpu" and body["checkpoint_devices"] == {"english": "cpu"}, body
+print("ok")
+''' % ROOT
+ out = subprocess.run([sys.executable, "-c", probe], capture_output=True, text=True)
+ assert out.returncode == 0, out.stderr[-2000:]
+ assert "ok" in out.stdout, out.stdout
+
+
+def test_build_router_strips_the_device_before_torch_sees_it(monkeypatch):
+ """A value pasted from a Dockerfile or a `.env` file carries a trailing newline."""
+ import laya.router
+ import laya.serve
+
+ seen = {}
+
+ class RecordingRouter:
+ def __init__(self, device=None, **kwargs):
+ seen["device"] = device
+
+ def preload(self, names=None):
+ seen["preloaded"] = names
+
+ monkeypatch.setattr(laya.router, "Router", RecordingRouter)
+ monkeypatch.setenv("LAYA_PRELOAD", "0")
+ for raw, want in ((" cpu\n", "cpu"), ("cuda ", "cuda"), (" ", None), ("cpu", "cpu")):
+ monkeypatch.setenv("LAYA_DEVICE", raw)
+ laya.serve.build_router()
+ assert seen["device"] == want, "%r -> %r" % (raw, seen["device"])
+ monkeypatch.delenv("LAYA_DEVICE")
+ laya.serve.build_router()
+ assert seen["device"] is None, repr(seen["device"]) # unset means auto
+
+
def test_health_supports_router_without_loaded_revisions(monkeypatch):
# FakeRouter deliberately has no loaded_revisions attribute. Injected test or
# embedding routers predating revision reporting must remain health-compatible.