Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
fix(serve): report where inference actually runs, not the requested device
Aashish64e91f15406a
diff --git a/docs/docker.md b/docs/docker.mdindex 27487236..785b2b15 100644--- a/docs/docker.md+++ b/docs/docker.md@@ -212,6 +212,13 @@ The service has a healthcheck on `/health`. The server preloads before it startslistening, so with `LAYA_PRELOAD=1` a healthy container has its checkpoints loaded.`docker compose ... up -d --wait laya-serve` returns once it is healthy.+`/health` reports `device` as the device a resident checkpoint actually computes+on, which is not always what `LAYA_DEVICE` asked for: a checkpoint that wants a+GPU it cannot get falls back to CPU silently and still answers correctly.+`checkpoint_devices` names each loaded checkpoint, and `device_is_preference` is+`true` only while nothing is resident, so a deployment that quietly lost its GPU+says so instead of echoing its own configuration back.+### Server configurationThese apply to the `laya-serve` service only.diff --git a/laya/serve.py b/laya/serve.pyindex 2cd30922..3bd40391 100644--- a/laya/serve.py+++ b/laya/serve.py@@ -30,6 +30,11 @@ env var meaning defaultgets 503 (see below)====================== ============================================ =========+``LAYA_DEVICE`` is a preference, not a guarantee: an ``Agent`` that asks for a+GPU it cannot get falls back to CPU silently and keeps answering correctly.+``/health`` reports where a resident checkpoint actually computes, using the+same helpers the MCP status tool uses (``laya.mcp.device``).+Imports of heavy dependencies (fastapi, uvicorn, torch via Router) are alldeferred into the functions that need them, so ``import laya.serve`` stays cheapand touches no GPU -- which is what keeps the Nix ``pythonImportsCheck`` honest.@@ -229,10 +234,14 @@ def _apply_thread_limit():def build_router():"""Build a Router from the environment, preloading unless told otherwise."""+ from .mcp.device import env_devicefrom .router import Router_apply_thread_limit()- device = os.environ.get("LAYA_DEVICE") or None+ # `env_device` rather than a raw read: it strips, the way `LAYA_MODELS` two lines+ # below is stripped. A value copied out of a Dockerfile or a `.env` file carries a+ # trailing newline, and torch refuses that as an invalid device string.+ device = env_device()models_env = os.environ.get("LAYA_MODELS", "").strip()preload_names = [m.strip() for m in models_env.split(",") if m.strip()] or Nonerouter = Router(device=device, auto_task_detection=_env_bool("LAYA_AUTO_TASK", False))@@ -249,6 +258,8 @@ def create_app(router: Optional[Any] = None):from fastapi import FastAPI, Header, HTTPException, Request+ from .mcp.device import agent_device, resolve_device, router_agent+if router is None:router = build_router()api_key = os.environ.get("LAYA_API_KEY") or None@@ -306,11 +317,24 @@ def create_app(router: Optional[Any] = None):@app.get("/health")def health() -> Dict[str, Any]:+ # `device` is where a resident checkpoint really computes, not what was asked for:+ # `Agent.device` reflects the silent GPU -> CPU fallback, so a container that asked+ # for a GPU it did not get says so. With nothing resident it is the configured+ # preference, and `device_is_preference` tells the reader which of the two it is+ # looking at. Same convention, and the same three keys, as `laya_status` over MCP.+ checkpoint_devices: Dict[str, str] = {}+ for name in (router.loaded or []):+ device = agent_device(router_agent(router, name))+ if device:+ checkpoint_devices[name] = device+ actual = next(iter(checkpoint_devices.values()), None)return {"status": "ok","loaded": router.loaded,"revisions": getattr(router, "loaded_revisions", {}),- "device": os.environ.get("LAYA_DEVICE") or "auto",+ "device": actual or resolve_device(),+ "device_is_preference": actual is None,+ "checkpoint_devices": checkpoint_devices,}@app.post("/v1/systemone")diff --git a/tests/test_serve.py b/tests/test_serve.pyindex ae52aae2..b905afd1 100644--- a/tests/test_serve.py+++ b/tests/test_serve.py@@ -5,12 +5,18 @@ HTTP layer maps requests/responses and enforces auth as hs-jev expects."""import jsonimport logging+import os+import subprocess+import sys+from types import SimpleNamespaceimport pytestfastapi = pytest.importorskip("fastapi")from fastapi.testclient import TestClient # noqa: E402+ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))+from laya.serve import ( # noqa: E402MAX_BODY_BYTES,_apply_thread_limit,@@ -201,6 +207,119 @@ def test_body_read_preserves_parse_error_codes(monkeypatch):assert r.status_code == 400, (payload, r.status_code)+class DeviceRouter:+ """A Router with resident checkpoints whose real devices are known.++ `Agent.device` is a `torch.device`; `laya.mcp.device.agent_device` also accepts the+ plain string, which keeps this stub free of torch and of checkpoint weights.+ """++ def __init__(self, **devices):+ self._agents = {name: SimpleNamespace(device=device)+ for name, device in devices.items()}+ self.loaded = list(devices)++ def predict(self, state, questions, model=None):+ return {"model": "laya-rl-agent",+ "answers": {"dept": {"type": "choice", "choice": "billing",+ "probabilities": {"billing": 1.0}, "confidence": 1.0}},+ "usage": {"input_tokens": 1, "output_tokens": 0},+ "routing": {"model": (self.loaded or ["english"])[0]}}+++def test_health_reports_where_inference_actually_runs(monkeypatch):+ """`LAYA_DEVICE` is a request, not a fact: the Agent falls back to CPU silently."""+ monkeypatch.setenv("LAYA_DEVICE", "cuda")+ fake = DeviceRouter(english="cpu") # asked for cuda, ended up on cpu+ body = TestClient(create_app(router=fake)).get("/health").json()+ assert body["device"] == "cpu", body+ assert body["device_is_preference"] is False, body+ assert body["checkpoint_devices"] == {"english": "cpu"}, body+++def test_health_names_each_checkpoint_device(monkeypatch):+ monkeypatch.setenv("LAYA_DEVICE", "cuda")+ fake = DeviceRouter(english="cpu", multilingual="cuda")+ body = TestClient(create_app(router=fake)).get("/health").json()+ assert body["checkpoint_devices"] == {"english": "cpu", "multilingual": "cuda"}, body+ # The top-level answer is the first resident one, exactly as `laya_status` reports it.+ assert body["device"] == body["checkpoint_devices"][body["loaded"][0]], body+++def test_health_without_a_resident_checkpoint_flags_the_preference(monkeypatch):+ monkeypatch.setenv("LAYA_DEVICE", "cuda")+ body = TestClient(create_app(router=FakeRouter())).get("/health").json()+ assert body["device_is_preference"] is True, body+ assert body["checkpoint_devices"] == {}, body+ assert body["device"] == "cuda", body # what was asked for, labelled as such+++def test_health_agrees_with_the_mcp_status_tool(monkeypatch):+ """One fact about the device, reported the same way by both surfaces."""+ pytest.importorskip("mcp")+ from laya.mcp.tools import laya_status++ monkeypatch.setenv("LAYA_DEVICE", "cuda")+ fake = DeviceRouter(english="cpu")+ body = TestClient(create_app(router=fake)).get("/health").json()+ status = laya_status(router=fake, loaded=list(fake.loaded))+ assert body["device"] == status["device"], (body["device"], status["device"])+ assert body["checkpoint_devices"] == status["checkpoint_devices"], body+ assert body["device_is_preference"] == status["device_is_preference"], body+++def test_health_needs_neither_the_mcp_extra_nor_torch():+ """`laya.mcp.device` is documented as importable without `mcp`; prove the server agrees."""+ probe = r'''+import sys+class Blocker:+ def find_spec(self, name, path=None, target=None):+ if name == "mcp" or name.startswith("mcp."):+ raise ImportError("mcp blocked")+sys.meta_path.insert(0, Blocker())+sys.path.insert(0, %r)+from laya.mcp.device import agent_device, env_device, resolve_device, router_agent+assert agent_device(type("A", (), {"device": "cpu"})()) == "cpu"+assert env_device.__module__ == "laya.mcp.device"+import laya.serve+assert "mcp" not in sys.modules, "laya.mcp.device reached the mcp distribution"+from fastapi.testclient import TestClient+client = TestClient(laya.serve.create_app(router=type("R", (), {"loaded": ["english"],+ "_agents": {"english": type("A", (), {"device": "cpu"})()}})()))+body = client.get("/health").json()+assert body["device"] == "cpu" and body["checkpoint_devices"] == {"english": "cpu"}, body+print("ok")+''' % ROOT+ out = subprocess.run([sys.executable, "-c", probe], capture_output=True, text=True)+ assert out.returncode == 0, out.stderr[-2000:]+ assert "ok" in out.stdout, out.stdout+++def test_build_router_strips_the_device_before_torch_sees_it(monkeypatch):+ """A value pasted from a Dockerfile or a `.env` file carries a trailing newline."""+ import laya.router+ import laya.serve++ seen = {}++ class RecordingRouter:+ def __init__(self, device=None, **kwargs):+ seen["device"] = device++ def preload(self, names=None):+ seen["preloaded"] = names++ monkeypatch.setattr(laya.router, "Router", RecordingRouter)+ monkeypatch.setenv("LAYA_PRELOAD", "0")+ for raw, want in ((" cpu\n", "cpu"), ("cuda ", "cuda"), (" ", None), ("cpu", "cpu")):+ monkeypatch.setenv("LAYA_DEVICE", raw)+ laya.serve.build_router()+ assert seen["device"] == want, "%r -> %r" % (raw, seen["device"])+ monkeypatch.delenv("LAYA_DEVICE")+ laya.serve.build_router()+ assert seen["device"] is None, repr(seen["device"]) # unset means auto++def test_health_supports_router_without_loaded_revisions(monkeypatch):# FakeRouter deliberately has no loaded_revisions attribute. Injected test or# embedding routers predating revision reporting must remain health-compatible.