Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
fix(serve): log the inference failure the client is not allowed to see
PerryLink5d703df03b2e
diff --git a/laya/serve.py b/laya/serve.pyindex c0fc1bc3..b1522f17 100644--- a/laya/serve.py+++ b/laya/serve.py@@ -34,10 +34,17 @@ and touches no GPU -- which is what keeps the Nix ``pythonImportsCheck`` honest."""import hmacimport json+import loggingimport osfrom contextlib import asynccontextmanagerfrom typing import Any, Dict, Optional+# A failed inference is reported to the client as a fixed 500 so nothing about paths,+# weights or memory state leaks, which leaves the server log as the only place the+# actual cause can appear. Uvicorn configures the root logger, so a module logger+# propagates there without this module setting up any handlers.+_log = logging.getLogger("laya.serve")+# The three checkpoint names the router understands; used to decide whether a# client's `model` field names a Laya checkpoint (honour it) or is some other# Jev model id (ignore it and let the router auto-select).@@ -280,6 +287,11 @@ def create_app(router: Optional[Any] = None):# Question validation errors name the question and what to fix: safe for clients.raise HTTPException(status_code=422, detail=str(e))except Exception: # noqa: BLE001 -- never leak paths/weights/OOM text to clients+ # The client still learns nothing, but the operator gets the traceback. Without+ # this the container logs show only the 500, so a deterministic failure such as a+ # missing C compiler for triton's JIT (#365) is invisible from the running server+ # and has to be reproduced in-process to be diagnosed at all.+ _log.exception("inference failed for model=%s", model)raise HTTPException(status_code=500, detail="inference failed")return appdiff --git a/tests/test_serve.py b/tests/test_serve.pyindex 566660bb..1c057eb7 100644--- a/tests/test_serve.py+++ b/tests/test_serve.py@@ -4,6 +4,7 @@ A fake Router is injected so nothing loads a checkpoint; we only assert that theHTTP layer maps requests/responses and enforces auth as hs-jev expects."""import json+import loggingimport pytest@@ -315,3 +316,73 @@ def test_health_stays_available_during_inference(monkeypatch):assert seen["slow"] == 200assert seen["health"] == 200 and seen["payload"]["status"] == "ok"assert fake.threads and "MainThread" not in fake.threads, fake.threads+++class ExplodingRouter:+ """Fails the way a container missing triton's C compiler does (#365).++ The message is the shape a real failure takes: it names a path and a tool, which is+ exactly what must not reach the client and exactly what the operator needs.+ """++ loaded = ["multilingual"]++ def __init__(self, message):+ self.message = message++ def predict(self, state, questions, model=None):+ raise RuntimeError(self.message)+++def test_inference_failure_is_logged_and_not_leaked(monkeypatch, caplog):+ """A failed inference still returns a bare 500, but the cause reaches the log.++ The client-facing message is deliberately fixed, so the server log is the only place+ the real exception can appear. Before this, the log carried nothing at all: a+ deterministic failure was visible only as `POST /v1/systemone HTTP/1.1" 500`, and the+ cause had to be reproduced in-process to be found.+ """+ secret = ("Failed to find C compiler. Please specify via CC environment variable "+ "or set triton.knobs.build.impl (/opt/venv/lib/python3.11/site-packages/triton)")+ monkeypatch.delenv("LAYA_API_KEY", raising=False)+ client = TestClient(create_app(router=ExplodingRouter(secret)), raise_server_exceptions=False)++ with caplog.at_level(logging.ERROR, logger="laya.serve"):+ response = client.post("/v1/systemone", json=REQ)++ assert response.status_code == 500+ assert response.json() == {"detail": "inference failed"}+ for leaked in ("C compiler", "triton", "/opt/venv", "site-packages"):+ assert leaked not in response.text, response.text++ logged = "\n".join(r.getMessage() if isinstance(r.getMessage(), str) else str(r.msg)+ for r in caplog.records)+ assert any(r.levelno == logging.ERROR for r in caplog.records), caplog.records+ # the traceback has to be in the record, not only the summary line+ assert any(r.exc_info for r in caplog.records), "no exc_info on the failure record"+ assert "inference failed" in logged+++def test_validation_errors_are_not_logged_as_failures(monkeypatch, caplog):+ """A 422 is the caller's mistake and must not be logged as a server error.++ `ValueError` from the router is mapped to 422 with its message intact, because those+ messages name the question and what to fix. Only the bare `except Exception` below it+ reports a server fault, so only that branch logs.+ """+ class RejectingRouter:+ loaded = ["english"]++ def predict(self, state, questions, model=None):+ raise ValueError("question 'q': a choice question needs at least one criterion")++ monkeypatch.delenv("LAYA_API_KEY", raising=False)+ client = TestClient(create_app(router=RejectingRouter()), raise_server_exceptions=False)++ with caplog.at_level(logging.ERROR, logger="laya.serve"):+ response = client.post("/v1/systemone", json=REQ)++ assert response.status_code == 422, response.text+ assert "at least one criterion" in response.text, response.text+ assert not [r for r in caplog.records if r.levelno >= logging.ERROR], caplog.records+