Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(serve): log the inference failure the client is not allowed to see

PerryLink5d703df03b2e
diff --git a/laya/serve.py b/laya/serve.py
index c0fc1bc3..b1522f17 100644
--- a/laya/serve.py
+++ b/laya/serve.py
@@ -34,10 +34,17 @@ and touches no GPU -- which is what keeps the Nix ``pythonImportsCheck`` honest.
"""
import hmac
import json
+import logging
import os
from contextlib import asynccontextmanager
from typing import Any, Dict, Optional
+# A failed inference is reported to the client as a fixed 500 so nothing about paths,
+# weights or memory state leaks, which leaves the server log as the only place the
+# actual cause can appear. Uvicorn configures the root logger, so a module logger
+# propagates there without this module setting up any handlers.
+_log = logging.getLogger("laya.serve")
+
# The three checkpoint names the router understands; used to decide whether a
# client's `model` field names a Laya checkpoint (honour it) or is some other
# Jev model id (ignore it and let the router auto-select).
@@ -280,6 +287,11 @@ def create_app(router: Optional[Any] = None):
# Question validation errors name the question and what to fix: safe for clients.
raise HTTPException(status_code=422, detail=str(e))
except Exception: # noqa: BLE001 -- never leak paths/weights/OOM text to clients
+ # The client still learns nothing, but the operator gets the traceback. Without
+ # this the container logs show only the 500, so a deterministic failure such as a
+ # missing C compiler for triton's JIT (#365) is invisible from the running server
+ # and has to be reproduced in-process to be diagnosed at all.
+ _log.exception("inference failed for model=%s", model)
raise HTTPException(status_code=500, detail="inference failed")
return app
diff --git a/tests/test_serve.py b/tests/test_serve.py
index 566660bb..1c057eb7 100644
--- a/tests/test_serve.py
+++ b/tests/test_serve.py
@@ -4,6 +4,7 @@ A fake Router is injected so nothing loads a checkpoint; we only assert that the
HTTP layer maps requests/responses and enforces auth as hs-jev expects.
"""
import json
+import logging
import pytest
@@ -315,3 +316,73 @@ def test_health_stays_available_during_inference(monkeypatch):
assert seen["slow"] == 200
assert seen["health"] == 200 and seen["payload"]["status"] == "ok"
assert fake.threads and "MainThread" not in fake.threads, fake.threads
+
+
+class ExplodingRouter:
+ """Fails the way a container missing triton's C compiler does (#365).
+
+ The message is the shape a real failure takes: it names a path and a tool, which is
+ exactly what must not reach the client and exactly what the operator needs.
+ """
+
+ loaded = ["multilingual"]
+
+ def __init__(self, message):
+ self.message = message
+
+ def predict(self, state, questions, model=None):
+ raise RuntimeError(self.message)
+
+
+def test_inference_failure_is_logged_and_not_leaked(monkeypatch, caplog):
+ """A failed inference still returns a bare 500, but the cause reaches the log.
+
+ The client-facing message is deliberately fixed, so the server log is the only place
+ the real exception can appear. Before this, the log carried nothing at all: a
+ deterministic failure was visible only as `POST /v1/systemone HTTP/1.1" 500`, and the
+ cause had to be reproduced in-process to be found.
+ """
+ secret = ("Failed to find C compiler. Please specify via CC environment variable "
+ "or set triton.knobs.build.impl (/opt/venv/lib/python3.11/site-packages/triton)")
+ monkeypatch.delenv("LAYA_API_KEY", raising=False)
+ client = TestClient(create_app(router=ExplodingRouter(secret)), raise_server_exceptions=False)
+
+ with caplog.at_level(logging.ERROR, logger="laya.serve"):
+ response = client.post("/v1/systemone", json=REQ)
+
+ assert response.status_code == 500
+ assert response.json() == {"detail": "inference failed"}
+ for leaked in ("C compiler", "triton", "/opt/venv", "site-packages"):
+ assert leaked not in response.text, response.text
+
+ logged = "\n".join(r.getMessage() if isinstance(r.getMessage(), str) else str(r.msg)
+ for r in caplog.records)
+ assert any(r.levelno == logging.ERROR for r in caplog.records), caplog.records
+ # the traceback has to be in the record, not only the summary line
+ assert any(r.exc_info for r in caplog.records), "no exc_info on the failure record"
+ assert "inference failed" in logged
+
+
+def test_validation_errors_are_not_logged_as_failures(monkeypatch, caplog):
+ """A 422 is the caller's mistake and must not be logged as a server error.
+
+ `ValueError` from the router is mapped to 422 with its message intact, because those
+ messages name the question and what to fix. Only the bare `except Exception` below it
+ reports a server fault, so only that branch logs.
+ """
+ class RejectingRouter:
+ loaded = ["english"]
+
+ def predict(self, state, questions, model=None):
+ raise ValueError("question 'q': a choice question needs at least one criterion")
+
+ monkeypatch.delenv("LAYA_API_KEY", raising=False)
+ client = TestClient(create_app(router=RejectingRouter()), raise_server_exceptions=False)
+
+ with caplog.at_level(logging.ERROR, logger="laya.serve"):
+ response = client.post("/v1/systemone", json=REQ)
+
+ assert response.status_code == 422, response.text
+ assert "at least one criterion" in response.text, response.text
+ assert not [r for r in caplog.records if r.levelno >= logging.ERROR], caplog.records
+