Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
fix(serve): align doc table, add fallback warning log, and preserve 3-arg router regression pin
Swatantra Yadavcc2ed5b79269
diff --git a/laya/serve.py b/laya/serve.pyindex 2c6f19d0..0ef6fa60 100644--- a/laya/serve.py+++ b/laya/serve.py@@ -11,25 +11,25 @@ probe.Configuration is entirely via environment variables so the same entry pointserves a laptop dev run and a systemd unit:-====================== ============================================ =========-env var meaning default-====================== ============================================ =========-``LAYA_HOST`` bind address 0.0.0.0-``LAYA_PORT`` bind port 8000-``LAYA_DEVICE`` torch device for every checkpoint (auto)-``LAYA_PRELOAD`` build the checkpoints at startup, not lazily 1-``LAYA_MODELS`` comma list to preload (english,multilingual, (all)- typed-decisions); empty = every checkpoint-``LAYA_THREADS`` cap torch intra-op threads (CPU inference). (torch- Keep <= physical cores; oversubscribing the default)- logical/hyperthread count is a large regression.-``LAYA_AUTO_TASK`` auto-route to the typed-decisions checkpoint 0-``LAYA_API_KEY`` if set, require ``Authorization: Bearer <it>`` (none)-``LAYA_LOG_LEVEL`` uvicorn log level info-``LAYA_MAX_CONCURRENT`` cap on requests past auth at once; excess 16- gets 503 (see below)+========================= ============================================ =========+env var meaning default+========================= ============================================ =========+``LAYA_HOST`` bind address 0.0.0.0+``LAYA_PORT`` bind port 8000+``LAYA_DEVICE`` torch device for every checkpoint (auto)+``LAYA_PRELOAD`` build the checkpoints at startup, not lazily 1+``LAYA_MODELS`` comma list to preload (english,multilingual, (all)+ typed-decisions); empty = every checkpoint+``LAYA_THREADS`` cap torch intra-op threads (CPU inference). (torch+ Keep <= physical cores; oversubscribing the default)+ logical/hyperthread count is a large regression.+``LAYA_AUTO_TASK`` auto-route to the typed-decisions checkpoint 0+``LAYA_API_KEY`` if set, require ``Authorization: Bearer <it>`` (none)+``LAYA_LOG_LEVEL`` uvicorn log level info+``LAYA_MAX_CONCURRENT`` cap on requests past auth at once; excess 16+ gets 503 (see below)``LAYA_MAX_TOKEN_BUDGET`` cap on per-request max_len / head_max_len 8192-====================== ============================================ =========+========================= ============================================ =========Imports of heavy dependencies (fastapi, uvicorn, torch via Router) are alldeferred into the functions that need them, so ``import laya.serve`` stays cheap@@ -126,8 +126,12 @@ def _resolve_max_token_budget() -> int:try:n = int(str(raw).strip())except (TypeError, ValueError):+ _log.warning("invalid LAYA_MAX_TOKEN_BUDGET %r; falling back to %d", raw, DEFAULT_MAX_TOKEN_BUDGET)return DEFAULT_MAX_TOKEN_BUDGET- return n if n > 0 else DEFAULT_MAX_TOKEN_BUDGET+ if n <= 0:+ _log.warning("LAYA_MAX_TOKEN_BUDGET must be positive (got %d); falling back to %d", n, DEFAULT_MAX_TOKEN_BUDGET)+ return DEFAULT_MAX_TOKEN_BUDGET+ return ndef _validate_budget_param(body: Dict[str, Any], key: str, max_cap: int) -> Optional[int]:diff --git a/tests/test_serve.py b/tests/test_serve.pyindex e152f7a2..cc381dfc 100644--- a/tests/test_serve.py+++ b/tests/test_serve.py@@ -12,9 +12,11 @@ fastapi = pytest.importorskip("fastapi")from fastapi.testclient import TestClient # noqa: E402from laya.serve import ( # noqa: E402+ DEFAULT_MAX_TOKEN_BUDGET,MAX_BODY_BYTES,_apply_thread_limit,_env_bool,+ _resolve_max_token_budget,_resolve_model,create_app,)@@ -28,10 +30,8 @@ class FakeRouter:def __init__(self):self.calls = []- def predict(self, state, questions, model=None, **kwargs):- call_info = {"state": state, "questions": questions, "model": model}- call_info.update(kwargs)- self.calls.append(call_info)+ def predict(self, state, questions, model=None):+ self.calls.append({"state": state, "questions": questions, "model": model})return {"model": "laya-rl-agent","answers": {@@ -43,6 +43,15 @@ class FakeRouter:}+class BudgetRouter(FakeRouter):+ """A router whose predict() takes token-budget keywords and records them."""++ def predict(self, state, questions, model=None, **kwargs):+ out = super().predict(state, questions, model=model)+ self.calls[-1].update(kwargs)+ return out++def _client(monkeypatch, api_key=None):if api_key is None:monkeypatch.delenv("LAYA_API_KEY", raising=False)@@ -52,6 +61,16 @@ def _client(monkeypatch, api_key=None):return TestClient(create_app(router=fake)), fake+def _budget_client(monkeypatch, api_key=None):+ if api_key is None:+ monkeypatch.delenv("LAYA_API_KEY", raising=False)+ else:+ monkeypatch.setenv("LAYA_API_KEY", api_key)+ fake = BudgetRouter()+ return TestClient(create_app(router=fake)), fake+++REQ = {"model": "jev-1", # a non-Laya model id -> should be ignored, router auto-routes"state": {"body": "billed twice, refund please"},@@ -481,10 +500,10 @@ class GatedRouter(FakeRouter):self.entered = threading.Event()self.release = threading.Event()- def predict(self, state, questions, model=None, **kwargs):+ def predict(self, state, questions, model=None):self.entered.set()assert self.release.wait(timeout=10), "test did not release the router"- return super().predict(state, questions, model=model, **kwargs)+ return super().predict(state, questions, model=model)def test_admission_bound_refuses_with_503_when_full(monkeypatch):@@ -533,8 +552,16 @@ def test_admission_slot_is_released_after_inference(monkeypatch):assert client.post("/v1/systemone", json=REQ).status_code == 200-def test_token_budget_forwarded(monkeypatch):+def test_no_budget_keeps_the_call_unchanged(monkeypatch):+ """An injected router whose predict() takes no kwargs continues to work when body sends no budget."""client, fake = _client(monkeypatch)+ for body in (REQ, dict(REQ, max_len=None, head_max_len=None)):+ assert client.post("/v1/systemone", json=body).status_code == 200+ assert len(fake.calls) == 2+++def test_token_budget_forwarded(monkeypatch):+ client, fake = _budget_client(monkeypatch)r = client.post("/v1/systemone", json={**REQ, "max_len": 4096, "head_max_len": 256})assert r.status_code == 200assert fake.calls[0]["max_len"] == 4096@@ -543,7 +570,7 @@ def test_token_budget_forwarded(monkeypatch):@pytest.mark.parametrize("bad_budget", ["fast", True, 3.14])def test_token_budget_validation_type(monkeypatch, bad_budget):- client, fake = _client(monkeypatch)+ client, fake = _budget_client(monkeypatch)r = client.post("/v1/systemone", json={**REQ, "max_len": bad_budget})assert r.status_code == 422assert "must be an integer" in r.json()["detail"]@@ -551,14 +578,14 @@ def test_token_budget_validation_type(monkeypatch, bad_budget):@pytest.mark.parametrize("bad_val", [0, -10])def test_token_budget_validation_positive(monkeypatch, bad_val):- client, fake = _client(monkeypatch)+ client, fake = _budget_client(monkeypatch)r = client.post("/v1/systemone", json={**REQ, "max_len": bad_val})assert r.status_code == 422assert "must be a positive integer" in r.json()["detail"]def test_token_budget_exceeds_server_cap(monkeypatch):- client, fake = _client(monkeypatch)+ client, fake = _budget_client(monkeypatch)r = client.post("/v1/systemone", json={**REQ, "max_len": 9000})assert r.status_code == 422assert "exceeds server limit" in r.json()["detail"]@@ -566,7 +593,7 @@ def test_token_budget_exceeds_server_cap(monkeypatch):def test_token_budget_head_max_len_equal_to_max_len(monkeypatch):"""Core accepts head_max_len == max_len; serve forwards both without artificial restriction."""- client, fake = _client(monkeypatch)+ client, fake = _budget_client(monkeypatch)r = client.post("/v1/systemone", json={**REQ, "max_len": 512, "head_max_len": 512})assert r.status_code == 200assert fake.calls[0]["max_len"] == 512@@ -575,7 +602,7 @@ def test_token_budget_head_max_len_equal_to_max_len(monkeypatch):def test_token_budget_env_cap_override(monkeypatch):monkeypatch.setenv("LAYA_MAX_TOKEN_BUDGET", "2048")- client, fake = _client(monkeypatch)+ client, fake = _budget_client(monkeypatch)r = client.post("/v1/systemone", json={**REQ, "max_len": 4096})assert r.status_code == 422assert "exceeds server limit" in r.json()["detail"]@@ -583,3 +610,14 @@ def test_token_budget_env_cap_override(monkeypatch):r2 = client.post("/v1/systemone", json={**REQ, "max_len": 2048})assert r2.status_code == 200assert fake.calls[0]["max_len"] == 2048+++def test_resolve_max_token_budget_fallback(monkeypatch, caplog):+ monkeypatch.delenv("LAYA_MAX_TOKEN_BUDGET", raising=False)+ assert _resolve_max_token_budget() == DEFAULT_MAX_TOKEN_BUDGET+ for bad in ("abc", "-10", "0"):+ monkeypatch.setenv("LAYA_MAX_TOKEN_BUDGET", bad)+ assert _resolve_max_token_budget() == DEFAULT_MAX_TOKEN_BUDGET+ assert "invalid LAYA_MAX_TOKEN_BUDGET" in caplog.text+ assert "LAYA_MAX_TOKEN_BUDGET must be positive" in caplog.text+