Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
fix(serve): serve again after the app's lifespan restarts
hemanthrayuduu37cdb5cb4d91
diff --git a/laya/serve.py b/laya/serve.pyindex cde2ee09..c34de10e 100644--- a/laya/serve.py+++ b/laya/serve.py@@ -885,6 +885,7 @@ def create_app(router: Optional[Any] = None):# predictions can share a checkpoint -- a GPU-shaped choice this endpoint does not# rely on). `loop.run_in_executor` is the API the issue asked for.pool = ThreadPoolExecutor(max_workers=1, thread_name_prefix="laya-infer")+ pool_closed = False# Created on first request, not here: an `asyncio.Lock` binds to the loop that is# running when it is first awaited, and `create_app` may be called before that loop# exists (module scope, TestClient startup, a preload script).@@ -934,6 +935,15 @@ def create_app(router: Optional[Any] = None):@asynccontextmanagerasync def lifespan(_app: FastAPI):+ nonlocal pool, pool_closed, gate, admission+ # A host can run one app's lifespan more than once (a TestClient per test, an embedding+ # server that restarts it). The previous shutdown closed the pool, and the gate and the+ # admission semaphore bind to the loop they first waited on, which a restart replaces.+ if pool_closed:+ pool = ThreadPoolExecutor(max_workers=1, thread_name_prefix="laya-infer")+ pool_closed = False+ gate = None+ admission = Nonereaper = asyncio.create_task(_idle_reaper()) if idle_unload_seconds else Nonetry:yield@@ -947,6 +957,7 @@ def create_app(router: Optional[Any] = None):# TestClient, embedded ASGI apps, and process supervisors all need# the executor to drain when the app stops.pool.shutdown(wait=True, cancel_futures=True)+ pool_closed = Trueapp = FastAPI(title="laya-serve",diff --git a/tests/test_serve.py b/tests/test_serve.pyindex 26f7a152..79919a4f 100644--- a/tests/test_serve.py+++ b/tests/test_serve.py@@ -3344,6 +3344,74 @@ def test_idle_unload_retries_failure_and_stops_at_shutdown(monkeypatch):assert not router.unloaded.wait(0.15)+# -------------------------------- an app can be started again after its lifespan stops+# The lifespan shuts the inference pool down (#319), and a host can run one app's lifespan more+# than once: a TestClient per test module, an embedding server that restarts it. The pool was made+# once in create_app, so after a second startup every request was a 500 "inference failed" from+# "cannot schedule new futures after shutdown", and the idle reaper logged the same each tick.[email protected]("batch", [False, True])+def test_an_app_serves_again_after_its_lifespan_restarts(monkeypatch, batch):+ monkeypatch.delenv("LAYA_API_KEY", raising=False)+ monkeypatch.delenv("LAYA_IDLE_UNLOAD_SECONDS", raising=False)+ app = create_app(FakeRouter())+ path = "/v1/systemone/batch" if batch else "/v1/systemone"+ body = {"states": [REQ["state"]], "questions": REQ["questions"]} if batch else REQ+ statuses = []+ for _ in range(2):+ with TestClient(app) as client:+ statuses.append(client.post(path, json=body).status_code)+ assert statuses == [200, 200]+++def test_idle_unload_runs_again_after_a_restart(monkeypatch):+ monkeypatch.setenv("LAYA_IDLE_UNLOAD_SECONDS", "0.05")+ monkeypatch.delenv("LAYA_API_KEY", raising=False)+ router = IdleRouter()+ app = create_app(router)+ with TestClient(app):+ assert router.unloaded.wait(2.0)+ router.loaded = ["english"]+ router.unloaded.clear()+ with TestClient(app):+ assert router.unloaded.wait(2.0)+++def test_a_restarted_app_still_queues_concurrent_requests(monkeypatch):+ # The inference gate binds to the event loop the first time a request has to wait on it, and+ # a restarted lifespan runs on a new loop. A gate carried over from the first run made the+ # second run's queued request fail with "is bound to a different event loop".+ import threading+ import time+ from concurrent.futures import ThreadPoolExecutor++ monkeypatch.delenv("LAYA_API_KEY", raising=False)+ monkeypatch.delenv("LAYA_IDLE_UNLOAD_SECONDS", raising=False)++ class SlowRouter(FakeRouter):+ def __init__(self):+ super().__init__()+ self.entered = threading.Event()+ self.release = threading.Event()++ def predict(self, *args, **kwargs):+ self.entered.set()+ assert self.release.wait(3.0)+ return super().predict(*args, **kwargs)++ router = SlowRouter()+ app = create_app(router)+ for _ in range(2):+ router.entered.clear()+ router.release.clear()+ with TestClient(app) as client, ThreadPoolExecutor(max_workers=2) as callers:+ first = callers.submit(client.post, "/v1/systemone", json=REQ)+ assert router.entered.wait(2.0)+ second = callers.submit(client.post, "/v1/systemone", json=REQ)+ time.sleep(0.2) # the second request is now waiting on the gate+ router.release.set()+ assert [first.result(timeout=5).status_code, second.result(timeout=5).status_code] == [200, 200]++# -------------------------------- docs/typescript-sdk.md must list the keys the /health handler returns# The SDK page's contract sentence used to say "public `/health` returns `status`, `loaded`,# and `device`" while `laya/serve.py`'s authorized branch returns seven keys -- `status`,