Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

feat(serve): unload idle checkpoints on the inference worker

Rishit Sharma81b3af82dcb7
diff --git a/README.md b/README.md
index af65dae7..061732d3 100644
--- a/README.md
+++ b/README.md
@@ -662,7 +662,8 @@ language evidence falls back to, `english` by default; set it to `multilingual`
your traffic is not English), `LAYA_MAX_LOADED` (checkpoints resident at
once, 2 by default; raise it to 3 when `LAYA_AUTO_TASK` makes a third one
reachable on demand, or the server rebuilds one every time routing switches),
-and `LAYA_API_KEY` (when set, clients must
+`LAYA_IDLE_UNLOAD_SECONDS` (unload idle checkpoints; `0` disables it, `300` frees device memory
+after five minutes and makes the next request pay a cold load), and `LAYA_API_KEY` (when set, clients must
send `Authorization: Bearer <key>`). A client's `model` field is honoured when it
names a Laya checkpoint (`english`/`multilingual`/`typed-decisions`), otherwise
the router auto-selects by script/language. The same body can also carry `task`,
diff --git a/docs/http-api.md b/docs/http-api.md
index 6ab7bdb9..5d83b51f 100644
--- a/docs/http-api.md
+++ b/docs/http-api.md
@@ -33,6 +33,7 @@ Everything is environment variables, so one image serves a laptop dev run and a
| `LAYA_MODELS` | comma list to preload (`english,multilingual,typed-decisions`); empty = all | all |
| `LAYA_THREADS` | cap torch intra-op threads on CPU; keep it <= physical cores -- oversubscribing logical cores is a large regression | torch default |
| `LAYA_AUTO_TASK` | auto-route to the typed-decisions checkpoint | `0` |
+| `LAYA_IDLE_UNLOAD_SECONDS` | unload resident checkpoints after this many idle seconds; the next request loads its checkpoint again. Zero disables unloading | `0` |
| `LAYA_DEFAULT_MODEL` | checkpoint a state with no language evidence falls back to; aliases such as `ml` resolve the way core resolves them, and an unresolvable name stops the server at startup | `english` |
| `LAYA_API_KEY` | if set, require `Authorization: Bearer <key>` | none |
| `LAYA_LOG_LEVEL` | uvicorn log level | `info` |
@@ -46,6 +47,12 @@ strip `/laya` before forwarding requests to Laya; the app's routes remain `/heal
For containers, including CUDA and ARM64 images, see [Docker quickstart](docker.md).
+For bursty local use, set `LAYA_IDLE_UNLOAD_SECONDS=300`. Inference and unload run on the same
+worker, and the idle window starts again when a single or batch forward pass finishes, including
+failed requests. The next prediction pays a cold load. Unloading releases model references and
+device caches, including Metal; the process allocator may retain RAM pages, so process RSS need
+not fall by the size of the checkpoint.
+
## Endpoints
### `GET /health`
@@ -69,6 +76,10 @@ One server's answer, so the blocks agree with each other: every key of `revision
`checkpoint_devices` and `cpu_fallbacks` is a name in `loaded`. `tests/test_serve.py` holds this
sample to the handler that produces it, field by field.
+With idle unloading enabled, authenticated health responses also include `idle_unload_seconds`
+(the configured window) and `idle_seconds` (time since the last inference request or completion).
+Health probes do not reset that clock. An empty `loaded` list is normal after an idle unload.
+
- `status` is `ok` whenever the process answers at all. It says nothing about the checkpoints.
- `loaded` lists the checkpoints resident in memory. It is empty until a request builds one, which is
what `LAYA_PRELOAD=0` leaves the process doing.
diff --git a/laya/serve.py b/laya/serve.py
index 502d0bc1..49c713d2 100644
--- a/laya/serve.py
+++ b/laya/serve.py
@@ -28,6 +28,8 @@ env var meaning defaul
language evidence; aliases like ml work
``LAYA_MAX_LOADED`` checkpoints kept resident at once. Below what 2
routing can choose, one reloads per switch.
+``LAYA_IDLE_UNLOAD_SECONDS`` unload checkpoints after this many idle 0 (off)
+ seconds; the next request loads them again.
``LAYA_API_KEY`` if set, require ``Authorization: Bearer <it>`` (none)
``LAYA_ROOT_PATH`` public URL prefix behind a reverse proxy (none)
``LAYA_LOG_LEVEL`` uvicorn log level info
@@ -53,6 +55,7 @@ and touches no GPU -- which is what keeps the Nix ``pythonImportsCheck`` honest.
import hmac
import json
import logging
+import math
import os
import re
import time
@@ -210,6 +213,21 @@ def _resolve_max_concurrent() -> int:
return n if n > 0 else DEFAULT_MAX_CONCURRENT
+def _resolve_idle_unload_seconds() -> float:
+ """Idle window from LAYA_IDLE_UNLOAD_SECONDS; zero disables unloading."""
+ raw = os.environ.get("LAYA_IDLE_UNLOAD_SECONDS", "").strip()
+ if not raw:
+ return 0.0
+ try:
+ seconds = float(raw)
+ except ValueError:
+ seconds = float("nan")
+ if not math.isfinite(seconds):
+ _log.warning("invalid LAYA_IDLE_UNLOAD_SECONDS %r; idle unload disabled", raw)
+ return 0.0
+ return seconds if seconds > 0 else 0.0
+
+
def _resolve_max_token_budget() -> int:
"""Server-side cap on per-request max_len from LAYA_MAX_TOKEN_BUDGET."""
raw = os.environ.get("LAYA_MAX_TOKEN_BUDGET")
@@ -739,6 +757,7 @@ def create_app(router: Optional[Any] = None):
is built from the environment (and preloaded) at app-creation time."""
import asyncio
from concurrent.futures import ThreadPoolExecutor
+ from functools import partial
from fastapi import FastAPI, Header, HTTPException, Request
@@ -768,11 +787,53 @@ def create_app(router: Optional[Any] = None):
max_concurrent = _resolve_max_concurrent()
admission: Optional[asyncio.Semaphore] = None
+ idle_unload_seconds = _resolve_idle_unload_seconds()
+ last_request = time.monotonic()
+
+ def _mark_request():
+ nonlocal last_request
+ last_request = time.monotonic()
+
+ def _run_inference(fn):
+ try:
+ return fn()
+ finally:
+ # On the worker, including failures and cancelled HTTP requests whose forward
+ # pass continues after the event loop releases the gate.
+ _mark_request()
+
+ def _unload_if_idle():
+ # Recheck on the inference worker: a queued unload must see any forward pass that
+ # finished after the reaper checked the gate. Unload and inference never overlap.
+ idle_for = time.monotonic() - last_request
+ if idle_for >= idle_unload_seconds and router.loaded:
+ router.unload()
+ _log.info("idle for %.1fs: unloaded resident checkpoints", idle_for)
+
+ async def _idle_reaper():
+ interval = max(0.05, min(idle_unload_seconds / 4.0, 5.0))
+ loop = asyncio.get_running_loop()
+ while True:
+ await asyncio.sleep(interval)
+ if gate is not None and gate.locked():
+ continue
+ try:
+ await loop.run_in_executor(pool, _unload_if_idle)
+ except Exception: # noqa: BLE001 -- a failed unload must not stop future attempts
+ _log.exception("idle unload failed")
+
@asynccontextmanager
async def lifespan(_app: FastAPI):
+ reaper = asyncio.create_task(_idle_reaper()) if idle_unload_seconds else None
try:
yield
finally:
+ if reaper is not None:
+ reaper.cancel()
+ try:
+ await reaper
+ except asyncio.CancelledError:
+ pass
# TestClient, embedded ASGI apps, and process supervisors all need
# the executor to drain when the app stops.
pool.shutdown(wait=True, cancel_futures=True)
@@ -839,6 +900,8 @@ def create_app(router: Optional[Any] = None):
agent = router_agent(router, name)
fallbacks[name] = {"count": getattr(agent, "cpu_fallback_count", 0),
"last_reason": getattr(agent, "last_fallback_reason", None)}
+ idle = ({"idle_unload_seconds": idle_unload_seconds,
+ "idle_seconds": round(time.monotonic() - last_request, 1)} if idle_unload_seconds else {})
return {
"status": "ok",
"loaded": router.loaded,
@@ -847,6 +910,7 @@ def create_app(router: Optional[Any] = None):
"device_is_preference": actual is None,
"checkpoint_devices": checkpoint_devices,
"cpu_fallbacks": fallbacks,
+ **idle,
}
@asynccontextmanager
@@ -875,6 +939,7 @@ def create_app(router: Optional[Any] = None):
async def _systemone_inner(request: Request):
nonlocal gate
+ _mark_request()
body = await _read_json_body(request)
if not isinstance(body, dict) or "questions" not in body:
raise HTTPException(status_code=400, detail="request body must be an object with a 'questions' field")
@@ -922,12 +987,8 @@ def create_app(router: Optional[Any] = None):
async with gate:
loop = asyncio.get_running_loop()
t0 = time.perf_counter()
- if predict_kwargs:
- result = await loop.run_in_executor(
- pool, lambda: router.predict(state, questions, model=model, **predict_kwargs))
- else:
- result = await loop.run_in_executor(
- pool, lambda: router.predict(state, questions, model=model))
+ result = await loop.run_in_executor(
+ pool, _run_inference, partial(router.predict, state, questions, model=model, **predict_kwargs))
infer_ms = (time.perf_counter() - t0) * 1000.0
if _env_bool("LAYA_JEV_STRICT", False):
result = _project_jev_strict(result)
@@ -964,6 +1025,7 @@ def create_app(router: Optional[Any] = None):
async def _systemone_batch_inner(request: Request):
nonlocal gate
+ _mark_request()
body = await _read_json_body(request)
if not isinstance(body, dict) or "questions" not in body or "states" not in body:
raise HTTPException(
@@ -1055,7 +1117,7 @@ def create_app(router: Optional[Any] = None):
}
t0 = time.perf_counter()
- batch_res = await loop.run_in_executor(pool, _do_batch)
+ batch_res = await loop.run_in_executor(pool, _run_inference, _do_batch)
infer_ms = (time.perf_counter() - t0) * 1000.0
if _env_bool("LAYA_JEV_STRICT", False):
batch_res["results"] = [_project_jev_strict(item) for item in batch_res["results"]]
diff --git a/tests/test_serve.py b/tests/test_serve.py
index 33200783..576047c1 100644
--- a/tests/test_serve.py
+++ b/tests/test_serve.py
@@ -33,6 +33,7 @@ from laya.serve import ( # noqa: E402
_env_bool,
_LONE_SURROGATE_DETAIL,
_resolve_max_token_budget,
+ _resolve_idle_unload_seconds,
_resolve_max_loaded,
_resolve_model,
create_app,
@@ -2834,3 +2835,164 @@ def test_http_api_page_documents_the_gate_report_on_an_answer():
assert set(answer) == discriminator | rows[answer["type"]] | rows["all"], (
"the sample's %s answer says %s, its table row plus the shared row say %s" % (
qid, sorted(answer), sorted(discriminator | rows[answer["type"]] | rows["all"])))
+
+
+class IdleRouter(FakeRouter):
+ def __init__(self):
+ import threading
+ super().__init__()
+ self.loaded = ["english"]
+ self.unloaded = threading.Event()
+ self.unload_threads = []
+
+ def unload(self):
+ import threading
+ self.unload_threads.append(threading.current_thread().name)
+ self.loaded = []
+ self.unloaded.set()
+
+ def predict(self, *args, **kwargs):
+ self.loaded = ["english"]
+ return super().predict(*args, **kwargs)
+
+
[email protected]("raw, expected", [
+ (None, 0.0), ("", 0.0), (" ", 0.0), ("0", 0.0), ("-5", 0.0), ("abc", 0.0),
+ ("nan", 0.0), ("inf", 0.0), ("-inf", 0.0), ("1e999", 0.0),
+ ("300", 300.0), (" 0.5 ", 0.5), ("1e2", 100.0),
+])
+def test_idle_unload_setting_is_finite_and_opt_in(monkeypatch, raw, expected):
+ if raw is None:
+ monkeypatch.delenv("LAYA_IDLE_UNLOAD_SECONDS", raising=False)
+ else:
+ monkeypatch.setenv("LAYA_IDLE_UNLOAD_SECONDS", raw)
+ assert _resolve_idle_unload_seconds() == expected
+
+
+def test_idle_unload_is_off_by_default(monkeypatch):
+ monkeypatch.delenv("LAYA_IDLE_UNLOAD_SECONDS", raising=False)
+ router = IdleRouter()
+ with TestClient(create_app(router)) as client:
+ assert not router.unloaded.wait(0.15)
+ assert router.loaded == ["english"]
+ assert "idle_seconds" not in client.get("/health").json()
+
+
+def test_idle_unload_reloads_on_the_next_request(monkeypatch):
+ monkeypatch.setenv("LAYA_IDLE_UNLOAD_SECONDS", "0.1")
+ monkeypatch.delenv("LAYA_API_KEY", raising=False)
+ router = IdleRouter()
+ with TestClient(create_app(router)) as client:
+ first = client.post("/v1/systemone", json=REQ).json()
+ assert router.unloaded.wait(2.0)
+ health = client.get("/health").json()
+ assert health["loaded"] == [] and health["idle_unload_seconds"] == 0.1
+ assert health["idle_seconds"] >= 0.1
+ assert client.post("/v1/systemone", json=REQ).json() == first
+ assert router.loaded == ["english"]
+ assert all(name.startswith("laya-infer") for name in router.unload_threads)
+
+
+def test_idle_details_stay_authenticated(monkeypatch):
+ monkeypatch.setenv("LAYA_IDLE_UNLOAD_SECONDS", "300")
+ monkeypatch.setenv("LAYA_API_KEY", "key")
+ with TestClient(create_app(IdleRouter())) as client:
+ assert client.get("/health").json() == {"status": "ok"}
+ assert client.get("/health", headers={"Authorization": "Bearer key"}).json()["idle_unload_seconds"] == 300
+
+
[email protected]("batch", [False, True])
[email protected]("cancel", [False, True])
[email protected]("fail", [False, True])
+def test_idle_window_starts_after_worker_finishes_even_on_cancel_or_failure(monkeypatch, batch, cancel, fail):
+ import threading
+ import time
+ from concurrent.futures import ThreadPoolExecutor
+
+ monkeypatch.setenv("LAYA_IDLE_UNLOAD_SECONDS", "0.2")
+ monkeypatch.delenv("LAYA_API_KEY", raising=False)
+
+ class BlockingRouter(IdleRouter):
+ def __init__(self):
+ super().__init__()
+ self.started = threading.Event()
+ self.release = threading.Event()
+ self.finished = threading.Event()
+ self.active = False
+
+ def predict(self, *args, **kwargs):
+ self.active = True
+ self.started.set()
+ try:
+ assert self.release.wait(3.0)
+ if fail:
+ raise ValueError("test failure")
+ return super().predict(*args, **kwargs)
+ finally:
+ self.active = False
+ self.finished.set()
+
+ def unload(self):
+ assert not self.active, "unload overlapped inference"
+ super().unload()
+
+ router = BlockingRouter()
+ path = "/v1/systemone/batch" if batch else "/v1/systemone"
+ body = {"states": [REQ["state"]], "questions": REQ["questions"]} if batch else REQ
+ app = create_app(router)
+ if cancel:
+ import httpx
+
+ async def exercise():
+ async with app.router.lifespan_context(app):
+ async with httpx.AsyncClient(transport=httpx.ASGITransport(app), base_url="http://test") as client:
+ task = asyncio.create_task(client.post(path, json=body))
+ try:
+ assert await asyncio.to_thread(router.started.wait, 2.0)
+ task.cancel()
+ with pytest.raises(asyncio.CancelledError):
+ await task
+ # A reaper can now queue behind the still-running forward pass.
+ await asyncio.sleep(0.3)
+ assert not router.unloaded.is_set()
+ router.release.set()
+ assert await asyncio.to_thread(router.finished.wait, 2.0)
+ assert not await asyncio.to_thread(router.unloaded.wait, 0.1)
+ assert await asyncio.to_thread(router.unloaded.wait, 2.0)
+ finally:
+ router.release.set()
+ asyncio.run(exercise())
+ else:
+ with TestClient(app) as client, ThreadPoolExecutor(max_workers=1) as callers:
+ request = callers.submit(client.post, path, json=body)
+ try:
+ assert router.started.wait(2.0)
+ time.sleep(0.3)
+ assert not router.unloaded.is_set()
+ router.release.set()
+ assert request.result(timeout=2.0).status_code == (422 if fail else 200)
+ assert not router.unloaded.wait(0.1)
+ assert router.unloaded.wait(2.0)
+ finally:
+ router.release.set()
+
+
+def test_idle_unload_retries_failure_and_stops_at_shutdown(monkeypatch):
+ monkeypatch.setenv("LAYA_IDLE_UNLOAD_SECONDS", "0.05")
+
+ class RetryRouter(IdleRouter):
+ attempts = 0
+
+ def unload(self):
+ self.attempts += 1
+ if self.attempts == 1:
+ raise RuntimeError("test unload failure")
+ super().unload()
+
+ router = RetryRouter()
+ with TestClient(create_app(router)):
+ assert router.unloaded.wait(2.0)
+ assert router.attempts == 2
+ router.loaded = ["english"]
+ router.unloaded.clear()
+ assert not router.unloaded.wait(0.15)