Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

Merge pull request #603 from aashish254/fix-serve-max-loaded

Nandakishor98e3fa9036a0
diff --cc docs/docker.md
index 3ffe38a4,ae6c704f..90bc52f0
--- a/docs/docker.md
+++ b/docs/docker.md
@@@ -236,10 -225,9 +236,11 @@@ These apply to the `laya-serve` servic
| `LAYA_MODELS` | (all) | comma list to preload: `english,multilingual,typed-decisions` |
| `LAYA_THREADS` | `OMP_NUM_THREADS` | caps torch intra-op threads; keep at or below physical cores |
| `LAYA_AUTO_TASK` | `0` | `1` lets the router reach `typed-decisions` automatically |
+ | `LAYA_MAX_LOADED` | `2` | Checkpoints kept resident; `LAYA_AUTO_TASK` makes a third reachable on demand, and a cap below what routing chooses rebuilds one per switch |
+| `LAYA_MAX_CONCURRENT` | `16` | requests admitted at once; later ones get `503` (a value that does not parse, or is not positive, falls back to `16`) |
| `LAYA_LOG_LEVEL` | `info` | uvicorn log level |
| `LAYA_API_KEY` | (none) | when set, requires `Authorization: Bearer <key>` |
+| `LAYA_MAX_TOKEN_BUDGET` | `8192` | cap on per-request `max_len` and `head_max_len` overrides |
`LAYA_PRELOAD` defaults to `0` here rather than the package default of `1`, because
preloading makes the first boot download all three checkpoints. Set it to `1` for a
diff --cc laya/serve.py
index a751f5b5,012ed2e2..e3c0d7c9
--- a/laya/serve.py
+++ b/laya/serve.py
@@@ -11,30 -11,26 +11,32 @@@ probe
Configuration is entirely via environment variables so the same entry point
serves a laptop dev run and a systemd unit:
-====================== ============================================ =========
-env var meaning default
-====================== ============================================ =========
-``LAYA_HOST`` bind address 0.0.0.0
-``LAYA_PORT`` bind port 8000
-``LAYA_DEVICE`` torch device for every checkpoint (auto)
-``LAYA_PRELOAD`` build the checkpoints at startup, not lazily 1
-``LAYA_MODELS`` comma list to preload (english,multilingual, (all)
- typed-decisions); empty = every checkpoint
-``LAYA_THREADS`` cap torch intra-op threads (CPU inference). (torch
- Keep <= physical cores; oversubscribing the default)
- logical/hyperthread count is a large regression.
-``LAYA_AUTO_TASK`` auto-route to the typed-decisions checkpoint 0
-``LAYA_MAX_LOADED`` checkpoints kept resident at once. Below what 2
- routing can choose, one reloads per switch.
-``LAYA_API_KEY`` if set, require ``Authorization: Bearer <it>`` (none)
-``LAYA_LOG_LEVEL`` uvicorn log level info
-``LAYA_MAX_CONCURRENT`` cap on requests past auth at once; excess 16
- gets 503 (see below)
-====================== ============================================ =========
+========================= ============================================ =========
+env var meaning default
+========================= ============================================ =========
+``LAYA_HOST`` bind address 0.0.0.0
+``LAYA_PORT`` bind port 8000
+``LAYA_DEVICE`` torch device for every checkpoint (auto)
+``LAYA_PRELOAD`` build the checkpoints at startup, not lazily 1
+``LAYA_MODELS`` comma list to preload (english,multilingual, (all)
+ typed-decisions); empty = every checkpoint
+``LAYA_THREADS`` cap torch intra-op threads (CPU inference). (torch
+ Keep <= physical cores; oversubscribing the default)
+ logical/hyperthread count is a large regression.
+``LAYA_AUTO_TASK`` auto-route to the typed-decisions checkpoint 0
++``LAYA_MAX_LOADED`` checkpoints kept resident at once. Below what 2
++ routing can choose, one reloads per switch.
+``LAYA_API_KEY`` if set, require ``Authorization: Bearer <it>`` (none)
+``LAYA_LOG_LEVEL`` uvicorn log level info
+``LAYA_MAX_CONCURRENT`` cap on requests past auth at once; excess 16
+ gets 503 (see below)
+``LAYA_MAX_TOKEN_BUDGET`` cap on per-request max_len / head_max_len 8192
+========================= ============================================ =========
+
+``LAYA_DEVICE`` is a preference, not a guarantee: an ``Agent`` that asks for a
+GPU it cannot get falls back to CPU silently and keeps answering correctly.
+``/health`` reports where a resident checkpoint actually computes, using the
+same helpers the MCP status tool uses (``laya.mcp.device``).
Imports of heavy dependencies (fastapi, uvicorn, torch via Router) are all
deferred into the functions that need them, so ``import laya.serve`` stays cheap
@@@ -123,43 -117,31 +125,68 @@@ def _resolve_max_concurrent() -> int
return n if n > 0 else DEFAULT_MAX_CONCURRENT
+def _resolve_max_token_budget() -> int:
+ """Server-side cap on per-request max_len from LAYA_MAX_TOKEN_BUDGET."""
+ raw = os.environ.get("LAYA_MAX_TOKEN_BUDGET")
+ if not raw:
+ return DEFAULT_MAX_TOKEN_BUDGET
+ try:
+ n = int(str(raw).strip())
+ except (TypeError, ValueError):
+ _log.warning("invalid LAYA_MAX_TOKEN_BUDGET %r; falling back to %d", raw, DEFAULT_MAX_TOKEN_BUDGET)
+ return DEFAULT_MAX_TOKEN_BUDGET
+ if n <= 0:
+ _log.warning("LAYA_MAX_TOKEN_BUDGET must be positive (got %d); falling back to %d", n, DEFAULT_MAX_TOKEN_BUDGET)
+ return DEFAULT_MAX_TOKEN_BUDGET
+ return n
+
+
+def _validate_budget_param(body: Dict[str, Any], key: str, max_cap: int) -> Optional[int]:
+ """Validate optional integer budget parameters (max_len / head_max_len) from request body (422)."""
+ from fastapi import HTTPException
+
+ if key not in body:
+ return None
+ val = body[key]
+ if val is None:
+ return None
+ if not isinstance(val, int) or isinstance(val, bool):
+ raise HTTPException(status_code=422, detail="%s must be an integer" % key)
+ if val <= 0:
+ raise HTTPException(status_code=422, detail="%s must be a positive integer" % key)
+ if val > max_cap:
+ raise HTTPException(
+ status_code=422,
+ detail="%s exceeds server limit (%d > %d)" % (key, val, max_cap),
+ )
+ return val
+
+
+ def _resolve_max_loaded() -> Optional[int]:
+ """Resident-checkpoint cap from ``LAYA_MAX_LOADED``; ``None`` leaves it to ``Router``.
+
+ ``Router`` keeps two checkpoints resident, which is exactly the number automatic routing
+ picks between. ``LAYA_AUTO_TASK`` adds `typed-decisions` as a third possibility, and a cap
+ below the number of checkpoints in play unloads the least recently used one on every
+ switch -- #172 measured 20-23 s per request rebuilding a checkpoint on CPU against
+ 49-136 ms with it resident. The value is left out of the constructor when unset rather
+ than defaulted to a copy of ``Router``'s own default here, so the two cannot drift.
+ Unparseable or non-positive input falls back the way ``_resolve_max_concurrent`` does: a
+ typo in a deployment file should not stop the server from starting.
+
+ Only checkpoints reached on demand are bounded by it: ``Router.preload()`` raises the cap
+ to hold whatever it builds, so a lower value never evicts a preloaded checkpoint.
+ """
+ raw = os.environ.get("LAYA_MAX_LOADED")
+ if not raw:
+ return None
+ try:
+ n = int(raw)
+ except ValueError:
+ return None
+ return n if n > 0 else None
+
+
def _resolve_port() -> int:
"""Port from LAYA_PORT, validated. Exits with a message instead of a traceback."""
raw = os.environ.get("LAYA_PORT", "8000")
@@@ -278,13 -259,14 +305,17 @@@ def build_router()
from .router import Router
_apply_thread_limit()
- device = os.environ.get("LAYA_DEVICE") or None
+ # `env_device` rather than a raw read: it strips, the way `LAYA_MODELS` two lines
+ # below is stripped. A value copied out of a Dockerfile or a `.env` file carries a
+ # trailing newline, and torch refuses that as an invalid device string.
+ device = env_device()
models_env = os.environ.get("LAYA_MODELS", "").strip()
preload_names = [m.strip() for m in models_env.split(",") if m.strip()] or None
- router = Router(device=device, auto_task_detection=_env_bool("LAYA_AUTO_TASK", False))
+ options = {"device": device, "auto_task_detection": _env_bool("LAYA_AUTO_TASK", False)}
+ max_loaded = _resolve_max_loaded()
+ if max_loaded is not None:
+ options["max_loaded"] = max_loaded
+ router = Router(**options)
if _env_bool("LAYA_PRELOAD", True):
router.preload(preload_names)
return router
diff --cc tests/test_serve.py
index fb7575dd,c8d00662..e985611e
--- a/tests/test_serve.py
+++ b/tests/test_serve.py
@@@ -22,7 -15,7 +22,8 @@@ from laya.serve import ( # noqa: E40
MAX_BODY_BYTES,
_apply_thread_limit,
_env_bool,
+ _resolve_max_token_budget,
+ _resolve_max_loaded,
_resolve_model,
create_app,
)