Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

feat(serve): register extra checkpoints from LAYA_EXTRA_MODELS

Jose Eduardo Rojas Jimenezee0af9607b21
diff --git a/README.md b/README.md
index f748de58..8c4777e7 100644
--- a/README.md
+++ b/README.md
@@ -728,9 +728,12 @@ your traffic is not English), `LAYA_MAX_LOADED` (checkpoints resident at
once, 2 by default; raise it to 3 when `LAYA_AUTO_TASK` makes a third one
reachable on demand, or the server rebuilds one every time routing switches),
`LAYA_IDLE_UNLOAD_SECONDS` (unload idle checkpoints; `0` disables it, `300` frees device memory
-after five minutes and makes the next request pay a cold load), and `LAYA_API_KEY` (when set, clients must
+after five minutes and makes the next request pay a cold load), `LAYA_EXTRA_MODELS` (a JSON object
+`{"my-checkpoint": "/path/to/it"}` registering extra checkpoints — Hub repo ids, local directories
+or `["repo", "subfolder"]` pairs — beside the bundled ones), and `LAYA_API_KEY` (when set, clients must
send `Authorization: Bearer <key>`). A client's `model` field is honoured when it
-names a Laya checkpoint (`english`/`multilingual`/`typed-decisions` or a published Hub id).
+names a Laya checkpoint (`english`/`multilingual`/`typed-decisions`, a `LAYA_EXTRA_MODELS` name,
+or a published Hub id).
`jev-1` and `convaiinnovations/laya` still mean the router auto-selects by script/language.
A path or an unpublished Hub id (`/path/to/checkpoint`, `org/repo`) is a 422 on both
`/v1/systemone` and `/v1/systemone/batch` instead of being answered by another checkpoint.
diff --git a/docs/http-api.md b/docs/http-api.md
index bc6b580a..8bb76aa3 100644
--- a/docs/http-api.md
+++ b/docs/http-api.md
@@ -35,6 +35,7 @@ Everything is environment variables, so one image serves a laptop dev run and a
| `LAYA_AUTO_TASK` | auto-route to the typed-decisions checkpoint | `0` |
| `LAYA_IDLE_UNLOAD_SECONDS` | unload resident checkpoints after this many idle seconds; the next request loads its checkpoint again. Zero disables unloading | `0` |
| `LAYA_DEFAULT_MODEL` | checkpoint a state with no language evidence falls back to; aliases such as `ml` resolve the way core resolves them, and an unresolvable name stops the server at startup | `english` |
+| `LAYA_EXTRA_MODELS` | JSON object `{name: source}` registering extra checkpoints beside the bundled ones: a Hub repo id or local checkpoint directory as a string, or a `["repo", "subfolder"]` pair. Names get the same `model=` pin a built-in does; a malformed value or bad name stops the server at startup | none |
| `LAYA_API_KEY` | if set, require `Authorization: Bearer <key>` | none |
| `LAYA_LOG_LEVEL` | uvicorn log level | `info` |
| `LAYA_MAX_CONCURRENT` | requests admitted past auth at once; excess gets `503` | `16` |
@@ -141,7 +142,9 @@ now get the same answer.
`model` is accepted so a Jev client can keep sending one. The public Hugging Face ids
(`convaiinnovations/laya-multilingual`, `convaiinnovations/laya-typed-decisions`), the checkpoint
-names (`english`, `multilingual`, `typed-decisions`) and their aliases select a checkpoint.
+names (`english`, `multilingual`, `typed-decisions`) and their aliases select a checkpoint, and so
+does every name `LAYA_EXTRA_MODELS` registered on this server's Router -- including a name that
+re-points a built-in, which is how a fine-tune is served under `english` without a code change.
`convaiinnovations/laya`, and any other value that is not a path or a Hub repo id -- including a
Jev id like `jev-1` -- means "let the router choose", and the response's `routing` block records
what was chosen and why. A value that looks like a filesystem path or an unpublished Hub id
diff --git a/laya/serve.py b/laya/serve.py
index cde2ee09..1ceef3a6 100644
--- a/laya/serve.py
+++ b/laya/serve.py
@@ -35,6 +35,11 @@ env var meaning defaul
``LAYA_LOG_LEVEL`` uvicorn log level info
``LAYA_MAX_CONCURRENT`` cap on requests past auth at once; excess 16
gets 503 (see below)
+``LAYA_EXTRA_MODELS`` JSON object ``{name: source}`` registering (none)
+ extra checkpoints beside the bundled ones --
+ a Hub repo id, a local checkpoint directory, or
+ a ``["repo", "subfolder"]`` pair. Served under
+ explicit ``model=`` like a built-in.
``LAYA_MAX_TOKEN_BUDGET`` cap on per-request max_len / head_max_len 8192
``LAYA_JEV_STRICT`` if set, serve the strict Jev wire contract: no 0
root `routing`, no per-answer `action` /
@@ -227,7 +232,7 @@ def _names_unpublished_source(text: str) -> bool:
return "/" in text or "\\" in text
-def _resolve_model(model: Optional[str]) -> Optional[str]:
+def _resolve_model(model: Optional[str], resolve=None) -> Optional[str]:
"""Map a client's `model` field onto a Laya checkpoint, or None to auto-route.
A Jev id such as ``jev-1``, and the bundle id ``convaiinnovations/laya``, stay None:
@@ -247,15 +252,19 @@ def _resolve_model(model: Optional[str]) -> Optional[str]:
# It contains a slash, so the path check below would otherwise refuse it.
if text.lower() == BUNDLE_REPO:
return None
- # normalise_name raises ValueError on anything that is not a known checkpoint
- # or alias, and returns a name from router.DEFAULT_MODELS when it does accept one --
- # so it is the only list of accepted names this needs. A Jev client's `model` field
- # (e.g. "jev-1") is expected to miss; treat that as "no explicit checkpoint" and let
- # the router auto-select. A path or unpublished Hub id is not that miss: the caller
- # named a checkpoint this server cannot load, the same refusal `validate_model` gives
- # an MCP client, reported as this endpoint's 422.
+ # The resolver raises ValueError on anything that is not a known checkpoint or
+ # alias. `resolve` defaults to the built-in table's `normalise_name`; the request
+ # handlers pass the app's `Router.resolve`, so a checkpoint registered on the
+ # Router -- `Router.register`, `Router(models=...)`, or `LAYA_EXTRA_MODELS` --
+ # pins here too, and its "unknown model" error lists the full set the server
+ # actually serves. A Jev client's `model` field (e.g. "jev-1") is expected to
+ # miss; treat that as "no explicit checkpoint" and let the router auto-select.
+ # A path or unpublished Hub id is not that miss: the caller named a checkpoint
+ # this server cannot load, the same refusal `validate_model` gives an MCP
+ # client, reported as this endpoint's 422.
+ resolver = normalise_name if resolve is None else resolve
try:
- return normalise_name(model)
+ return resolver(model)
except ValueError as error:
if _names_unpublished_source(text):
from fastapi import HTTPException
@@ -483,7 +492,44 @@ def _resolve_max_loaded() -> Optional[int]:
return n if n > 0 else None
-def _default_model_option() -> Dict[str, str]:
+def _extra_models_option() -> Dict[str, Any]:
+ """Checkpoint registrations from ``LAYA_EXTRA_MODELS``, as the ``Router`` ``models`` keyword.
+
+ The variable is a JSON object mapping a checkpoint name to its source -- the same
+ shapes ``Router.register`` accepts: a Hub repo id or local checkpoint directory as a
+ plain string, or a ``["repo", "subfolder"]`` pair. ``Router._add`` applies the same
+ name rules as ``register`` (lowercase letters, digits, ``.``, ``_``, ``-``; never
+ ``auto``), and a name that matches a bundled checkpoint re-points it -- which is how
+ an operator serves a fine-tune under ``english`` without a code change.
+
+ As with ``_default_model_option``, a malformed value has no harmless fallback: a
+ checkpoint silently missing from the registry would auto-route its requests to a
+ different one, so this raises ``SystemExit`` and the caller refuses to start.
+ """
+ raw = os.environ.get("LAYA_EXTRA_MODELS", "").strip()
+ if not raw:
+ return {}
+ try:
+ data = json.loads(raw)
+ except ValueError as error:
+ raise SystemExit("invalid LAYA_EXTRA_MODELS: not a JSON object (%s)" % error) from None
+ if not isinstance(data, dict):
+ raise SystemExit("invalid LAYA_EXTRA_MODELS: a JSON object "
+ "{name: source}, got %s" % type(data).__name__)
+ out = {}
+ for name, source in data.items():
+ if isinstance(source, list):
+ out[name] = tuple(source) # JSON has no tuple: [repo, subfolder]
+ elif isinstance(source, str):
+ out[name] = source
+ else:
+ raise SystemExit("invalid LAYA_EXTRA_MODELS: source for %r must be a repo "
+ "id, a local path, or a [repo, subfolder] pair; got %s"
+ % (name, type(source).__name__))
+ return out
+
+
+def _default_model_option(extra_models: Optional[Dict[str, Any]] = None) -> Dict[str, str]:
"""Routing fallback from ``LAYA_DEFAULT_MODEL``, as a ``Router`` keyword; unset sends nothing.
``Router.default`` answers the two states that carry no language evidence at all: no letters,
@@ -493,22 +539,28 @@ def _default_model_option() -> Dict[str, str]:
of the constructor when unset, so the value cannot drift from ``Router``'s own default -- the
same reasoning as ``_resolve_max_loaded`` above.
- The name goes through ``normalise_name``, so the accepted set and its aliases are core's and
- not a list restated here. Unlike the numeric knobs, a typo here has no harmless fallback: a
- silently-ignored value would keep routing the ambiguous states to the checkpoint the operator
- just said cannot read them, so this raises and the caller refuses to start rather than serve a
- configuration nobody asked for. ``laya.mcp.server`` turns the same error into a ``ToolError``,
- because a stdio server has no startup to refuse.
+ The name is checked against the built-in table and its aliases plus any
+ ``LAYA_EXTRA_MODELS`` registrations, which is the set the Router ends up
+ serving -- not a list restated here. Unlike the numeric knobs, a typo here has
+ no harmless fallback: a silently-ignored value would keep routing the
+ ambiguous states to the checkpoint the operator just said cannot read them,
+ so this raises and the caller refuses to start rather than serve a
+ configuration nobody asked for. ``laya.mcp.server`` turns the same error into
+ a ``ToolError``, because a stdio server has no startup to refuse.
"""
raw = os.environ.get("LAYA_DEFAULT_MODEL")
if raw is None or not raw.strip():
return {}
- from .router import normalise_name
+ from .router import canonical_name, normalise_name
try:
name = normalise_name(raw)
except ValueError as error:
- raise ValueError("invalid LAYA_DEFAULT_MODEL %r: %s" % (raw.strip(), error)) from None
+ # A LAYA_EXTRA_MODELS name is a valid default once the Router registers it.
+ key = canonical_name(raw)
+ if not extra_models or key not in extra_models:
+ raise ValueError("invalid LAYA_DEFAULT_MODEL %r: %s" % (raw.strip(), error)) from None
+ name = key
return {"default": name}
@@ -852,11 +904,17 @@ def build_router():
options["max_loaded"] = max_loaded
# Resolved before the Router is built: a name `Router` would reject is a configuration error,
# and `_resolve_port`'s idiom applies -- exit with the message, not a traceback.
+ extra_models = _extra_models_option()
try:
- options.update(_default_model_option())
+ options.update(_default_model_option(extra_models))
except ValueError as error:
raise SystemExit(str(error)) from None
- router = Router(**options)
+ if extra_models:
+ options["models"] = extra_models
+ try:
+ router = Router(**options)
+ except (ValueError, TypeError) as error:
+ raise SystemExit(str(error)) from None
if _env_bool("LAYA_PRELOAD", True):
router.preload(preload_names)
return router
@@ -1065,7 +1123,11 @@ def create_app(router: Optional[Any] = None):
# the time `json.loads` is done, so only lone ones are rejected here.
if _has_lone_surrogate(body):
raise HTTPException(status_code=400, detail=_LONE_SURROGATE_DETAIL)
- model = _resolve_model(body.get("model"))
+ # `getattr` rather than `router.resolve`: test doubles injected through
+ # `create_app(router=...)` predate the registry-aware resolver, and a fake
+ # that cannot resolve names falls back to the built-in table, which is the
+ # registry such a fake is asserting anyway.
+ model = _resolve_model(body.get("model"), getattr(router, "resolve", None))
max_budget_cap = _resolve_max_token_budget()
max_len = _validate_budget_param(body, "max_len", max_budget_cap)
head_max_len = _validate_budget_param(body, "head_max_len", max_budget_cap)
@@ -1159,7 +1221,9 @@ def create_app(router: Optional[Any] = None):
# and `MAX_QUESTIONS` bound what the walk can reach.
if _has_lone_surrogate(body):
raise HTTPException(status_code=400, detail=_LONE_SURROGATE_DETAIL)
- model = _resolve_model(body.get("model"))
+ # Same resolver fallback as `_systemone_inner`: a test double without a
+ # registry-aware `resolve` keeps the built-in table it was written against.
+ model = _resolve_model(body.get("model"), getattr(router, "resolve", None))
max_budget_cap = _resolve_max_token_budget()
max_len = _validate_budget_param(body, "max_len", max_budget_cap)
head_max_len = _validate_budget_param(body, "head_max_len", max_budget_cap)
diff --git a/tests/test_serve.py b/tests/test_serve.py
index 26f7a152..aa3b172d 100644
--- a/tests/test_serve.py
+++ b/tests/test_serve.py
@@ -662,23 +662,29 @@ class _StubAgent:
def system_one(self, state, questions):
return {"model": self.name, "answers": {}, "usage": {}}
+ def predict_batch(self, states, questions, batch_size=None, **kwargs):
+ return [self.system_one(state, questions) for state in states]
+
def _server_router(monkeypatch, **env):
"""The Router `laya-serve` builds for `env`, with loads recorded instead of performed."""
- from laya.router import normalise_name
from laya.serve import build_router
monkeypatch.setenv("LAYA_PRELOAD", "0") # nothing may download
monkeypatch.setenv("LAYA_AUTO_TASK", "1") # the config that puts three checkpoints in play
monkeypatch.delenv("LAYA_MAX_LOADED", raising=False)
monkeypatch.delenv("LAYA_DEFAULT_MODEL", raising=False)
+ monkeypatch.delenv("LAYA_EXTRA_MODELS", raising=False)
for name, value in env.items():
monkeypatch.setenv(name, value)
router = build_router()
built = []
def load(name):
- key = normalise_name(name)
+ # `router.resolve`, not `normalise_name`: `predict` hands `load` the
+ # decision's key, which can be a LAYA_EXTRA_MODELS registration that the
+ # built-in table does not know.
+ key = router.resolve(name)
if key in router._agents:
router._touch(key)
return router._agents[key]
@@ -3505,3 +3511,131 @@ def test_jev_strict_projection_page_names_the_projected_keys():
# The noul bullet must not shrink the discriminator away.
assert "noul only" not in page, "pre-fix 'noul only' wording is still on the page"
+
+# Serving a fine-tune today means rebuilding the package: `Router` has grown
+# `register` / `models=`, but `laya-serve` reads its whole configuration from the
+# environment, so an operator with a local checkpoint had no way to reach either.
+# LAYA_EXTRA_MODELS is the JSON-object form of `Router(models=...)`, parsed once
+# at startup; the request handlers then resolve `model` against the Router's own
+# registry rather than the built-in table, so a registered name pins its
+# checkpoint the same way `english` does.
+def test_extra_models_option_parses_the_json_object(monkeypatch):
+ from laya.serve import _extra_models_option
+
+ # Unset and blank are "not asked for", like every other option here.
+ monkeypatch.delenv("LAYA_EXTRA_MODELS", raising=False)
+ assert _extra_models_option() == {}
+ for blank in ("", " ", "\n"):
+ monkeypatch.setenv("LAYA_EXTRA_MODELS", blank)
+ assert _extra_models_option() == {}, repr(blank)
+
+ # The two source shapes Router.register accepts: a plain string (Hub repo id
+ # or local directory) and a [repo, subfolder] pair, which arrives as a tuple
+ # because JSON has no tuple.
+ monkeypatch.setenv("LAYA_EXTRA_MODELS", json.dumps({
+ "my-ckpt": "/tmp/ckpt",
+ "nested": ["org/repo", "folder"],
+ "hub-only": "org/repo",
+ }))
+ parsed = _extra_models_option()
+ assert parsed == {
+ "my-ckpt": "/tmp/ckpt",
+ "nested": ("org/repo", "folder"),
+ "hub-only": "org/repo",
+ }, parsed
+
+
+def test_extra_models_option_refuses_a_malformed_value(monkeypatch):
+ from laya.serve import _extra_models_option
+
+ # The fail-closed rule is `_default_model_option`'s: a checkpoint silently
+ # absent from the registry would auto-route its traffic to a different one,
+ # which is the wrong answer -- so the server refuses to start and the
+ # message names the variable and the offending shape.
+ for raw in ("{", "not json", '"english"', '["english"]', "42", "null"):
+ monkeypatch.setenv("LAYA_EXTRA_MODELS", raw)
+ with pytest.raises(SystemExit) as raised:
+ _extra_models_option()
+ assert "invalid LAYA_EXTRA_MODELS" in str(raised.value), raw
+
+ # A source that is not a string or a pair fails the same way, naming the
+ # checkpoint it was meant to register.
+ monkeypatch.setenv("LAYA_EXTRA_MODELS", json.dumps({"my-ckpt": 7}))
+ with pytest.raises(SystemExit) as raised:
+ _extra_models_option()
+ message = str(raised.value)
+ assert "invalid LAYA_EXTRA_MODELS" in message
+ assert "'my-ckpt'" in message, message
+
+
+def test_extra_models_reach_the_router_the_server_builds(monkeypatch):
+ # Registration happens before preload, so the names resolve on the Router
+ # itself and `registered` reports them beside the bundled checkpoints.
+ router, built = _server_router(monkeypatch, LAYA_EXTRA_MODELS=json.dumps({
+ "my-ckpt": "/tmp/ckpt",
+ "nested": ["org/repo", "folder"],
+ }))
+ assert router.resolve("my-ckpt") == "my-ckpt"
+ assert router.resolve("nested") == "nested"
+ assert router.registered["my-ckpt"]["source"] == "/tmp/ckpt"
+ assert built == [] # registering is lazy: nothing loads until a request names it
+
+ # The built-ins are untouched, and an extra name is accepted as
+ # LAYA_DEFAULT_MODEL now that the Router knows it.
+ assert set(router.registered) == {"my-ckpt", "nested"}
+ defaulted, _ = _server_router(
+ monkeypatch,
+ LAYA_EXTRA_MODELS='{"my-ckpt": "/tmp/ckpt"}',
+ LAYA_DEFAULT_MODEL="my-ckpt",
+ )
+ assert defaulted.default == "my-ckpt"
+
+
+def test_extra_models_refuse_the_names_router_refuses(monkeypatch):
+ # Validation is Router._add's: the env var is only the delivery mechanism,
+ # so a name that cannot be a checkpoint is fatal with core's own message.
+ for name in ("auto", "Bad Name", "not/a/checkpoint"):
+ with pytest.raises(SystemExit) as raised:
+ _server_router(monkeypatch, LAYA_EXTRA_MODELS=json.dumps({name: "/tmp/ckpt"}))
+ assert "checkpoint name" in str(raised.value), name
+
+ # And a default that names nothing registered is still fatal, unchanged.
+ from laya.serve import build_router
+
+ monkeypatch.setenv("LAYA_PRELOAD", "0")
+ monkeypatch.setenv("LAYA_EXTRA_MODELS", '{"my-ckpt": "/tmp/ckpt"}')
+ monkeypatch.setenv("LAYA_DEFAULT_MODEL", "other-ckpt")
+ with pytest.raises(SystemExit) as raised:
+ build_router()
+ assert "invalid LAYA_DEFAULT_MODEL" in str(raised.value)
+
+
+def test_extra_models_pin_over_http_and_do_not_leak_into_routing(monkeypatch):
+ # The end-to-end contract: `model=my-ckpt` reaches `predict` as the pin, a
+ # Jev id still means "let the router choose", and a path is still a 422.
+ router, built = _server_router(monkeypatch, LAYA_EXTRA_MODELS='{"my-ckpt": "/tmp/ckpt"}')
+ client = TestClient(create_app(router=router))
+
+ pinned = client.post("/v1/systemone", json={**REQ, "model": "my-ckpt"})
+ assert pinned.status_code == 200, pinned.text
+ assert built == ["my-ckpt"], built
+ assert pinned.json()["routing"]["model"] == "my-ckpt", pinned.json()
+
+ batch = client.post("/v1/systemone/batch", json={
+ "states": ["one"], "questions": REQ["questions"], "model": "my-ckpt"})
+ assert batch.status_code == 200, batch.text
+
+ unrouted = client.post("/v1/systemone", json={**REQ, "model": "jev-1"})
+ assert unrouted.status_code == 200, unrouted.text
+ assert unrouted.json()["routing"]["model"] in ("english", "multilingual"), unrouted.json()
+
+ refused = client.post("/v1/systemone", json={**REQ, "model": "/path/to/other"})
+ assert refused.status_code == 422, refused.text
+ assert "unknown model" in refused.json()["detail"]
+
+ # A registered name is a pin, not a routing target: the extra checkpoint is
+ # never picked for a state that did not name it.
+ plain = client.post("/v1/systemone", json=REQ)
+ assert plain.status_code == 200, plain.text
+ assert plain.json()["routing"]["model"] in ("english", "multilingual", "typed-decisions")
+ assert "my-ckpt" not in built[1:] or built[1:] == [], built