Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
test(serve): the cap must reach the Router the server builds, and change rebuilds
Aashish799f57c84b4c
diff --git a/tests/test_packaging.py b/tests/test_packaging.pyindex 66971a35..798cb064 100644--- a/tests/test_packaging.py+++ b/tests/test_packaging.py@@ -173,6 +173,10 @@ if port_map:check("compose.http/both sides use the same variable", port_map.group(1), port_map.group(3))check_true("compose.http/the server reads the same variable",'LAYA_PORT: "${LAYA_PORT:-8000}"' in http)+# The block enumerates what it forwards, so a variable the server reads and the file omits is+# unreachable for the documented compose path -- `docker run -e` still works, compose does not.+check_true("compose.http/forwards the resident-checkpoint cap",+ 'LAYA_MAX_LOADED: "${LAYA_MAX_LOADED:-}"' in http)check_true("compose.http/shares the model cache","model-cache:/home/laya/.cache/huggingface" in http)# The base service is what `docker compose run --rm laya` uses; publishing it a port ordiff --git a/tests/test_serve.py b/tests/test_serve.pyindex ae52aae2..c8d00662 100644--- a/tests/test_serve.py+++ b/tests/test_serve.py@@ -15,10 +15,15 @@ from laya.serve import ( # noqa: E402MAX_BODY_BYTES,_apply_thread_limit,_env_bool,+ _resolve_max_loaded,_resolve_model,create_app,)+# Read from the router rather than copied here: the workload below has to reach the+# typed-decisions checkpoint the same way a request does.+from laya.router import _TYPED_DECISION_WORKFLOWS # noqa: E402+class FakeRouter:"""Records the last predict() call and returns a Jev-shaped payload."""@@ -235,6 +240,116 @@ def test_thread_limit(monkeypatch):assert torch.get_num_threads() == 8+# README's own answer to a checkpoint-rebuild storm is a constructor argument --+# `Router(max_loaded=3) # keep all three hot, e.g. with auto_task_detection` -- and #172+# measured what ignoring it costs: 20-23 s per request reloading a checkpoint on CPU against+# 49-136 ms with it resident. The server builds its own Router from the environment and had no+# way to pass it, so the one configuration that needs the knob (auto task routing, which adds a+# third checkpoint reached on demand) could not use it. These drive `build_router()` itself,+# with the loader replaced by a stub, so nothing is downloaded.+class _StubAgent:+ def __init__(self, name):+ self.name = name++ def system_one(self, state, questions):+ return {"model": self.name, "answers": {}, "usage": {}}+++def _server_router(monkeypatch, **env):+ """The Router `laya-serve` builds for `env`, with loads recorded instead of performed."""+ from laya.router import normalise_name+ from laya.serve import build_router++ monkeypatch.setenv("LAYA_PRELOAD", "0") # nothing may download+ monkeypatch.setenv("LAYA_AUTO_TASK", "1") # the config that puts three checkpoints in play+ monkeypatch.delenv("LAYA_MAX_LOADED", raising=False)+ for name, value in env.items():+ monkeypatch.setenv(name, value)+ router = build_router()+ built = []++ def load(name):+ key = normalise_name(name)+ if key in router._agents:+ router._touch(key)+ return router._agents[key]+ built.append(key)+ router._agents[key] = _StubAgent(key)+ router._order.append(key)+ router._evict()+ return router._agents[key]++ router.load = load+ return router, built+++# One request per checkpoint, so a cap of 2 cannot hold them all.+_WORKLOAD = [+ ({"body": "I was charged twice, please refund the duplicate"},+ {"issue": {"type": "choice", "options": ["billing", "other"]}}),+ ({"body": "Der Kunde wurde zweimal belastet und moechte eine Rueckerstattung"},+ {"issue": {"type": "choice", "options": ["billing", "other"]}}),+ ({"body": "Invoice 4411 was paid twice. Please refund the duplicate line."},+ {qid: {"type": "choice", "options": ["yes", "no"]}+ for qid in sorted(_TYPED_DECISION_WORKFLOWS["customer_service"])}),+]+++def _run_workload(router, cycles):+ for _ in range(cycles):+ for state, questions in _WORKLOAD:+ router.predict(state, questions)+++def test_max_loaded_reaches_the_router_the_server_builds(monkeypatch):+ from laya.router import Router++ # Unset has to be reported as "not set", not as a copy of Router's default, or the two+ # numbers drift the day the default moves. Checked at the resolver, because a copy of 2 and+ # the real default are otherwise indistinguishable at the Router.+ monkeypatch.delenv("LAYA_MAX_LOADED", raising=False)+ assert _resolve_max_loaded() is None+ for raw in ("abc", "0", "-2", "2.5", ""):+ monkeypatch.setenv("LAYA_MAX_LOADED", raw)+ assert _resolve_max_loaded() is None, raw+ monkeypatch.setenv("LAYA_MAX_LOADED", "3")+ assert _resolve_max_loaded() == 3++ # And it reaches the Router the server actually builds. The literal below is the value the+ # docs quote, so moving Router's default has to move those too.+ router, _ = _server_router(monkeypatch)+ assert router.max_loaded == Router().max_loaded+ assert router.max_loaded == 2+ for raw, want in (("3", 3), ("1", 1), (" 4 ", 4)):+ router, _ = _server_router(monkeypatch, LAYA_MAX_LOADED=raw)+ assert router.max_loaded == want, raw+ # A bad value falls back the way LAYA_MAX_CONCURRENT's does: it must not stop the server+ # and must not be read as "no limit" or "one".+ for raw in ("abc", "0", "-2", "2.5", ""):+ router, _ = _server_router(monkeypatch, LAYA_MAX_LOADED=raw)+ assert router.max_loaded == 2, raw+++def test_raising_the_cap_stops_the_server_rebuilding_a_checkpoint(monkeypatch):+ from laya.router import DEFAULT_MODELS++ cycles = 3+ router, built = _server_router(monkeypatch)+ _run_workload(router, cycles)+ # The instrument has to be the workload the clause is about: three checkpoints in play,+ # and the default cap that cannot hold them.+ assert len(set(built)) == 3, built+ assert router.max_loaded == 2+ assert len(built) == 9, built # every request after the second rebuilds one++ roomy, built3 = _server_router(monkeypatch, LAYA_MAX_LOADED="3")+ _run_workload(roomy, cycles)+ assert roomy.max_loaded == 3+ assert len(built3) == 3, built3 # each checkpoint once, then they stay resident+ assert sorted(roomy.loaded) == sorted(DEFAULT_MODELS)+++# The endpoint is `async def` and inference is synchronous torch, which on CPU takes# hundreds of milliseconds to seconds. Calling it from the coroutine puts that work on# the event loop, so every other client -- `GET /health` included -- waits for it.