Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
Merge pull request #811 from aashish254/feat-health-field-contract
Nandakishor42e173bc2ac3
diff --cc tests/test_serve.pyindex 7edb1a31,047f3d41..6184d290--- a/tests/test_serve.py+++ b/tests/test_serve.py@@@ -3,7 -3,7 +3,8 @@@A fake Router is injected so nothing loads a checkpoint; we only assert that theHTTP layer maps requests/responses and enforces auth as hs-jev expects."""+ import ast+import asyncioimport inspectimport jsonimport logging@@@ -1871,300 -1750,92 +1872,388 @@@ def test_http_api_page_documents_exactlassert "`%s`" % key in prose, "%s is not named where the refusal is explained" % key+# --------------------------------------------------------------------- batch endpoint per-call controls+#+# `Router.predict_batch` reads two kinds of control: call-level kwargs that apply to the whole+# batch (chunking, padding, abstention) and per-request keys it lifts off each item of ``requests``+# (checkpoint, task, language, token budget). Before this PR, the HTTP batch endpoint read neither+# -- it always synthesized a plain ``{state, questions, model}`` dict and called ``predict_batch``+# with no kwargs, so a caller could not send any of the controls the single endpoint already+# forwards. The suite below pins the two split lists against core's own signature, verifies every+# control reaches the right side of the Router call, and refuses a hook control the same way+# ``/v1/systemone`` already does.+++class BatchRecordingRouter(FakeRouter):+ """Records ``predict_batch``'s full call: the requests list and every call kwarg."""++ def __init__(self):+ super().__init__()+ self.batch_calls = []++ def predict_batch(self, requests, **kwargs):+ self.batch_calls.append(([dict(r) for r in requests], dict(kwargs)))+ return [+ {+ "model": "laya-rl-agent",+ "answers": {+ "dept": {+ "type": "choice",+ "choice": "billing",+ "probabilities": {"billing": 0.94, "tech": 0.06},+ "confidence": 0.94,+ }+ },+ "usage": {"input_tokens": 42, "output_tokens": 0},+ "routing": {"model": "english", "reason": "English Latin text"},+ }+ for _ in requests+ ]+++class StrictBatchRouter(BatchRecordingRouter):+ """``predict_batch(self, requests)`` -- takes no call kwargs.++ Witness against an always-forward mutation: if the handler unconditionally passed ``batch_size``+ or ``min_confidence``, this Router would raise ``TypeError`` and 500 on a bare body.+ """++ def predict_batch(self, requests): # noqa: D401 -- strict on purpose+ return super().predict_batch(requests)+++BATCH_CALL_VALUES = {"batch_size": 4, "min_confidence": 0.9, "sort_by_length": True}+BATCH_ITEM_VALUES = {"max_len": 64, "head_max_len": 32, "task": "typed-decisions",+ "lang": "de", "lang_guess": "de"}+++def _batch_client(monkeypatch):+ monkeypatch.delenv("LAYA_API_KEY", raising=False)+ fake = BatchRecordingRouter()+ return TestClient(create_app(router=fake)), fake+++def test_batch_call_controls_are_exactly_predict_batch_kwargs_or_refusal():+ """A control added to ``Router.predict_batch`` must be placed on one side of the line.++ Either serve forwards it (``BATCH_BODY_CALL_CONTROLS``) or it refuses it (``hooks_timeout``,+ the one hook-execution arg that ``predict_batch`` takes). Silently ignoring a new kwarg is the+ failure mode this pin exists to catch: the same reasoning that made ``/v1/systemone`` refuse+ ``hooks_timeout`` in #724 applies to the batch path -- a caller cannot shorten the deadline+ on which the operator's own hooks execute.+ """+ from laya.router import Router++ taken = set(inspect.signature(Router.predict_batch).parameters) - {"self", "requests"}+ declared = set(BATCH_BODY_CALL_CONTROLS) | {"hooks_timeout"}+ assert taken == declared, "predict_batch() takes %s; serve declares %s" % (+ sorted(taken), sorted(declared))+++def test_batch_controls_cover_the_single_endpoints_set_plus_batch_only():+ """Everything the single endpoint forwards must reach the batch path too, and vice versa.++ The batch forwards the same per-call controls ``/v1/systemone`` does -- split across the item+ dict (``Router.predict_batch`` reads them per item) and the call kwargs (min_confidence) --+ plus the batch-only chunking and padding knobs (``batch_size``, ``sort_by_length``) that+ ``predict_batch`` accepts and ``predict`` does not. ``model`` still goes into every item, as+ it has since the batch endpoint shipped. If a control moves across either boundary, this+ test names it.+ """+ batch_side = set(BATCH_BODY_ITEM_CONTROLS) | set(BATCH_BODY_CALL_CONTROLS)+ single_side = (set(BODY_CONTROLS) | {"batch_size", "sort_by_length"}) - {"model"}+ assert batch_side == single_side, (+ "batch forwards %s; single endpoint + batch-only says %s"+ % (sorted(batch_side), sorted(single_side)))++[email protected]("key", sorted(BATCH_ITEM_VALUES))+def test_each_batch_item_control_reaches_every_request(monkeypatch, key):+ client, fake = _batch_client(monkeypatch)+ r = client.post("/v1/systemone/batch", json={**BATCH_REQ, key: BATCH_ITEM_VALUES[key]})+ assert r.status_code == 200, r.text+ requests, _ = fake.batch_calls[0]+ assert len(requests) == len(BATCH_REQ["states"])+ for item in requests:+ assert item.get(key) == BATCH_ITEM_VALUES[key], (key, item)++[email protected]("key", sorted(BATCH_CALL_VALUES))+def test_each_batch_call_control_reaches_predict_batch_kwargs(monkeypatch, key):+ client, fake = _batch_client(monkeypatch)+ r = client.post("/v1/systemone/batch", json={**BATCH_REQ, key: BATCH_CALL_VALUES[key]})+ assert r.status_code == 200, r.text+ _, kwargs = fake.batch_calls[0]+ assert kwargs.get(key) == BATCH_CALL_VALUES[key], (key, kwargs)+++def test_batch_unset_controls_are_not_sent_as_none(monkeypatch):+ """An absent control must stay absent in either the item dict or the call kwargs.++ Core reads an absent argument as "inherit what the Router was built with", so sending+ ``min_confidence=None`` would override a deployment's ``Router(min_confidence=...)``, and+ sending a per-item key with value ``None`` would break a Router whose ``predict_batch``+ predates the key (#294 shape).+ """+ client, fake = _batch_client(monkeypatch)+ assert client.post("/v1/systemone/batch", json=BATCH_REQ).status_code == 200+ requests, kwargs = fake.batch_calls[0]+ assert kwargs == {}+ for item in requests:+ for key in BATCH_BODY_ITEM_CONTROLS:+ assert key not in item, (key, item)++[email protected]("key", sorted(set(BATCH_ITEM_VALUES) | set(BATCH_CALL_VALUES)))+def test_batch_explicit_null_is_no_control(monkeypatch, key):+ """``{"lang_guess": null}`` means "no hint" -- same shape the single endpoint honours.++ A Jev client that serialises its absent fields must keep working; only a value says the+ caller asked for something.+ """+ client, fake = _batch_client(monkeypatch)+ assert client.post("/v1/systemone/batch", json={**BATCH_REQ, key: None}).status_code == 200+ requests, kwargs = fake.batch_calls[0]+ assert "min_confidence" not in kwargs and "batch_size" not in kwargs+ assert "sort_by_length" not in kwargs+ for item in requests:+ assert key not in item+++def test_batch_min_confidence_zero_is_still_a_threshold(monkeypatch):+ """``0.0`` is falsy but is a value the caller chose; ``if min_confidence:`` would drop it."""+ client, fake = _batch_client(monkeypatch)+ assert client.post("/v1/systemone/batch",+ json={**BATCH_REQ, "min_confidence": 0.0}).status_code == 200+ _, kwargs = fake.batch_calls[0]+ assert kwargs["min_confidence"] == 0.0+++def test_batch_sort_by_length_false_is_not_forwarded(monkeypatch):+ """``predict_batch``'s own default is ``False``, so ``False`` is what the caller did NOT ask.++ Forwarding ``False`` explicitly would still work on ``Router`` but silently disappear on any+ attached agent whose ``predict_batch`` predates the knob (#294) -- which is why the MCP tool+ makes the same choice. The endpoint does accept the value; it just does not translate it into+ a kwarg.+ """+ client, fake = _batch_client(monkeypatch)+ assert client.post("/v1/systemone/batch",+ json={**BATCH_REQ, "sort_by_length": False}).status_code == 200+ _, kwargs = fake.batch_calls[0]+ assert "sort_by_length" not in kwargs+++def test_batch_strict_predict_batch_router_answers_a_quiet_body(monkeypatch):+ """An attached Router whose ``predict_batch(self, requests)`` takes nothing else must still work.++ The endpoint forwards kwargs only when the caller asks. If the handler always passed+ ``batch_size=None`` (or any call kwarg) unconditionally, this Router would raise ``TypeError``+ and the endpoint would 500 on a body that sent nothing -- which is exactly the mutation this+ witness kills.+ """+ monkeypatch.delenv("LAYA_API_KEY", raising=False)+ fake = StrictBatchRouter()+ client = TestClient(create_app(router=fake))+ r = client.post("/v1/systemone/batch", json=BATCH_REQ)+ assert r.status_code == 200, r.text+ assert len(fake.batch_calls) == 1+ _, kwargs = fake.batch_calls[0]+ assert kwargs == {}++[email protected]("key,value", [+ ("lang", True),+ ("lang", 5),+ ("lang_guess", ["de"]),+ ("lang_guess", {"code": "de"}),+ ("max_len", 0),+ ("max_len", -1),+ ("max_len", True),+ ("max_len", "64"),+ ("head_max_len", "32"),+ ("min_confidence", 1.5),+ ("min_confidence", -0.1),+ ("min_confidence", "0.9"),+ ("batch_size", 0),+ ("batch_size", -3),+ ("batch_size", True),+ ("batch_size", 1.5),+ ("batch_size", "4"),+ ("sort_by_length", "yes"),+ ("sort_by_length", 1),+])+def test_bad_control_on_batch_is_refused_before_inference(monkeypatch, key, value):+ client, fake = _batch_client(monkeypatch)+ r = client.post("/v1/systemone/batch", json={**BATCH_REQ, key: value})+ assert r.status_code == 422, (key, value, r.text)+ assert not fake.batch_calls, "a refused body must not reach predict_batch"++[email protected]("key", BODY_REFUSALS)+def test_batch_hook_control_is_refused_not_dropped(monkeypatch, key):+ """The batch endpoint must honour the same refusal policy the single endpoint already states.++ Without ``_refuse_body_refusals``, a batch body would silently hand a caller a shorter+ deadline for the operator's hooks -- ``predict_batch`` *does* take ``hooks_timeout`` as a+ call arg, so this is not a theoretical refusal here. The other four are refused for the same+ reason they are on ``/v1/systemone``: they run inside the server process.+ """+ value = {"hooks": [], "on_predict_start": "cache", "on_predict_end": "audit",+ "hooks_raise": False, "hooks_timeout": 5.0}[key]+ client, fake = _batch_client(monkeypatch)+ r = client.post("/v1/systemone/batch", json={**BATCH_REQ, key: value})+ assert r.status_code == 422, r.text+ detail = r.json()["detail"]+ assert key in detail+ assert "cannot be sent to this endpoint" in detail+ assert not fake.batch_calls, "the refusal must come before inference"+++def test_batch_predict_fallback_forwards_every_control(monkeypatch):+ """When the attached Router has no ``predict_batch``, every control must reach ``predict()``.++ The batch path synthesizes per-state ``predict()`` calls with the same kwargs the single+ endpoint uses -- ``max_len``/``head_max_len``/``task``/``lang``/``lang_guess``/``min_confidence``+ all apply per call. A caller that sends ``min_confidence`` on a batch body must see the+ abstention gate apply per item on the fallback path, not silently skip it because+ ``predict_batch`` was unavailable.+ """+ monkeypatch.delenv("LAYA_API_KEY", raising=False)+ fake = BudgetRouter()+ client = TestClient(create_app(router=fake))+ body = {**BATCH_REQ, "max_len": 64, "head_max_len": 32, "task": "typed-decisions",+ "lang": "de", "lang_guess": "de", "min_confidence": 0.5}+ r = client.post("/v1/systemone/batch", json=body)+ assert r.status_code == 200, r.text+ assert len(fake.calls) == len(BATCH_REQ["states"])+ for call in fake.calls:+ assert call["max_len"] == 64+ assert call["head_max_len"] == 32+ assert call["task"] == "typed-decisions"+ assert call["lang"] == "de"+ assert call["lang_guess"] == "de"+ assert call["min_confidence"] == 0.5+++def test_batch_predict_fallback_sends_no_kwargs_when_body_is_quiet(monkeypatch):+ """A quiet batch body on the fallback path must call ``predict()`` with no kwargs at all.++ An attached Router whose ``predict()`` does not take the new keywords -- the ``FakeRouter``+ class this suite has always used -- must keep answering on a bare batch. Passing ``None``+ would 500 on that shape and would override a deployment's own ``Router(lang_guess=...)``+ on a modern one.+ """+ monkeypatch.delenv("LAYA_API_KEY", raising=False)+ fake = FakeRouter()+ client = TestClient(create_app(router=fake))+ r = client.post("/v1/systemone/batch", json=BATCH_REQ)+ assert r.status_code == 200, r.text+ assert len(fake.calls) == len(BATCH_REQ["states"])+++def test_batch_controls_combine_on_one_call(monkeypatch):+ """Call kwargs and item overrides land on the same call, not in isolation.++ A caller that sends a token budget and a chunking cap together must see them on the same+ ``predict_batch`` invocation -- not the budget on the item and the chunk cap dropped, and+ not the reverse. This is the whole-point-of-the-PR shape.+ """+ client, fake = _batch_client(monkeypatch)+ r = client.post("/v1/systemone/batch",+ json={**BATCH_REQ, "batch_size": 2, "sort_by_length": True,+ "min_confidence": 0.8, "task": "typed-decisions", "max_len": 128})+ assert r.status_code == 200, r.text+ requests, kwargs = fake.batch_calls[0]+ assert kwargs == {"min_confidence": 0.8, "batch_size": 2, "sort_by_length": True}+ for item in requests:+ assert item["task"] == "typed-decisions"+ assert item["max_len"] == 128+ def _health_return_keys():+ """The keys `health()` hands back, read out of its own `return` statement."""+ path = os.path.join(ROOT, "laya", "serve.py")+ with open(path, encoding="utf-8") as handle:+ tree = ast.parse(handle.read(), filename=path)+ for node in ast.walk(tree):+ if isinstance(node, ast.FunctionDef) and node.name == "health":+ for stmt in node.body:+ if isinstance(stmt, ast.Return) and isinstance(stmt.value, ast.Dict):+ return node, [k.value for k in stmt.value.keys+ if isinstance(k, ast.Constant) and isinstance(k.value, str)]+ raise AssertionError("no `def health()` returning a dict literal in laya/serve.py")+++ def test_http_api_page_documents_exactly_the_health_fields():+ """The payload `/health` documents has to be the payload the handler builds.++ The page showed four keys while the handler returned seven: `device_is_preference`,+ `checkpoint_devices` and `cpu_fallbacks` arrived with the device-fact reporting and #574's+ fallback counters, and both were written up in docs/docker.md -- the page a reader opens before+ pointing a probe at a server was the one page nobody updated. Its sample also printed+ `"device": "auto"`, which is what `LAYA_DEVICE` defaults to, not a value the handler can return.++ So the key list is read out of `health()`'s `return` rather than typed here, in both directions+ like the request-body gate above, and the sample is checked for internal coherence too: it is one+ server's answer, so every per-checkpoint block is keyed by the names in `loaded`, and+ `device_is_preference` and `device` follow the two rules the handler states.+ """+ health, returned = _health_return_keys()+ assert returned, "health() returns no literal keys; retarget this"++ page = open(os.path.join(ROOT, "docs", "http-api.md"), encoding="utf-8").read()+ section = page[page.index("### `GET /health`"):page.index("### `POST")]+ sample = json.loads(section.split("```json", 1)[1].split("```", 1)[0])++ assert sorted(sample) == sorted(returned), "the sample says %s, health() returns %s" % (+ sorted(sample), sorted(returned))+ for key in returned:+ assert "\n- `%s` " % key in section, (+ "%s is in the sample with no bullet defining it, so the field is listed and not "+ "explained" % key)++ # The per-checkpoint blocks all key on the same resident checkpoints. A fourth dict-valued key+ # means the handler grew one and this list has to grow with it, deliberately.+ resident = set(sample["loaded"])+ assert resident, "the sample shows no resident checkpoint, so the blocks below prove nothing"+ blocks = [k for k, v in sample.items() if isinstance(v, dict) and v]+ assert len(blocks) == 3, "dict-valued blocks are %s; retarget this if the payload grew" % blocks+ for block in blocks:+ assert set(sample[block]) == resident, "%s is keyed %r, loaded says %r" % (+ block, sorted(sample[block]), sorted(resident))++ # `device` is a device label, not the configuration word: the labels come from the only function+ # that can produce one when nothing is resident.+ dev_path = os.path.join(ROOT, "laya", "mcp", "device.py")+ with open(dev_path, encoding="utf-8") as handle:+ dev_tree = ast.parse(handle.read(), filename=dev_path)+ labels = set()+ for node in ast.walk(dev_tree):+ if isinstance(node, ast.FunctionDef) and node.name == "resolve_device":+ labels = {stmt.value.value for stmt in ast.walk(node)+ if isinstance(stmt, ast.Return) and isinstance(stmt.value, ast.Constant)+ and isinstance(stmt.value.value, str)}+ assert labels, "resolve_device() returns no literal device label; retarget this"+ assert sample["device"] in labels, "%r is not a device label resolve_device can return (%s)" % (+ sample["device"], sorted(labels))++ # The handler's two rules about those fields: the preference flag is true exactly while nothing+ # is resident, and the top-level answer is the first resident checkpoint's own device.+ assert sample["device_is_preference"] == (not sample["checkpoint_devices"]), (+ "device_is_preference says %r with checkpoint_devices %r" % (+ sample["device_is_preference"], sample["checkpoint_devices"]))+ if sample["checkpoint_devices"]:+ assert sample["device"] == next(iter(sample["checkpoint_devices"].values())), (+ "device must be the first resident checkpoint's device, as serve.py computes it")++ # And the shape of a fallback entry, which no page has ever spelled out: read from the dict the+ # handler builds per checkpoint.+ counters = None+ for node in ast.walk(health):+ if isinstance(node, ast.Assign) and isinstance(node.targets[0], ast.Subscript) \+ and isinstance(node.value, ast.Dict):+ counters = [k.value for k in node.value.keys if isinstance(k, ast.Constant)]+ assert counters, "health() builds no per-checkpoint dict literal; retarget this"+ for name in resident:+ assert sorted(sample["cpu_fallbacks"][name]) == sorted(counters), (+ "cpu_fallbacks entries say %s, the handler builds %s" % (+ sorted(sample["cpu_fallbacks"][name]), sorted(counters)))-