Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(serve): a caller batch_size may shrink a pass, never widen one

thread3d5e37171ab802
diff --git a/docs/http-api.md b/docs/http-api.md
index c9d805b8..0b2e946e 100644
--- a/docs/http-api.md
+++ b/docs/http-api.md
@@ -370,7 +370,7 @@ Rather than refuse a large batch, the endpoint **splits** it: when that product
inside the budget, and `Router.predict_batch` runs the batch in several passes. Every state is still
answered and the response is unchanged. A request whose rows already fit is passed no `batch_size` at
all, so it behaves exactly as before -- which matters because batch shape can move floating-point
-results. A `batch_size` the caller sends always wins: it asked for a shape.
+results. A `batch_size` the caller sends may make a pass *smaller* than that plan -- it asked for a shape -- but never larger: the budget is a property of one forward pass, so a value above the plan is capped at it.
At the default, 256 rows go through in one pass -- 64 states of 4 questions, or 8 of 32. Larger
batches are split, and raising `max_len` makes each pass narrower rather than costing 16 times the
diff --git a/laya/serve.py b/laya/serve.py
index 788b1f4b..b51c9212 100644
--- a/laya/serve.py
+++ b/laya/serve.py
@@ -1492,17 +1492,19 @@ def create_app(router: Optional[Any] = None):
if min_confidence is not None:
call_kwargs["min_confidence"] = min_confidence
batch_size = _validate_batch_size_param(body)
+ # The token budget is a property of ONE forward pass, so the caller may shape the passes but
+ # never widen one past the budget: a body is capped at 64 states x 64 questions x 8192 tokens,
+ # and a caller-supplied `batch_size` used to be forwarded verbatim, so `batch_size=10**18`
+ # meant "collate all of it at once" and opted out of the only guard that bounds a pass. Take
+ # the smaller of the two. `planned` is `None` when the request already fits, which is the
+ # case where nothing was going to be passed anyway.
+ planned = _batch_chunk_size(
+ len(states), len(questions) if isinstance(questions, dict) else 0,
+ body.get("max_len"))
if batch_size is not None:
- call_kwargs["batch_size"] = batch_size
- else:
- # The caller's own `batch_size` always wins -- they asked for a shape. Otherwise split the
- # batch so one forward pass stays inside the token budget. `None` when it already fits, so
- # nothing is passed and the call is the one it has always been.
- planned = _batch_chunk_size(
- len(states), len(questions) if isinstance(questions, dict) else 0,
- body.get("max_len"))
- if planned is not None:
- call_kwargs["batch_size"] = planned
+ call_kwargs["batch_size"] = batch_size if planned is None else min(batch_size, planned)
+ elif planned is not None:
+ call_kwargs["batch_size"] = planned
sort_by_length = _validate_sort_by_length_param(body)
if sort_by_length:
# Only `True` is forwarded: `predict_batch`'s own default is `False`, and an attached
diff --git a/tests/test_serve.py b/tests/test_serve.py
index 6ba279bf..177d3eae 100644
--- a/tests/test_serve.py
+++ b/tests/test_serve.py
@@ -2008,15 +2008,39 @@ def test_batch_a_raised_max_len_chunks_harder(monkeypatch):
assert _sent_batch_size(wide_fake) == max(1, rows_per_pass // (rows // 8))
-def test_batch_an_explicit_batch_size_is_never_overridden(monkeypatch):
- """The caller asked for a shape. Planning one on top would silently change what they requested."""
+def test_batch_an_explicit_batch_size_shapes_a_pass_but_cannot_widen_one(monkeypatch):
+ """The caller asked for a shape, not for an unbounded pass.
+
+ `batch_size` is the only knob that decides what ONE forward pass collates, and the plan above is
+ the only thing that keeps that pass inside `LAYA_MAX_BATCH_TOKENS`. Forwarding the caller's value
+ verbatim made the budget opt-out: `batch_size=10**18` on a 4 KB body meant one pass over 524 288
+ tokens (64 states x 8192) with every documented limit satisfied. A smaller value still wins,
+ because asking for a *narrower* pass is asking for less work than the plan allows.
+ """
import laya.serve as serve_mod
+ rows_that_do_not_fit = {"states": ["hi"] * serve_mod.MAX_BATCH_STATES,
+ "questions": _noul_questions(serve_mod.MAX_QUESTIONS)}
+ planned = serve_mod._batch_chunk_size(serve_mod.MAX_BATCH_STATES, serve_mod.MAX_QUESTIONS, None)
+ assert planned is not None and planned < serve_mod.MAX_BATCH_STATES
+
+ # Narrower than the plan: the caller's own shape.
+ client, fake = _batch_env_client(monkeypatch)
+ r = client.post("/v1/systemone/batch", json={**rows_that_do_not_fit, "batch_size": 1})
+ assert r.status_code == 200, r.text
+ assert _sent_batch_size(fake) == 1
+
+ # Wider than the plan, including absurdly wider: capped at the plan.
+ for asked in (planned + 1, 7 if 7 > planned else planned + 1, 10 ** 18):
+ client, fake = _batch_env_client(monkeypatch)
+ r = client.post("/v1/systemone/batch", json={**rows_that_do_not_fit, "batch_size": asked})
+ assert r.status_code == 200, r.text
+ assert _sent_batch_size(fake) == planned, (asked, planned)
+
+ # A batch that already fits is passed no plan at all, so the caller's value is forwarded as-is.
client, fake = _batch_env_client(monkeypatch)
r = client.post("/v1/systemone/batch",
- json={"states": ["hi"] * serve_mod.MAX_BATCH_STATES,
- "questions": _noul_questions(serve_mod.MAX_QUESTIONS),
- "batch_size": 7})
+ json={"states": ["hi"] * 8, "questions": _noul_questions(2), "batch_size": 7})
assert r.status_code == 200, r.text
assert _sent_batch_size(fake) == 7