Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

Merge pull request #611 from aashish254/feat-demo-predict-batch

Nandakishorfe210ebd3fd1
diff --cc tests/test_example_server_limits.py
index 8287584a,74c0f0b5..62443a88
--- a/tests/test_example_server_limits.py
+++ b/tests/test_example_server_limits.py
@@@ -227,150 -137,141 +227,285 @@@ def main()
ok("the page and health endpoints are unaffected",
client.get("/").status_code == 200 and client.get("/health").status_code == 200)
+ # --- the two surfaces must reach the same verdict on the same numbers ---------
+ # The demo reads `Question` instances where serve reads plain dicts, so the counting
+ # lives behind a shape branch that could drift from serve's. Pin the verdicts against
+ # each other rather than restating either one: serve's validator is called with the
+ # plain dicts it actually receives, the demo through HTTP with the models it actually
+ # receives, and the two have to agree.
+ import laya.serve as _serve_mod # noqa: E402
+
+ def serve_verdict(state, questions):
+ try:
+ _serve_mod._check_request_limits(state, questions)
+ except Exception: # noqa: BLE001 -- HTTPException carries the 413
+ return "refused"
+ return "accepted"
+
+ def demo_verdict(state, questions):
+ return "refused" if client.post("/predict", json={"state": state,
+ "questions": questions}).status_code == 413 else "accepted"
+
+ noul_crit = {"type": "noul", "instructions": "True?",
+ "criteria": {"false": "no", "true": "yes"}}
+ parity_cases = [
+ ("101 choice options", "hi", choice(MAX_CHOICE_OPTIONS + 1)),
+ ("33 score levels", "hi", score(MAX_SCORE_LEVELS + 1)),
+ ("600 options total", "hi",
+ dict(("q%d" % i, choice(20, "q%d" % i)["q%d" % i]) for i in range(30))),
+ ("exactly 100 choice options", "hi", choice(MAX_CHOICE_OPTIONS)),
+ ("exactly 32 score levels", "hi", score(MAX_SCORE_LEVELS)),
+ ("noul criteria only, near the total", "hi",
+ dict([("c%d" % i, choice(MAX_CHOICE_OPTIONS, "c%d" % i)["c%d" % i]) for i in range(5)]
+ + [("last", choice(12, "last")["last"])]
+ + [("n%d" % i, noul_crit) for i in range(30)])),
+ ("noul only", "hi", {"n%d" % i: noul_crit for i in range(10)}),
+ ("an ordinary request", {"body": "billed twice"}, one),
+ ]
+ for label, st, qs in parity_cases:
+ s, d = serve_verdict(st, qs), demo_verdict(st, qs)
+ ok("parity with laya.serve: %s" % label, s == d, "serve=%s demo=%s" % (s, d))
+
+ # --- the resident-checkpoint cap: derived, not copied -------------------
+ # examples/server.py used to build its Router with `max_loaded=1`, a copy of a default
+ # laya/router.py retired in #180. A cap of one cannot hold both english and
+ # multilingual, so the demo ran the churn #172 measured and fixed -- one checkpoint
+ # rebuilt per alternating-language request -- while the library and laya.serve did not.
+ # What is asserted here is that the demo asks for nothing unless asked to, and that
+ # what it then reports is the number the running Router holds.
+ from laya.router import Router
+
+ ok("an unset LAYA_MAX_LOADED means 'not asked for', not a number of this file's own",
+ demo._CFG["max_loaded"] is None, repr(demo._CFG["max_loaded"]))
+ ok("so the key never reaches the constructor",
+ "max_loaded" not in demo._router_kwargs(demo._CFG),
+ repr(demo._router_kwargs(demo._CFG)))
+ # Before a Router exists the page has no resident count to state. This is the check
+ # that fails if the page goes back to guessing one (`cfg.get('max_loaded', 1)`).
+ ok("and while loading, the page states no resident count",
+ "kept in memory" not in client.get("/health", headers={"accept": "text/html"}).text,
+ client.get("/health", headers={"accept": "text/html"}).text[:200])
+
+ with client: # the lifespan builds the Router; preload is off
+ # The witness that this arm needed no weights: LAYA_PRELOAD=0 above means the
+ # Router the app built holds no checkpoint yet, so the cap is the only thing here
+ # that could have been loaded.
+ ok("building it downloaded nothing",
+ not demo.ROUTER._agents, repr(sorted(demo.ROUTER._agents)))
+ cap = demo.ROUTER.max_loaded
+ ok("the running Router holds laya's own default",
+ cap == Router().max_loaded, "demo %r vs laya %r" % (cap, Router().max_loaded))
+ ok("and laya's own default is more than one checkpoint",
+ cap > 1, repr(cap))
+ ok("/health reports the cap the Router really holds",
+ client.get("/health").json()["config"]["max_loaded"] == cap, repr(cap))
+ ok("and the HTML page prints that same number",
+ ("up to %d kept in memory" % cap)
+ in client.get("/health", headers={"accept": "text/html"}).text, repr(cap))
+ # The number cannot come from the requested config, because the config and the
+ # Router are allowed to disagree: `Router(preload=True)` raises its own cap to fit
+ # what it preloads (laya/router.py:275 -> :372), so the demo's default mode has run
+ # with three resident while its env said one. Move the Router and the page must move.
+ demo.ROUTER.max_loaded = cap + 1
+ moved = client.get("/health")
+ ok("and both follow the Router when the Router raises its own cap",
+ moved.json()["config"]["max_loaded"] == cap + 1
+ and ("up to %d kept in memory" % (cap + 1))
+ in client.get("/health", headers={"accept": "text/html"}).text,
+ repr(moved.json()["config"]))
+
+ def with_cap(value):
+ """Re-import the demo the way uvicorn starts it, with LAYA_MAX_LOADED set to `value`."""
+ if value is None:
+ os.environ.pop("LAYA_MAX_LOADED", None)
+ else:
+ os.environ["LAYA_MAX_LOADED"] = value
+ return importlib.reload(demo)
+
+ # The default moving must not take the operator's own number away with it.
+ for raw, want in (("3", 3), ("1", 1), (" 4 ", 4)):
+ fresh = with_cap(raw)
+ with TestClient(fresh.app) as c:
+ ok("LAYA_MAX_LOADED=%r still reaches the Router" % raw,
+ fresh.ROUTER.max_loaded == want, repr(fresh.ROUTER.max_loaded))
+ ok("LAYA_MAX_LOADED=%r still shows up in /health" % raw,
+ c.get("/health").json()["config"]["max_loaded"] == want, raw)
+ fresh = with_cap(None)
+ from laya.serve import build_router
+
+ ok("the demo and laya.serve leave the cap to the same place",
+ fresh._router_kwargs(fresh._CFG).get("max_loaded", Router().max_loaded)
+ == build_router().max_loaded,
+ "%r vs %r" % (fresh._router_kwargs(fresh._CFG), build_router().max_loaded))
+
+ # --- the --reload env push has to survive the round trip ----------------
+ # With reload=True uvicorn re-imports `server:app` in a child process and only the
+ # environment crosses over. Writing str(None) for "not asked for" would land on the
+ # int() that reads LAYA_MAX_LOADED in that child and stop the server at import, so the
+ # unset case must push nothing at all. uvicorn is replaced so main() never binds a port.
+ real_uvicorn = sys.modules.get("uvicorn")
+ started = {}
+ fake = types.ModuleType("uvicorn")
+ fake.run = lambda target, **kw: started.update(target=target)
+ sys.modules["uvicorn"] = fake
+ argv = sys.argv
+
+ def start(args):
+ with_cap(None)
+ sys.argv = ["server.py"] + args
+ demo.main()
+ return os.environ.get("LAYA_MAX_LOADED", "(absent)")
+
+ try:
+ ok("--reload with no flag pushes no cap, so the reimport can read it",
+ start(["--reload"]) == "(absent)", repr(started))
+ ok("--reload --max-loaded 3 pushes 3",
+ start(["--reload", "--max-loaded", "3"]) == "3", repr(started))
+ ok("without --reload nothing is pushed at all",
+ start([]) == "(absent)", repr(started))
+ finally:
+ sys.argv = argv
+ if real_uvicorn is not None:
+ sys.modules["uvicorn"] = real_uvicorn
+ else:
+ sys.modules.pop("uvicorn", None)
+ with_cap(None) # leave the module as the rest of the file found it
+
+ # --- /predict/batch must make ONE Router call, not one per state ---------
+ #
+ # README teaches `Router.predict_batch` as the way to answer many states with shared forward
+ # passes ("routes the full workload first, groups requests by checkpoint ... results are
+ # restored to the original request order"), and this endpoint is the batch-shaped surface the
+ # README points at for trying Laya without writing code. The handler used to call
+ # `Router.predict` once per state anyway, so 64 states were 64 forwards. These drive the real
+ # app -- validation, `_questions()`, the handler body, the JSON envelope -- over a recording
+ # stand-in at `demo.ROUTER`, so no weights are needed to count the calls.
+
+ import inspect
+
+ from laya.router import Router as CoreRouter
+
+ class RecordingRouter:
+ """Answers like the Router does, and remembers how it was asked."""
+
+ def __init__(self, fail_on=None):
+ self.predict_calls = []
+ self.batch_calls = []
+ self.fail_on = fail_on
+
+ def predict(self, state, questions, **kw):
+ self.predict_calls.append((state, dict(kw)))
+ return self._answer(state, questions)
+
+ def predict_batch(self, requests, **kw):
+ self.batch_calls.append((list(requests), dict(kw)))
+ return [self._answer(r["state"], r["questions"]) for r in requests]
+
+ def _answer(self, state, questions):
+ if self.fail_on and self.fail_on in str(state):
+ raise ValueError("simulated failure for " + self.fail_on)
+ return {"answers": {"a": {"type": "noul", "choice": False}}, "state": state,
+ "questions": questions}
+
+ states = ["ticket %d" % i for i in range(8)]
+
+ class LegacyRouter(RecordingRouter):
+ """A Router-like object whose `predict_batch` predates the batch path entirely."""
+
+ predict_batch = None
+
+ # A Router that predates `predict_batch` must still be usable through the fallback.
+ older = LegacyRouter()
+ demo.ROUTER = older
+ legacy = TestClient(demo.app, raise_server_exceptions=False).post(
+ "/predict/batch", json={"states": states, "questions": one})
+ ok("a router without predict_batch still answers every state",
+ legacy.status_code == 200 and len(legacy.json()["results"]) == 8
+ and not [r for r in legacy.json()["results"] if "error" in r],
+ "%s / %s" % (legacy.status_code, json.dumps(legacy.json())[:200]))
+
+ router = RecordingRouter()
+ demo.ROUTER = router
+ batched = TestClient(demo.app, raise_server_exceptions=False).post(
+ "/predict/batch", json={"states": states, "questions": one})
+ body = batched.json()
+ ok("an 8-state batch is ONE predict_batch call and zero predict calls",
+ len(router.batch_calls) == 1 and not router.predict_calls,
+ "predict_batch=%d predict=%d" % (len(router.batch_calls), len(router.predict_calls)))
+ ok("the batch envelope is unchanged: count and one result per state, in order",
+ body.get("count") == 8 and [r.get("state") for r in body.get("results", [])] == states)
+ # Read defensively: an endpoint that never batches has nothing to inspect, and the checks below
+ # say so by name instead of letting this script die at the unpack.
+ requests_sent, call_kwargs = (router.batch_calls[0] if router.batch_calls else ([], {}))
+ ok("each request carries state + questions, and the questions map is the same one",
+ len(requests_sent) == 8
+ and all(sorted(r) == ["questions", "state"] for r in requests_sent)
+ and all(r["questions"] == requests_sent[0]["questions"] for r in requests_sent))
+ controls_sent = [k for r in requests_sent for k in r if k in ("model", "task", "lang")]
+ ok("unset controls are absent, not sent as null",
+ len(requests_sent) == 8 and not call_kwargs and not controls_sent,
+ "call kwargs=%r controls=%r" % (call_kwargs, controls_sent))
+ # Every key the endpoint puts in a request must be one core actually reads out of it, and the
+ # set of those keys is derived from `Router.route_batch`'s own source, so a rename or a new
+ # override in core shows up here rather than silently stopping reaching the router.
+ import re
+
+ route_src = inspect.getsource(CoreRouter.route_batch)
+ read = ({"state", "questions"}
+ | set(re.findall(r'request\["(\w+)"\]', route_src))
+ | set(re.findall(r'request\.get\("(\w+)"', route_src)))
+ sent_keys = {k for r in requests_sent for k in r}
+ ok("the request keys the endpoint sends are ones core reads",
+ len(requests_sent) == 8 and read >= sent_keys,
+ "core reads %r, endpoint sends %r" % (sorted(read), sorted(sent_keys)))
+ ok("predict_batch is still a one-positional-list call",
+ list(inspect.signature(CoreRouter.predict_batch).parameters)[1] == "requests")
+
+ pinned = RecordingRouter()
+ demo.ROUTER = pinned
+ TestClient(demo.app, raise_server_exceptions=False).post(
+ "/predict/batch", json={"states": states, "questions": one,
+ "model": "multilingual", "lang": "de"})
+ sent = pinned.batch_calls[0][0] if pinned.batch_calls else []
+ ok("a pinned model/lang travels with every request",
+ len(sent) == 8
+ and all(r["model"] == "multilingual" and r["lang"] == "de" for r in sent)
+ and not any("task" in r for r in sent),
+ json.dumps(sent[:1])[:200])
+
+ # One state failing must not cost its neighbours their answer: the endpoint's published
+ # contract is per-item errors inside a 200.
+ partial = RecordingRouter(fail_on="ticket 3")
+ demo.ROUTER = partial
+ poison = TestClient(demo.app, raise_server_exceptions=False).post(
+ "/predict/batch", json={"states": states, "questions": one}).json()
+ errors = [r for r in poison["results"] if "error" in r]
+ ok("one failing state yields one error entry, not an empty batch",
+ len(poison["results"]) == 8 and [r["index"] for r in errors] == [3]
+ and "simulated failure" in errors[0]["error"],
+ json.dumps(poison)[:240])
+ ok("the failing batch retried per state, so its neighbours still answered",
+ len(partial.batch_calls) == 1 and len(partial.predict_calls) == 8)
+
+ # The single-state surface must keep going through predict(), and `/predict/batch` with one
+ # state must still batch -- otherwise the two endpoints diverge on where hooks fire.
+ single = RecordingRouter()
+ demo.ROUTER = single
+ one_state = TestClient(demo.app, raise_server_exceptions=False)
+ one_state.post("/predict", json={"state": "ticket 0", "questions": one})
+ ok("/predict still calls Router.predict once",
+ len(single.predict_calls) == 1 and not single.batch_calls)
+
+ demo.ROUTER = None
+ not_ready = TestClient(demo.app, raise_server_exceptions=False).post(
+ "/predict/batch", json={"states": states, "questions": one})
+ ok("an unready router keeps answering 200 with per-item 503s",
+ not_ready.status_code == 200
+ and len(not_ready.json()["results"]) == 8
+ and all("503" in r.get("error", "") for r in not_ready.json()["results"]),
+ json.dumps(not_ready.json())[:200])
+ demo.ROUTER = None
+
print("\n%d passed, %d failed" % (len(PASS), len(FAIL)))
for f in FAIL:
print(" FAIL " + f)