Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
feat(agent): add predict_batch for shared-forward-pass batch inference (#47)
AlKor13ffaae82e10ca
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.ymlindex ab7b9ace..bdb76d7f 100644--- a/.github/workflows/ci.yml+++ b/.github/workflows/ci.yml@@ -58,6 +58,7 @@ jobs:run: |python tests/test_router.pypython tests/test_criteria.py+ python tests/test_batch.pypython tests/test_download.pypython tests/test_shortlist.pypython tests/test_decision_model.pydiff --git a/.github/workflows/release.yml b/.github/workflows/release.ymlindex 0fcdd5b9..f89264f0 100644--- a/.github/workflows/release.yml+++ b/.github/workflows/release.yml@@ -39,6 +39,7 @@ jobs:run: |python tests/test_router.pypython tests/test_criteria.py+ python tests/test_batch.pypython tests/test_email.pypython tests/test_download.pypython tests/test_shortlist.pydiff --git a/README.md b/README.mdindex fbf9a5fa..46dfd8b3 100644--- a/README.md+++ b/README.md@@ -406,6 +406,29 @@ Passing an empty question dictionary to `agent.predict(state, {})` orand `"usage": {"input_tokens": 0, "output_tokens": 0}`. The state is not tokenizedand no model forward pass runs.+### Batch Mode: score many states in one forward pass++`predict` handles one state per call, which leaves most of the GPU's batch dimension idle. When you+have a list of items to score against the *same* questions — a backlog of tickets, a table of rows,+a log slice — `predict_batch` packs them into shared forward passes:++```python+states = [{"body": t} for t in ticket_texts] # a list of states++results = agent.predict_batch(states, questions) # one forward pass for the whole list+# results[i] is exactly what agent.predict(states[i], questions) would return++# Bound peak memory when the list (or the texts) are large — chunk into passes of N:+results = agent.predict_batch(states, questions, batch_size=64)+```++Results are aligned with `states` by index and identical in shape to `predict`. Decisions match the+one-at-a-time path exactly (numbers are bit-identical on CPU; on GPU they can differ in the 4th+decimal because fp16 autocast reorders reductions across padding widths). Batching is a **GPU+throughput win** — on an RTX 5060 Ti, per-decision latency drops from ~10 ms one-by-one to ~1 ms+batched (measured ~9–10×). On CPU the model is already compute-bound, so batching does not speed it+up; use it there only for API convenience.+---## GPU Fast Path (TileLang)diff --git a/laya/agent.py b/laya/agent.pyindex e5e33728..7a1ad6fa 100644--- a/laya/agent.py+++ b/laya/agent.py@@ -3,7 +3,7 @@ import jsonimport osimport threadingimport warnings-from typing import Any, Dict, Optional, Union+from typing import Any, Dict, List, Optional, Unionimport numpy as npimport torch@@ -416,89 +416,59 @@ class Agent:q["labels"] = qdef["labels"]return q- @torch.no_grad()- def system_one(self, state: Union[str, dict, list], questions: Dict[str, Dict[str, Any]]) -> Dict[str, Any]:- """Evaluate typed questions across state in a single, parallel forward pass.-- Args:- state: Text string, JSON dict, or conversation turn list.- questions: Dictionary mapping question_id -> question definition.- - choice: {"type": "choice", "instructions": "...", "criteria": {"optA": "...", ...}}- - score: {"type": "score", "instructions": "...", "criteria": ["lvl0", "lvl1", ...]}- - noul: {"type": "noul", "instructions": "...",- "criteria": {"false": "...", "true": "..."},- "labels": {"false": "B", "true": "A"}}-- Noul criteria and labels are optional. Labels only control the text shown to the- model; their keys retain false/true semantics, and the returned `noul` value is- always P(true). Labels default to false/true for compatibility.-- Returns:- Dictionary with answers, probabilities, calibrated confidence, and token usage.- Empty questions return empty answers and zero token usage without tokenization- or a model forward pass.- """- ids = list(questions.keys())- if not ids:- return {- "model": "laya-rl-agent",- "answers": {},- "usage": {"input_tokens": 0, "output_tokens": 0},- }- items = []+ def _encode_state(self, state: Union[str, dict, list], ids: List[str], internal: Dict[str, Dict]) -> List[Dict]:+ """Tokenize one state against every (already validated + normalized) question."""max_len = self.cfg.get("max_len", 512)head_max_len = self.cfg.get("head_max_len", 192)# A chronological conversation list is serialized newest-last, so the default# right-truncation (st[:room]) would silently drop the newest turn. Truncate# from the left for lists so the most recent intent is preserved.truncate_left = isinstance(state, list)-+ items = []for qid in ids:- self._check_question(qid, questions[qid])- q = self._to_internal(questions[qid])+ q = internal[qid]seq, markers = build_sequence(self.tok, state, q, max_len, head_max_len,truncate_left=truncate_left)if len(markers) != len(render_options(q)):raise ValueError("question %r options exceed head_max_len=%d" % (qid, head_max_len))items.append({"ids": seq, "markers": markers, "qtype": QTYPES[q["t"]]})+ return items- b = collate_items([items], self.tok.pad_token_id)- use_amp = self.device.type == "cuda"-- try:+ def _forward(self, b: Dict):+ """Run the model on a collated batch, with the GPU->CPU OOM fallback, and return numpy outputs."""+ def run():+ use_amp = self.device.type == "cuda"with torch.autocast(device_type=self.device.type, dtype=self.dtype, enabled=use_amp):- logits, act = self.model(+ return self.model(b["input_ids"].to(self.device),b["attention_mask"].to(self.device),b["marker_pos"].to(self.device),b["marker_mask"].to(self.device),b["qtype"].to(self.device),)++ try:+ logits, act = run()except (RuntimeError, torch.cuda.OutOfMemoryError) as e:if self.device.type != "cpu" and ("memory" in str(e).lower() or "cuda" in str(e).lower()):print("Warning: GPU memory exceeded during inference. Falling back to CPU...")self.device = torch.device("cpu")self.dtype = torch.float32self.model.to(self.device)- logits, act = self.model(- b["input_ids"].to(self.device),- b["attention_mask"].to(self.device),- b["marker_pos"].to(self.device),- b["marker_mask"].to(self.device),- b["qtype"].to(self.device),- )+ logits, act = run()else:raise e- logits = logits.float().cpu().numpy()- act = torch.softmax(act.float(), -1).cpu().numpy()+ return logits.float().cpu().numpy(), torch.softmax(act.float(), -1).cpu().numpy()+ def _decode_answers(self, logits, act, items: List[Dict], ids: List[str],+ internal: Dict[str, Dict], offset: int) -> Dict[str, Any]:+ """Turn one state's logit rows (starting at `offset`) into typed answers."""answers = {}- n_tokens = int(b["attention_mask"].sum())-- for r, qid in enumerate(ids):- q = self._to_internal(questions[qid])- k = len(items[r]["markers"])+ for j, qid in enumerate(ids):+ r = offset + j+ q = internal[qid]+ k = len(items[j]["markers"])qt = QTYPES[q["t"]]t_scale = self.temperature_by_options.get(temp_bucket(qt, k), self.temperature[qt])z = logits[r, :k] / t_scale@@ -534,12 +504,97 @@ class Agent:"confidence": round(max(float(p[1]), 1.0 - float(p[1])), 4),"action": ext,}+ return answers++ @torch.no_grad()+ def predict_batch(self, states: List[Union[str, dict, list]], questions: Dict[str, Dict[str, Any]],+ batch_size: Optional[int] = None) -> List[Dict[str, Any]]:+ """Evaluate the same questions over many states, packing them into shared forward passes.++ This is the throughput path. `system_one`/`predict` handle one state per forward pass; on a+ GPU that leaves most of the batch dimension idle. `predict_batch` collates several states'+ question rows into one tensor, so a call that would take N sequential forward passes takes+ one (or `ceil(len(states) / batch_size)`), which is several times faster per decision on GPU.++ Args:+ states: A list of states (each a text string, JSON dict, or conversation turn list).+ The same `questions` are evaluated against every state.+ questions: Question definitions, exactly as accepted by `system_one`.+ batch_size: Optional cap on states per forward pass. `None` sends them all in one pass;+ set it to bound peak memory when batching many or long states.++ Returns:+ A list of per-state result dicts, each identical in shape to `system_one`'s output and+ aligned with `states` by index.+ """+ if isinstance(states, (str, bytes, dict)):+ raise TypeError(+ "predict_batch expects a list of states; pass a single state to predict()/system_one()."+ )+ states = list(states)+ if not states:+ return []++ ids = list(questions.keys())+ # Empty questions: empty answers, zero usage, no tokenization or forward (matches system_one).+ if not ids:+ return [{"model": "laya-rl-agent", "answers": {}, "usage": {"input_tokens": 0, "output_tokens": 0}}+ for _ in states]++ # Validate + normalize each question once (state-independent), preserving system_one's checks.+ for qid in ids:+ self._check_question(qid, questions[qid])+ internal = {qid: self._to_internal(questions[qid]) for qid in ids}+ chunk = batch_size if (batch_size and batch_size > 0) else len(states)++ results: List[Dict[str, Any]] = []+ for start in range(0, len(states), chunk):+ part = states[start:start + chunk]+ per_state_items = [self._encode_state(st, ids, internal) for st in part]++ b = collate_items(per_state_items, self.tok.pad_token_id)+ logits, act = self._forward(b)+ att = b["attention_mask"]++ row = 0+ for items in per_state_items:+ nrows = len(items)+ n_tokens = int(att[row:row + nrows].sum())+ answers = self._decode_answers(logits, act, items, ids, internal, row)+ results.append({+ "model": "laya-rl-agent",+ "answers": answers,+ "usage": {"input_tokens": n_tokens, "output_tokens": 0},+ })+ row += nrows++ return results++ @torch.no_grad()+ def system_one(self, state: Union[str, dict, list], questions: Dict[str, Dict[str, Any]]) -> Dict[str, Any]:+ """Evaluate typed questions across state in a single, parallel forward pass.- return {- "model": "laya-rl-agent",- "answers": answers,- "usage": {"input_tokens": n_tokens, "output_tokens": 0},- }+ Args:+ state: Text string, JSON dict, or conversation turn list.+ questions: Dictionary mapping question_id -> question definition.+ - choice: {"type": "choice", "instructions": "...", "criteria": {"optA": "...", ...}}+ - score: {"type": "score", "instructions": "...", "criteria": ["lvl0", "lvl1", ...]}+ - noul: {"type": "noul", "instructions": "...",+ "criteria": {"false": "...", "true": "..."},+ "labels": {"false": "B", "true": "A"}}++ Noul criteria and labels are optional. Labels only control the text shown to the+ model; their keys retain false/true semantics, and the returned `noul` value is+ always P(true). Labels default to false/true for compatibility.++ Returns:+ Dictionary with answers, probabilities, calibrated confidence, and token usage.+ Empty questions return empty answers and zero token usage without tokenization+ or a model forward pass.++ To score many states at once, see `predict_batch`, which shares forward passes across them.+ """+ return self.predict_batch([state], questions)[0]def __enter__(self):return selfdiff --git a/tests/test_batch.py b/tests/test_batch.pynew file mode 100644index 00000000..ca9041e5--- /dev/null+++ b/tests/test_batch.py@@ -0,0 +1,125 @@+"""Batch inference: predict_batch packs many states into shared forward passes.++These tests need no model weights. They cover two things:+ 1. Contract — predict_batch exists and system_one is defined in terms of it, so the single-state+ and batched paths can never numerically drift apart.+ 2. Orchestration — empty input, bad input, per-state row mapping, and batch_size chunking, all+ exercised against a fake whose forward is stubbed. The numerical equivalence of the real+ model path (predict_batch(states) == [system_one(s) for s in states]) is checked with weights+ in tests/test_local_e2e.py.+"""+import inspect+import os+import sys++sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))++import numpy as np # noqa: E402++from laya import agent as _agent # noqa: E402+from laya.agent import Agent # noqa: E402++PASS, FAIL = [], []+++def check(name, got, want):+ if got == want:+ PASS.append(name)+ else:+ FAIL.append("%s:\n got %r\n want %r" % (name, got, want))+++def check_true(name, cond, detail=""):+ if cond:+ PASS.append(name)+ else:+ FAIL.append("%s %s" % (name, detail))+++def check_raises(name, exc, fn):+ try:+ fn()+ except exc:+ PASS.append(name)+ except Exception as e: # noqa: BLE001+ FAIL.append("%s: raised %r, expected %s" % (name, e, exc.__name__))+ else:+ FAIL.append("%s: did not raise %s" % (name, exc.__name__))+++# --------------------------------------------------------------- contract (source inspection)+check_true("contract/predict_batch is defined", callable(getattr(Agent, "predict_batch", None)))++_so = inspect.getsource(Agent.system_one)+check_true("contract/system_one delegates to predict_batch", "self.predict_batch(" in _so)+check_true("contract/predict alias preserved", Agent.predict is Agent.system_one)++_sig = inspect.signature(Agent.predict_batch)+check_true("contract/predict_batch has batch_size kwarg", "batch_size" in _sig.parameters)+++# --------------------------------------------------------------- orchestration (weight-free fake)+# We drive the real predict_batch method but stub the three helpers it composes, so no encoder,+# tokenizer vocabulary, or weights are needed. collate_items runs for real on the fake items.+NQ = 2+QUESTIONS = {"a": {"type": "noul", "instructions": "?"}, "b": {"type": "noul", "instructions": "?"}}+++def make_fake():+ fake = _agent.Agent.__new__(_agent.Agent)+ fake.tok = type("Tok", (), {"pad_token_id": 0})()+ fake._to_internal = staticmethod(Agent._to_internal).__func__ # reuse the real normalizer+ fake._forward_calls = []++ def _encode_state(state, ids, internal):+ # one 3-token, 2-marker item per question; content is irrelevant to the mapping test+ return [{"ids": [1, 2, 3], "markers": [0, 1], "qtype": 2} for _ in ids]++ def _forward(b):+ n = b["input_ids"].shape[0]+ fake._forward_calls.append(n)+ # deterministic logits so decode is stable; act pre-softmaxed as _forward would return it+ return np.zeros((n, 2), dtype=np.float32), np.full((n, 2), 0.5, dtype=np.float32)++ def _decode_answers(logits, act, items, ids, internal, offset):+ # record the row offset this state was decoded from, to verify per-state alignment+ return {"_offset": offset}++ fake._encode_state = _encode_state+ fake._forward = _forward+ fake._decode_answers = _decode_answers+ return fake+++f = make_fake()+check("orchestration/empty states -> empty list", f.predict_batch([], QUESTIONS), [])++f = make_fake()+res = f.predict_batch(["s0", "s1", "s2"], QUESTIONS)+check("orchestration/one result per state", len(res), 3)+check("orchestration/state 0 decoded from row 0", res[0]["answers"]["_offset"], 0)+check("orchestration/state 1 decoded from row NQ", res[1]["answers"]["_offset"], NQ)+check("orchestration/state 2 decoded from row 2*NQ", res[2]["answers"]["_offset"], 2 * NQ)+check_true("orchestration/result shape matches system_one",+ all(set(r) == {"model", "answers", "usage"} for r in res))+check("orchestration/single forward pass by default", f._forward_calls, [6]) # 3 states x 2 questions++f = make_fake()+f.predict_batch(["s0", "s1", "s2"], QUESTIONS, batch_size=1)+check("orchestration/batch_size=1 chunks into three passes", f._forward_calls, [2, 2, 2])++f = make_fake()+f.predict_batch(["s0", "s1", "s2", "s3", "s4"], QUESTIONS, batch_size=2)+check("orchestration/batch_size=2 chunks 5 states as 2+2+1", f._forward_calls, [4, 4, 2])++f = make_fake()+check_raises("orchestration/bare string rejected", TypeError, lambda: f.predict_batch("just a string", QUESTIONS))+f = make_fake()+check_raises("orchestration/bare dict rejected", TypeError, lambda: f.predict_batch({"body": "x"}, QUESTIONS))+++# --------------------------------------------------------------------------- report+print("\n%d passed, %d failed" % (len(PASS), len(FAIL)))+for f_ in FAIL:+ print(" FAIL " + f_)+sys.exit(1 if FAIL else 0)diff --git a/tests/test_local_e2e.py b/tests/test_local_e2e.pyindex 20556bb5..7c03f02f 100644--- a/tests/test_local_e2e.py+++ b/tests/test_local_e2e.py@@ -256,6 +256,81 @@ ok("a rewritten tokenizer config forces a fresh parse", refreshed.tok is not fir"an edited on-disk tokenizer must not be masked by the cache")del first, again, multi, refreshed++# ------------------------------------------------- 7. Batch inference matches one-by-one+head("7. predict_batch matches system_one, decision-for-decision (real forward passes)")+BATCH_STATES = [+ {"message": "I was charged twice for invoice 4411, please refund it today."},+ {"message": "The dashboard has been down for an hour and my team is blocked."},+ {"message": "What is the price of the enterprise plan? No rush at all."},+ {"message": "Cannot reset my password, the email never arrives."},+ {"message": "Thanks, everything is working great now!"},+]+# The English checkpoint was freed above; load a fresh agent for this section.+ba = laya.load(LOCAL["multilingual"], device=DEVICE)+# fp16 autocast on GPU reorders reductions across padding widths, so numbers can wobble in the+# 4th decimal; CPU fp32 is exact. Decisions (argmax) must be identical either way.+ATOL = 5e-3 if str(ba.device) != "cpu" else 0.0++single = [ba.predict(s, QD) for s in BATCH_STATES]+batched = ba.predict_batch(BATCH_STATES, QD)+chunked = ba.predict_batch(BATCH_STATES, QD, batch_size=2)++ok("batch returns one result per state", len(batched) == len(BATCH_STATES),+ "got %d" % len(batched))+ok("empty batch returns []", ba.predict_batch([], QD) == [])+try:+ ba.predict_batch("a bare string", QD)+ ok("bare state rejected", False, "no TypeError raised")+except TypeError:+ ok("bare state rejected", True)+++def _num_close(x, y, atol):+ return abs(x - y) <= atol+++def _answers_agree(a, b, atol):+ if set(a) != set(b):+ return False, "question ids differ"+ for qid in a:+ x, y = a[qid], b[qid]+ if x["type"] != y["type"]:+ return False, "%s: type" % qid+ if x["type"] == "choice":+ if x["choice"] != y["choice"]:+ return False, "%s: choice %s vs %s" % (qid, x["choice"], y["choice"])+ for kk in x["probabilities"]:+ if not _num_close(x["probabilities"][kk], y["probabilities"][kk], atol):+ return False, "%s: prob[%s]" % (qid, kk)+ elif x["type"] == "score":+ if not _num_close(x["score"], y["score"], atol):+ return False, "%s: score %s vs %s" % (qid, x["score"], y["score"])+ elif x["type"] == "noul":+ if not _num_close(x["noul"], y["noul"], atol):+ return False, "%s: noul %s vs %s" % (qid, x["noul"], y["noul"])+ return True, ""+++bad = 0+for i, (s, b) in enumerate(zip(single, batched)):+ agree, why = _answers_agree(s["answers"], b["answers"], ATOL)+ if not agree:+ bad += 1+ print(" FAIL state %d: %s" % (i, why), flush=True)+ if s["usage"]["input_tokens"] != b["usage"]["input_tokens"]:+ bad += 1+ print(" FAIL state %d: token count %d vs %d"+ % (i, s["usage"]["input_tokens"], b["usage"]["input_tokens"]), flush=True)+ok("batched == one-by-one (decisions + numbers within atol %.0e)" % ATOL, bad == 0,+ "%d states diverged" % bad)++bad_chunk = sum(0 if _answers_agree(b["answers"], c["answers"], ATOL)[0] else 1+ for b, c in zip(batched, chunked))+ok("batch_size chunking matches one big pass", bad_chunk == 0, "%d diverged" % bad_chunk)+print(" verified %d states across full / chunked / one-by-one paths" % len(BATCH_STATES), flush=True)+del ba+# ---------------------------------------------------------------- summaryhead("SUMMARY")for n in NOTES: