Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

feat(agent): add predict_batch for shared-forward-pass batch inference (#47)

AlKor13ffaae82e10ca
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index ab7b9ace..bdb76d7f 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -58,6 +58,7 @@ jobs:
run: |
python tests/test_router.py
python tests/test_criteria.py
+ python tests/test_batch.py
python tests/test_download.py
python tests/test_shortlist.py
python tests/test_decision_model.py
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index 0fcdd5b9..f89264f0 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -39,6 +39,7 @@ jobs:
run: |
python tests/test_router.py
python tests/test_criteria.py
+ python tests/test_batch.py
python tests/test_email.py
python tests/test_download.py
python tests/test_shortlist.py
diff --git a/README.md b/README.md
index fbf9a5fa..46dfd8b3 100644
--- a/README.md
+++ b/README.md
@@ -406,6 +406,29 @@ Passing an empty question dictionary to `agent.predict(state, {})` or
and `"usage": {"input_tokens": 0, "output_tokens": 0}`. The state is not tokenized
and no model forward pass runs.
+### Batch Mode: score many states in one forward pass
+
+`predict` handles one state per call, which leaves most of the GPU's batch dimension idle. When you
+have a list of items to score against the *same* questions — a backlog of tickets, a table of rows,
+a log slice — `predict_batch` packs them into shared forward passes:
+
+```python
+states = [{"body": t} for t in ticket_texts] # a list of states
+
+results = agent.predict_batch(states, questions) # one forward pass for the whole list
+# results[i] is exactly what agent.predict(states[i], questions) would return
+
+# Bound peak memory when the list (or the texts) are large — chunk into passes of N:
+results = agent.predict_batch(states, questions, batch_size=64)
+```
+
+Results are aligned with `states` by index and identical in shape to `predict`. Decisions match the
+one-at-a-time path exactly (numbers are bit-identical on CPU; on GPU they can differ in the 4th
+decimal because fp16 autocast reorders reductions across padding widths). Batching is a **GPU
+throughput win** — on an RTX 5060 Ti, per-decision latency drops from ~10 ms one-by-one to ~1 ms
+batched (measured ~9–10×). On CPU the model is already compute-bound, so batching does not speed it
+up; use it there only for API convenience.
+
---
## GPU Fast Path (TileLang)
diff --git a/laya/agent.py b/laya/agent.py
index e5e33728..7a1ad6fa 100644
--- a/laya/agent.py
+++ b/laya/agent.py
@@ -3,7 +3,7 @@ import json
import os
import threading
import warnings
-from typing import Any, Dict, Optional, Union
+from typing import Any, Dict, List, Optional, Union
import numpy as np
import torch
@@ -416,89 +416,59 @@ class Agent:
q["labels"] = qdef["labels"]
return q
- @torch.no_grad()
- def system_one(self, state: Union[str, dict, list], questions: Dict[str, Dict[str, Any]]) -> Dict[str, Any]:
- """Evaluate typed questions across state in a single, parallel forward pass.
-
- Args:
- state: Text string, JSON dict, or conversation turn list.
- questions: Dictionary mapping question_id -> question definition.
- - choice: {"type": "choice", "instructions": "...", "criteria": {"optA": "...", ...}}
- - score: {"type": "score", "instructions": "...", "criteria": ["lvl0", "lvl1", ...]}
- - noul: {"type": "noul", "instructions": "...",
- "criteria": {"false": "...", "true": "..."},
- "labels": {"false": "B", "true": "A"}}
-
- Noul criteria and labels are optional. Labels only control the text shown to the
- model; their keys retain false/true semantics, and the returned `noul` value is
- always P(true). Labels default to false/true for compatibility.
-
- Returns:
- Dictionary with answers, probabilities, calibrated confidence, and token usage.
- Empty questions return empty answers and zero token usage without tokenization
- or a model forward pass.
- """
- ids = list(questions.keys())
- if not ids:
- return {
- "model": "laya-rl-agent",
- "answers": {},
- "usage": {"input_tokens": 0, "output_tokens": 0},
- }
- items = []
+ def _encode_state(self, state: Union[str, dict, list], ids: List[str], internal: Dict[str, Dict]) -> List[Dict]:
+ """Tokenize one state against every (already validated + normalized) question."""
max_len = self.cfg.get("max_len", 512)
head_max_len = self.cfg.get("head_max_len", 192)
# A chronological conversation list is serialized newest-last, so the default
# right-truncation (st[:room]) would silently drop the newest turn. Truncate
# from the left for lists so the most recent intent is preserved.
truncate_left = isinstance(state, list)
-
+ items = []
for qid in ids:
- self._check_question(qid, questions[qid])
- q = self._to_internal(questions[qid])
+ q = internal[qid]
seq, markers = build_sequence(self.tok, state, q, max_len, head_max_len,
truncate_left=truncate_left)
if len(markers) != len(render_options(q)):
raise ValueError("question %r options exceed head_max_len=%d" % (qid, head_max_len))
items.append({"ids": seq, "markers": markers, "qtype": QTYPES[q["t"]]})
+ return items
- b = collate_items([items], self.tok.pad_token_id)
- use_amp = self.device.type == "cuda"
-
- try:
+ def _forward(self, b: Dict):
+ """Run the model on a collated batch, with the GPU->CPU OOM fallback, and return numpy outputs."""
+ def run():
+ use_amp = self.device.type == "cuda"
with torch.autocast(device_type=self.device.type, dtype=self.dtype, enabled=use_amp):
- logits, act = self.model(
+ return self.model(
b["input_ids"].to(self.device),
b["attention_mask"].to(self.device),
b["marker_pos"].to(self.device),
b["marker_mask"].to(self.device),
b["qtype"].to(self.device),
)
+
+ try:
+ logits, act = run()
except (RuntimeError, torch.cuda.OutOfMemoryError) as e:
if self.device.type != "cpu" and ("memory" in str(e).lower() or "cuda" in str(e).lower()):
print("Warning: GPU memory exceeded during inference. Falling back to CPU...")
self.device = torch.device("cpu")
self.dtype = torch.float32
self.model.to(self.device)
- logits, act = self.model(
- b["input_ids"].to(self.device),
- b["attention_mask"].to(self.device),
- b["marker_pos"].to(self.device),
- b["marker_mask"].to(self.device),
- b["qtype"].to(self.device),
- )
+ logits, act = run()
else:
raise e
- logits = logits.float().cpu().numpy()
- act = torch.softmax(act.float(), -1).cpu().numpy()
+ return logits.float().cpu().numpy(), torch.softmax(act.float(), -1).cpu().numpy()
+ def _decode_answers(self, logits, act, items: List[Dict], ids: List[str],
+ internal: Dict[str, Dict], offset: int) -> Dict[str, Any]:
+ """Turn one state's logit rows (starting at `offset`) into typed answers."""
answers = {}
- n_tokens = int(b["attention_mask"].sum())
-
- for r, qid in enumerate(ids):
- q = self._to_internal(questions[qid])
- k = len(items[r]["markers"])
+ for j, qid in enumerate(ids):
+ r = offset + j
+ q = internal[qid]
+ k = len(items[j]["markers"])
qt = QTYPES[q["t"]]
t_scale = self.temperature_by_options.get(temp_bucket(qt, k), self.temperature[qt])
z = logits[r, :k] / t_scale
@@ -534,12 +504,97 @@ class Agent:
"confidence": round(max(float(p[1]), 1.0 - float(p[1])), 4),
"action": ext,
}
+ return answers
+
+ @torch.no_grad()
+ def predict_batch(self, states: List[Union[str, dict, list]], questions: Dict[str, Dict[str, Any]],
+ batch_size: Optional[int] = None) -> List[Dict[str, Any]]:
+ """Evaluate the same questions over many states, packing them into shared forward passes.
+
+ This is the throughput path. `system_one`/`predict` handle one state per forward pass; on a
+ GPU that leaves most of the batch dimension idle. `predict_batch` collates several states'
+ question rows into one tensor, so a call that would take N sequential forward passes takes
+ one (or `ceil(len(states) / batch_size)`), which is several times faster per decision on GPU.
+
+ Args:
+ states: A list of states (each a text string, JSON dict, or conversation turn list).
+ The same `questions` are evaluated against every state.
+ questions: Question definitions, exactly as accepted by `system_one`.
+ batch_size: Optional cap on states per forward pass. `None` sends them all in one pass;
+ set it to bound peak memory when batching many or long states.
+
+ Returns:
+ A list of per-state result dicts, each identical in shape to `system_one`'s output and
+ aligned with `states` by index.
+ """
+ if isinstance(states, (str, bytes, dict)):
+ raise TypeError(
+ "predict_batch expects a list of states; pass a single state to predict()/system_one()."
+ )
+ states = list(states)
+ if not states:
+ return []
+
+ ids = list(questions.keys())
+ # Empty questions: empty answers, zero usage, no tokenization or forward (matches system_one).
+ if not ids:
+ return [{"model": "laya-rl-agent", "answers": {}, "usage": {"input_tokens": 0, "output_tokens": 0}}
+ for _ in states]
+
+ # Validate + normalize each question once (state-independent), preserving system_one's checks.
+ for qid in ids:
+ self._check_question(qid, questions[qid])
+ internal = {qid: self._to_internal(questions[qid]) for qid in ids}
+ chunk = batch_size if (batch_size and batch_size > 0) else len(states)
+
+ results: List[Dict[str, Any]] = []
+ for start in range(0, len(states), chunk):
+ part = states[start:start + chunk]
+ per_state_items = [self._encode_state(st, ids, internal) for st in part]
+
+ b = collate_items(per_state_items, self.tok.pad_token_id)
+ logits, act = self._forward(b)
+ att = b["attention_mask"]
+
+ row = 0
+ for items in per_state_items:
+ nrows = len(items)
+ n_tokens = int(att[row:row + nrows].sum())
+ answers = self._decode_answers(logits, act, items, ids, internal, row)
+ results.append({
+ "model": "laya-rl-agent",
+ "answers": answers,
+ "usage": {"input_tokens": n_tokens, "output_tokens": 0},
+ })
+ row += nrows
+
+ return results
+
+ @torch.no_grad()
+ def system_one(self, state: Union[str, dict, list], questions: Dict[str, Dict[str, Any]]) -> Dict[str, Any]:
+ """Evaluate typed questions across state in a single, parallel forward pass.
- return {
- "model": "laya-rl-agent",
- "answers": answers,
- "usage": {"input_tokens": n_tokens, "output_tokens": 0},
- }
+ Args:
+ state: Text string, JSON dict, or conversation turn list.
+ questions: Dictionary mapping question_id -> question definition.
+ - choice: {"type": "choice", "instructions": "...", "criteria": {"optA": "...", ...}}
+ - score: {"type": "score", "instructions": "...", "criteria": ["lvl0", "lvl1", ...]}
+ - noul: {"type": "noul", "instructions": "...",
+ "criteria": {"false": "...", "true": "..."},
+ "labels": {"false": "B", "true": "A"}}
+
+ Noul criteria and labels are optional. Labels only control the text shown to the
+ model; their keys retain false/true semantics, and the returned `noul` value is
+ always P(true). Labels default to false/true for compatibility.
+
+ Returns:
+ Dictionary with answers, probabilities, calibrated confidence, and token usage.
+ Empty questions return empty answers and zero token usage without tokenization
+ or a model forward pass.
+
+ To score many states at once, see `predict_batch`, which shares forward passes across them.
+ """
+ return self.predict_batch([state], questions)[0]
def __enter__(self):
return self
diff --git a/tests/test_batch.py b/tests/test_batch.py
new file mode 100644
index 00000000..ca9041e5
--- /dev/null
+++ b/tests/test_batch.py
@@ -0,0 +1,125 @@
+"""Batch inference: predict_batch packs many states into shared forward passes.
+
+These tests need no model weights. They cover two things:
+ 1. Contract — predict_batch exists and system_one is defined in terms of it, so the single-state
+ and batched paths can never numerically drift apart.
+ 2. Orchestration — empty input, bad input, per-state row mapping, and batch_size chunking, all
+ exercised against a fake whose forward is stubbed. The numerical equivalence of the real
+ model path (predict_batch(states) == [system_one(s) for s in states]) is checked with weights
+ in tests/test_local_e2e.py.
+"""
+import inspect
+import os
+import sys
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+import numpy as np # noqa: E402
+
+from laya import agent as _agent # noqa: E402
+from laya.agent import Agent # noqa: E402
+
+PASS, FAIL = [], []
+
+
+def check(name, got, want):
+ if got == want:
+ PASS.append(name)
+ else:
+ FAIL.append("%s:\n got %r\n want %r" % (name, got, want))
+
+
+def check_true(name, cond, detail=""):
+ if cond:
+ PASS.append(name)
+ else:
+ FAIL.append("%s %s" % (name, detail))
+
+
+def check_raises(name, exc, fn):
+ try:
+ fn()
+ except exc:
+ PASS.append(name)
+ except Exception as e: # noqa: BLE001
+ FAIL.append("%s: raised %r, expected %s" % (name, e, exc.__name__))
+ else:
+ FAIL.append("%s: did not raise %s" % (name, exc.__name__))
+
+
+# --------------------------------------------------------------- contract (source inspection)
+check_true("contract/predict_batch is defined", callable(getattr(Agent, "predict_batch", None)))
+
+_so = inspect.getsource(Agent.system_one)
+check_true("contract/system_one delegates to predict_batch", "self.predict_batch(" in _so)
+check_true("contract/predict alias preserved", Agent.predict is Agent.system_one)
+
+_sig = inspect.signature(Agent.predict_batch)
+check_true("contract/predict_batch has batch_size kwarg", "batch_size" in _sig.parameters)
+
+
+# --------------------------------------------------------------- orchestration (weight-free fake)
+# We drive the real predict_batch method but stub the three helpers it composes, so no encoder,
+# tokenizer vocabulary, or weights are needed. collate_items runs for real on the fake items.
+NQ = 2
+QUESTIONS = {"a": {"type": "noul", "instructions": "?"}, "b": {"type": "noul", "instructions": "?"}}
+
+
+def make_fake():
+ fake = _agent.Agent.__new__(_agent.Agent)
+ fake.tok = type("Tok", (), {"pad_token_id": 0})()
+ fake._to_internal = staticmethod(Agent._to_internal).__func__ # reuse the real normalizer
+ fake._forward_calls = []
+
+ def _encode_state(state, ids, internal):
+ # one 3-token, 2-marker item per question; content is irrelevant to the mapping test
+ return [{"ids": [1, 2, 3], "markers": [0, 1], "qtype": 2} for _ in ids]
+
+ def _forward(b):
+ n = b["input_ids"].shape[0]
+ fake._forward_calls.append(n)
+ # deterministic logits so decode is stable; act pre-softmaxed as _forward would return it
+ return np.zeros((n, 2), dtype=np.float32), np.full((n, 2), 0.5, dtype=np.float32)
+
+ def _decode_answers(logits, act, items, ids, internal, offset):
+ # record the row offset this state was decoded from, to verify per-state alignment
+ return {"_offset": offset}
+
+ fake._encode_state = _encode_state
+ fake._forward = _forward
+ fake._decode_answers = _decode_answers
+ return fake
+
+
+f = make_fake()
+check("orchestration/empty states -> empty list", f.predict_batch([], QUESTIONS), [])
+
+f = make_fake()
+res = f.predict_batch(["s0", "s1", "s2"], QUESTIONS)
+check("orchestration/one result per state", len(res), 3)
+check("orchestration/state 0 decoded from row 0", res[0]["answers"]["_offset"], 0)
+check("orchestration/state 1 decoded from row NQ", res[1]["answers"]["_offset"], NQ)
+check("orchestration/state 2 decoded from row 2*NQ", res[2]["answers"]["_offset"], 2 * NQ)
+check_true("orchestration/result shape matches system_one",
+ all(set(r) == {"model", "answers", "usage"} for r in res))
+check("orchestration/single forward pass by default", f._forward_calls, [6]) # 3 states x 2 questions
+
+f = make_fake()
+f.predict_batch(["s0", "s1", "s2"], QUESTIONS, batch_size=1)
+check("orchestration/batch_size=1 chunks into three passes", f._forward_calls, [2, 2, 2])
+
+f = make_fake()
+f.predict_batch(["s0", "s1", "s2", "s3", "s4"], QUESTIONS, batch_size=2)
+check("orchestration/batch_size=2 chunks 5 states as 2+2+1", f._forward_calls, [4, 4, 2])
+
+f = make_fake()
+check_raises("orchestration/bare string rejected", TypeError, lambda: f.predict_batch("just a string", QUESTIONS))
+f = make_fake()
+check_raises("orchestration/bare dict rejected", TypeError, lambda: f.predict_batch({"body": "x"}, QUESTIONS))
+
+
+# --------------------------------------------------------------------------- report
+print("\n%d passed, %d failed" % (len(PASS), len(FAIL)))
+for f_ in FAIL:
+ print(" FAIL " + f_)
+sys.exit(1 if FAIL else 0)
diff --git a/tests/test_local_e2e.py b/tests/test_local_e2e.py
index 20556bb5..7c03f02f 100644
--- a/tests/test_local_e2e.py
+++ b/tests/test_local_e2e.py
@@ -256,6 +256,81 @@ ok("a rewritten tokenizer config forces a fresh parse", refreshed.tok is not fir
"an edited on-disk tokenizer must not be masked by the cache")
del first, again, multi, refreshed
+
+# ------------------------------------------------- 7. Batch inference matches one-by-one
+head("7. predict_batch matches system_one, decision-for-decision (real forward passes)")
+BATCH_STATES = [
+ {"message": "I was charged twice for invoice 4411, please refund it today."},
+ {"message": "The dashboard has been down for an hour and my team is blocked."},
+ {"message": "What is the price of the enterprise plan? No rush at all."},
+ {"message": "Cannot reset my password, the email never arrives."},
+ {"message": "Thanks, everything is working great now!"},
+]
+# The English checkpoint was freed above; load a fresh agent for this section.
+ba = laya.load(LOCAL["multilingual"], device=DEVICE)
+# fp16 autocast on GPU reorders reductions across padding widths, so numbers can wobble in the
+# 4th decimal; CPU fp32 is exact. Decisions (argmax) must be identical either way.
+ATOL = 5e-3 if str(ba.device) != "cpu" else 0.0
+
+single = [ba.predict(s, QD) for s in BATCH_STATES]
+batched = ba.predict_batch(BATCH_STATES, QD)
+chunked = ba.predict_batch(BATCH_STATES, QD, batch_size=2)
+
+ok("batch returns one result per state", len(batched) == len(BATCH_STATES),
+ "got %d" % len(batched))
+ok("empty batch returns []", ba.predict_batch([], QD) == [])
+try:
+ ba.predict_batch("a bare string", QD)
+ ok("bare state rejected", False, "no TypeError raised")
+except TypeError:
+ ok("bare state rejected", True)
+
+
+def _num_close(x, y, atol):
+ return abs(x - y) <= atol
+
+
+def _answers_agree(a, b, atol):
+ if set(a) != set(b):
+ return False, "question ids differ"
+ for qid in a:
+ x, y = a[qid], b[qid]
+ if x["type"] != y["type"]:
+ return False, "%s: type" % qid
+ if x["type"] == "choice":
+ if x["choice"] != y["choice"]:
+ return False, "%s: choice %s vs %s" % (qid, x["choice"], y["choice"])
+ for kk in x["probabilities"]:
+ if not _num_close(x["probabilities"][kk], y["probabilities"][kk], atol):
+ return False, "%s: prob[%s]" % (qid, kk)
+ elif x["type"] == "score":
+ if not _num_close(x["score"], y["score"], atol):
+ return False, "%s: score %s vs %s" % (qid, x["score"], y["score"])
+ elif x["type"] == "noul":
+ if not _num_close(x["noul"], y["noul"], atol):
+ return False, "%s: noul %s vs %s" % (qid, x["noul"], y["noul"])
+ return True, ""
+
+
+bad = 0
+for i, (s, b) in enumerate(zip(single, batched)):
+ agree, why = _answers_agree(s["answers"], b["answers"], ATOL)
+ if not agree:
+ bad += 1
+ print(" FAIL state %d: %s" % (i, why), flush=True)
+ if s["usage"]["input_tokens"] != b["usage"]["input_tokens"]:
+ bad += 1
+ print(" FAIL state %d: token count %d vs %d"
+ % (i, s["usage"]["input_tokens"], b["usage"]["input_tokens"]), flush=True)
+ok("batched == one-by-one (decisions + numbers within atol %.0e)" % ATOL, bad == 0,
+ "%d states diverged" % bad)
+
+bad_chunk = sum(0 if _answers_agree(b["answers"], c["answers"], ATOL)[0] else 1
+ for b, c in zip(batched, chunked))
+ok("batch_size chunking matches one big pass", bad_chunk == 0, "%d diverged" % bad_chunk)
+print(" verified %d states across full / chunked / one-by-one paths" % len(BATCH_STATES), flush=True)
+del ba
+
# ---------------------------------------------------------------- summary
head("SUMMARY")
for n in NOTES: