Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

feat(structured): add decide_batch, the throughput form of decide

Aashish130a8574dac0
diff --git a/laya/__init__.py b/laya/__init__.py
index b6462e54..39b24301 100644
--- a/laya/__init__.py
+++ b/laya/__init__.py
@@ -12,7 +12,7 @@ from .presets import (
triage_questions,
)
from .router import DEFAULT_MODELS, RouteDecision, Router
-from .structured import DecisionResult, decide
+from .structured import DecisionResult, decide, decide_batch
__version__ = "0.3.20"
@@ -99,6 +99,7 @@ __all__ = [
"BaseHook",
"AsyncHook",
"decide",
+ "decide_batch",
"DecisionResult",
"__version__",
]
diff --git a/laya/agent.py b/laya/agent.py
index 1b58de6f..64353cb8 100644
--- a/laya/agent.py
+++ b/laya/agent.py
@@ -1121,6 +1121,21 @@ class Agent(HookRegistry):
return _decide(self, state, schema, questions=questions,
return_details=return_details, **predict_kwargs)
+ def decide_batch(self, states: List[Union[str, dict, list]], schema: Any = None, *,
+ questions: Optional[Dict[str, Any]] = None,
+ return_details: bool = False, **predict_kwargs) -> List[Any]:
+ """Answer many states against one schema (JSON schema or pydantic model) in one batched call.
+
+ The throughput form of :meth:`decide`: the schema is planned once and its questions
+ run over every state through :meth:`predict_batch` (shared forward passes, results
+ in input order), then each state's answers are projected as ``decide`` does. Extra
+ keyword arguments (``batch_size=``, ``lang=``, ``hooks=``, ...) are forwarded to
+ ``predict_batch``. See `laya.structured`.
+ """
+ from .structured import decide_batch as _decide_batch
+ return _decide_batch(self, states, schema, questions=questions,
+ return_details=return_details, **predict_kwargs)
+
def __repr__(self) -> str:
return "Agent(model_id=%r, device=%s)" % (self.model_id, getattr(self, "device", None))
diff --git a/laya/router.py b/laya/router.py
index 7dd999eb..34642b56 100644
--- a/laya/router.py
+++ b/laya/router.py
@@ -648,6 +648,21 @@ class Router(HookRegistry):
return _decide(self, state, schema, questions=questions,
return_details=return_details, **predict_kwargs)
+ def decide_batch(self, states: Sequence[Any], schema: Any = None, *,
+ questions: Optional[Dict[str, Any]] = None,
+ return_details: bool = False, **predict_kwargs) -> List[Any]:
+ """Answer many states against one schema (JSON schema or pydantic model) in one batched call.
+
+ The throughput form of :meth:`decide`: the schema is planned once and its questions
+ run over every state through :meth:`predict_batch` (grouped forward passes, results in
+ input order), then each state's answers are projected as ``decide`` does. Extra keyword
+ arguments (``batch_size=``, ``model=``, ``hooks=``, ...) are forwarded to
+ ``predict_batch``. See `laya.structured`.
+ """
+ from .structured import decide_batch as _decide_batch
+ return _decide_batch(self, states, schema, questions=questions,
+ return_details=return_details, **predict_kwargs)
+
def __enter__(self):
return self
diff --git a/laya/structured.py b/laya/structured.py
index 61005c35..12914b38 100644
--- a/laya/structured.py
+++ b/laya/structured.py
@@ -16,6 +16,7 @@ rejected with an error that names the path.
"""
from __future__ import annotations
+from collections.abc import Sequence as SequenceABC
from dataclasses import dataclass, field
from typing import Any, Dict, List, Optional, Sequence, Tuple
@@ -269,12 +270,71 @@ def decide(runner, state: Any, schema: Any = None, *, questions: Optional[Dict[s
return values
+def decide_batch(runner, states: Sequence[Any], schema: Any = None, *,
+ questions: Optional[Dict[str, Any]] = None,
+ return_details: bool = False, **predict_kwargs) -> List[Any]:
+ """Answer many states against one schema in one batched call, in input order.
+
+ The throughput form of :meth:`decide`: the schema is planned once and its questions
+ are evaluated over every state through ``runner.predict_batch`` (the same
+ shared-forward-pass path as :meth:`Agent.predict_batch` /
+ :meth:`Router.predict_batch`), then each state's answers are projected exactly as
+ ``decide`` does. Pass exactly one of ``schema`` or ``questions``; extra keyword
+ arguments (``batch_size=``, ``model=``, ``hooks=``, ...) are forwarded to
+ ``runner.predict_batch``. With ``return_details=True`` each item is a
+ ``DecisionResult``.
+
+ Both batch calling conventions are handled: an ``Agent``-like runner receives
+ ``(states, questions)``, while a ``Router``-like runner (one exposing
+ ``route_batch``) receives one ``{"state": ..., "questions": ...}`` request per
+ state, so states may route to different checkpoints.
+
+ Not every runner batches: ``ONNXAgent`` has no ``predict_batch`` yet, so passing one
+ raises ``TypeError`` here rather than silently degrading to N sequential ``decide``
+ calls -- loop ``decide`` yourself when the runner cannot batch.
+ """
+ if (schema is None) == (questions is None):
+ raise ValueError("pass exactly one of schema= or questions=")
+ if isinstance(states, (str, bytes)) or not isinstance(states, SequenceABC):
+ raise TypeError("states must be a sequence of states, not %s" % type(states).__name__)
+
+ fields: Optional[List[_Field]] = None
+ if schema is not None:
+ fields = plan_from_json_schema(_schema_of(schema))
+ questions = {f.name: f.question for f in fields}
+
+ predict_batch = getattr(runner, "predict_batch", None)
+ if predict_batch is None:
+ raise TypeError(
+ "%s has no predict_batch; loop decide() over the states instead"
+ % type(runner).__name__)
+
+ if hasattr(runner, "route_batch"):
+ # Router convention: one request dict per state, each carrying the shared
+ # questions, so it routes, groups by checkpoint and restores input order.
+ results = predict_batch([{"state": s, "questions": questions} for s in states],
+ **predict_kwargs)
+ else:
+ # Agent convention: a list of states evaluated against one question set.
+ results = predict_batch(list(states), questions, **predict_kwargs)
+
+ def _one(r: Dict[str, Any]) -> Any:
+ answers = r.get("answers", {}) or {}
+ values = _project(answers, fields) if fields is not None else dict(answers)
+ if return_details:
+ return _details(values, answers, r)
+ return values
+
+ return [_one(r) for r in results]
+
+
__all__ = [
"DecisionResult",
"SchemaError",
"answers_to_json",
"answer_to_pydantic",
"decide",
+ "decide_batch",
"plan_from_json_schema",
"questions_from_json_schema",
"questions_from_pydantic",
diff --git a/tests/test_structured.py b/tests/test_structured.py
index 6bc73eb4..9329d4be 100644
--- a/tests/test_structured.py
+++ b/tests/test_structured.py
@@ -13,6 +13,7 @@ from laya.structured import ( # noqa: E402
answers_to_json,
answer_to_pydantic,
decide,
+ decide_batch,
plan_from_json_schema,
questions_from_json_schema,
questions_from_pydantic,
@@ -174,6 +175,119 @@ decide(runner, "s", schema=SCHEMA, hooks_raise=False)
check("decide/forwards predict kwargs", runner.calls[0]["kwargs"], {"hooks_raise": False})
+# --------------------------------------------------------------- decide_batch
+class FakeBatchRunner:
+ """predict_batch(states, questions, **kwargs) like Agent's/Router's: one
+ result dict per state, in input order."""
+
+ def __init__(self, answers_by_state):
+ self.answers_by_state = answers_by_state
+ self.calls = []
+
+ def predict_batch(self, states, questions, **kwargs):
+ self.calls.append({"states": states, "questions": questions, "kwargs": kwargs})
+ # like Router.predict_batch: empty answers only for a state that was
+ # genuinely skipped by a hook -- the fake keys them off an unknown text.
+ return [{"answers": self.answers_by_state.get(s, {}), "routing": {"model": "english"}}
+ for s in states]
+
+
+STATES = ["first ticket", "second ticket"]
+BATCH_ANSWERS = {
+ "first ticket": ANSWERS,
+ "second ticket": {
+ "department": {"type": "choice", "choice": "sales", "confidence": 0.8,
+ "probabilities": {"billing": 0.0, "support": 0.1, "sales": 0.9}},
+ "urgency": {"type": "score", "score": 0.4, "confidence": 0.5,
+ "probabilities": {"0": 0.7, "1": 0.2, "2": 0.1}},
+ "needs_human": {"type": "noul", "noul": 0.1, "confidence": 0.8},
+ "priority": {"type": "choice", "choice": "1", "confidence": 0.6,
+ "probabilities": {"1": 0.6, "2": 0.3, "3": 0.1}},
+ },
+}
+
+batch_runner = FakeBatchRunner(BATCH_ANSWERS)
+outs = decide_batch(batch_runner, STATES, schema=SCHEMA)
+check("batch/one predict_batch call", len(batch_runner.calls), 1)
+check("batch/states forwarded", batch_runner.calls[0]["states"], STATES)
+check("batch/questions planned once", batch_runner.calls[0]["questions"],
+ {f.name: f.question for f in plan_from_json_schema(SCHEMA)})
+check("batch/input order", [o["department"] for o in outs], ["billing", "sales"])
+check("batch/projection matches decide", outs[0], decide(FakeRunner(ANSWERS), "x", schema=SCHEMA))
+check("batch/per-state values", outs[1],
+ {"department": "sales", "urgency": 0, "needs_human": False, "priority": 1})
+
+# kwargs are forwarded to predict_batch (batch_size, model, hooks, ...)
+batch_runner = FakeBatchRunner(BATCH_ANSWERS)
+decide_batch(batch_runner, STATES, schema=SCHEMA, batch_size=4, model="english")
+check("batch/forwards predict kwargs", batch_runner.calls[0]["kwargs"],
+ {"batch_size": 4, "model": "english"})
+
+# return_details: one DecisionResult per state with the usual fields
+details = decide_batch(FakeBatchRunner(BATCH_ANSWERS), STATES, schema=SCHEMA, return_details=True)
+check_true("batch/details are DecisionResult", all(isinstance(d, laya.DecisionResult) for d in details))
+check("batch/details values", [d.values["department"] for d in details], ["billing", "sales"])
+check("batch/details confidence", details[0].confidence["department"], 0.9)
+check("batch/details routing", details[0].routing, {"model": "english"})
+
+# questions= is the raw-answers pass-through, exactly like decide()
+raw = decide_batch(FakeBatchRunner({"a": {"x": {"type": "noul", "noul": 0.9, "confidence": 0.9}}}),
+ ["a"], questions={"x": {"type": "noul", "instructions": "?"}})
+check("batch/questions pass-through", raw, [{"x": {"type": "noul", "noul": 0.9, "confidence": 0.9}}])
+
+
+class FakeRouterLike:
+ """predict_batch(requests, **kwargs) like Router's: one request dict per
+ state (no positional questions), routed through route_batch."""
+
+ def __init__(self, answers_by_state):
+ self.answers_by_state = answers_by_state
+ self.calls = []
+
+ def route_batch(self, requests):
+ return [{"model": "english"} for _ in requests]
+
+ def predict_batch(self, requests, **kwargs):
+ self.calls.append({"requests": requests, "kwargs": kwargs})
+ return [{"answers": self.answers_by_state.get(r["state"], {}),
+ "routing": {"model": "english"}} for r in requests]
+
+
+router_like = FakeRouterLike(BATCH_ANSWERS)
+router_outs = decide_batch(router_like, STATES, schema=SCHEMA, batch_size=3)
+expected_questions = {f.name: f.question for f in plan_from_json_schema(SCHEMA)}
+check("batch/router one call", len(router_like.calls), 1)
+check("batch/router request dicts", [r["state"] for r in router_like.calls[0]["requests"]], STATES)
+check("batch/router questions per request",
+ all(r["questions"] == expected_questions for r in router_like.calls[0]["requests"]), True)
+check("batch/router kwargs forwarded", router_like.calls[0]["kwargs"], {"batch_size": 3})
+check("batch/router projection matches agent path", router_outs, outs)
+
+# a string is a sequence of states only accidentally: refuse it like route_batch does
+check_raises("batch/rejects string states", TypeError,
+ lambda: decide_batch(FakeBatchRunner({}), "abc", schema=SCHEMA))
+check_raises("batch/requires schema or questions", ValueError,
+ lambda: decide_batch(FakeBatchRunner({}), ["a"]))
+check_raises("batch/rejects both", ValueError,
+ lambda: decide_batch(FakeBatchRunner({}), ["a"], schema=SCHEMA, questions={}))
+check_raises("batch/rejects bad schema", SchemaError,
+ lambda: decide_batch(FakeBatchRunner({}), ["a"],
+ schema={"type": "object", "properties": {"s": {"type": "string"}}}))
+
+
+class NoBatchRunner(FakeRunner):
+ pass # has predict() but no predict_batch
+
+
+check_raises("batch/runner without predict_batch", TypeError,
+ lambda: decide_batch(NoBatchRunner(ANSWERS), ["a"], schema=SCHEMA))
+
+# a runner that answers nothing (hook-skipped states surface as empty values)
+noop_runner = FakeBatchRunner({})
+check("batch/skipped states", decide_batch(noop_runner, ["unmatched"], schema=SCHEMA), [{}])
+check("batch/noop still batches", len(noop_runner.calls), 1)
+
+
# --------------------------------------------------------------- pydantic (optional)
try:
from typing import Literal
@@ -203,9 +317,12 @@ except ImportError:
# --------------------------------------------------------------- exports and methods
check("export/decide", callable(laya.decide), True)
+check("export/decide_batch", callable(laya.decide_batch), True)
check("export/DecisionResult", hasattr(laya, "DecisionResult"), True)
check("method/Agent.decide", callable(getattr(laya.Agent, "decide", None)), True)
+check("method/Agent.decide_batch", callable(getattr(laya.Agent, "decide_batch", None)), True)
check("method/Router.decide", callable(getattr(laya.Router, "decide", None)), True)
+check("method/Router.decide_batch", callable(getattr(laya.Router, "decide_batch", None)), True)
from laya.onnx_agent import ONNXAgent # noqa: E402
check("method/ONNXAgent.decide", callable(getattr(ONNXAgent, "decide", None)), True)