Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
feat(structured): add decide_batch, the throughput form of decide
Aashish130a8574dac0
diff --git a/laya/__init__.py b/laya/__init__.pyindex b6462e54..39b24301 100644--- a/laya/__init__.py+++ b/laya/__init__.py@@ -12,7 +12,7 @@ from .presets import (triage_questions,)from .router import DEFAULT_MODELS, RouteDecision, Router-from .structured import DecisionResult, decide+from .structured import DecisionResult, decide, decide_batch__version__ = "0.3.20"@@ -99,6 +99,7 @@ __all__ = ["BaseHook","AsyncHook","decide",+ "decide_batch","DecisionResult","__version__",]diff --git a/laya/agent.py b/laya/agent.pyindex 1b58de6f..64353cb8 100644--- a/laya/agent.py+++ b/laya/agent.py@@ -1121,6 +1121,21 @@ class Agent(HookRegistry):return _decide(self, state, schema, questions=questions,return_details=return_details, **predict_kwargs)+ def decide_batch(self, states: List[Union[str, dict, list]], schema: Any = None, *,+ questions: Optional[Dict[str, Any]] = None,+ return_details: bool = False, **predict_kwargs) -> List[Any]:+ """Answer many states against one schema (JSON schema or pydantic model) in one batched call.++ The throughput form of :meth:`decide`: the schema is planned once and its questions+ run over every state through :meth:`predict_batch` (shared forward passes, results+ in input order), then each state's answers are projected as ``decide`` does. Extra+ keyword arguments (``batch_size=``, ``lang=``, ``hooks=``, ...) are forwarded to+ ``predict_batch``. See `laya.structured`.+ """+ from .structured import decide_batch as _decide_batch+ return _decide_batch(self, states, schema, questions=questions,+ return_details=return_details, **predict_kwargs)+def __repr__(self) -> str:return "Agent(model_id=%r, device=%s)" % (self.model_id, getattr(self, "device", None))diff --git a/laya/router.py b/laya/router.pyindex 7dd999eb..34642b56 100644--- a/laya/router.py+++ b/laya/router.py@@ -648,6 +648,21 @@ class Router(HookRegistry):return _decide(self, state, schema, questions=questions,return_details=return_details, **predict_kwargs)+ def decide_batch(self, states: Sequence[Any], schema: Any = None, *,+ questions: Optional[Dict[str, Any]] = None,+ return_details: bool = False, **predict_kwargs) -> List[Any]:+ """Answer many states against one schema (JSON schema or pydantic model) in one batched call.++ The throughput form of :meth:`decide`: the schema is planned once and its questions+ run over every state through :meth:`predict_batch` (grouped forward passes, results in+ input order), then each state's answers are projected as ``decide`` does. Extra keyword+ arguments (``batch_size=``, ``model=``, ``hooks=``, ...) are forwarded to+ ``predict_batch``. See `laya.structured`.+ """+ from .structured import decide_batch as _decide_batch+ return _decide_batch(self, states, schema, questions=questions,+ return_details=return_details, **predict_kwargs)+def __enter__(self):return selfdiff --git a/laya/structured.py b/laya/structured.pyindex 61005c35..12914b38 100644--- a/laya/structured.py+++ b/laya/structured.py@@ -16,6 +16,7 @@ rejected with an error that names the path."""from __future__ import annotations+from collections.abc import Sequence as SequenceABCfrom dataclasses import dataclass, fieldfrom typing import Any, Dict, List, Optional, Sequence, Tuple@@ -269,12 +270,71 @@ def decide(runner, state: Any, schema: Any = None, *, questions: Optional[Dict[sreturn values+def decide_batch(runner, states: Sequence[Any], schema: Any = None, *,+ questions: Optional[Dict[str, Any]] = None,+ return_details: bool = False, **predict_kwargs) -> List[Any]:+ """Answer many states against one schema in one batched call, in input order.++ The throughput form of :meth:`decide`: the schema is planned once and its questions+ are evaluated over every state through ``runner.predict_batch`` (the same+ shared-forward-pass path as :meth:`Agent.predict_batch` /+ :meth:`Router.predict_batch`), then each state's answers are projected exactly as+ ``decide`` does. Pass exactly one of ``schema`` or ``questions``; extra keyword+ arguments (``batch_size=``, ``model=``, ``hooks=``, ...) are forwarded to+ ``runner.predict_batch``. With ``return_details=True`` each item is a+ ``DecisionResult``.++ Both batch calling conventions are handled: an ``Agent``-like runner receives+ ``(states, questions)``, while a ``Router``-like runner (one exposing+ ``route_batch``) receives one ``{"state": ..., "questions": ...}`` request per+ state, so states may route to different checkpoints.++ Not every runner batches: ``ONNXAgent`` has no ``predict_batch`` yet, so passing one+ raises ``TypeError`` here rather than silently degrading to N sequential ``decide``+ calls -- loop ``decide`` yourself when the runner cannot batch.+ """+ if (schema is None) == (questions is None):+ raise ValueError("pass exactly one of schema= or questions=")+ if isinstance(states, (str, bytes)) or not isinstance(states, SequenceABC):+ raise TypeError("states must be a sequence of states, not %s" % type(states).__name__)++ fields: Optional[List[_Field]] = None+ if schema is not None:+ fields = plan_from_json_schema(_schema_of(schema))+ questions = {f.name: f.question for f in fields}++ predict_batch = getattr(runner, "predict_batch", None)+ if predict_batch is None:+ raise TypeError(+ "%s has no predict_batch; loop decide() over the states instead"+ % type(runner).__name__)++ if hasattr(runner, "route_batch"):+ # Router convention: one request dict per state, each carrying the shared+ # questions, so it routes, groups by checkpoint and restores input order.+ results = predict_batch([{"state": s, "questions": questions} for s in states],+ **predict_kwargs)+ else:+ # Agent convention: a list of states evaluated against one question set.+ results = predict_batch(list(states), questions, **predict_kwargs)++ def _one(r: Dict[str, Any]) -> Any:+ answers = r.get("answers", {}) or {}+ values = _project(answers, fields) if fields is not None else dict(answers)+ if return_details:+ return _details(values, answers, r)+ return values++ return [_one(r) for r in results]++__all__ = ["DecisionResult","SchemaError","answers_to_json","answer_to_pydantic","decide",+ "decide_batch","plan_from_json_schema","questions_from_json_schema","questions_from_pydantic",diff --git a/tests/test_structured.py b/tests/test_structured.pyindex 6bc73eb4..9329d4be 100644--- a/tests/test_structured.py+++ b/tests/test_structured.py@@ -13,6 +13,7 @@ from laya.structured import ( # noqa: E402answers_to_json,answer_to_pydantic,decide,+ decide_batch,plan_from_json_schema,questions_from_json_schema,questions_from_pydantic,@@ -174,6 +175,119 @@ decide(runner, "s", schema=SCHEMA, hooks_raise=False)check("decide/forwards predict kwargs", runner.calls[0]["kwargs"], {"hooks_raise": False})+# --------------------------------------------------------------- decide_batch+class FakeBatchRunner:+ """predict_batch(states, questions, **kwargs) like Agent's/Router's: one+ result dict per state, in input order."""++ def __init__(self, answers_by_state):+ self.answers_by_state = answers_by_state+ self.calls = []++ def predict_batch(self, states, questions, **kwargs):+ self.calls.append({"states": states, "questions": questions, "kwargs": kwargs})+ # like Router.predict_batch: empty answers only for a state that was+ # genuinely skipped by a hook -- the fake keys them off an unknown text.+ return [{"answers": self.answers_by_state.get(s, {}), "routing": {"model": "english"}}+ for s in states]+++STATES = ["first ticket", "second ticket"]+BATCH_ANSWERS = {+ "first ticket": ANSWERS,+ "second ticket": {+ "department": {"type": "choice", "choice": "sales", "confidence": 0.8,+ "probabilities": {"billing": 0.0, "support": 0.1, "sales": 0.9}},+ "urgency": {"type": "score", "score": 0.4, "confidence": 0.5,+ "probabilities": {"0": 0.7, "1": 0.2, "2": 0.1}},+ "needs_human": {"type": "noul", "noul": 0.1, "confidence": 0.8},+ "priority": {"type": "choice", "choice": "1", "confidence": 0.6,+ "probabilities": {"1": 0.6, "2": 0.3, "3": 0.1}},+ },+}++batch_runner = FakeBatchRunner(BATCH_ANSWERS)+outs = decide_batch(batch_runner, STATES, schema=SCHEMA)+check("batch/one predict_batch call", len(batch_runner.calls), 1)+check("batch/states forwarded", batch_runner.calls[0]["states"], STATES)+check("batch/questions planned once", batch_runner.calls[0]["questions"],+ {f.name: f.question for f in plan_from_json_schema(SCHEMA)})+check("batch/input order", [o["department"] for o in outs], ["billing", "sales"])+check("batch/projection matches decide", outs[0], decide(FakeRunner(ANSWERS), "x", schema=SCHEMA))+check("batch/per-state values", outs[1],+ {"department": "sales", "urgency": 0, "needs_human": False, "priority": 1})++# kwargs are forwarded to predict_batch (batch_size, model, hooks, ...)+batch_runner = FakeBatchRunner(BATCH_ANSWERS)+decide_batch(batch_runner, STATES, schema=SCHEMA, batch_size=4, model="english")+check("batch/forwards predict kwargs", batch_runner.calls[0]["kwargs"],+ {"batch_size": 4, "model": "english"})++# return_details: one DecisionResult per state with the usual fields+details = decide_batch(FakeBatchRunner(BATCH_ANSWERS), STATES, schema=SCHEMA, return_details=True)+check_true("batch/details are DecisionResult", all(isinstance(d, laya.DecisionResult) for d in details))+check("batch/details values", [d.values["department"] for d in details], ["billing", "sales"])+check("batch/details confidence", details[0].confidence["department"], 0.9)+check("batch/details routing", details[0].routing, {"model": "english"})++# questions= is the raw-answers pass-through, exactly like decide()+raw = decide_batch(FakeBatchRunner({"a": {"x": {"type": "noul", "noul": 0.9, "confidence": 0.9}}}),+ ["a"], questions={"x": {"type": "noul", "instructions": "?"}})+check("batch/questions pass-through", raw, [{"x": {"type": "noul", "noul": 0.9, "confidence": 0.9}}])+++class FakeRouterLike:+ """predict_batch(requests, **kwargs) like Router's: one request dict per+ state (no positional questions), routed through route_batch."""++ def __init__(self, answers_by_state):+ self.answers_by_state = answers_by_state+ self.calls = []++ def route_batch(self, requests):+ return [{"model": "english"} for _ in requests]++ def predict_batch(self, requests, **kwargs):+ self.calls.append({"requests": requests, "kwargs": kwargs})+ return [{"answers": self.answers_by_state.get(r["state"], {}),+ "routing": {"model": "english"}} for r in requests]+++router_like = FakeRouterLike(BATCH_ANSWERS)+router_outs = decide_batch(router_like, STATES, schema=SCHEMA, batch_size=3)+expected_questions = {f.name: f.question for f in plan_from_json_schema(SCHEMA)}+check("batch/router one call", len(router_like.calls), 1)+check("batch/router request dicts", [r["state"] for r in router_like.calls[0]["requests"]], STATES)+check("batch/router questions per request",+ all(r["questions"] == expected_questions for r in router_like.calls[0]["requests"]), True)+check("batch/router kwargs forwarded", router_like.calls[0]["kwargs"], {"batch_size": 3})+check("batch/router projection matches agent path", router_outs, outs)++# a string is a sequence of states only accidentally: refuse it like route_batch does+check_raises("batch/rejects string states", TypeError,+ lambda: decide_batch(FakeBatchRunner({}), "abc", schema=SCHEMA))+check_raises("batch/requires schema or questions", ValueError,+ lambda: decide_batch(FakeBatchRunner({}), ["a"]))+check_raises("batch/rejects both", ValueError,+ lambda: decide_batch(FakeBatchRunner({}), ["a"], schema=SCHEMA, questions={}))+check_raises("batch/rejects bad schema", SchemaError,+ lambda: decide_batch(FakeBatchRunner({}), ["a"],+ schema={"type": "object", "properties": {"s": {"type": "string"}}}))+++class NoBatchRunner(FakeRunner):+ pass # has predict() but no predict_batch+++check_raises("batch/runner without predict_batch", TypeError,+ lambda: decide_batch(NoBatchRunner(ANSWERS), ["a"], schema=SCHEMA))++# a runner that answers nothing (hook-skipped states surface as empty values)+noop_runner = FakeBatchRunner({})+check("batch/skipped states", decide_batch(noop_runner, ["unmatched"], schema=SCHEMA), [{}])+check("batch/noop still batches", len(noop_runner.calls), 1)++# --------------------------------------------------------------- pydantic (optional)try:from typing import Literal@@ -203,9 +317,12 @@ except ImportError:# --------------------------------------------------------------- exports and methodscheck("export/decide", callable(laya.decide), True)+check("export/decide_batch", callable(laya.decide_batch), True)check("export/DecisionResult", hasattr(laya, "DecisionResult"), True)check("method/Agent.decide", callable(getattr(laya.Agent, "decide", None)), True)+check("method/Agent.decide_batch", callable(getattr(laya.Agent, "decide_batch", None)), True)check("method/Router.decide", callable(getattr(laya.Router, "decide", None)), True)+check("method/Router.decide_batch", callable(getattr(laya.Router, "decide_batch", None)), True)from laya.onnx_agent import ONNXAgent # noqa: E402check("method/ONNXAgent.decide", callable(getattr(ONNXAgent, "decide", None)), True)