Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(onnx,structured): keep ONNXAgent and decide_batch in step after the batch merges

Nandakishorbd1513e92486
diff --git a/README.md b/README.md
index 4cd6755e..d28fe059 100644
--- a/README.md
+++ b/README.md
@@ -790,8 +790,7 @@ vs 723 ms batched (**3.6×**); 16 mixed English/German states through a `Router`
1903 ms (**1.6×**). On CPU the same workloads gave **1.6×** and **2.0×**. Projected values matched
the one-by-one loop 8/8 and 16/16 on both devices, as expected wherever argmax is not at a
threshold — verify against your own decision boundaries. `laya.decide_batch(runner, states, ...)`
-is the function form. `ONNXAgent` has no `predict_batch` yet, so it raises `TypeError` instead of
-silently looping.
+is the function form, and `ONNXAgent.decide_batch` runs the same thing on an ONNX export.
---
diff --git a/docs/structured.md b/docs/structured.md
index 30863c53..6ac170db 100644
--- a/docs/structured.md
+++ b/docs/structured.md
@@ -104,9 +104,9 @@ Limits: `MAX_PROPERTIES = 32`, `MAX_OPTIONS = 32`, `MAX_SCORE_LEVELS = 10`.
| function | purpose |
|---|---|
-| `laya.decide(runner, state, schema=..., *, questions=..., return_details=..., **predict_kwargs)` | the free function, works for `Agent` and `Router` |
+| `laya.decide(runner, state, schema=..., *, questions=..., return_details=..., min_confidence=..., **predict_kwargs)` | the free function, works for `Agent` and `Router` |
| `agent.decide(state, schema=..., ...)` / `router.decide(state, schema=..., ...)` | convenience methods |
-| `laya.decide_batch(runner, states, schema=..., ...)` / `runner.decide_batch(...)` | the same over many states, one batched call |
+| `laya.decide_batch(runner, states, schema=..., ...)` / `agent.decide_batch(...)` / `router.decide_batch(...)` | the same over many states, one batched call |
| `questions_from_json_schema(schema)` | schema to Laya questions |
| `questions_from_pydantic(model)` | pydantic model to questions (requires pydantic) |
| `answers_to_json(answers, schema)` | project raw answers onto schema values |
@@ -135,8 +135,8 @@ results = router.decide_batch(states, schema=Ticket, return_details=True, batch_
On a `Router` each state is still routed on its own, so one call can span checkpoints. Keyword
arguments reach `predict_batch`, so `batch_size=`, `model=` and hooks work as they do for `decide`.
-A runner without `predict_batch` (such as `ONNXAgent`) raises `TypeError` rather than silently
-falling back to a loop — call `decide` per state there. Batching can shift borderline argmaxes the
+`Agent`, `ONNXAgent` and `Router` all have it; a runner without `predict_batch` raises
+`TypeError` rather than silently falling back to a loop — call `decide` per state there. Batching can shift borderline argmaxes the
same way `predict_batch` does; the README records the measured speedups for both devices.
## Confidence and probabilities
diff --git a/laya/agent.py b/laya/agent.py
index d1ae4aa4..5ca3861c 100644
--- a/laya/agent.py
+++ b/laya/agent.py
@@ -1356,7 +1356,8 @@ class Agent(HookRegistry):
def decide_batch(self, states: List[Union[str, dict, list]], schema: Any = None, *,
questions: Optional[Dict[str, Any]] = None,
- return_details: bool = False, **predict_kwargs) -> List[Any]:
+ return_details: bool = False, min_confidence: Optional[float] = None,
+ **predict_kwargs) -> List[Any]:
"""Answer many states against one schema (JSON schema or pydantic model) in one batched call.
The throughput form of :meth:`decide`: the schema is planned once and its questions
@@ -1367,7 +1368,8 @@ class Agent(HookRegistry):
"""
from .structured import decide_batch as _decide_batch
return _decide_batch(self, states, schema, questions=questions,
- return_details=return_details, **predict_kwargs)
+ return_details=return_details, min_confidence=min_confidence,
+ **predict_kwargs)
def __repr__(self) -> str:
return "Agent(model_id=%r, device=%s)" % (self.model_id, getattr(self, "device", None))
diff --git a/laya/onnx_agent.py b/laya/onnx_agent.py
index 65e10574..79e3a863 100644
--- a/laya/onnx_agent.py
+++ b/laya/onnx_agent.py
@@ -370,7 +370,10 @@ class ONNXAgent(HookRegistry):
def predict_long(self, state: Union[str, dict, list], questions: Dict[str, Dict[str, Any]],
window: Optional[int] = None, stride: Optional[int] = None,
aggregate: str = "auto", batch_size: Optional[int] = None,
- lang: Optional[str] = None) -> Dict[str, Any]:
+ lang: Optional[str] = None,
+ hooks=None, on_predict_start=None, on_predict_end=None,
+ hooks_raise: Optional[bool] = None,
+ hooks_timeout: Optional[float] = None) -> Dict[str, Any]:
"""Evaluate questions over a state longer than the context window, scanning it in
overlapping windows and aggregating per question.
@@ -402,10 +405,21 @@ class ONNXAgent(HookRegistry):
aggregate: "auto" (the per-type rules above) is the only mode for now.
batch_size: Cap on windows per session run, to bound peak memory on very long states.
lang: Per-language temperature selection, as in `system_one`.
+ hooks, on_predict_start, on_predict_end, hooks_raise, hooks_timeout: per-call hooks,
+ with the same contract as `Agent.predict_long`: they wrap the inference that
+ answers the state, a start hook that answers with `ctx.skip(...)` gets
+ `usage["windows"] == 0` and no window attribution, and a rewritten scan is
+ aggregated without `answer["window"]`.
Returns a single result dict, the same shape as `system_one`, with `usage["windows"]`
added.
"""
+ from .agent import _start_evidence, _with_start_probe
+ from .hooks import aggregate_usage
+
+ hook_kwargs = {"hooks": hooks, "on_predict_start": on_predict_start,
+ "on_predict_end": on_predict_end, "hooks_raise": hooks_raise,
+ "hooks_timeout": hooks_timeout}
if aggregate != "auto":
raise ValueError("predict_long: only aggregate='auto' is supported")
max_len = self.cfg.get("max_len", 512)
@@ -417,9 +431,14 @@ class ONNXAgent(HookRegistry):
serialize_state(state).replace(self.tok.mask_token, " "),
add_special_tokens=False,
)["input_ids"]
- # Fits in one window: identical to a plain call, no windowing overhead.
+ # Fits in one window: identical to a plain call, no windowing overhead. `windows` is still
+ # written, as on the torch Agent: 1 for a window the model read, 0 for a hook's answer.
if len(state_ids) <= budget:
- return self.system_one(state, questions, lang=lang)
+ probe, evidence = _start_evidence()
+ single = dict(self.system_one(state, questions, lang=lang,
+ **_with_start_probe(hook_kwargs, probe)))
+ single["usage"] = {**(single.get("usage") or {}), "windows": 0 if evidence["answered"] else 1}
+ return single
step = stride if (stride and stride > 0) else max(1, budget // 2)
windows, starts = [], []
@@ -433,7 +452,36 @@ class ONNXAgent(HookRegistry):
break
i += step
- results = self.predict_batch(windows, questions, batch_size=batch_size, lang=lang)
+ probe, evidence = _start_evidence()
+ # A copy, so a start hook that mutates `ctx.states` in place cannot shift `starts`.
+ results = self.predict_batch(list(windows), questions, batch_size=batch_size, lang=lang,
+ **_with_start_probe(hook_kwargs, probe))
+
+ if evidence["answered"]:
+ # A hook answered the document before any window was scored: pass that answer
+ # through unattributed, with no window counted (the torch Agent's rule).
+ if len(results) != 1:
+ raise ValueError(
+ "predict_long: a start hook answered this state with %d results; ctx.skip()"
+ " takes one result for the document, not one per window" % len(results))
+ warnings.warn("laya: predict_long: a hook answered the state before it was scanned, so "
+ "no window decided the result and none is reported", RuntimeWarning,
+ stacklevel=2)
+ document = dict(results[0])
+ document["usage"] = dict(document.get("usage") or {})
+ document["usage"]["windows"] = 0
+ return document
+
+ rewritten = evidence["states"] is not None and evidence["states"] != windows
+ if len(results) != len(windows) and not rewritten:
+ raise ValueError(
+ "predict_long: the state was split into %d windows and the call returned %d results"
+ % (len(windows), len(results)))
+ if not results:
+ warnings.warn("laya: predict_long: a start hook left no states to score, so the call"
+ " aggregated nothing and returns no answers", RuntimeWarning, stacklevel=2)
+ return {"model": "laya-rl-agent-onnx", "answers": {},
+ "usage": {**aggregate_usage(results), "windows": 0}}
ids = list(questions.keys())
internal = {qid: self._to_internal(questions[qid]) for qid in ids}
@@ -450,9 +498,10 @@ class ONNXAgent(HookRegistry):
best = max(range(len(per)), key=lambda j: float(per[j]["answer_confidence"]))
ans = per[best]
# Name the window that decided; the probability is that window's, not the document's.
- ans["window"] = {"index": best, "token_start": starts[best],
- "token_end": min(starts[best] + budget, len(state_ids)),
- "count": len(windows)}
+ if not rewritten:
+ ans["window"] = {"index": best, "token_start": starts[best],
+ "token_end": min(starts[best] + budget, len(state_ids)),
+ "count": len(results)}
answers[qid] = ans
# Aggregate usage generically so fields predict_batch may grow later are propagated
# rather than silently dropped, then record the window count.
@@ -461,7 +510,7 @@ class ONNXAgent(HookRegistry):
for key, val in r["usage"].items():
usage[key] = (usage.get(key, 0) + val) if isinstance(val, (int, float)) else val
usage["output_tokens"] = 0
- usage["windows"] = len(windows)
+ usage["windows"] = len(results)
return {"model": "laya-rl-agent-onnx", "answers": answers, "usage": usage}
def _infer(self, state: Union[str, dict, list], questions: Dict[str, Dict[str, Any]],
@@ -654,4 +703,14 @@ class ONNXAgent(HookRegistry):
return _decide(self, state, schema, questions=questions,
return_details=return_details, min_confidence=min_confidence, **predict_kwargs)
+ def decide_batch(self, states: List[Union[str, dict, list]], schema: Any = None, *,
+ questions: Optional[Dict[str, Dict[str, Any]]] = None,
+ return_details: bool = False, min_confidence: Optional[float] = None,
+ **predict_kwargs) -> List[Any]:
+ """Answer many states against one schema through `predict_batch`; see `laya.structured`."""
+ from laya.structured import decide_batch as _decide_batch
+ return _decide_batch(self, states, schema, questions=questions,
+ return_details=return_details, min_confidence=min_confidence,
+ **predict_kwargs)
+
predict = system_one
diff --git a/laya/router.py b/laya/router.py
index 404f700b..9a2f3cc8 100644
--- a/laya/router.py
+++ b/laya/router.py
@@ -875,7 +875,8 @@ class Router(HookRegistry):
def decide_batch(self, states: Sequence[Any], schema: Any = None, *,
questions: Optional[Dict[str, Any]] = None,
- return_details: bool = False, **predict_kwargs) -> List[Any]:
+ return_details: bool = False, min_confidence: Optional[float] = None,
+ **predict_kwargs) -> List[Any]:
"""Answer many states against one schema (JSON schema or pydantic model) in one batched call.
The throughput form of :meth:`decide`: the schema is planned once and its questions
@@ -886,7 +887,8 @@ class Router(HookRegistry):
"""
from .structured import decide_batch as _decide_batch
return _decide_batch(self, states, schema, questions=questions,
- return_details=return_details, **predict_kwargs)
+ return_details=return_details, min_confidence=min_confidence,
+ **predict_kwargs)
def __enter__(self):
return self
diff --git a/laya/structured.py b/laya/structured.py
index 84e68095..30e5d3ba 100644
--- a/laya/structured.py
+++ b/laya/structured.py
@@ -292,7 +292,8 @@ def decide(runner, state: Any, schema: Any = None, *, questions: Optional[Dict[s
def decide_batch(runner, states: Sequence[Any], schema: Any = None, *,
questions: Optional[Dict[str, Any]] = None,
- return_details: bool = False, **predict_kwargs) -> List[Any]:
+ return_details: bool = False, min_confidence: Optional[float] = None,
+ **predict_kwargs) -> List[Any]:
"""Answer many states against one schema in one batched call, in input order.
The throughput form of :meth:`decide`: the schema is planned once and its questions
@@ -302,14 +303,15 @@ def decide_batch(runner, states: Sequence[Any], schema: Any = None, *,
``decide`` does. Pass exactly one of ``schema`` or ``questions``; extra keyword
arguments (``batch_size=``, ``model=``, ``hooks=``, ...) are forwarded to
``runner.predict_batch``. With ``return_details=True`` each item is a
- ``DecisionResult``.
+ ``DecisionResult``. ``min_confidence`` works as in ``decide``: a field whose answer falls
+ below it comes back as ``None``, with the answer kept in the details.
Both batch calling conventions are handled: an ``Agent``-like runner receives
``(states, questions)``, while a ``Router``-like runner (one exposing
``route_batch``) receives one ``{"state": ..., "questions": ...}`` request per
state, so states may route to different checkpoints.
- Not every runner batches: ``ONNXAgent`` has no ``predict_batch`` yet, so passing one
+ ``Agent``, ``ONNXAgent`` and ``Router`` all batch. A runner with no ``predict_batch``
raises ``TypeError`` here rather than silently degrading to N sequential ``decide``
calls -- loop ``decide`` yourself when the runner cannot batch.
"""
@@ -318,6 +320,8 @@ def decide_batch(runner, states: Sequence[Any], schema: Any = None, *,
if isinstance(states, (str, bytes)) or not isinstance(states, SequenceABC):
raise TypeError("states must be a sequence of states, not %s" % type(states).__name__)
+ mc = check_min_confidence(min_confidence) if min_confidence is not None else None
+
fields: Optional[List[_Field]] = None
if schema is not None:
fields = plan_from_json_schema(_schema_of(schema))
@@ -338,6 +342,11 @@ def decide_batch(runner, states: Sequence[Any], schema: Any = None, *,
# Agent convention: a list of states evaluated against one question set.
results = predict_batch(list(states), questions, **predict_kwargs)
+ if mc is not None:
+ # Flagged here rather than passed down, so a runner whose predict_batch predates the
+ # keyword still gets the same projection.
+ flag_low_confidence([r for r in results if isinstance(r, dict)], mc)
+
def _one(r: Dict[str, Any]) -> Any:
answers = r.get("answers", {}) or {}
values = _project(answers, fields) if fields is not None else dict(answers)
diff --git a/tests/test_onnx_long.py b/tests/test_onnx_long.py
index 855e1ad6..035dbc7f 100644
--- a/tests/test_onnx_long.py
+++ b/tests/test_onnx_long.py
@@ -124,11 +124,17 @@ short_result = short.predict_long("aa", QUESTIONS)
check("short/one session run for the two questions", short.session.calls, [2])
check_true("short/no window fields added",
all("window" not in a for a in short_result["answers"].values()))
+# `usage["windows"]` is total on every path since #577 (1 for a state that fit one window), so the
+# comparison is system_one's payload with that one key added.
+def _with_one_window(result):
+ return dict(result, usage=dict(result["usage"], windows=1))
+
+
check("short/delegates byte-for-byte to system_one",
- short_result, _bare_onnx().system_one("aa", QUESTIONS))
+ short_result, _with_one_window(_bare_onnx().system_one("aa", QUESTIONS)))
check("short/lang forwards to system_one",
_bare_onnx().predict_long("aa", QUESTIONS, lang="de"),
- _bare_onnx().system_one("aa", QUESTIONS, lang="de"))
+ _with_one_window(_bare_onnx().system_one("aa", QUESTIONS, lang="de")))
# ---------------------------------------------------------------- long state windows and aggregates
diff --git a/tests/test_structured_docs.py b/tests/test_structured_docs.py
index c06035de..28a1a82f 100644
--- a/tests/test_structured_docs.py
+++ b/tests/test_structured_docs.py
@@ -385,7 +385,7 @@ def main() -> int:
check("decide/keyword-only arguments",
[p.name for p in inspect.signature(S.decide).parameters.values()
if p.kind is p.KEYWORD_ONLY],
- ["questions", "return_details"])
+ ["questions", "return_details", "min_confidence"])
check_true("decide/forwards predict kwargs",
any(p.kind is p.VAR_KEYWORD for p in inspect.signature(S.decide).parameters.values()),
"the page promises `**predict_kwargs`")