Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
fix(agent): report truncation from predict_batch on current main
Som Tripathi16ecd6033050
diff --git a/laya/agent.py b/laya/agent.pyindex 5ca3861c..3ff2af34 100644--- a/laya/agent.py+++ b/laya/agent.py@@ -774,12 +774,13 @@ class Agent(HookRegistry):items = []for qid in ids:q = internal[qid]- seq, markers, stats = build_sequence(self.tok, state, q, max_len, head_max_len,- truncate_left=truncate_left, state_ids=state_ids,- return_stats=True)+ seq, markers, stats, state_stats = build_sequence(self.tok, state, q, max_len, head_max_len,+ truncate_left=truncate_left, state_ids=state_ids,+ return_stats=True, return_truncation_stats=True)if len(markers) != len(render_options(q)):raise ValueError("question %r options exceed head_max_len=%d" % (qid, head_max_len))- items.append({"ids": seq, "markers": markers, "qtype": QTYPES[q["t"]], "options": stats})+ items.append({"ids": seq, "markers": markers, "qtype": QTYPES[q["t"]], "options": stats,+ "state_stats": state_stats})return itemsdef _amp_enabled_for(self, rows: int) -> bool:@@ -1047,7 +1048,21 @@ class Agent(HookRegistry):n_tokens = int(att[row:row + nrows].sum())answers = self._decode_answers(logits, act, items, ids, internal, row,**({"lang": lang} if lang else {}))- usage = {"input_tokens": n_tokens, "output_tokens": 0}+ # Truncation is a token budget that moves with max_len, head_max_len+ # and each question's head, so only build_sequence knows it (#174).+ stats = [item["state_stats"] for item in items]+ dropped = max(s["state_tokens_dropped"] for s in stats)+ usage = {+ "input_tokens": n_tokens,+ "output_tokens": 0,+ "state_tokens": stats[0]["state_tokens"],+ # worst case: the questions share one state, not one head budget+ "state_tokens_dropped": dropped,+ "truncated": dropped > 0,+ "truncated_questions": [+ qid for qid, s in zip(ids, stats) if s["truncated"]+ ],+ }# Only when a question actually lost options to the head# budget: an answer chosen from 42 distinguishable spans of# 58 has a ceiling the caller cannot otherwise see, and a@@ -1313,6 +1328,11 @@ class Agent(HookRegistry):58 has a ceiling that is the budget's and not the model's. Questions whose optionsall survive are absent, so a request that collapses nothing is unchanged.+ `usage` also reports whether the state fit: `truncated`, `state_tokens`,+ `state_tokens_dropped`, and `truncated_questions` (the questions whose head left+ too little room). A caller that cares whether the answer saw the whole state should+ read `usage["truncated"]` rather than estimate from the length of what it sent.+To score many states at once, see `predict_batch`, which shares forward passes across them."""return self.predict_batch([state], questions, lang=lang, hooks=hooks,diff --git a/laya/structured.py b/laya/structured.pyindex 30e5d3ba..b152613f 100644--- a/laya/structured.py+++ b/laya/structured.py@@ -43,7 +43,7 @@ class DecisionResult:confidence: Dict[str, float]probabilities: Dict[str, Dict[str, Any]]answers: Dict[str, Any]- usage: Optional[Dict[str, int]] = None+ usage: Optional[Dict[str, Any]] = Nonerouting: Optional[Dict[str, Any]] = Nonediff --git a/tests/test_batch.py b/tests/test_batch.pyindex 47b3b845..b92f62d6 100644--- a/tests/test_batch.py+++ b/tests/test_batch.py@@ -67,6 +67,10 @@ NQ = 2QUESTIONS = {"a": {"type": "noul", "instructions": "?"}, "b": {"type": "noul", "instructions": "?"}}+# `_encode_state` items carry the state truncation counts that `predict_batch` reports in `usage` (#174)+NO_STATE_STATS = {"state_tokens": 0, "state_tokens_used": 0, "state_tokens_dropped": 0, "truncated": False}++def make_fake():fake = _agent.Agent.__new__(_agent.Agent)fake.tok = type("Tok", (), {"pad_token_id": 0})()@@ -75,7 +79,7 @@ def make_fake():def _encode_state(state, ids, internal):# one 3-token, 2-marker item per question; content is irrelevant to the mapping test- return [{"ids": [1, 2, 3], "markers": [0, 1], "qtype": 2} for _ in ids]+ return [{"ids": [1, 2, 3], "markers": [0, 1], "qtype": 2, "state_stats": NO_STATE_STATS} for _ in ids]def _forward(b):n = b["input_ids"].shape[0]@@ -105,7 +109,8 @@ def make_length_fake():fake.encoded.append(state)limit = overrides.get("max_len", 100)# Question rows have different lengths; sorting must use the longest.- return [{"ids": [state["id"] + 1] * min(limit, size), "markers": [0, 1], "qtype": 2}+ return [{"ids": [state["id"] + 1] * min(limit, size), "markers": [0, 1], "qtype": 2,+ "state_stats": NO_STATE_STATS}for size in (3, state["length"])]def forward(batch):diff --git a/tests/test_hooks.py b/tests/test_hooks.pyindex 1b18a7cc..870cded7 100644--- a/tests/test_hooks.py+++ b/tests/test_hooks.py@@ -55,6 +55,10 @@ NQ = 2QUESTIONS = {"a": {"type": "noul", "instructions": "?"}, "b": {"type": "noul", "instructions": "?"}}+# `_encode_state` items carry the state truncation counts that `predict_batch` reports in `usage` (#174)+NO_STATE_STATS = {"state_tokens": 0, "state_tokens_used": 0, "state_tokens_dropped": 0, "truncated": False}++def make_fake():"""A real `predict_batch` with the three composed helpers stubbed out."""fake = Agent.__new__(Agent)@@ -65,7 +69,7 @@ def make_fake():def _encode_state(state, ids, internal):fake._encode_states.append(state)- return [{"ids": [1, 2, 3], "markers": [0, 1], "qtype": 2} for _ in ids]+ return [{"ids": [1, 2, 3], "markers": [0, 1], "qtype": 2, "state_stats": NO_STATE_STATS} for _ in ids]def _forward(b):n = b["input_ids"].shape[0]@@ -392,7 +396,7 @@ def make_len_fake():def _encode(state, ids, internal, max_len=None, head_max_len=None):fake._seen.append((max_len, head_max_len))- return [{"ids": [1, 2, 3], "markers": [0, 1], "qtype": 2} for _ in ids]+ return [{"ids": [1, 2, 3], "markers": [0, 1], "qtype": 2, "state_stats": NO_STATE_STATS} for _ in ids]fake._encode_state = _encodereturn fakediff --git a/tests/test_hooks_api.py b/tests/test_hooks_api.pyindex 6ca39ec8..3262076a 100644--- a/tests/test_hooks_api.py+++ b/tests/test_hooks_api.py@@ -295,6 +295,7 @@ for label, cls in (("LayaRouter", LayaRouter), ("LayaGuardrail", LayaGuardrail),from laya.common import build_sequence, collapsed_options # noqa: E402check_param("build_sequence", build_sequence, "return_stats", False)+check_param("build_sequence", build_sequence, "return_truncation_stats", False)check("collapsed_options/nothing collapsed is empty",collapsed_options(["q"], [{"options": {"options": 3, "options_distinct": 3,"tokens_per_option": None}}]), {})diff --git a/tests/test_predict_long.py b/tests/test_predict_long.pyindex 67d54d69..9abe1df9 100644--- a/tests/test_predict_long.py+++ b/tests/test_predict_long.py@@ -74,6 +74,10 @@ def make_agent(batch_result_fn):return a+# `_encode_state` items carry the state truncation counts that `predict_batch` reports in `usage` (#174)+NO_STATE_STATS = {"state_tokens": 0, "state_tokens_used": 0, "state_tokens_dropped": 0, "truncated": False}++def make_real_agent():"""`predict_long` on the real `predict_batch`/`system_one`, with only the three composedhelpers stubbed -- the harness `tests/test_hooks.py` uses, so no weights are involved.@@ -97,7 +101,7 @@ def make_real_agent():def _encode_state(state, ids, internal, **overrides):a._encoded.append(state)- return [{"ids": [1, 2, 3], "markers": [0, 1], "qtype": 2} for _ in ids]+ return [{"ids": [1, 2, 3], "markers": [0, 1], "qtype": 2, "state_stats": NO_STATE_STATS} for _ in ids]def _forward(b):n = b["input_ids"].shape[0]diff --git a/tests/test_truncation.py b/tests/test_truncation.pyindex 886e61c0..e52985f4 100644--- a/tests/test_truncation.py+++ b/tests/test_truncation.py@@ -42,8 +42,10 @@ class StubTok:sep_token_id = 3pad_token_id = 0- def __call__(self, text, add_special_tokens=True):- return {"input_ids": [10 + (len(w) % 50) for w in text.split()]}+ def __call__(self, text, add_special_tokens=True, truncation=False, max_length=None):+ ids = [10 + (len(w) % 50) for w in text.split()]+ # build_sequence caps each option at the tokenizer (truncation=True, max_length=48)+ return {"input_ids": ids[:max_length] if truncation and max_length is not None else ids}TOK = StubTok()@@ -150,7 +152,7 @@ import inspect # noqa: E402from laya import agent as _agent # noqa: E402-_src = inspect.getsource(_agent.Agent.system_one)+_src = inspect.getsource(_agent.Agent._encode_state) + inspect.getsource(_agent.Agent.predict_batch)check_true("usage/asks build_sequence for the stats", "return_truncation_stats=True" in _src)check_true("usage/publishes the flag", '"truncated": dropped > 0' in _src)check_true("usage/publishes the dropped count", '"state_tokens_dropped": dropped' in _src)