Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(agent): report truncation from predict_batch on current main

Som Tripathi16ecd6033050
diff --git a/laya/agent.py b/laya/agent.py
index 5ca3861c..3ff2af34 100644
--- a/laya/agent.py
+++ b/laya/agent.py
@@ -774,12 +774,13 @@ class Agent(HookRegistry):
items = []
for qid in ids:
q = internal[qid]
- seq, markers, stats = build_sequence(self.tok, state, q, max_len, head_max_len,
- truncate_left=truncate_left, state_ids=state_ids,
- return_stats=True)
+ seq, markers, stats, state_stats = build_sequence(self.tok, state, q, max_len, head_max_len,
+ truncate_left=truncate_left, state_ids=state_ids,
+ return_stats=True, return_truncation_stats=True)
if len(markers) != len(render_options(q)):
raise ValueError("question %r options exceed head_max_len=%d" % (qid, head_max_len))
- items.append({"ids": seq, "markers": markers, "qtype": QTYPES[q["t"]], "options": stats})
+ items.append({"ids": seq, "markers": markers, "qtype": QTYPES[q["t"]], "options": stats,
+ "state_stats": state_stats})
return items
def _amp_enabled_for(self, rows: int) -> bool:
@@ -1047,7 +1048,21 @@ class Agent(HookRegistry):
n_tokens = int(att[row:row + nrows].sum())
answers = self._decode_answers(logits, act, items, ids, internal, row,
**({"lang": lang} if lang else {}))
- usage = {"input_tokens": n_tokens, "output_tokens": 0}
+ # Truncation is a token budget that moves with max_len, head_max_len
+ # and each question's head, so only build_sequence knows it (#174).
+ stats = [item["state_stats"] for item in items]
+ dropped = max(s["state_tokens_dropped"] for s in stats)
+ usage = {
+ "input_tokens": n_tokens,
+ "output_tokens": 0,
+ "state_tokens": stats[0]["state_tokens"],
+ # worst case: the questions share one state, not one head budget
+ "state_tokens_dropped": dropped,
+ "truncated": dropped > 0,
+ "truncated_questions": [
+ qid for qid, s in zip(ids, stats) if s["truncated"]
+ ],
+ }
# Only when a question actually lost options to the head
# budget: an answer chosen from 42 distinguishable spans of
# 58 has a ceiling the caller cannot otherwise see, and a
@@ -1313,6 +1328,11 @@ class Agent(HookRegistry):
58 has a ceiling that is the budget's and not the model's. Questions whose options
all survive are absent, so a request that collapses nothing is unchanged.
+ `usage` also reports whether the state fit: `truncated`, `state_tokens`,
+ `state_tokens_dropped`, and `truncated_questions` (the questions whose head left
+ too little room). A caller that cares whether the answer saw the whole state should
+ read `usage["truncated"]` rather than estimate from the length of what it sent.
+
To score many states at once, see `predict_batch`, which shares forward passes across them.
"""
return self.predict_batch([state], questions, lang=lang, hooks=hooks,
diff --git a/laya/structured.py b/laya/structured.py
index 30e5d3ba..b152613f 100644
--- a/laya/structured.py
+++ b/laya/structured.py
@@ -43,7 +43,7 @@ class DecisionResult:
confidence: Dict[str, float]
probabilities: Dict[str, Dict[str, Any]]
answers: Dict[str, Any]
- usage: Optional[Dict[str, int]] = None
+ usage: Optional[Dict[str, Any]] = None
routing: Optional[Dict[str, Any]] = None
diff --git a/tests/test_batch.py b/tests/test_batch.py
index 47b3b845..b92f62d6 100644
--- a/tests/test_batch.py
+++ b/tests/test_batch.py
@@ -67,6 +67,10 @@ NQ = 2
QUESTIONS = {"a": {"type": "noul", "instructions": "?"}, "b": {"type": "noul", "instructions": "?"}}
+# `_encode_state` items carry the state truncation counts that `predict_batch` reports in `usage` (#174)
+NO_STATE_STATS = {"state_tokens": 0, "state_tokens_used": 0, "state_tokens_dropped": 0, "truncated": False}
+
+
def make_fake():
fake = _agent.Agent.__new__(_agent.Agent)
fake.tok = type("Tok", (), {"pad_token_id": 0})()
@@ -75,7 +79,7 @@ def make_fake():
def _encode_state(state, ids, internal):
# one 3-token, 2-marker item per question; content is irrelevant to the mapping test
- return [{"ids": [1, 2, 3], "markers": [0, 1], "qtype": 2} for _ in ids]
+ return [{"ids": [1, 2, 3], "markers": [0, 1], "qtype": 2, "state_stats": NO_STATE_STATS} for _ in ids]
def _forward(b):
n = b["input_ids"].shape[0]
@@ -105,7 +109,8 @@ def make_length_fake():
fake.encoded.append(state)
limit = overrides.get("max_len", 100)
# Question rows have different lengths; sorting must use the longest.
- return [{"ids": [state["id"] + 1] * min(limit, size), "markers": [0, 1], "qtype": 2}
+ return [{"ids": [state["id"] + 1] * min(limit, size), "markers": [0, 1], "qtype": 2,
+ "state_stats": NO_STATE_STATS}
for size in (3, state["length"])]
def forward(batch):
diff --git a/tests/test_hooks.py b/tests/test_hooks.py
index 1b18a7cc..870cded7 100644
--- a/tests/test_hooks.py
+++ b/tests/test_hooks.py
@@ -55,6 +55,10 @@ NQ = 2
QUESTIONS = {"a": {"type": "noul", "instructions": "?"}, "b": {"type": "noul", "instructions": "?"}}
+# `_encode_state` items carry the state truncation counts that `predict_batch` reports in `usage` (#174)
+NO_STATE_STATS = {"state_tokens": 0, "state_tokens_used": 0, "state_tokens_dropped": 0, "truncated": False}
+
+
def make_fake():
"""A real `predict_batch` with the three composed helpers stubbed out."""
fake = Agent.__new__(Agent)
@@ -65,7 +69,7 @@ def make_fake():
def _encode_state(state, ids, internal):
fake._encode_states.append(state)
- return [{"ids": [1, 2, 3], "markers": [0, 1], "qtype": 2} for _ in ids]
+ return [{"ids": [1, 2, 3], "markers": [0, 1], "qtype": 2, "state_stats": NO_STATE_STATS} for _ in ids]
def _forward(b):
n = b["input_ids"].shape[0]
@@ -392,7 +396,7 @@ def make_len_fake():
def _encode(state, ids, internal, max_len=None, head_max_len=None):
fake._seen.append((max_len, head_max_len))
- return [{"ids": [1, 2, 3], "markers": [0, 1], "qtype": 2} for _ in ids]
+ return [{"ids": [1, 2, 3], "markers": [0, 1], "qtype": 2, "state_stats": NO_STATE_STATS} for _ in ids]
fake._encode_state = _encode
return fake
diff --git a/tests/test_hooks_api.py b/tests/test_hooks_api.py
index 6ca39ec8..3262076a 100644
--- a/tests/test_hooks_api.py
+++ b/tests/test_hooks_api.py
@@ -295,6 +295,7 @@ for label, cls in (("LayaRouter", LayaRouter), ("LayaGuardrail", LayaGuardrail),
from laya.common import build_sequence, collapsed_options # noqa: E402
check_param("build_sequence", build_sequence, "return_stats", False)
+check_param("build_sequence", build_sequence, "return_truncation_stats", False)
check("collapsed_options/nothing collapsed is empty",
collapsed_options(["q"], [{"options": {"options": 3, "options_distinct": 3,
"tokens_per_option": None}}]), {})
diff --git a/tests/test_predict_long.py b/tests/test_predict_long.py
index 67d54d69..9abe1df9 100644
--- a/tests/test_predict_long.py
+++ b/tests/test_predict_long.py
@@ -74,6 +74,10 @@ def make_agent(batch_result_fn):
return a
+# `_encode_state` items carry the state truncation counts that `predict_batch` reports in `usage` (#174)
+NO_STATE_STATS = {"state_tokens": 0, "state_tokens_used": 0, "state_tokens_dropped": 0, "truncated": False}
+
+
def make_real_agent():
"""`predict_long` on the real `predict_batch`/`system_one`, with only the three composed
helpers stubbed -- the harness `tests/test_hooks.py` uses, so no weights are involved.
@@ -97,7 +101,7 @@ def make_real_agent():
def _encode_state(state, ids, internal, **overrides):
a._encoded.append(state)
- return [{"ids": [1, 2, 3], "markers": [0, 1], "qtype": 2} for _ in ids]
+ return [{"ids": [1, 2, 3], "markers": [0, 1], "qtype": 2, "state_stats": NO_STATE_STATS} for _ in ids]
def _forward(b):
n = b["input_ids"].shape[0]
diff --git a/tests/test_truncation.py b/tests/test_truncation.py
index 886e61c0..e52985f4 100644
--- a/tests/test_truncation.py
+++ b/tests/test_truncation.py
@@ -42,8 +42,10 @@ class StubTok:
sep_token_id = 3
pad_token_id = 0
- def __call__(self, text, add_special_tokens=True):
- return {"input_ids": [10 + (len(w) % 50) for w in text.split()]}
+ def __call__(self, text, add_special_tokens=True, truncation=False, max_length=None):
+ ids = [10 + (len(w) % 50) for w in text.split()]
+ # build_sequence caps each option at the tokenizer (truncation=True, max_length=48)
+ return {"input_ids": ids[:max_length] if truncation and max_length is not None else ids}
TOK = StubTok()
@@ -150,7 +152,7 @@ import inspect # noqa: E402
from laya import agent as _agent # noqa: E402
-_src = inspect.getsource(_agent.Agent.system_one)
+_src = inspect.getsource(_agent.Agent._encode_state) + inspect.getsource(_agent.Agent.predict_batch)
check_true("usage/asks build_sequence for the stats", "return_truncation_stats=True" in _src)
check_true("usage/publishes the flag", '"truncated": dropped > 0' in _src)
check_true("usage/publishes the dropped count", '"state_tokens_dropped": dropped' in _src)