Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(predict_long): skip the budget recompute when nothing the scan was sized against moved

Adeshina Lasisi5ebf326760fa
diff --git a/laya/agent.py b/laya/agent.py
index f336e44a..700325e9 100644
--- a/laya/agent.py
+++ b/laya/agent.py
@@ -293,7 +293,7 @@ def _start_evidence():
return probe, evidence
-def _check_scan_budget(agent, evidence, sized, cfg_max_len, cfg_head_max_len):
+def _check_scan_budget(agent, evidence, sized, cfg_max_len, cfg_head_max_len, asked=None):
"""Raise when the budget or questions in force leave less room than the scan was sized for.
`predict_long` sizes its windows from the agent's config, before any hook has run. A start hook
@@ -323,6 +323,13 @@ def _check_scan_budget(agent, evidence, sized, cfg_max_len, cfg_head_max_len):
eff_head = evidence.get("head_max_len")
eff_max_len = cfg_max_len if eff_max_len is None else eff_max_len
eff_head = cfg_head_max_len if eff_head is None else eff_head
+ if eff_max_len == cfg_max_len and eff_head == cfg_head_max_len and (
+ asked is None or questions == asked):
+ # Nothing the scan was sized against moved, so there is nothing to recompute -- and
+ # recomputing anyway costs a tokenization of every question head, which `window_budget`
+ # already paid and `_encode_state` will pay again. That showed up as a third head
+ # tokenization in `test_question_token_reuse`, which asserts two.
+ return
try:
internal = [agent._to_internal(q) for q in questions.values()]
except Exception:
@@ -1486,7 +1493,7 @@ class Agent(HookRegistry):
# fit one window was silently truncated by a re-budgeting hook and still reported
# `windows: 1`, i.e. "the model read all of it" -- measured, 138 of 240 state tokens
# never reached the model, while a longer document on the identical input hard-failed.
- _check_scan_budget(self, evidence, budget, max_len, head_max_len)
+ _check_scan_budget(self, evidence, budget, max_len, head_max_len, questions)
single["usage"] = {**(single.get("usage") or {}), "windows": 0 if evidence["answered"] else 1}
return single
@@ -1512,7 +1519,7 @@ class Agent(HookRegistry):
batch_size)
results = self.predict_batch(list(windows), questions, batch_size=cap, lang=lang,
**_with_start_probe(hook_kwargs, probe))
- _check_scan_budget(self, evidence, budget, max_len, head_max_len)
+ _check_scan_budget(self, evidence, budget, max_len, head_max_len, questions)
if evidence["answered"]:
# The hook replaced the call before any window was scored. Aggregating over its payload
diff --git a/laya/onnx_agent.py b/laya/onnx_agent.py
index 23ec2ccd..08d79842 100644
--- a/laya/onnx_agent.py
+++ b/laya/onnx_agent.py
@@ -490,7 +490,7 @@ class ONNXAgent(HookRegistry):
# fit one window was silently truncated by a re-budgeting hook and still reported
# `windows: 1`, i.e. "the model read all of it" -- measured, 138 of 240 state tokens
# never reached the model, while a longer document on the identical input hard-failed.
- _check_scan_budget(self, evidence, budget, max_len, head_max_len)
+ _check_scan_budget(self, evidence, budget, max_len, head_max_len, questions)
single["usage"] = {**(single.get("usage") or {}), "windows": 0 if evidence["answered"] else 1}
return single
@@ -519,7 +519,7 @@ class ONNXAgent(HookRegistry):
# `ctx.max_len`/`ctx.head_max_len` identically), so leaving it off here meant the bug was
# fully live on this path while the other agent refused the identical input -- measured,
# 34.6% of a document reaching no model.
- _check_scan_budget(self, evidence, budget, max_len, head_max_len)
+ _check_scan_budget(self, evidence, budget, max_len, head_max_len, questions)
if evidence["answered"]:
# A hook answered the document before any window was scored: pass that answer
diff --git a/tests/test_predict_long.py b/tests/test_predict_long.py
index 6ea2a7ff..30f10c92 100644
--- a/tests/test_predict_long.py
+++ b/tests/test_predict_long.py
@@ -20,7 +20,6 @@ import numpy as np # noqa: E402
import laya.agent as agent_mod
from laya.agent import Agent, _check_scan_budget, _start_evidence
-from laya import common
from laya.onnx_agent import ONNXAgent # noqa: E402
PASS, FAIL = [], []
@@ -894,9 +893,13 @@ def _budget_message():
check_raises("hook budget/a hook that shrinks the room is refused", ValueError,
lambda: _check_scan_budget(_probe_agent, _evidence(head_max_len=60), 72, *_CFG))
+# `asked` is what predict_long was called with; the recorded questions are what the chain left.
check_raises("hook budget/rewriting ctx.questions is caught too", ValueError,
lambda: _check_scan_budget(_probe_agent, _evidence(questions={"a": q_many(16)}),
- 72, *_CFG))
+ 72, *_CFG, asked={"a": q_many(2)}))
+check_true("hook budget/questions untouched skips the recompute entirely",
+ _check_scan_budget(_probe_agent, _evidence(questions={"a": q_many(16)}), 72, *_CFG,
+ asked={"a": q_many(16)}) is None, "nothing moved")
check_true("hook budget/an unchanged budget is allowed",
_check_scan_budget(_probe_agent, _evidence(), 72, *_CFG) is None, "no hook")
check_true("hook budget/a budget widened together is allowed",
@@ -917,7 +920,8 @@ check_true("hook budget/the refusal names both budgets and both rooms",
_seen_budget = []
_real_check = agent_mod._check_scan_budget
try:
- agent_mod._check_scan_budget = lambda ag, ev, sized, ml, hm: _seen_budget.append((sized, ml, hm))
+ agent_mod._check_scan_budget = (
+ lambda ag, ev, sized, ml, hm, asked=None: _seen_budget.append((sized, ml, hm)))
_wired = make_agent(lambda states, qs: [
{"model": "m", "answers": {"a": {"choice": "x", "answer_confidence": 0.5}},
"usage": {"input_tokens": 1}} for _ in states])
@@ -932,7 +936,8 @@ check("hook budget/the torch scan checks its budget exactly once", len(_seen_bud
# them under a widened head (both 36) makes the mutant survive, which is how it slipped through.
check_raises("hook budget/the smallest room wins across questions", ValueError,
lambda: _check_scan_budget(
- _probe_agent, _evidence(questions={"a": q_many(2), "b": q_many(16)}), 72, *_CFG))
+ _probe_agent, _evidence(questions={"a": q_many(2), "b": q_many(16)}), 72, *_CFG,
+ asked={"a": q_many(2)}))
# What the probe RECORDS is half the check: synthesising an evidence dict in the tests above leaves
# the recording itself unpinned, and dropping either field made the checker silently inert.
@@ -964,7 +969,8 @@ check_raises("questions/the torch scan validates before _to_internal", ValueErro
# says the contract binds both agents.
_onnx_src = inspect.getsource(ONNXAgent.predict_long)
check_true("hook budget/the ONNX scan checks its budget too",
- "_check_scan_budget(self, evidence, budget, max_len, head_max_len)" in _onnx_src, "")
+ "_check_scan_budget(self, evidence, budget, max_len, head_max_len, questions)" in _onnx_src,
+ "")
check("hook budget/both agents check it on the single-window path too",
(inspect.getsource(Agent.predict_long).count("_check_scan_budget("),
_onnx_src.count("_check_scan_budget(")), (2, 2))