Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(agent): read a state predict_long's questions leave room for in one pass

Jeel Gajera3e9bda301793
diff --git a/README.md b/README.md
index 600b184e..1019711c 100644
--- a/README.md
+++ b/README.md
@@ -957,7 +957,8 @@ result = agent.predict_long(state, questions, hooks=[AuditLog()]) # the scan,
- `choice` / `score` take the most-confident window — averaging over a long, mostly-neutral
document lets the neutral majority out-vote the one window that saw the deciding span.
- A state that already fits one window is passed straight to `system_one` (identical output, plus
- `usage["windows"] = 1`). The key is total: `1` single window, `N` scanned windows, `0` a hook
+ `usage["windows"] = 1`). Without an explicit `window`, that is any state the questions leave
+ room for, since `system_one` reads it whole. The key is total: `1` single window, `N` scanned windows, `0` a hook
answered the document before the model read any of it.
- Hooks wrap the inference that answers the state, so on a scanned document `on_predict_start`
fires once with `ctx.states` holding the decoded windows, not the state you passed in (it was
diff --git a/laya/agent.py b/laya/agent.py
index 7714f1fc..1e035c16 100644
--- a/laya/agent.py
+++ b/laya/agent.py
@@ -1627,7 +1627,8 @@ class Agent(HookRegistry):
That span is the one the model read, not merely the one asked for: the window is capped at
the room the questions leave, so what is handed to `predict_batch` is not cut short again.
- A state that already fits one window is passed straight to `system_one` (identical output).
+ A state the questions leave room for is passed straight to `system_one` (identical output),
+ since `system_one` reads it whole; with an explicit `window`, a state that fits that window.
The hooks wrap the inference that answers the state, which for a document needing several
windows is the one shared `predict_batch` over them: `on_predict_start` fires once, and
@@ -1716,8 +1717,8 @@ class Agent(HookRegistry):
for qid in ids:
self._check_question(qid, questions[qid])
internal = {qid: self._to_internal(questions[qid]) for qid in ids}
- budget, step, _ = window_budget(self.tok, [internal[qid] for qid in ids], max_len,
- head_max_len, window=window, stride=stride)
+ budget, step, room = window_budget(self.tok, [internal[qid] for qid in ids], max_len,
+ head_max_len, window=window, stride=stride)
# Snapshot the questions the scan was just sized against, BEFORE any start hook can
# rewrite them, and compare against this instead of `questions` itself. `==` over the
# mapping cannot see an in-place rewrite: a hook that adds options to
@@ -1735,7 +1736,10 @@ class Agent(HookRegistry):
# ask "how much of the document did the model read?" without handling a KeyError on the
# shortest, most common inputs. The result is copied first: a start hook that answers with
# `ctx.skip(...)` hands back its own payload dict, and it may be a cached object.
- if len(state_ids) <= budget:
+ # "Fits" is the room the questions leave, not the default window: `system_one` reads a
+ # state up to that room whole, so windowing one between the two only re-reads it in pieces
+ # and lets the per-window max inflate the answer. An explicit `window` still scans.
+ if len(state_ids) <= (budget if window and window > 0 else room):
probe, evidence = _start_evidence()
single = dict(self.system_one(state, questions, lang=lang,
**_with_start_probe(hook_kwargs, probe)))
@@ -1746,7 +1750,10 @@ class Agent(HookRegistry):
# fit one window was silently truncated by a re-budgeting hook and still reported
# `windows: 1`, i.e. "the model read all of it" -- measured, 138 of 240 state tokens
# never reached the model, while a longer document on the identical input hard-failed.
- _check_scan_budget(self, evidence, budget, max_len, head_max_len, asked)
+ # Sized for the whole state when it is longer than the window, so a hook that narrows
+ # the room below it is refused rather than cutting its tail.
+ _check_scan_budget(self, evidence, max(budget, len(state_ids)), max_len, head_max_len,
+ asked)
single["usage"] = {**(single.get("usage") or {}), "windows": 0 if evidence["answered"] else 1}
return single
diff --git a/laya/onnx_agent.py b/laya/onnx_agent.py
index 34c94194..242e617c 100644
--- a/laya/onnx_agent.py
+++ b/laya/onnx_agent.py
@@ -421,7 +421,8 @@ class ONNXAgent(HookRegistry):
`answer["window"]` -- the deciding window's `index`, `token_start`/`token_end` into the
tokenized state, and the window `count`.
- A state that already fits one window is passed straight to `system_one` (identical output).
+ A state the questions leave room for is passed straight to `system_one` (identical output),
+ since `system_one` reads it whole; with an explicit `window`, a state that fits that window.
Args:
state: Text string, JSON dict, or conversation turn list.
@@ -480,8 +481,8 @@ class ONNXAgent(HookRegistry):
for qid in ids:
_Agent._check_question(qid, questions[qid])
internal = {qid: self._to_internal(questions[qid]) for qid in ids}
- budget, step_default, _ = window_budget(self.tok, [internal[qid] for qid in ids], max_len,
- head_max_len, window=window, stride=stride)
+ budget, step_default, room = window_budget(self.tok, [internal[qid] for qid in ids], max_len,
+ head_max_len, window=window, stride=stride)
# Snapshot the questions the scan was just sized against, BEFORE any start hook can
# rewrite them, as the torch `predict_long` does (5f29170). `==` over the caller's mapping
# cannot see an in-place rewrite: a hook that adds options to `ctx.questions[q]["criteria"]`
@@ -498,7 +499,8 @@ class ONNXAgent(HookRegistry):
)["input_ids"]
# Fits in one window: identical to a plain call, no windowing overhead. `windows` is still
# written, as on the torch Agent: 1 for a window the model read, 0 for a hook's answer.
- if len(state_ids) <= budget:
+ # "Fits" is the room the questions leave, not the default window, as on the torch Agent.
+ if len(state_ids) <= (budget if window and window > 0 else room):
probe, evidence = _start_evidence()
single = dict(self.system_one(state, questions, lang=lang,
**_with_start_probe(hook_kwargs, probe)))
@@ -506,7 +508,8 @@ class ONNXAgent(HookRegistry):
# fit one window was silently truncated by a re-budgeting hook and still reported
# `windows: 1`, i.e. "the model read all of it" -- measured, 138 of 240 state tokens
# never reached the model, while a longer document on the identical input hard-failed.
- _check_scan_budget(self, evidence, budget, max_len, head_max_len, asked)
+ _check_scan_budget(self, evidence, max(budget, len(state_ids)), max_len, head_max_len,
+ asked)
single["usage"] = {**(single.get("usage") or {}), "windows": 0 if evidence["answered"] else 1}
return single
diff --git a/tests/test_onnx_long.py b/tests/test_onnx_long.py
index c5f55826..407f5bfb 100644
--- a/tests/test_onnx_long.py
+++ b/tests/test_onnx_long.py
@@ -139,6 +139,28 @@ check("short/lang forwards to system_one",
_with_one_window(_bare_onnx().system_one("aa", QUESTIONS, lang="de")))
+# ---------------------------------------------------------------- a state the questions leave room for
+# "Fits" is the room the questions leave, not the default window: system_one reads a state up to that
+# room whole, so windowing one between the two re-read it in pieces and moved the answer.
+def _roomy_onnx():
+ a = _bare_onnx()
+ a.cfg = {"max_len": 256, "head_max_len": 96} # default window 152
+ return a
+
+
+_win, _, _room_q = window_budget(_FakeTok(), [Agent._to_internal(q) for q in QUESTIONS.values()], 256, 96)
+check_true("room/these questions leave more room than the default window", _room_q > _win, (_win, _room_q))
+_at_room = "".join(chr(65 + (k % 11)) for k in range(_room_q))
+roomy = _roomy_onnx()
+check("room/a state at the room delegates byte-for-byte to system_one",
+ roomy.predict_long(_at_room, QUESTIONS), _with_one_window(_roomy_onnx().system_one(_at_room, QUESTIONS)))
+check("room/in one session run", roomy.session.calls, [2])
+check_true("room/one token past the room is still scanned",
+ _roomy_onnx().predict_long(_at_room + "A", QUESTIONS)["usage"]["windows"] > 1)
+check_true("room/an explicit window still scans a state wider than it",
+ _roomy_onnx().predict_long(_at_room, QUESTIONS, window=_win)["usage"]["windows"] > 1)
+
+
# ---------------------------------------------------------------- long state windows and aggregates
# Record the windows predict_long feeds predict_batch, then score the same windows directly so
# every aggregation claim is checked against real per-window answers, not a re-implementation.
diff --git a/tests/test_predict_long.py b/tests/test_predict_long.py
index d2986620..c8638d8d 100644
--- a/tests/test_predict_long.py
+++ b/tests/test_predict_long.py
@@ -862,6 +862,40 @@ check("docs/README says the window is capped at the room the questions leave",
check("docs/predict_long's docstring documents the cap",
"capped at the room the" in (Agent.predict_long.__doc__ or ""), True)
+# 12h. "fits in one window" is the room the questions leave, not the default window. A plain call
+# reads a state up to that room whole, so windowing one between the two only re-read it in pieces,
+# and the max over windows moved answers `predict` had already given, at twice the forward passes.
+ROOM_Q = min(room_for(q) for q in Q.values())
+check("fits/these questions leave more room than the default window", ROOM_Q > CONFIG_BUDGET, True)
+_pad = len(serialize_state({"body": ""}))
+AT_ROOM, PAST_ROOM = {"body": "x" * (ROOM_Q - _pad)}, {"body": "x" * (ROOM_Q - _pad + 1)}
+check("fits/the state under test is exactly the room",
+ len(TOK(serialize_state(AT_ROOM))["input_ids"]), ROOM_Q)
+a = make_agent(canned)
+fits = a.predict_long(AT_ROOM, Q)
+check("fits/a state at the room goes to system_one", fits["answers"], {"_via": "system_one"})
+check("fits/and is not windowed", a._calls["batch_states"], None)
+check("fits/one window is reported", fits["usage"].get("windows", "<absent>"), 1)
+a = make_agent(canned)
+a.predict_long(PAST_ROOM, Q)
+check_true("fits/one token past the room is still scanned",
+ len(a._calls["batch_states"] or []) > 1, a._calls["batch_states"])
+a = make_agent(canned)
+a.predict_long(AT_ROOM, Q, window=CONFIG_BUDGET)
+check_true("fits/an explicit window still scans a state wider than it",
+ len(a._calls["batch_states"] or []) > 1, a._calls["batch_states"])
+# The one-pass branch still refuses a start hook that narrows the room under the state: sized at the
+# default window, the check would pass and the tail of the state would be cut without a word.
+a = make_real_agent()
+_, narrowed = _attempt(lambda: a.predict_long(
+ AT_ROOM, Q, on_predict_start=lambda ctx: setattr(ctx, "max_len", MAX_LEN - 1)))
+check("fits/a hook that narrows the room under the state is refused", _kind(narrowed), "ValueError")
+a = make_real_agent()
+whole, exc = _attempt(lambda: a.predict_long(AT_ROOM, Q))
+check("fits/unhooked, the real path reads it in one pass",
+ (_kind(exc), a._forward_calls, ((whole or {}).get("usage") or {}).get("windows")),
+ (None, [2], 1))
+
# --- findings from an adversarial review -----------------------------------------------------