Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(agent): preserve deciding-window attribution after end hooks

Rishit Sharmab2047e18254a
diff --git a/README.md b/README.md
index dae60d74..5bf64a1f 100644
--- a/README.md
+++ b/README.md
@@ -918,6 +918,8 @@ result = agent.predict_long(state, questions, hooks=[AuditLog()]) # the scan,
reported when the scan reached inference unchanged. A hook that means to answer the document
calls `ctx.skip(...)` instead: its result comes back with no `answer["window"]` and
`usage["windows"]` at 0, because no window scored it.
+ End-hook result reordering keeps an unchanged answer's original span; cloned, replaced or edited
+ answers omit attribution. Reliability annotations are retained.
A smaller `window` isolates a short deciding span better (it becomes a larger fraction of its
window); the default (`max(64, max_len - head_max_len - 8)` — 312 tokens on the English checkpoint
diff --git a/docs/hooks/api.md b/docs/hooks/api.md
index e02758f3..1a901472 100644
--- a/docs/hooks/api.md
+++ b/docs/hooks/api.md
@@ -360,6 +360,12 @@ agent.predict(...) # alias of system_one
`usage["windows"]` is total over every path, including the one where a state already fit a single
window, so a cached answer never reads as a window the model read.
+ End hooks may reorder result rows: an unchanged answer keeps the index and token span of its
+ original window. Attribution requires the original answer object and unchanged prediction fields
+ (type, choice/noul/score, probabilities, legend and confidence). Cloned, replaced or edited answers
+ omit `answer["window"]`; reliability and action annotations are retained. Adding the window to
+ the returned answer does not mutate the per-window answer a hook may have cached.
+
### Router
```python
diff --git a/laya/agent.py b/laya/agent.py
index acc8cf2b..8ebf2c06 100644
--- a/laya/agent.py
+++ b/laya/agent.py
@@ -336,7 +336,25 @@ def _option_count(qdef: Dict) -> int:
return len(crit) if isinstance(crit, (dict, list, tuple)) else 0
-def _start_evidence():
+def _scan_answer_values(answer):
+ # Reliability/action annotations do not change which prediction the model made. Keep the
+ # prediction fields independent of in-place edits to probabilities or a score's legend.
+ return {key: answer[key] for key in ("type", "choice", "noul", "score", "legend",
+ "probabilities", "confidence", "answer_confidence")
+ if key in answer}
+
+
+def _scan_answer_index(evidence, qid, answer, fallback):
+ recorded = evidence["answers"]
+ if recorded is None:
+ return fallback # a replacement batch never dispatched hooks
+ record = recorded.get((qid, id(answer)))
+ if record is not None and record[0] is answer and record[2] == _scan_answer_values(answer):
+ return record[1]
+ return None # replaced, edited, or shared across windows
+
+
+def _start_evidence(track_answers=False):
"""A recorder for `predict_long`: what the start-hook chain left for inference to run on.
Why a recorder at all: `ctx.skip()` only assigns `ctx.results` (see `laya.hooks.PredictContext`),
@@ -354,9 +372,22 @@ def _start_evidence():
inference, `states` is a snapshot of the states that reached it (`None` if the probe never ran,
which means `predict_batch` was replaced and no hook chain was dispatched).
"""
- evidence = {"answered": False, "states": None, "question_types": None,
+ evidence = {"answered": False, "states": None, "question_types": None, "answers": None,
"max_len": None, "head_max_len": None, "questions": None}
+ def record_results(ctx):
+ if evidence["answered"]:
+ return
+ recorded = {}
+ for index, result in enumerate(ctx.results):
+ for qid, answer in result["answers"].items():
+ key = (qid, id(answer))
+ # Keep the object alive as well as its id. One object emitted for more than
+ # one window has no unique origin, even when all prediction fields agree.
+ recorded[key] = (None if key in recorded else
+ (answer, index, copy.deepcopy(_scan_answer_values(answer))))
+ evidence["answers"] = recorded
+
def probe(ctx):
evidence["answered"] = ctx.results is not None
evidence["states"] = list(ctx.states)
@@ -376,6 +407,10 @@ def _start_evidence():
evidence["max_len"] = ctx.max_len
evidence["head_max_len"] = ctx.head_max_len
evidence["questions"] = dict(ctx.questions) if ctx.questions else {}
+ if track_answers:
+ # predict_batch calls this before ANY end hook, including defaults and installed
+ # hooks. An end observer appended to the hook list would already see reordered rows.
+ ctx._record_scan_results = record_results
return probe, evidence
@@ -1589,6 +1624,9 @@ class Agent(HookRegistry):
ctx.usage = aggregate_usage(ctx.results)
apply_confidence_gate(ctx.results, mc)
try:
+ record_scan = getattr(ctx, "_record_scan_results", None)
+ if ctx.error is None and ctx.results is not None and record_scan is not None:
+ record_scan(ctx)
dispatch(active, "on_predict_end", ctx, raise_errors=raise_errors, lock=self._hooks_lock, timeout=timeout)
except BaseException as hook_exc:
# End hooks run on the failure path too; do not let one mask the real error.
@@ -1644,6 +1682,10 @@ class Agent(HookRegistry):
the states that were scored, but no answer carries `answer["window"]` -- the offsets
above describe this method's windows, not the text the model read
+ End hooks may reorder results without changing an answer's original window. Attribution
+ follows the original answer object, provided its prediction fields are unchanged. Cloned,
+ replaced or edited answers have no window attribution; added reliability metadata is kept.
+
Args:
window: state tokens per window. Defaults to the checkpoint's state budget,
`max(64, max_len - head_max_len - 8)` -- the 64 is a floor, so widening
@@ -1772,7 +1814,7 @@ class Agent(HookRegistry):
break
i += step
- probe, evidence = _start_evidence()
+ probe, evidence = _start_evidence(track_answers=True)
# `list(windows)`, not `windows`: `ctx.states` is the list the hook receives, so a hook that
# mutates it in place (`append`, `sort`) would otherwise also grow the split this call
# attributes answers to, and the counts would agree while `starts` no longer lined up.
@@ -1849,12 +1891,15 @@ class Agent(HookRegistry):
# neutral document lets the neutral majority out-vote the one window that saw the
# deciding span; the single most-confident window preserves a localized signal.
best = max(range(len(per)), key=lambda j: float(per[j]["answer_confidence"]))
- ans = per[best]
+ original = per[best]
+ index = None if rewritten else _scan_answer_index(evidence, qid, original, best)
+ ans = dict(original)
+ ans.pop("window", None)
# Name the window that decided, so a caller can check the deciding span itself. The
# probability here is that window's, NOT a document-level calibrated number.
- if not rewritten:
- ans["window"] = {"index": best, "token_start": starts[best],
- "token_end": min(starts[best] + budget, len(state_ids)),
+ if index is not None:
+ ans["window"] = {"index": index, "token_start": starts[index],
+ "token_end": min(starts[index] + budget, len(state_ids)),
"count": len(results)}
answers[qid] = ans
# Aggregate usage generically so fields predict_batch may grow later (e.g. the
diff --git a/laya/onnx_agent.py b/laya/onnx_agent.py
index 7e8b9677..24ee289d 100644
--- a/laya/onnx_agent.py
+++ b/laya/onnx_agent.py
@@ -436,6 +436,9 @@ class ONNXAgent(HookRegistry):
ctx.usage = aggregate_usage(ctx.results)
apply_confidence_gate(ctx.results, mc)
try:
+ record_scan = getattr(ctx, "_record_scan_results", None)
+ if ctx.error is None and ctx.results is not None and record_scan is not None:
+ record_scan(ctx)
dispatch(active, "on_predict_end", ctx, raise_errors=raise_errors, lock=self._hooks_lock, timeout=timeout)
except BaseException as hook_exc:
if ctx.error is not None:
@@ -490,7 +493,8 @@ class ONNXAgent(HookRegistry):
follow `Agent.predict_long`'s contract: they wrap the inference that answers
the state, a start hook that answers with `ctx.skip(...)` gets
`usage["windows"] == 0` and no window attribution, and a rewritten scan is
- aggregated without `answer["window"]`.
+ aggregated without `answer["window"]`. End-hook result reordering keeps an
+ unchanged answer's original span; cloned, replaced or edited answers omit it.
on_predict_start (PredictHookArg): A per-call start hook, as in `system_one`.
on_predict_end (PredictHookArg): A per-call end hook, as in `system_one`.
hooks_raise: Override the agent's `hooks_raise` for this call.
@@ -501,7 +505,7 @@ class ONNXAgent(HookRegistry):
in `Agent.predict_long`: `truncated` is a window count and `truncated_questions` is the
last window's list, so `truncated` can be above 0 while the list is empty.
"""
- from .agent import _check_scan_budget, _start_evidence, _with_start_probe
+ from .agent import _check_scan_budget, _scan_answer_index, _start_evidence, _with_start_probe
from .hooks import aggregate_usage
hook_kwargs = {"hooks": hooks, "on_predict_start": on_predict_start,
@@ -577,7 +581,7 @@ class ONNXAgent(HookRegistry):
break
i += step
- probe, evidence = _start_evidence()
+ probe, evidence = _start_evidence(track_answers=True)
# A copy, so a start hook that mutates `ctx.states` in place cannot shift `starts`.
# Bound one forward pass to what the un-capped scan would have used; see
# `window_batch_cap`. An explicit batch_size is honoured untouched.
@@ -640,11 +644,14 @@ class ONNXAgent(HookRegistry):
# choice / score: the most-confident window wins, preserving a localized signal
# that averaging over a mostly-neutral document would drown.
best = max(range(len(per)), key=lambda j: float(per[j]["answer_confidence"]))
- ans = per[best]
+ original = per[best]
+ index = None if rewritten else _scan_answer_index(evidence, qid, original, best)
+ ans = dict(original)
+ ans.pop("window", None)
# Name the window that decided; the probability is that window's, not the document's.
- if not rewritten:
- ans["window"] = {"index": best, "token_start": starts[best],
- "token_end": min(starts[best] + budget, len(state_ids)),
+ if index is not None:
+ ans["window"] = {"index": index, "token_start": starts[index],
+ "token_end": min(starts[index] + budget, len(state_ids)),
"count": len(results)}
answers[qid] = ans
# Aggregate usage generically so fields predict_batch may grow later are propagated
diff --git a/tests/_long_attribution.py b/tests/_long_attribution.py
new file mode 100644
index 00000000..1001f624
--- /dev/null
+++ b/tests/_long_attribution.py
@@ -0,0 +1,160 @@
+"""Shared control-flow cases for the two predict_long script suites; no model weights."""
+import copy
+import math
+
+from laya.hooks import default_hooks, set_default_hooks
+
+
+def check_long_attribution(factory, check):
+ questions = {"dept": {"type": "choice", "instructions": "?", "criteria": ["a", "b"]},
+ "flag": {"type": "noul", "instructions": "?"},
+ "severity": {"type": "score", "instructions": "?", "criteria": ["low", "high"]}}
+ state = "".join(chr(65 + (i * i % 23)) for i in range(300))
+ winners = {"dept": 8, "flag": 6, "severity": 3}
+
+ def run(transform=None, tier="callback", reverse_start=False, shared=False, ties=False):
+ agent = factory()
+ agent.cfg = {"max_len": 128, "head_max_len": 20}
+ original, shared_answers = [], {}
+
+ def decode(logits, act, items, ids, internal, row, **kwargs):
+ index = row // len(ids)
+ answers = {}
+ for qid, q in questions.items():
+ conf = 0.9 if index == winners[qid] or ties else 0.6
+ entropy_conf = 1 + (conf * math.log(conf) + (1 - conf) * math.log(1 - conf)) / math.log(2)
+ answer = {"type": q["type"], "confidence": entropy_conf, "answer_confidence": conf,
+ "action": {"act_probability": 0.75}}
+ if qid == "flag":
+ answer["noul"] = 0.9 if index == winners[qid] else 0.1
+ answer["confidence"] = answer["answer_confidence"] = 0.9
+ elif qid == "dept":
+ choice = "b" if index == winners[qid] else "a"
+ answer.update(choice=choice,
+ probabilities={key: conf if key == choice else 1 - conf for key in ("a", "b")})
+ else:
+ score = conf if index == winners[qid] else 1 - conf
+ answer.update(score=score, legend={"0": "low", "1": "high"},
+ probabilities={"0": 1 - score, "1": score})
+ answers[qid] = shared_answers.setdefault(qid, answer) if shared else answer
+ return answers
+
+ agent._decode_answers = decode
+
+ def end(ctx):
+ original.extend(ctx.results)
+ if transform is not None:
+ transform(ctx)
+
+ class End:
+ def on_predict_end(self, ctx):
+ end(ctx)
+
+ async def async_end(ctx):
+ end(ctx)
+
+ kwargs = {}
+ defaults = default_hooks()
+ if tier == "installed":
+ agent.hooks = [End()]
+ elif tier == "default":
+ set_default_hooks([End()])
+ elif tier == "per-call":
+ kwargs["hooks"] = [End()]
+ else:
+ kwargs["on_predict_end"] = async_end if tier == "async" else end
+ if reverse_start:
+ kwargs["on_predict_start"] = lambda ctx: ctx.states.reverse()
+ try:
+ result = agent.predict_long(state, copy.deepcopy(questions), window=64, stride=32,
+ batch_size=100, **kwargs)
+ finally:
+ if tier == "default":
+ set_default_hooks(defaults)
+ return result, original
+
+ baseline, original = run()
+ check("attribution/nine windows were scored", len(original), 9)
+ for qid, index in winners.items():
+ check("attribution/%s fixture has its own winning span" % qid,
+ baseline["answers"][qid]["window"],
+ {"index": index, "token_start": index * 32,
+ "token_end": min(index * 32 + 64, 300), "count": 9})
+ check("attribution/%s does not annotate the hook's result object" % qid,
+ "window" in original[index]["answers"][qid], False)
+
+ for tier in ("callback", "per-call", "installed", "default", "async"):
+ result, _ = run(lambda ctx: ctx.results.reverse(), tier=tier)
+ check("attribution/%s reordering preserves answers and original spans" % tier,
+ result, baseline)
+
+ def reorder_wrappers(ctx):
+ ctx.results = [dict(row) for row in reversed(ctx.results)]
+
+ result, _ = run(reorder_wrappers)
+ check("attribution/new result wrappers retain the original answers' spans", result, baseline)
+
+ def move_answers(ctx):
+ ctx.results[0]["answers"], ctx.results[-1]["answers"] = (
+ ctx.results[-1]["answers"], ctx.results[0]["answers"])
+
+ result, _ = run(move_answers)
+ check("attribution/answers moved between result wrappers retain their spans",
+ result["answers"], baseline["answers"])
+
+ result, _ = run(lambda ctx: ctx.results.reverse(), ties=True)
+ check("attribution/a tie still selects the first final result, with its original span",
+ result["answers"]["dept"]["window"]["index"], 8)
+
+ def annotate(ctx):
+ for row in ctx.results:
+ for answer in row["answers"].values():
+ answer["reliability"] = {"status": "observed"}
+
+ result, _ = run(annotate)
+ for qid in questions:
+ check("attribution/%s permits reliability annotations" % qid,
+ result["answers"][qid], {**baseline["answers"][qid],
+ "reliability": {"status": "observed"}})
+
+ def clone(ctx):
+ ctx.results = copy.deepcopy(ctx.results)
+ for row in ctx.results:
+ for answer in row["answers"].values():
+ answer["window"] = {"index": 999}
+
+ for name, transform, reverse_start, shared in (
+ ("cloned answers", clone, False, False),
+ ("rewritten start states", None, True, False),
+ ("ambiguous shared answers", None, False, True)):
+ result, _ = run(transform, reverse_start=reverse_start, shared=shared)
+ check("attribution/%s names no span" % name,
+ [qid for qid, answer in result["answers"].items() if "window" in answer], [])
+ check("attribution/%s still reports scored windows" % name, result["usage"]["windows"], 9)
+
+ for qid, field, value in (("dept", "choice", "edited"),
+ ("dept", "answer_confidence", 0.95),
+ ("dept", "confidence", 0.95),
+ ("flag", "noul", 0.95),
+ ("severity", "score", 0.95),
+ ("severity", "legend", {"0": "edited", "1": "high"})):
+ def edit(ctx):
+ answer = ctx.results[winners[qid]]["answers"][qid]
+ answer[field] = value
+ answer["window"] = {"index": 999}
+
+ result, _ = run(edit)
+ check("attribution/an edited %s.%s names no span" % (qid, field),
+ "window" in result["answers"][qid], False)
+ check("attribution/an edited %s.%s is still returned" % (qid, field),
+ result["answers"][qid][field], value)
+ for other in questions.keys() - {qid}:
+ check("attribution/editing %s.%s keeps %s's span" % (qid, field, other),
+ result["answers"][other], baseline["answers"][other])
+
+ def edit_nested(ctx):
+ ctx.results[winners["dept"]]["answers"]["dept"]["probabilities"]["b"] = 0.89
+
+ result, _ = run(edit_nested)
+ check("attribution/an in-place probability edit names no span",
+ "window" in result["answers"]["dept"], False)
diff --git a/tests/test_onnx_long.py b/tests/test_onnx_long.py
index 616b40b6..2c926cdb 100644
--- a/tests/test_onnx_long.py
+++ b/tests/test_onnx_long.py
@@ -22,6 +22,7 @@ import numpy as np # noqa: E402
from laya.agent import Agent # noqa: E402
from laya.common import window_budget # noqa: E402
from laya.hooks import OptionStabilityHook # noqa: E402
+from _long_attribution import check_long_attribution
from laya.onnx_agent import ONNXAgent # noqa: E402
PASS, FAIL = [], []
@@ -102,6 +103,9 @@ def _bare_onnx():
return a
+check_long_attribution(_bare_onnx, check)
+
+
QUESTIONS = {
"dept": {"type": "choice", "instructions": "Which team?",
"criteria": {"billing": "money", "support": "help", "sales": "buy"}},
diff --git a/tests/test_predict_long.py b/tests/test_predict_long.py
index 71cd7a42..75fe6ee9 100644
--- a/tests/test_predict_long.py
+++ b/tests/test_predict_long.py
@@ -21,6 +21,7 @@ import numpy as np # noqa: E402
import laya.agent as agent_mod
from laya.agent import Agent, _check_scan_budget, _start_evidence
from laya.hooks import OptionStabilityHook
+from _long_attribution import check_long_attribution
from laya.onnx_agent import ONNXAgent # noqa: E402
@@ -136,6 +137,8 @@ def make_real_agent():
Q = {"dept": {"type": "choice", "instructions": "?", "criteria": {"a": "x", "b": "y"}},
"flag": {"type": "noul", "instructions": "?"}}
+check_long_attribution(make_real_agent, check)
+
# 1. fits in one window -> delegates to system_one, no windowing
a = make_agent(lambda s, q: [])
short = a.predict_long({"body": "x" * 50}, Q) # 50 tokens <= budget 72