Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(agent): handle removed long-scan questions

boulea7502bd88c5f34
diff --git a/laya/agent.py b/laya/agent.py
index aef82cd8..acc8cf2b 100644
--- a/laya/agent.py
+++ b/laya/agent.py
@@ -1831,6 +1831,14 @@ class Agent(HookRegistry):
question_types = {qid: self._to_internal(qdef)["t"] for qid, qdef in questions.items()}
answers = {}
for qid, qtype in question_types.items():
+ missing = sum(qid not in r["answers"] for r in results)
+ if missing:
+ # End hooks may remove start-added questions, but must not hide a missing
+ # caller answer or an inconsistent set of answers across windows.
+ if missing == len(results) and qid not in asked and evidence["question_types"] is not None:
+ continue
+ raise ValueError("predict_long: answer %r is missing from %d of %d windows after prediction hooks"
+ % (qid, missing, len(results)))
per = [r["answers"][qid] for r in results]
if qtype == "noul":
# Evidence anywhere: the strongest window decides. Its own P(true) and confidence
diff --git a/laya/onnx_agent.py b/laya/onnx_agent.py
index dfa983b1..7e8b9677 100644
--- a/laya/onnx_agent.py
+++ b/laya/onnx_agent.py
@@ -623,6 +623,14 @@ class ONNXAgent(HookRegistry):
question_types = {qid: self._to_internal(qdef)["t"] for qid, qdef in questions.items()}
answers = {}
for qid, qtype in question_types.items():
+ missing = sum(qid not in r["answers"] for r in results)
+ if missing:
+ # End hooks may remove start-added questions, but must not hide a missing
+ # caller answer or an inconsistent set of answers across windows.
+ if missing == len(results) and qid not in asked and evidence["question_types"] is not None:
+ continue
+ raise ValueError("predict_long: answer %r is missing from %d of %d windows after prediction hooks"
+ % (qid, missing, len(results)))
per = [r["answers"][qid] for r in results]
if qtype == "noul":
# Evidence anywhere: the strongest window decides, carrying its own P(true),
diff --git a/tests/test_onnx_long.py b/tests/test_onnx_long.py
index 407f5bfb..616b40b6 100644
--- a/tests/test_onnx_long.py
+++ b/tests/test_onnx_long.py
@@ -21,6 +21,7 @@ import numpy as np # noqa: E402
from laya.agent import Agent # noqa: E402
from laya.common import window_budget # noqa: E402
+from laya.hooks import OptionStabilityHook # noqa: E402
from laya.onnx_agent import ONNXAgent # noqa: E402
PASS, FAIL = [], []
@@ -399,6 +400,87 @@ except Exception as exc:
else:
check("questions/a first-window end annotation preserves the scan's answers", actual, expected)
+# Exercise the real encoder, typed decoder and stability hook with the existing CPU session.
+def _stability_hook():
+ return OptionStabilityHook(accept=None, escalate=None, probes="reorder",
+ variants=["reversed"], seed="long-regression")
+
+
+stability_questions = {"dept": {"type": "choice", "instructions": "?", "criteria": {"a": "x", "b": "y"}},
+ "urgent": QUESTIONS["urgent"]}
+for name, method, states in (("ordinary", "system_one", "aa"),
+ ("short", "predict_long", "aa"),
+ ("batch", "predict_batch", ["aa", "bb"])):
+ hook = _stability_hook()
+ result = getattr(_bare_onnx(), method)(states, stability_questions, hooks=[hook])
+ rows = result if isinstance(result, list) else [result]
+ check("stability/%s retains user questions" % name,
+ [list(row["answers"]) for row in rows], [list(stability_questions)] * len(rows))
+ check("stability/%s runs the real reorder probe" % name,
+ [row["answers"]["dept"]["reliability"]["probes"] for row in rows], [["reorder"]] * len(rows))
+ check("stability/%s releases its run plan" % name, hook._plans, {})
+
+stability_windows = []
+_bare_onnx().predict_long(LONG_STATE, stability_questions, window=16, stride=8,
+ on_predict_start=lambda ctx: stability_windows.extend(ctx.states))
+reference_rows = _bare_onnx().predict_batch(stability_windows, stability_questions, hooks=[_stability_hook()])
+hook = _stability_hook()
+try:
+ actual = _bare_onnx().predict_long(LONG_STATE, stability_questions, window=16, stride=8, hooks=[hook])
+except Exception as exc:
+ FAIL.append("stability/long raised %r after probe removal" % exc)
+else:
+ check("stability/long retains exactly the user questions", list(actual["answers"]), list(stability_questions))
+ for qid in stability_questions:
+ answer = dict(actual["answers"][qid])
+ window = answer.pop("window")
+ check("stability/%s retains its exact winning batch answer and metadata" % qid,
+ answer, reference_rows[window["index"]]["answers"][qid])
+ check("stability/%s attributes the original token window" % qid,
+ (window["token_start"], window["token_end"], window["count"]),
+ (window["index"] * 8, min(window["index"] * 8 + 16, len(LONG_STATE)), len(stability_windows)))
+ check("stability/long counts the scored windows", actual["usage"]["windows"], len(stability_windows))
+check("stability/long releases its run plan", hook._plans, {})
+
+
+def _remove_dept(ctx):
+ for row in ctx.results:
+ del row["answers"]["dept"]
+
+
+check_raises("end removal/a caller question cannot silently disappear from every window", ValueError,
+ lambda: _bare_onnx().predict_long(LONG_STATE, stability_questions, window=16, stride=8,
+ on_predict_end=_remove_dept))
+
+for policy in (True, False):
+ check_raises("end removal/partial removal is refused with hooks_raise=%s" % policy, ValueError,
+ lambda: _bare_onnx().predict_long(
+ LONG_STATE, stability_questions, window=16, stride=8, hooks_raise=policy,
+ on_predict_end=lambda ctx: ctx.results[0]["answers"].pop("dept")))
+
+
+def _add_temporary(ctx):
+ ctx.questions["temporary"] = dict(stability_questions["urgent"])
+
+
+def _remove_temporary(ctx):
+ for row in ctx.results:
+ del row["answers"]["temporary"]
+
+
+try:
+ actual = _bare_onnx().predict_long(LONG_STATE, dict(stability_questions), window=16, stride=8,
+ on_predict_start=_add_temporary, on_predict_end=_remove_temporary)
+except Exception as exc:
+ FAIL.append("end removal/an in-place start addition raised %r after removal" % exc)
+else:
+ check("end removal/an in-place start addition can be uniformly removed", actual["answers"],
+ _bare_onnx().predict_long(LONG_STATE, stability_questions, window=16, stride=8)["answers"])
+check_raises("end removal/a start-added question missing in only one window is refused", ValueError,
+ lambda: _bare_onnx().predict_long(
+ LONG_STATE, dict(stability_questions), window=16, stride=8, on_predict_start=_add_temporary,
+ on_predict_end=lambda ctx: ctx.results[0]["answers"].pop("temporary")))
+
for malformed in (None, [], {"bad": None}, {"bad": {}}, {"bad": {"type": "unknown"}}):
errors = []
for method in ("system_one", "predict_long"):
diff --git a/tests/test_predict_long.py b/tests/test_predict_long.py
index 961078f2..71cd7a42 100644
--- a/tests/test_predict_long.py
+++ b/tests/test_predict_long.py
@@ -20,6 +20,7 @@ import numpy as np # noqa: E402
import laya.agent as agent_mod
from laya.agent import Agent, _check_scan_budget, _start_evidence
+from laya.hooks import OptionStabilityHook
from laya.onnx_agent import ONNXAgent # noqa: E402
@@ -584,6 +585,97 @@ actual, exc = _attempt(lambda: _question_agent().predict_long(LONG, Q, on_predic
check("questions/a first-window end annotation preserves the scan's answers", actual, expected)
check("questions/a first-window end annotation does not require other windows to match", _kind(exc), None)
+# The public stability hook adds probe questions at start and removes their answers at end.
+# Ordinary inference is the reference for the returned labels, probabilities and reliability;
+# a long scan must retain the same user-question mapping and its deciding window's metadata.
+def _stability_hook():
+ return OptionStabilityHook(accept=None, escalate=None, probes="reorder",
+ variants=["reversed"], seed="long-regression")
+
+
+for name, method, states in (("ordinary", "system_one", "short"),
+ ("short", "predict_long", "short"),
+ ("batch", "predict_batch", ["first", "second"])):
+ hook = _stability_hook()
+ result = getattr(_question_agent(), method)(states, Q, hooks=[hook])
+ rows = result if isinstance(result, list) else [result]
+ check("stability/%s returns only user questions" % name,
+ [list(row["answers"]) for row in rows], [list(Q)] * len(rows))
+ check("stability/%s ran the real reorder probe" % name,
+ [row["answers"]["dept"]["reliability"]["probes"] for row in rows],
+ [["reorder"]] * len(rows))
+ check("stability/%s releases its run plan" % name, hook._plans, {})
+
+reference_agent = _question_agent()
+reference_agent.predict_long(LONG, Q)
+reference_rows = _question_agent().predict_batch(list(reference_agent._encoded), Q,
+ hooks=[_stability_hook()])
+check_true("stability/the first window has the most confident original choice",
+ reference_rows[0]["answers"]["dept"]["answer_confidence"] >
+ reference_rows[1]["answers"]["dept"]["answer_confidence"])
+check("stability/the original choice keeps its own label", reference_rows[0]["answers"]["dept"]["choice"], "a")
+hook = _stability_hook()
+actual, exc = _attempt(lambda: _question_agent().predict_long(LONG, Q, hooks=[hook]))
+check("stability/long returns after probe answers are removed", _kind(exc), None)
+check("stability/long retains the final user schema", list((actual or {}).get("answers", {})), list(Q))
+if actual is not None:
+ for qid in Q:
+ answer = dict(actual["answers"][qid])
+ window = answer.pop("window")
+ check("stability/%s keeps the winning batch answer and all metadata" % qid,
+ answer, reference_rows[0]["answers"][qid])
+ check("stability/%s keeps the original winning window" % qid, window,
+ {"index": 0, "token_start": 0, "token_end": 72, "count": len(reference_rows)})
+ check("stability/usage counts every scored window", actual["usage"]["windows"], len(reference_rows))
+check("stability/long releases its run plan", hook._plans, {})
+
+routed_agent = _question_agent()
+routed_agent.hooks = [_stability_hook()]
+routed, exc = _attempt(lambda: _router(routed_agent).predict_long(LONG, Q, model="english"))
+check("stability/router reaches the hooked long scan", _kind(exc), None)
+if routed is not None and actual is not None:
+ check("stability/router preserves the scan's answers", routed["answers"], actual["answers"])
+ check("stability/router retains routing metadata", routed["routing"]["model"], "english")
+
+
+def _remove_dept(ctx):
+ for row in ctx.results:
+ del row["answers"]["dept"]
+
+
+check_raises("end removal/a caller question cannot silently disappear from every window", ValueError,
+ lambda: _question_agent().predict_long(LONG, Q, on_predict_end=_remove_dept))
+
+for policy in (True, False):
+ check_raises("end removal/a partial removal is refused with hooks_raise=%s" % policy, ValueError,
+ lambda: _question_agent().predict_long(
+ LONG, Q, hooks_raise=policy,
+ on_predict_end=lambda ctx: ctx.results[0]["answers"].pop("dept")))
+
+check_raises("end removal/a replacement batch cannot silently omit every answer", ValueError,
+ lambda: make_agent(lambda states, q: [{"answers": {}, "usage": {}} for _ in states])
+ .predict_long(LONG, Q))
+
+
+def _add_temporary(ctx):
+ ctx.questions["temporary"] = dict(Q["flag"])
+
+
+def _remove_temporary(ctx):
+ for row in ctx.results:
+ del row["answers"]["temporary"]
+
+
+actual, exc = _attempt(lambda: _question_agent().predict_long(
+ LONG, dict(Q), on_predict_start=_add_temporary, on_predict_end=_remove_temporary))
+check("end removal/an in-place start addition returns after removal", _kind(exc), None)
+check("end removal/an in-place start addition can be uniformly removed",
+ (actual or {}).get("answers"), _question_agent().predict_long(LONG, Q)["answers"])
+check_raises("end removal/a start-added question missing in only one window is refused", ValueError,
+ lambda: _question_agent().predict_long(
+ LONG, dict(Q), on_predict_start=_add_temporary,
+ on_predict_end=lambda ctx: ctx.results[0]["answers"].pop("temporary")))
+
# The recorder must leave invalid input to predict_batch's existing question validation.
for malformed in (None, [], {"bad": None}, {"bad": {}}, {"bad": {"type": "unknown"}}):
_, direct_exc = _attempt(lambda: _question_agent().system_one(LONG, malformed))