Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

Merge pull request #668 from somtri/fix/truncation-from-applied-budget

Nandakishorf3f1ea94d04a
diff --cc laya/agent.py
index 8a801b98,7cfc61fd..c98e6859
--- a/laya/agent.py
+++ b/laya/agent.py
@@@ -881,28 -774,13 +881,29 @@@ class Agent(HookRegistry)
items = []
for qid in ids:
q = internal[qid]
- seq, markers, stats = build_sequence(self.tok, state, q, max_len, head_max_len,
- option_order=q.get("option_order"),
- truncate_left=truncate_left, state_ids=state_ids,
- return_stats=True)
+ seq, markers, stats, state_stats = build_sequence(self.tok, state, q, max_len, head_max_len,
++ option_order=q.get("option_order"),
+ truncate_left=truncate_left, state_ids=state_ids,
+ return_stats=True, return_truncation_stats=True)
- if len(markers) != len(render_options(q)):
- raise ValueError("question %r options exceed head_max_len=%d" % (qid, head_max_len))
+ n_opts = len(render_options(q))
+ if len(markers) != n_opts:
+ # The markers are placed at absolute positions and `build_sequence` then drops the
+ # ones past `max_len`, so this is about the question fitting in the sequence --
+ # `head_max_len` is how much of it the options were given, and `max_len` is the
+ # ceiling that dropped them. Naming only `head_max_len` pointed at the wrong knob
+ # in both directions: lowering it shortens the option block and can make the
+ # call succeed, while raising it makes the overflow worse.
+ #
+ # The count reported is the markers that SURVIVED, not `len(seq)`: `build_sequence`
+ # truncates to `max_len` first, so `len(seq)` is always exactly `max_len` here and
+ # would state the ceiling as though it were the requirement.
+ raise ValueError(
+ "question %r: only %d of its %d option markers fit in max_len=%d with "
+ "head_max_len=%d spent on the question; lower head_max_len, raise max_len, "
+ "or use fewer options"
+ % (qid, len(markers), n_opts, max_len, head_max_len))
- items.append({"ids": seq, "markers": markers, "qtype": QTYPES[q["t"]], "options": stats})
+ items.append({"ids": seq, "markers": markers, "qtype": QTYPES[q["t"]], "options": stats,
+ "state_stats": state_stats})
return items
def _amp_enabled_for(self, rows: int) -> bool:
@@@ -1334,12 -1172,16 +1349,21 @@@
the `N` a start hook rewrote them to), and `0` when a start hook answered the document, or
left no states to score, before any window was read -- on either path, so a cached answer
never reads as a window the model read.
+
+ Across several windows the truncation keys are combined like every other `usage` field:
+ `truncated`, `state_tokens` and `state_tokens_dropped` are summed (so `truncated` is the
+ number of windows that were cut, and the token counts include the overlap), and
+ `truncated_questions` is the last window's list. The two can disagree: when only an
+ earlier window was cut, `truncated` is above 0 and `truncated_questions` is empty. A
+ window is cut when it is larger than the room a question's head leaves, from a `window`
+ above the default or a start hook that narrows `max_len` / `head_max_len`. Test
+ `usage["truncated"] > 0` here, not `is True`.
"""
+ if state is None:
+ raise TypeError("state must not be None; pass a string, dict, or list")
+ if not isinstance(questions, dict):
+ raise TypeError("questions must be a dict of question id -> definition, got %s"
+ % type(questions).__name__)
if aggregate != "auto":
raise ValueError("predict_long: only aggregate='auto' is supported")
hook_kwargs = {"hooks": hooks, "on_predict_start": on_predict_start,
diff --cc laya/onnx_agent.py
index c49bea5b,fa3bf502..560ca6bf
--- a/laya/onnx_agent.py
+++ b/laya/onnx_agent.py
@@@ -567,25 -542,26 +569,27 @@@ class ONNXAgent(HookRegistry)
items = []
for qid in ids:
q = internal[qid]
- seq, markers, stats = build_sequence(
+ seq, markers, stats, state_stats = build_sequence(
self.tok, state, q, max_len, head_max_len,
+ option_order=q.get("option_order"),
truncate_left=truncate_left, state_ids=state_ids, return_stats=True,
+ return_truncation_stats=True,
)
n_opts = len(render_options(q))
if len(markers) != n_opts:
- # The markers are placed at absolute positions and `build_sequence` then drops the
- # ones past `max_len`, so this is about the question fitting in the sequence --
- # `head_max_len` is how much of it the options were given, and `max_len` is the
- # ceiling that dropped them. Naming only `head_max_len` pointed at the wrong knob
- # in both directions: lowering it shortens the option block and can make the
- # call succeed, while raising it makes the overflow worse.
+ # Same diagnosis as `Agent._encode_state`, so both backends report the same thing.
+ # The markers are placed at absolute positions and `build_sequence` drops the ones
+ # past `max_len`, so `head_max_len` is how much of the sequence the options were
+ # given and `max_len` is the ceiling that dropped them -- naming only
+ # `head_max_len` pointed at the wrong knob in both directions. The count is the
+ # markers that survived, not `len(seq)`, which is always exactly `max_len` here.
raise ValueError(
- "question %r: its %d options and question text need %d tokens, more than "
- "max_len=%d allows once head_max_len=%d is spent on them; lower head_max_len, "
- "raise max_len, or use fewer options"
- % (qid, n_opts, len(seq), max_len, head_max_len))
+ "question %r: only %d of its %d option markers fit in max_len=%d with "
+ "head_max_len=%d spent on the question; lower head_max_len, raise max_len, "
+ "or use fewer options"
+ % (qid, len(markers), n_opts, max_len, head_max_len))
- items.append({"ids": seq, "markers": markers, "qtype": QTYPES[q["t"]], "options": stats})
+ items.append({"ids": seq, "markers": markers, "qtype": QTYPES[q["t"]], "options": stats,
+ "state_stats": state_stats})
return items
def _decode_answers(self, logits, act, items: List[Dict[str, Any]], ids: List[str],
diff --cc laya/structured.py
index db4d2942,b152613f..9e75dcec
--- a/laya/structured.py
+++ b/laya/structured.py
@@@ -59,11 -43,8 +59,11 @@@ class DecisionResult
confidence: Dict[str, float]
probabilities: Dict[str, Dict[str, Any]]
answers: Dict[str, Any]
- usage: Optional[Dict[str, int]] = None
+ usage: Optional[Dict[str, Any]] = None
routing: Optional[Dict[str, Any]] = None
+ # Appended with a default so every existing construction of this dataclass keeps working, and
+ # the first six fields keep their positions.
+ answer_confidence: Dict[str, Optional[float]] = field(default_factory=dict)
@dataclass