Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
fix(predict_long): a hard clamp of the default window must not be silent
Adeshina Lasisi0c394be7454d
diff --git a/laya/common.py b/laya/common.pyindex 7a1b97e3..919e504a 100644--- a/laya/common.py+++ b/laya/common.py@@ -259,6 +259,12 @@ def state_room(tok, q: Dict, max_len: int = 512, head_max_len: int = 192) -> intreturn max(0, max_len - len(head) - 1) # -1 for the [SEP] that closes the state+#: How far the DEFAULT window may be cut before `window_budget` says so. A small clamp is ordinary+#: and warning about it would be noise; a large one multiplies the number of forward passes by the+#: same factor, which a caller needs to be told about because nothing in their code implies it. 2x+#: is the point where the scan costs at least twice what the caller would estimate from `max_len`.+_WINDOW_CLAMP_WARN_RATIO = 2+# How much capping the window may multiply the window count before `window_batch_cap` starts# chunking. Under this, chunking would cost the single-shared-pass property for no real protection._WINDOW_BATCH_BLOWUP = 2@@ -295,7 +301,8 @@ def window_budget(tok, questions, max_len: int = 512, head_max_len: int = 192,if room <= 0:raise ValueError("predict_long: the questions' options fill the whole sequence (max_len=%d,"- " head_max_len=%d), leaving no room for the state; no window can carry any of it"+ " head_max_len=%d), leaving no room for the state; no window can carry any of it."+ " A label set this large is what laya.shortlist.predict_shortlist is for"% (max_len, head_max_len))if size > room:if window and window > 0:@@ -304,6 +311,27 @@ def window_budget(tok, questions, max_len: int = 512, head_max_len: int = 192," leave inside max_len=%d, so every window would be truncated to %d on the way to"" the model; scanning with window=%d instead" % (size, room, max_len, room, room),RuntimeWarning, stacklevel=3)+ elif size >= room * _WINDOW_CLAMP_WARN_RATIO:+ # The DEFAULT window was cut, and cut hard. Capping it is what stops the tail of every+ # window reaching no model, but it is not free and it must not be silent: the scan now+ # needs about `size / room` times as many windows, each one a full forward pass, and+ # nothing in the caller's code says why. Measured on the English checkpoint with 100+ # four-word options: room 102 of max_len 512, so the question head alone is 409 tokens,+ # the window falls 312 -> 102 and the scan goes 11 windows -> 36, a 3.1x wall-clock+ # increase (1902 ms -> 5858 ms) for the same document.+ #+ # The cost is not the capping, it is the shape of the request: 80% of every sequence is+ # the question, and because the encoder is bidirectional the head cannot be computed+ # once and reused -- its representations depend on the state it is paired with. So the+ # warning names the real remedy rather than only reporting the clamp.+ warnings.warn(+ "laya: predict_long: these questions leave only %d of max_len=%d for the state"+ " (their heads take the rest), so the scan window is capped %d -> %d and roughly"+ " %.1fx as many windows -- each a full forward pass -- are needed to read the"+ " document. Fewer or shorter options, a larger max_len, or"+ " laya.shortlist.predict_shortlist for a large label set will all cost less than"+ " scanning at this width" % (room, max_len, size, room, size / room),+ RuntimeWarning, stacklevel=3)size = roomstep = stride if (stride and stride > 0) else max(1, size // 2)if step > size:diff --git a/tests/test_predict_long.py b/tests/test_predict_long.pyindex c4672132..31759262 100644--- a/tests/test_predict_long.py+++ b/tests/test_predict_long.py@@ -662,6 +662,7 @@ check("docs/predict_long's docstring says the key is total",# the window was cut short on the way to the model while `answer["window"]["token_end"]` still# reported the whole span, and once the room fell under the stride, spans of the document were# read by no window at all.+from laya import common as common_mod # noqa: E402from laya.common import build_sequence, serialize_state # noqa: E402TOK = _Tok()@@ -1057,6 +1058,65 @@ check_true("no room/a question that leaves room is still scanned",room_for(q_many(2)) > 0 and len(scan({"a": q_many(2)})[1]) > 0, "")+# ------------------------------------------- a hard clamp of the DEFAULT window must not be silent+# Capping the window at the room is what stops the tail of every window reaching no model, but it is+# not free: the scan needs about `requested / room` times as many windows, each a full forward pass.+# Measured on the English checkpoint with 100 four-word options -- room 102 of max_len 512, window+# 312 -> 102, 11 windows -> 36, 1902 ms -> 5858 ms. Nothing in the caller's code implies that, so+# `window_budget` says so. Only on a HARD clamp: warning about every small one would be noise, and+# noise is how a warning that matters gets filtered out.+check_true("clamp warning/the threshold is a ratio above 1 and not absurd",+ 1 < common_mod._WINDOW_CLAMP_WARN_RATIO <= 4, common_mod._WINDOW_CLAMP_WARN_RATIO)++_default_window = max(64, MAX_LEN - HEAD_MAX_LEN - 8)+++def _clamp_warnings(qdef, **kw):+ with warnings.catch_warnings(record=True) as caught:+ warnings.simplefilter("always")+ window_budget(TOK, [Agent._to_internal(qdef)], MAX_LEN, HEAD_MAX_LEN,+ window=kw.get("window"), stride=kw.get("stride"))+ return [str(c.message) for c in caught]+++# A question whose head leaves less than half the default window: found by search rather than by+# hard-coding an option count, so a change to the fake tokenizer cannot quietly stop this reaching+# the branch.+_hard = next((n for n in range(2, 400)+ if 0 < room_for(q_many(n)) * common_mod._WINDOW_CLAMP_WARN_RATIO <= _default_window),+ None)+check_true("clamp warning/a hard-clamp fixture exists", _hard is not None, "")+# ... and one whose clamp is mild or absent, which must stay silent.+_mild = next((n for n in range(2, 400) if room_for(q_many(n)) >= _default_window), None)+check_true("clamp warning/a mild fixture exists", _mild is not None, "")++if _hard is not None and _mild is not None:+ _hard_msgs = _clamp_warnings(q_many(_hard))+ check("clamp warning/fires once on a hard clamp of the default window", len(_hard_msgs), 1)+ _m = _hard_msgs[0] if _hard_msgs else ""+ check_true("clamp warning/names the room it was cut to", str(room_for(q_many(_hard))) in _m, _m)+ check_true("clamp warning/names max_len", "max_len=%d" % MAX_LEN in _m, _m)+ check_true("clamp warning/names how much more scanning it costs", "as many windows" in _m, _m)+ check_true("clamp warning/points at the tool for a large label set",+ "predict_shortlist" in _m, _m)++ check("clamp warning/stays silent when the clamp is mild or absent",+ _clamp_warnings(q_many(_mild)), [])++ # An explicit `window=` keeps its own message and does not also get this one -- the caller who+ # named a width is told their width was reduced, which is a different statement.+ _explicit = _clamp_warnings(q_many(_hard), window=_default_window)+ check("clamp warning/an explicit window gets exactly one message", len(_explicit), 1)+ check_true("clamp warning/and it is the explicit-window one",+ _explicit and "is wider than the" in _explicit[0], _explicit)++# The refusal for a question with no room at all points at the same remedy.+_no_room_msg = _attempt(lambda: window_budget(TOK, [Agent._to_internal(q_many(400))],+ MAX_LEN, HEAD_MAX_LEN, window=None, stride=None))+check_true("clamp warning/the no-room refusal names predict_shortlist too",+ "predict_shortlist" in str(_no_room_msg), str(_no_room_msg)[:140])++print("\n%d passed, %d failed" % (len(PASS), len(FAIL)))for f in FAIL:print(" FAIL " + f)