Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(common): report state truncation from the token budget that applies

Ajit kumar8a47eff91dac
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index d4f38072..2c8b41e6 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -115,6 +115,7 @@ jobs:
python tests/test_shortlist.py
python tests/test_decision_model.py
python tests/test_head_checkpointing.py
+ python tests/test_truncation.py
python tests/test_packaging.py
python tests/test_compose_env.py
python tests/test_audit_scope.py
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index c5356a86..500b9a03 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -54,6 +54,7 @@ jobs:
python tests/test_shortlist.py
python tests/test_decision_model.py
python tests/test_head_checkpointing.py
+ python tests/test_truncation.py
python tests/test_packaging.py
python tests/test_compose_env.py
python tests/test_doc_tables.py
diff --git a/laya/common.py b/laya/common.py
index e5339320..72bb69b9 100644
--- a/laya/common.py
+++ b/laya/common.py
@@ -142,6 +142,7 @@ def build_sequence(
truncate_left: bool = False,
state_ids: Optional[List[int]] = None,
return_stats: bool = False,
+ return_truncation_stats: bool = False,
):
"""Format: [CLS] <type> instructions [SEP] [MASK] opt0 [MASK] opt1 ... [SEP] state [SEP].
@@ -151,6 +152,19 @@ def build_sequence(
`return_stats` adds a third return value describing what the head budget did to the options:
`options` (how many the question defines), `options_distinct` (how many still have a token
span of their own) and `tokens_per_option` (the cap applied to each, or None when none was).
+
+ The state is clamped to whatever room is left after the head, so a long state loses tokens
+ here silently. `return_truncation_stats=True` adds one more return value, after the option
+ stats when both are asked for, reporting that clamp:
+
+ {"state_tokens": int, "state_tokens_used": int, "state_tokens_dropped": int,
+ "truncated": bool}
+
+ Callers cannot reconstruct this from the outside. The budget is in tokens, not characters,
+ and the room left for the state depends on `max_len`, `head_max_len`, the instruction and
+ the rendered options - so it moves per checkpoint and per question. A caller guessing with a
+ fixed character threshold is wrong in both directions: it reports truncation that did not
+ happen, and stays silent while evidence is being dropped (issue #174).
"""
mask_tok = tok.mask_token
opts = render_options(q)
@@ -192,19 +206,28 @@ def build_sequence(
st = state_ids[max(0, len(state_ids) - room):] if truncate_left else state_ids[:room]
ids = ids + st + [tok.sep_token_id]
ids, markers = ids[:max_len], [m for m in markers if m < max_len]
+ extra = ()
+ if return_truncation_stats:
+ # `room` leaves space for the closing [SEP], so every token in `st` survives the [:max_len] clamp
+ extra = ({
+ "state_tokens": len(state_ids),
+ "state_tokens_used": len(st),
+ "state_tokens_dropped": len(state_ids) - len(st),
+ "truncated": len(st) < len(state_ids),
+ },)
if not return_stats:
- return ids, markers
+ return (ids, markers) + extra
# Two options that share a prefix can come out of the cut as the same token span: the marker
# count still matches the option count, so the guard in `Agent._encode_state` passes and
# nothing downstream can tell that the question lost the ability to name them apart. Counted
# on the capped option ids, before assembly: re-slicing the finished sequence cannot close
# the last option's span -- it runs on into the serialized state, which differs per request,
# so the last option always looks distinguishable however it collided (#538).
- return ids, markers, {
+ return (ids, markers, {
"options": len(opt_ids),
"options_distinct": len({tuple(o) for o in opt_ids}),
"tokens_per_option": per_option,
- }
+ }) + extra
def collapsed_options(qids, items) -> Dict[str, Dict[str, Optional[int]]]:
diff --git a/tests/test_truncation.py b/tests/test_truncation.py
new file mode 100644
index 00000000..886e61c0
--- /dev/null
+++ b/tests/test_truncation.py
@@ -0,0 +1,165 @@
+"""State truncation must be reported from the token budget that actually applies (issue #174).
+
+`build_sequence` clamps the state to whatever room the head leaves, and before this the clamp was
+silent: `system_one` returned a token count and nothing about what it had dropped. A caller could
+only guess, and the guess reported in #174 - a fixed character threshold - is wrong in both
+directions, because the budget is in tokens and moves with `max_len`, `head_max_len` and the
+rendered head of each question.
+
+These use a stub tokenizer (one token per whitespace word) so they run offline and the expected
+token counts are exact, the same approach as tests/test_shortlist.py.
+"""
+import os
+import sys
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from laya.common import build_sequence # noqa: E402
+
+PASS, FAIL = [], []
+
+
+def check(name, got, want):
+ if got == want:
+ PASS.append(name)
+ else:
+ FAIL.append("%s:\n got %r\n want %r" % (name, got, want))
+
+
+def check_true(name, cond, detail=""):
+ if cond:
+ PASS.append(name)
+ else:
+ FAIL.append("%s %s" % (name, detail))
+
+
+class StubTok:
+ """One token per whitespace-separated word, so token counts are exact and readable."""
+
+ mask_token = "[MASK]"
+ mask_token_id = 1
+ cls_token_id = 2
+ sep_token_id = 3
+ pad_token_id = 0
+
+ def __call__(self, text, add_special_tokens=True):
+ return {"input_ids": [10 + (len(w) % 50) for w in text.split()]}
+
+
+TOK = StubTok()
+Q = {"t": "noul", "ins": "Does the user ask for a refund?", "crit": None}
+
+
+def stats_for(state, max_len=512, head_max_len=192, truncate_left=False, q=Q):
+ _, _, st = build_sequence(
+ TOK, state, q, max_len, head_max_len, truncate_left=truncate_left, return_truncation_stats=True)
+ return st
+
+
+# --------------------------------------------------------------- the reported gap: a silent clamp
+# A state that fits loses nothing, and says so.
+short = stats_for("the customer was billed twice")
+check("short/not truncated", short["truncated"], False)
+check("short/nothing dropped", short["state_tokens_dropped"], 0)
+check("short/all tokens used", short["state_tokens_used"], short["state_tokens"])
+
+# A state that does not fit reports the exact shortfall rather than staying silent.
+long_state = " ".join("word%d" % i for i in range(4000))
+clamped = stats_for(long_state, max_len=512)
+check("long/truncated", clamped["truncated"], True)
+check("long/counts the whole state", clamped["state_tokens"], 4000)
+check_true("long/used less than sent", clamped["state_tokens_used"] < 4000)
+check("long/dropped is the remainder",
+ clamped["state_tokens_dropped"], 4000 - clamped["state_tokens_used"])
+check_true("long/used fits the window", clamped["state_tokens_used"] <= 512)
+
+
+# --------------------------------------------------------------- the flag tracks the real window
+# #174: the same state was reported identically on a 512-token and a 1024-token checkpoint, while
+# the actual clamp differed by more than 2x. The reported numbers must move with the window.
+w512 = stats_for(long_state, max_len=512)
+w1024 = stats_for(long_state, max_len=1024)
+check_true("window/1024 keeps strictly more than 512",
+ w1024["state_tokens_used"] > w512["state_tokens_used"],
+ "(512 -> %d, 1024 -> %d)" % (w512["state_tokens_used"], w1024["state_tokens_used"]))
+check_true("window/1024 drops strictly less than 512",
+ w1024["state_tokens_dropped"] < w512["state_tokens_dropped"])
+
+# A state sized to fit the larger window only: truncated on one checkpoint, intact on the other.
+mid = " ".join("word%d" % i for i in range(600))
+check("window/512 truncates the mid state", stats_for(mid, max_len=512)["truncated"], True)
+check("window/1024 does not", stats_for(mid, max_len=1024)["truncated"], False)
+
+# The head is part of the budget too, so two questions over one state can disagree. A longer
+# instruction and richer options leave less room for the state.
+wide = {
+ "t": "choice",
+ "ins": "Which team should own this request, given the account tier and the contract terms?",
+ "crit": {"billing": "invoices, payments, refunds, chargebacks and duplicate charges",
+ "technical": "bugs, outages, degraded performance and system errors",
+ "sales": "pricing, quotes, renewals and new contracts",
+ "other": "everything that does not belong to the three above"},
+}
+narrow = {"t": "noul", "ins": "Refund?", "crit": None}
+check_true("head/a wider head leaves less room for the state",
+ stats_for(long_state, q=wide)["state_tokens_used"]
+ < stats_for(long_state, q=narrow)["state_tokens_used"])
+
+
+# --------------------------------------------------------------- left truncation
+left = stats_for(long_state, max_len=512, truncate_left=True)
+check("left/truncated", left["truncated"], True)
+check_true("left/keeps as much as the right-hand clamp",
+ left["state_tokens_used"] == w512["state_tokens_used"])
+
+# With no room at all, `st[-0:]` is the whole state and only the final [:max_len] clamp drops it.
+# Counting `kept` would report nothing dropped while every state token was in fact discarded.
+no_room = stats_for(long_state, max_len=8, head_max_len=8, truncate_left=True)
+check("no-room/reports the drop", no_room["truncated"], True)
+check_true("no-room/never claims more than the window",
+ no_room["state_tokens_used"] <= 8,
+ "(used %d)" % no_room["state_tokens_used"])
+
+
+# --------------------------------------------------------------- reported counts match the sequence
+# The accounting must describe the sequence that is actually returned, not an estimate beside it.
+for name, ml, hml in [("512", 512, 192), ("1024", 1024, 512), ("tight", 64, 32)]:
+ seq, markers, st = build_sequence(TOK, long_state, Q, ml, hml, return_truncation_stats=True)
+ check_true("accounting/%s sequence fits max_len" % name, len(seq) <= ml,
+ "(len %d > %d)" % (len(seq), ml))
+ check_true("accounting/%s used tokens fit the sequence" % name, st["state_tokens_used"] <= len(seq))
+ check("accounting/%s used + dropped == sent" % name,
+ st["state_tokens_used"] + st["state_tokens_dropped"], st["state_tokens"])
+ check("accounting/%s truncated agrees with the count" % name,
+ st["truncated"], st["state_tokens_dropped"] > 0)
+
+
+# --------------------------------------------------------------- the stats are opt-in
+# Existing callers unpack two values (research/scripts/bench_local.py, the fine-tuning notebook).
+default = build_sequence(TOK, "a short state", Q)
+check("compat/default returns a pair", len(default), 2)
+seq_a, mk_a = build_sequence(TOK, long_state, Q, 512, 192)
+seq_b, mk_b, _ = build_sequence(TOK, long_state, Q, 512, 192, return_truncation_stats=True)
+check("compat/return_truncation_stats does not change the sequence", seq_a, seq_b)
+check("compat/return_truncation_stats does not change the markers", mk_a, mk_b)
+
+
+# --------------------------------------------------------------- system_one reports it in `usage`
+# Checked on the source: exercising it for real needs a checkpoint on disk (tests/test_local_e2e.py).
+import inspect # noqa: E402
+
+from laya import agent as _agent # noqa: E402
+
+_src = inspect.getsource(_agent.Agent.system_one)
+check_true("usage/asks build_sequence for the stats", "return_truncation_stats=True" in _src)
+check_true("usage/publishes the flag", '"truncated": dropped > 0' in _src)
+check_true("usage/publishes the dropped count", '"state_tokens_dropped": dropped' in _src)
+check_true("usage/names the questions that truncated", '"truncated_questions"' in _src)
+check_true("usage/keeps the existing token counts",
+ '"input_tokens": n_tokens' in _src and '"output_tokens": 0' in _src)
+
+
+print("\n%d passed, %d failed" % (len(PASS), len(FAIL)))
+for f in FAIL:
+ print(" FAIL " + f)
+sys.exit(1 if FAIL else 0)