Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

docs(examples): example 37 must teach the clamp its state's type really picks

Aashishf0f415662199
diff --git a/examples/37_long_document.py b/examples/37_long_document.py
index 8f85a8ff..a1cf609a 100644
--- a/examples/37_long_document.py
+++ b/examples/37_long_document.py
@@ -2,19 +2,23 @@
Laya gives every request a fixed window: the option head takes `head_max_len` and the state gets
the rest of `max_len`. A state larger than that is truncated. This example measures how much of
-a long multilingual document survives at the default, then raises the window at runtime.
+a long multilingual document survives at the default, raises the window at runtime, and then
+shows that which *end* of the state survives is decided by the state's own type.
"""
-from _common import banner, device_line, load
-from laya.common import build_sequence, serialize_state
+from _common import banner, device_line, heading, load
+from laya.common import build_sequence, serialize_state, state_room
banner("37", "Long documents and truncation", """
`laya-multilingual` defaults to max_len=1024 (head_max_len=256). The configured split
reserves up to 256 tokens for the option head, leaving 768 for the state in the worst case
(a small question uses less, so the measured room is a little larger). `build_sequence`
fills that window with `state[:room]` -- the *head* of the document is kept and the tail is
- dropped. `build_sequence` has a `truncate_left=True` switch to keep the tail instead, but
- `predict`/`system_one` does not expose it. Anything longer than the window is invisible to
- the model, and nothing warns you.
+ dropped. Which end that is comes from the state's own type, not from a parameter: `predict`
+ passes `truncate_left=isinstance(state, list)`, so a *list* of turns is clamped from the other
+ side and keeps its tail, while a string or dict keeps its head. `state_room` documents the same
+ split. And the cut is not invisible to the caller: every call reports it in `usage` --
+ `truncated`, `state_tokens`, `state_tokens_dropped`, `truncated_questions` -- which this
+ example prints below.
mmBERT is a RoPE encoder and supports up to 8192 tokens, so the window can be widened at
runtime: `agent.cfg` is a plain dict and `system_one` reads it on every call. We raise
@@ -83,6 +87,7 @@ def head_and_room(agent, question, max_len, head_max_len):
agent = load("multilingual")
device_line(agent)
+default_cfg = dict(agent.cfg)
doc = build_document()
doc_tokens = len(agent.tok(serialize_state(doc), add_special_tokens=False)["input_ids"])
print(" document: %d ticket records, %d characters, %d tokens (multilingual tokenizer)"
@@ -100,18 +105,68 @@ for head_max_len, max_len in ((256, 1024), (512, 8192)):
% (qid, head, room, visible, doc_tokens, 100 * visible // doc_tokens,
" TAIL DROPPED" if visible < doc_tokens else " full document"))
result = agent.predict(doc, QUESTIONS)
+ usage = result["usage"]
print(" usage: %d input tokens across %d questions (batch total), %d output tokens"
- % (result["usage"]["input_tokens"], len(QUESTIONS), result["usage"]["output_tokens"]))
+ % (usage["input_tokens"], len(QUESTIONS), usage["output_tokens"]))
for qid, a in result["answers"].items():
if a["type"] == "noul":
print(" %-12s noul=%.3f conf=%.3f" % (qid, a["noul"], a["confidence"]))
else:
print(" %-12s score=%.2f/3 conf=%.3f" % (qid, a["score"], a["confidence"]))
+ # The answers above do not look cut off -- `truncated` is the only thing that says so.
+ print(" the call reports the clamp: `truncated`=%s, `state_tokens`=%d,"
+ " `state_tokens_dropped`=%d, `truncated_questions`=%s"
+ % (usage["truncated"], usage["state_tokens"], usage["state_tokens_dropped"],
+ ", ".join("`%s`" % qid for qid in usage["truncated_questions"]) or "none"))
first, second = visible_by_config["needs_human"]
print("\n widening the window moved the visible document from %d to %d tokens: the tail is"
% (first, second))
-print(" only recoverable by setting the budget *before* the forward pass, not after.")
+print(" recoverable by setting the budget *before* the forward pass, not after -- and by")
+print(" handing the state over as a list, which is the section below.")
print(" the head stayed 36/48 tokens even at head_max_len=512 -- it is a ceiling, not a")
print(" reservation, and the unused head room flows back to the state.")
+# --- which end of the state survives: the same tokens, two different types -------------------
+heading("which end of the state survives")
+agent.cfg["head_max_len"], agent.cfg["max_len"] = default_cfg["head_max_len"], default_cfg["max_len"]
+max_len, head_max_len = agent.cfg["max_len"], agent.cfg["head_max_len"]
+
+CLOSING = "CLOSING NOTE: the last record of this export mentions the auditors."
+turns = [rec["body"] for rec in doc["records"]] + [CLOSING]
+turns_text = serialize_state(turns) # the exact characters a list state serializes to
+TAIL_Q = {"asked_last": {"type": "noul",
+ "instructions": "Does the closing note of this export mention the auditors?"}}
+# `predict` picks the clamp side with `truncate_left = isinstance(state, list)`, on both agents.
+# Rebuild that slice here so the two words that decide it -- first and last -- can be looked at.
+q = {"t": "noul", "ins": TAIL_Q["asked_last"]["instructions"], "crit": None}
+room = state_room(agent.tok, q, max_len, head_max_len)
+print(" %d state tokens, %d of them fit (`state_room`), so %d are dropped either way."
+ % (len(agent.tok(turns_text, add_special_tokens=False)["input_ids"]), room,
+ len(agent.tok(turns_text, add_special_tokens=False)["input_ids"]) - room))
+for label, state in (("a string", turns_text), ("a list", turns)):
+ tokens = agent.tok(serialize_state(state), add_special_tokens=False)["input_ids"]
+ left = isinstance(state, list)
+ kept = tokens[max(0, len(tokens) - room):] if left else tokens[:room]
+ answer = agent.predict(state, TAIL_Q)
+ usage = answer["usage"]
+ print(" the same %d tokens as %-9s: `truncate_left` = isinstance(state, list) -> %s,"
+ " so `state_tokens_dropped`=%d (my slice: %d)"
+ % (len(tokens), label, left, usage["state_tokens_dropped"], len(tokens) - len(kept)))
+ print(" %swhat the model actually read starts %r and ends %r"
+ % (" " * 24, agent.tok.decode(kept)[:44], agent.tok.decode(kept)[-44:]))
+ print(" %sthe closing note is in it: %s noul=%.3f conf=%.3f"
+ % (" " * 24, "auditors" in agent.tok.decode(kept),
+ answer["answers"]["asked_last"]["noul"], answer["answers"]["asked_last"]["confidence"]))
+
+print("""
+ Same characters, same `state_tokens`, same `state_tokens_dropped` -- the type alone decided
+ which end of it the model read. The string kept its head, so the auditors sentence never
+ reached the model; the list was clamped from the left, so it did. `truncate_left` is not a
+ parameter a caller passes: `predict` takes it from `isinstance(state, list)`, and `state_room`
+ is what tells you how much of either end fits.
+
+ And the answers disagree with that: both say yes, because a wall of angry tickets is enough to
+ make the head fire whatever survived in it. That is the reason the report exists. `truncated`
+ and `truncated_questions` are the only part of this result that says the evidence was cut.""")
+
diff --git a/tests/test_truncation.py b/tests/test_truncation.py
index cb499310..614aad33 100644
--- a/tests/test_truncation.py
+++ b/tests/test_truncation.py
@@ -161,17 +161,20 @@ check_true("usage/keeps the existing token counts",
'"input_tokens": n_tokens' in _src and '"output_tokens": 0' in _src)
-# --------------------------------------------------------------- example 15 teaches this exact dict
-# examples/15_truncation_basics.py is the page that teaches the report: it prints the dict off a call
-# that really truncated. tests/test_structured_docs.py already holds examples/03's drawn shape to the
-# same literal, so this covers only what that gate cannot see -- that 15 reads and names every key the
-# agents publish. The key set is read out of `laya/agent.py` by AST rather than repeated here, so a key
-# added to the dict has to be taught and one renamed fails the example still pointing at it.
+# --------------------------------------------------------- the truncation examples teach this dict
+# examples/15 and examples/37 are the two pages that teach the report: each prints it off a call that
+# really truncated. tests/test_structured_docs.py already holds examples/03's drawn shape to the same
+# literal, so this covers only what that gate cannot see -- that each example reads and names every
+# key the agents publish, and that the prose about which end of the state survives matches the code
+# that decides it. The key set is read out of `laya/agent.py` by AST rather than repeated here, so a
+# key added to the dict has to be taught and one renamed fails the example still pointing at it.
import ast # noqa: E402
import re # noqa: E402
ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
EX15 = "examples/15_truncation_basics.py"
+EX37 = "examples/37_long_document.py"
+EXAMPLES = (("15", EX15), ("37", EX37))
def _tree(rel):
@@ -222,23 +225,49 @@ report = tuple(k for k in published if k not in ("input_tokens", "output_tokens"
check_true("report/the report is more than the two token totals",
len(report) >= 4, "(%r)" % (report,))
-missing = tuple(k for k in report if k not in _usage_reads(EX15))
-check("15/prints every key of the report from a live call", missing, ())
-extra = tuple(k for k in _usage_reads(EX15) if k not in published)
-check("15/reads no key the agents do not publish", extra, ())
-prose = " ".join(_strings(EX15))
-for key in report:
- check_true("15/names `%s` so a reader can grep for it" % key, "`%s`" % key in prose)
-
-# The claim that made this example wrong when it was written: it told the reader truncation is silent
-# and "nothing warns you", one line after printing `truncated`. A sentence may still say the cut is
-# invisible -- the two answers really are indistinguishable -- but only if it names the key saying so.
UNREPORTED = re.compile(r"nothing warns|is silent|silent in|no warning|not reported"
r"|nothing says|never says|cannot be seen", re.I)
-for sentence in [s for text in _strings(EX15) for s in re.split(r"(?<=[.!?])\s+", text) if s.strip()]:
- if UNREPORTED.search(sentence):
- check_true("15/a claim that the cut is invisible must name the key that reports it",
- "`truncated`" in sentence, "(%r)" % sentence[:120])
+
+for tag, rel in EXAMPLES:
+ missing = tuple(k for k in report if k not in _usage_reads(rel))
+ check("%s/prints every key of the report from a live call" % tag, missing, ())
+ extra = tuple(k for k in _usage_reads(rel) if k not in published)
+ check("%s/reads no key the agents do not publish" % tag, extra, ())
+ prose = " ".join(_strings(rel))
+ for key in report:
+ check_true("%s/names `%s` so a reader can grep for it" % (tag, key), "`%s`" % key in prose)
+
+ # The claim that made 15 wrong when it was written: it told the reader truncation is silent and
+ # "nothing warns you", one line after printing `truncated`. A sentence may still say the cut is
+ # invisible -- the two answers really are indistinguishable -- but only if it names the key that
+ # says otherwise.
+ for text in _strings(rel):
+ for sentence in [s for s in re.split(r"(?<=[.!?])\s+", text) if s.strip()]:
+ if UNREPORTED.search(sentence):
+ check_true("%s/a claim that the cut is invisible must name the key that reports it" % tag,
+ "`truncated`" in sentence, "(%r)" % sentence[:120])
+ # Which end of the state survives is a property of the state's type, not of a parameter a
+ # caller passes. 37 said `predict` "does not expose" `truncate_left`, which is only true
+ # of the keyword: the rule is reachable, and it is `list`.
+ if "`truncate_left`" in sentence or "truncate_left=" in sentence:
+ check_true("%s/a sentence about `truncate_left` names the type that sets it" % tag,
+ "list" in sentence, "(%r)" % sentence[:120])
+
+# ------------------------------------------------- and the rule the examples teach is the one shipped
+# Both agents pick the clamp side the same way, from the state's type, because a conversation list
+# serializes newest-last and a left-keeping clamp is the only one that preserves the newest turn.
+# Read as a pattern rather than a string: #687 moves the same expression inline into the
+# `build_sequence` call, and the examples' claim has to survive that reformatting or keep its meaning.
+CLAMP = re.compile(r"truncate_left[^\n]*?isinstance\(state, ?list\)")
+for rel in ("laya/agent.py", "laya/onnx_agent.py"):
+ with open(os.path.join(ROOT, rel)) as fh:
+ src = fh.read()
+ check_true("clamp/%s clamps from the state's type" % rel, CLAMP.search(src) is not None,
+ "no `truncate_left = isinstance(state, list)`")
+# ...and 37 is the page that teaches it, so the rule above has something to police: a page that
+# dropped the name would pass every sentence check by never raising one.
+check_true("37/teaches the clamp by its real name",
+ any("truncate_left" in text for text in _strings(EX37)), "never mentions `truncate_left`")
print("\n%d passed, %d failed" % (len(PASS), len(FAIL)))