Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
perf(agent): reuse question tokenization within prediction batches
周诚宇(Zhou Chengyu)d868c9f2c066
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.ymlindex 7a9cca5f..e3e78d3d 100644--- a/.github/workflows/ci.yml+++ b/.github/workflows/ci.yml@@ -112,6 +112,7 @@ jobs:python tests/test_doc_tables.pypython tests/test_tokenizer_cache.pypython tests/test_tokenizer_concurrency.py+ python tests/test_question_token_reuse.pypython tests/test_lazy_import.pypython tests/test_runtime_fixes.pypython tests/test_lang_guess.py@@ -254,6 +255,10 @@ jobs:python tests/test_hooks_api.pyshell: bash+ - name: Question token reuse tests+ run: python tests/test_question_token_reuse.py+ shell: bash+# Same five pytest suites as the Linux job. They are named explicitly rather than added to# the loop above, because the loop runs each file as a script and that is exactly what does# not execute a pytest suite (#374).diff --git a/laya/agent.py b/laya/agent.pyindex 1b58de6f..cdfd4c16 100644--- a/laya/agent.py+++ b/laya/agent.py@@ -23,6 +23,8 @@ from .common import (answer_confidence,confidence_from_probs,_resolve_noul_labels,+ _reuse_question_tokens,+ _disable_question_token_reuse,encode_text,render_options,serialize_state,@@ -814,6 +816,7 @@ class Agent(HookRegistry):return answers@torch.no_grad()+ @_reuse_question_tokensdef predict_batch(self, states: List[Union[str, dict, list]], questions: Dict[str, Dict[str, Any]],batch_size: Optional[int] = None, lang: Optional[str] = None,hooks=None,@@ -873,6 +876,8 @@ class Agent(HookRegistry):"predict_batch expects a list of states; pass a single state to predict()/system_one().")states = list(states)+ if len(states) == 1:+ _disable_question_token_reuse()if not states:ctx.results = []else:diff --git a/laya/common.py b/laya/common.pyindex 91f702a5..3cf70571 100644--- a/laya/common.py+++ b/laya/common.py@@ -4,6 +4,8 @@ import mathimport osimport threadingfrom contextlib import nullcontext+from contextvars import ContextVar+from functools import wrapsfrom typing import Dict, List, Optional, Unionimport numpy as np@@ -22,6 +24,7 @@ _DEFAULT_NOUL_LABELS = {"false": "false", "true": "true"}# `RuntimeError: Already borrowed`. Serialise encoding instead: it is a small fraction of a call# next to the forward pass, and this keeps the cache's single parse._TOKENIZE_LOCK = threading.RLock()+_QUESTION_TOKEN_CACHE = ContextVar("laya_question_token_cache", default=None)def encode_text(tok, text, **kwargs):@@ -30,6 +33,43 @@ def encode_text(tok, text, **kwargs):return tok(text, **kwargs)+def _reuse_question_tokens(fn):+ @wraps(fn)+ def wrapped(*args, **kwargs):+ # One cache per prediction call: nested calls get their own scope, and exceptions+ # restore the outer scope. Nothing is retained on an Agent or shared across threads.+ scope = {"thread": threading.get_ident(), "tokens": {}}+ token = _QUESTION_TOKEN_CACHE.set(scope)+ try:+ return fn(*args, **kwargs)+ finally:+ # A timed hook can copy this context to a worker that outlives the call.+ scope["tokens"] = None+ _QUESTION_TOKEN_CACHE.reset(token)+ return wrapped+++def _disable_question_token_reuse():+ scope = _QUESTION_TOKEN_CACHE.get()+ if scope is not None:+ scope["tokens"] = None+++def _encode_question_text(tok, text, **kwargs):+ scope = _QUESTION_TOKEN_CACHE.get()+ cache = scope["tokens"] if scope is not None and scope["thread"] == threading.get_ident() else None+ if cache is None:+ return encode_text(tok, text, **kwargs)["input_ids"]+ # Key the rendered, mask-sanitized text and encoding settings, not a JSON question:+ # option order, structured criteria and custom noul labels must keep their meaning.+ key = (id(tok), text, tuple(kwargs.items()))+ if key not in cache:+ # Keep the tokenizer alive so its identity cannot be reused within this scope.+ cache[key] = (tok, tuple(encode_text(tok, text, **kwargs)["input_ids"]))+ # Sequence assembly must not mutate token lists retained for later states.+ return list(cache[key][1])++def serialize_state(state: Union[str, dict, list]) -> str:if isinstance(state, str):return state@@ -110,19 +150,19 @@ def build_sequence(opts = render_options(q)order = option_order if option_order is not None else list(range(len(opts)))ins = str(q["ins"]).replace(mask_tok, " ")- head_ids = encode_text(tok, "%s question: %s" % (q["t"], ins), add_special_tokens=False)["input_ids"]+ head_ids = _encode_question_text(tok, "%s question: %s" % (q["t"], ins), add_special_tokens=False)opt_ids = []for i in order:# Cap at the tokenizer, not after the fact: `[:48]` still makes the tokenizer process the# whole (possibly long) description. truncation=True, max_length=48 keeps the first 48# tokens, which is exactly what the previous slice produced.- opt_tokens = encode_text(+ opt_tokens = _encode_question_text(tok," " + opts[i].replace(mask_tok, " "),add_special_tokens=False,truncation=True,max_length=48,- )["input_ids"]+ )opt_ids.append([tok.mask_token_id] + opt_tokens)opt_budget = head_max_len - sum(len(o) for o in opt_ids)if opt_budget < 16:diff --git a/research/results/question_token_reuse_cuda.json b/research/results/question_token_reuse_cuda.jsonnew file mode 100644index 00000000..3ce02b42--- /dev/null+++ b/research/results/question_token_reuse_cuda.json@@ -0,0 +1,349 @@+{+ "base": "4066d5d5fbf08b66c6757ddeedbd797bd7655bc0",+ "model": "../models/laya",+ "model_revision": "55cf4c4ebb4ebe31b2550e8bdf3bd21b99753851",+ "torch": "2.7.1+cu128",+ "device": "cuda",+ "gpu": "NVIDIA GeForce RTX 4060 Laptop GPU",+ "threads": 4,+ "repeats": 16,+ "workloads": [+ {+ "name": "long_20_options_default",+ "windows": 30,+ "before_p50_ms": 770.7229144999985,+ "after_p50_ms": 741.8036320000141,+ "saving_percent": 3.752228194583475,+ "paired_saving_p50_ms": 25.38752499998509,+ "after_faster_pairs": 16,+ "question_encode_calls": {+ "before": 630,+ "after": 21+ },+ "pairs_ms": [+ {+ "before": 755.3662249999888,+ "after": 731.0198440000022+ },+ {+ "after": 734.5862699999941,+ "before": 756.0385290000227+ },+ {+ "before": 763.4225589999915,+ "after": 743.674795000004+ },+ {+ "after": 749.1004630000191,+ "before": 781.9324560000211+ },+ {+ "before": 771.513586999987,+ "after": 745.0849180000034+ },+ {+ "after": 739.2042749999916,+ "before": 775.3677320000065+ },+ {+ "before": 775.0609379999958,+ "after": 733.2346009999924+ },+ {+ "after": 742.5324480000199,+ "before": 773.6810950000006+ },+ {+ "before": 741.9111699999803,+ "after": 739.8703190000049+ },+ {+ "after": 741.0748160000082,+ "before": 769.993301999989+ },+ {+ "before": 766.7506409999874,+ "after": 766.6943709999998+ },+ {+ "after": 739.4848539999828,+ "before": 769.5560379999904+ },+ {+ "before": 779.1554670000096,+ "after": 750.8460010000135+ },+ {+ "after": 766.0764169999936,+ "before": 771.4525270000081+ },+ {+ "before": 775.2085010000087,+ "after": 755.3889690000233+ },+ {+ "after": 711.069574000021,+ "before": 725.6501040000103+ }+ ],+ "all_input_tensors_equal": true,+ "outputs_equal": true,+ "decision_logits_and_action_probabilities_equal": true+ },+ {+ "name": "long_20_options_batch8",+ "windows": 30,+ "before_p50_ms": 772.4074305000102,+ "after_p50_ms": 742.8103704999955,+ "saving_percent": 3.831793795776311,+ "paired_saving_p50_ms": 25.337741500010225,+ "after_faster_pairs": 16,+ "question_encode_calls": {+ "before": 630,+ "after": 21+ },+ "pairs_ms": [+ {+ "before": 747.2378940000226,+ "after": 722.010795000017+ },+ {+ "after": 724.8769889999949,+ "before": 749.1776330000448+ },+ {+ "before": 763.6030759999812,+ "after": 747.3942290000082+ },+ {+ "after": 737.9716530000451,+ "before": 762.8258909999772+ },+ {+ "before": 773.0705460000422,+ "after": 758.7429350000434+ },+ {+ "after": 741.4936989999887,+ "before": 774.9147319999565+ },+ {+ "before": 763.6511570000266,+ "after": 740.7228280000027+ },+ {+ "after": 743.3846420000236,+ "before": 772.8549529999782+ },+ {+ "before": 767.094568999994,+ "after": 743.7462590000337+ },+ {+ "after": 745.6122489999757,+ "before": 771.9599080000421+ },+ {+ "before": 774.0217000000484,+ "after": 742.2360989999675+ },+ {+ "after": 750.0837059999981,+ "before": 775.5320900000129+ },+ {+ "before": 776.6750619999812,+ "after": 741.2745449999534+ },+ {+ "after": 742.0869170000515,+ "before": 756.5714419999949+ },+ {+ "before": 784.0836970000282,+ "after": 758.568293000053+ },+ {+ "after": 778.9746190000528,+ "before": 816.0115910000059+ }+ ],+ "all_input_tensors_equal": true,+ "outputs_equal": true,+ "decision_logits_and_action_probabilities_equal": true+ },+ {+ "name": "long_4_options_default",+ "windows": 30,+ "before_p50_ms": 543.4064060000026,+ "after_p50_ms": 539.2311045000042,+ "saving_percent": 0.7683570627612957,+ "paired_saving_p50_ms": 12.141736000018,+ "after_faster_pairs": 13,+ "question_encode_calls": {+ "before": 150,+ "after": 5+ },+ "pairs_ms": [+ {+ "before": 597.1686920000252,+ "after": 570.3511920000324+ },+ {+ "after": 548.8087200000109,+ "before": 548.5951840000212+ },+ {+ "before": 563.6884250000094,+ "after": 541.0735230000228+ },+ {+ "after": 545.9787620000043,+ "before": 543.492614999991+ },+ {+ "before": 555.453449999959,+ "after": 549.3829319999577+ },+ {+ "after": 567.2547649999728,+ "before": 585.6271679999736+ },+ {+ "before": 585.101796999993,+ "after": 564.3718290000379+ },+ {+ "after": 577.6956290000044,+ "before": 599.3873820000317+ },+ {+ "before": 540.1400929999909,+ "after": 523.965282000006+ },+ {+ "after": 525.9081159999823,+ "before": 543.3201970000141+ },+ {+ "before": 525.2991820000261,+ "after": 524.15587400003+ },+ {+ "after": 528.3915110000521,+ "before": 533.8170489999925+ },+ {+ "before": 534.4393409999952,+ "after": 537.3886859999857+ },+ {+ "after": 526.5865289999851,+ "before": 541.0266940000383+ },+ {+ "before": 538.9093260000095,+ "after": 529.0660190000267+ },+ {+ "after": 531.238535,+ "before": 531.8594930000131+ }+ ],+ "all_input_tensors_equal": true,+ "outputs_equal": true,+ "decision_logits_and_action_probabilities_equal": true+ },+ {+ "name": "single_short_20_options",+ "windows": 1,+ "before_p50_ms": 27.625586999988627,+ "after_p50_ms": 27.79482599999028,+ "saving_percent": -0.6126168468446398,+ "paired_saving_p50_ms": -0.2940649999914058,+ "after_faster_pairs": 6,+ "question_encode_calls": {+ "before": 21,+ "after": 21+ },+ "pairs_ms": [+ {+ "before": 24.87685800002737,+ "after": 25.161067999988518+ },+ {+ "after": 25.549729000033494,+ "before": 32.6728030000254+ },+ {+ "before": 24.855655999999726,+ "after": 27.84138399999847+ },+ {+ "after": 33.07467999997016,+ "before": 24.60957400001007+ },+ {+ "before": 26.697087000002284,+ "after": 26.525146000039967+ },+ {+ "after": 28.13503100003345,+ "before": 33.05376600002319+ },+ {+ "before": 28.00941899999998,+ "after": 27.74826799998209+ },+ {+ "after": 31.101750999994238,+ "before": 27.477479999959087+ },+ {+ "before": 28.22349100000565,+ "after": 31.531492999988586+ },+ {+ "after": 27.645273000018733,+ "before": 25.38376799998332+ },+ {+ "before": 27.79949600000009,+ "after": 27.734253000005538+ },+ {+ "after": 28.282738000029894,+ "before": 27.978818000008232+ },+ {+ "before": 29.842222000013408,+ "after": 30.09515899998405+ },+ {+ "after": 26.838087000044197,+ "before": 26.429774000007455+ },+ {+ "before": 27.773694000018168,+ "after": 27.15556599997626+ },+ {+ "after": 30.651570999964406,+ "before": 25.120577999985017+ }+ ],+ "all_input_tensors_equal": true,+ "outputs_equal": true,+ "decision_logits_and_action_probabilities_equal": true+ }+ ],+ "python": "3.10.12",+ "transformers": "4.51.3",+ "os": "WSL2 Ubuntu-22.04",+ "gpu_memory_gib": 8,+ "amp": "bf16",+ "weights": "fp32",+ "warmup_per_mode": 3,+ "approximate_filler_tokens": 4800,+ "workload_note": "MASSIVE seed-13 English first case + bench_long_context filler is a latency workload, not an accuracy benchmark."+}diff --git a/research/scripts/bench_question_token_reuse.py b/research/scripts/bench_question_token_reuse.pynew file mode 100644index 00000000..b5b3d557--- /dev/null+++ b/research/scripts/bench_question_token_reuse.py@@ -0,0 +1,157 @@+"""Compare the actual patch against the original functions at the checked-out base commit."""+import argparse+import ast+import contextlib+import importlib+import json+import os+import pathlib+import sys+import statistics+import subprocess+import time+from unittest.mock import patch++ROOT = pathlib.Path(__file__).resolve().parents[2]+sys.path.insert(0, str(ROOT))+os.environ.setdefault("USE_TF", "0")+os.environ.setdefault("TOKENIZERS_PARALLELISM", "false")+import torch # noqa: E402+import laya # noqa: E402+from bench_long_context import FILLER, QUESTIONS # noqa: E402+import laya.common as common # noqa: E402++agent_module = importlib.import_module("laya.agent")+after_batch = laya.Agent.predict_batch+++def original_function(base, file, name, namespace, cls=None):+ # Execute only the original function from this checkout, with its original dependencies.+ source = subprocess.check_output(["git", "show", base + ":" + file], cwd=ROOT, text=True)+ tree = ast.parse(source)+ nodes = tree.body+ if cls:+ nodes = next(node for node in nodes if isinstance(node, ast.ClassDef) and node.name == cls).body+ fn = next(node for node in nodes if isinstance(node, ast.FunctionDef) and node.name == name)+ module = ast.Module(body=[fn], type_ignores=[])+ exec(compile(module, file + "@" + base, "exec"), namespace)+ return namespace[name]+++def mode(name, before_batch):+ with patch.object(laya.Agent, "predict_batch", before_batch if name == "before" else after_batch):+ yield+++def timed(fn, device):+ if device.type == "cuda":+ torch.cuda.synchronize(device)+ start = time.perf_counter()+ result = fn()+ if device.type == "cuda":+ torch.cuda.synchronize(device)+ return result, (time.perf_counter() - start) * 1000+++def capture(agent, fn):+ batches, raw = [], []+ forward = agent._forward++ def saved(batch):+ batches.append({key: val.clone() for key, val in batch.items() if isinstance(val, torch.Tensor)})+ result = forward(batch)+ raw.append(tuple(val.copy() for val in result))+ return result++ with patch.object(agent, "_forward", saved):+ result = fn()+ return result, batches, raw+++def main():+ parser = argparse.ArgumentParser(description=__doc__)+ parser.add_argument("--model", required=True, help="Checkpoint directory or Hugging Face repo")+ parser.add_argument("--revision", default=None)+ parser.add_argument("--baseline-ref", required=True, help="Trusted Git ref before question reuse")+ parser.add_argument("--device", default="cuda")+ parser.add_argument("--repeats", type=int, default=16)+ parser.add_argument("--out", type=pathlib.Path, required=True)+ args = parser.parse_args()+ if args.repeats < 1:+ parser.error("--repeats must be positive")+ torch.set_num_threads(4)+ before_build = original_function(args.baseline_ref, "laya/common.py", "build_sequence", dict(common.__dict__))+ before_batch = original_function(args.baseline_ref, "laya/agent.py", "predict_batch",+ dict(agent_module.__dict__, build_sequence=before_build), cls="Agent")+ agent = laya.Agent(args.model, revision=args.revision, device=args.device, fast=False, compile=False)+ # First seed-13 MASSIVE English regression case, combined with the long-context filler.+ # This is a latency workload; no dataset accuracy is claimed for the padded input.+ case = {"state": {"utterance": "wake me up at five am this week"},+ "instructions": "What is the user asking for in `utterance`?",+ "options": ["email_query", "takeaway_order", "qa_definition", "cooking_recipe", "qa_factoid",+ "calendar_set", "music_settings", "social_query", "general_quirky", "datetime_query",+ "qa_currency", "audio_volume_other", "transport_query", "alarm_remove", "general_greet",+ "transport_traffic", "email_querycontact", "alarm_set", "transport_ticket", "takeaway_query"]}+ questions20 = {"intent": {"type": "choice", "instructions": case["instructions"],+ "criteria": {key: key.replace("_", " ") for key in case["options"]}}}+ repeats = round(4800 / len(agent.tok(FILLER, add_special_tokens=False)["input_ids"]))+ state = FILLER * repeats + "\n\nActual request: " + common.serialize_state(case["state"])+ workloads = [("long_20_options_default", lambda: agent.predict_long(state, questions20)),+ ("long_20_options_batch8", lambda: agent.predict_long(state, questions20, batch_size=8)),+ ("long_4_options_default", lambda: agent.predict_long(state, QUESTIONS)),+ ("single_short_20_options", lambda: agent.system_one(case["state"], questions20))]+ report = {"base": args.baseline_ref, "model": args.model, "model_revision": args.revision,+ "torch": torch.__version__, "device": str(agent.device), "gpu": torch.cuda.get_device_name(agent.device) if agent.device.type == "cuda" else None, "threads": 4,+ "repeats": args.repeats, "workloads": []}+ for name, fn in workloads:+ for which in ("before", "after"):+ with mode(which, before_batch):+ for _ in range(3):+ fn()+ with mode("before", before_batch):+ expected, before_tensors, before_raw = capture(agent, fn)+ with mode("after", before_batch):+ actual, after_tensors, after_raw = capture(agent, fn)+ assert expected == actual+ assert len(before_tensors) == len(after_tensors)+ assert all(left[key].equal(right[key]) for left, right in zip(before_tensors, after_tensors) for key in left)+ assert all((x == y).all() for left, right in zip(before_raw, after_raw) for x, y in zip(left, right))+ counts = {}+ for which in ("before", "after"):+ seen = []+ encode = common.encode_text++ def counting(tok, text, **kwargs):+ seen.append(text)+ return encode(tok, text, **kwargs)++ # The original build_sequence has its own global namespace.+ with mode(which, before_batch), patch.object(common, "encode_text", counting), patch.dict(+ before_build.__globals__, encode_text=counting):+ fn()+ counts[which] = len(seen)+ pairs = []+ for i in range(args.repeats):+ pair = {}+ for which in (("before", "after") if i % 2 == 0 else ("after", "before")):+ with mode(which, before_batch):+ result, pair[which] = timed(fn, agent.device)+ assert result == expected+ assert agent.device.type == args.device.split(":")[0]+ pairs.append(pair)+ before = statistics.median(pair["before"] for pair in pairs)+ after = statistics.median(pair["after"] for pair in pairs)+ row = {"name": name, "windows": expected["usage"].get("windows", 1),+ "before_p50_ms": before, "after_p50_ms": after, "saving_percent": (before - after) / before * 100,+ "paired_saving_p50_ms": statistics.median(pair["before"] - pair["after"] for pair in pairs),+ "after_faster_pairs": sum(pair["after"] < pair["before"] for pair in pairs),+ "question_encode_calls": counts, "pairs_ms": pairs,+ "all_input_tensors_equal": True, "decision_logits_and_action_probabilities_equal": True, "outputs_equal": True}+ report["workloads"].append(row)+ args.out.write_text(json.dumps(report, indent=2))+ print(json.dumps({key: value for key, value in row.items() if key != "pairs_ms"}), flush=True)+++if __name__ == "__main__":+ main()diff --git a/tests/test_question_token_reuse.py b/tests/test_question_token_reuse.pynew file mode 100644index 00000000..a68e48e9--- /dev/null+++ b/tests/test_question_token_reuse.py@@ -0,0 +1,255 @@+"""Question encoding reuse: run with python tests/test_question_token_reuse.py (no weights)."""+import copy+import contextvars+import os+import sys+import threading+import unittest+from concurrent.futures import ThreadPoolExecutor++import numpy as np++sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))++from laya.agent import Agent # noqa: E402+from laya.common import _reuse_question_tokens, build_sequence # noqa: E402+++class Tokenizer:+ mask_token = "[MASK]"+ mask_token_id = 1+ cls_token_id = 2+ sep_token_id = 3+ pad_token_id = 0++ def __init__(self, offset=0):+ self.calls = []+ self.offset = offset++ def __call__(self, text, **kwargs):+ self.calls.append((threading.get_ident(), text, kwargs))+ ids = [ord(c) + 4 + self.offset for c in text]+ if kwargs.get("truncation"):+ ids = ids[:kwargs["max_length"]]+ return {"input_ids": ids}++ def decode(self, ids, **kwargs):+ return "".join(chr(i - 4 - self.offset) for i in ids)+++QUESTION = {"t": "choice", "ins": "Which team?", "crit": {"billing": "charges", "tech": "errors"}}+QUESTIONS = {"team": {"type": "choice", "instructions": QUESTION["ins"], "criteria": QUESTION["crit"]}}+++def agent_with(tok):+ agent = Agent.__new__(Agent)+ agent.tok = tok+ agent.cfg = {"max_len": 512, "head_max_len": 192}+ agent.batches = []++ def forward(batch):+ agent.batches.append({key: value.clone() for key, value in batch.items() if hasattr(value, "clone")})+ shape = batch["marker_pos"].shape+ return np.zeros(shape, dtype=np.float32), np.full((shape[0], 2), 0.5, dtype=np.float32)++ def decode(logits, act, items, ids, internal, offset, **kwargs):+ return {qid: {"ids": item["ids"], "markers": item["markers"], "answer_confidence": 0.5}+ for qid, item in zip(ids, items)}++ agent._forward = forward+ agent._decode_answers = decode+ return agent+++class QuestionTokenReuse(unittest.TestCase):+ def test_reuse_across_chunks_preserves_all_collated_tensors(self):+ states = ["state %d" % i for i in range(19)]+ tok = Tokenizer()+ agent = agent_with(tok)+ for grouped in (False, True):+ for batch_size in (None, 1, 3):+ with self.subTest(grouped=grouped, batch_size=batch_size):+ tok.calls.clear()+ agent.batches.clear()+ got = agent.predict_batch(states, QUESTIONS, batch_size=batch_size, sort_by_length=grouped)+ self.assertEqual(len(tok.calls), len(states) + 3)+ cached = agent.batches[:]+ agent.batches.clear()+ # The real undecorated method is the same orchestration without reuse.+ plain = Agent.predict_batch.__wrapped__.__wrapped__(+ agent, states, QUESTIONS, batch_size=batch_size, sort_by_length=grouped)+ self.assertEqual(got, plain)+ self.assertEqual(len(cached), len(agent.batches))+ for left, right in zip(cached, agent.batches):+ self.assertEqual(set(left), set(right))+ for key in left:+ self.assertTrue(left[key].equal(right[key]), key)++ def test_predict_long_uses_the_same_cache_for_all_windows(self):+ tok = Tokenizer()+ agent = agent_with(tok)+ agent.predict_long("x" * 1000, QUESTIONS, window=80, stride=50, batch_size=3)+ head = "choice question: " + QUESTION["ins"]+ self.assertEqual(sum(text == head for _, text, _ in tok.calls), 1)+ self.assertEqual(sum(text == " billing: charges" for _, text, _ in tok.calls), 1)+ self.assertGreater(len(agent.batches), 1)++ def test_sequence_parity_for_rendering_order_and_budgets(self):+ cases = [+ QUESTION,+ {"t": "choice", "ins": "a [MASK] b", "crit": {7: None, "zero": 0, "false": False}},+ {"t": "choice", "ins": "中文", "crit": {"a": {"desc": [1, "二"]}, "b": "z" * 200}},+ {"t": "score", "ins": "level", "crit": [{"desc": "one"}, ["two"], 0]},+ {"t": "noul", "ins": "holds?", "crit": {"false": {"reason": "no"}, "true": ["yes"]},+ "labels": {"false": "否", "true": "是"}},+ ]+ tok = Tokenizer()+ for q in cases:+ for max_len, head_len in ((512, 192), (128, 32), (16, 8)):+ for left in (False, True):+ for order in (None, list(reversed(range(len(q.get("crit", {})))))):+ with self.subTest(q=q, max_len=max_len, left=left, order=order):+ kwargs = dict(max_len=max_len, head_max_len=head_len,+ option_order=order, truncate_left=left)+ expected = build_sequence(tok, [{"text": "x" * 600}], q, **kwargs)++ @_reuse_question_tokens+ def repeated():+ first = build_sequence(tok, [{"text": "x" * 600}], q, **kwargs)+ second = build_sequence(tok, [{"text": "x" * 600}], q, **kwargs)+ return first, second++ self.assertEqual(repeated(), (expected, expected))++ def test_state_encodings_are_not_cached(self):+ tok = Tokenizer()+ agent_with(tok).predict_batch(["same"] * 20, QUESTIONS, batch_size=2)+ self.assertEqual(sum(text == "same" for _, text, _ in tok.calls), 20)+ self.assertEqual(len(tok.calls), 23)++ def test_single_state_skips_cache_but_a_hook_can_expand_it(self):+ tok = Tokenizer()+ agent = agent_with(tok)+ duplicate = dict(QUESTIONS, another=QUESTIONS["team"])+ agent.predict_batch(["state"], duplicate)+ self.assertEqual(len(tok.calls), 7) # one state plus both uncached three-tokenizer-call prefixes+ tok.calls.clear()++ def expand(ctx):+ ctx.states = ["state", "other"]++ agent.predict_batch(["state"], duplicate, on_predict_start=expand)+ self.assertEqual(len(tok.calls), 5) # two states plus one shared prefix++ def test_new_call_reencodes_and_sees_question_changes(self):+ tok = Tokenizer()+ agent = agent_with(tok)+ questions = copy.deepcopy(QUESTIONS)+ before = agent.predict_batch(["state"] * 2, questions)+ first_count = len(tok.calls)+ self.assertEqual(agent.predict_batch(["state"] * 2, questions), before)+ self.assertEqual(len(tok.calls), first_count * 2)+ questions["team"]["criteria"]["billing"] = {"rubric": "refunds"}+ after = agent.predict_batch(["state"] * 2, questions)+ self.assertNotEqual(before, after)+ self.assertTrue(any("refunds" in text for _, text, _ in tok.calls))++ def test_start_hook_question_rewrite_is_encoded(self):+ tok = Tokenizer()+ agent = agent_with(tok)++ def rewrite(ctx):+ ctx.questions = copy.deepcopy(QUESTIONS)+ ctx.questions["team"]["instructions"] = "Replacement"++ agent.predict_batch(["state"] * 2, QUESTIONS, on_predict_start=rewrite)+ self.assertEqual(sum(text == "choice question: Replacement" for _, text, _ in tok.calls), 1)+ self.assertFalse(any(text == "choice question: Which team?" for _, text, _ in tok.calls))++ def test_nested_call_restores_outer_cache_and_exception_discards_it(self):+ tok = Tokenizer()++ @_reuse_question_tokens+ def inner():+ build_sequence(tok, "state", QUESTION, state_ids=[])++ @_reuse_question_tokens+ def outer(fail=False):+ build_sequence(tok, "state", QUESTION, state_ids=[])+ inner()+ build_sequence(tok, "state", QUESTION, state_ids=[])+ if fail:+ raise RuntimeError("failure")++ outer()+ self.assertEqual(len(tok.calls), 6)+ with self.assertRaisesRegex(RuntimeError, "failure"):+ outer(fail=True)+ self.assertEqual(len(tok.calls), 12)+ build_sequence(tok, "state", QUESTION, state_ids=[])+ self.assertEqual(len(tok.calls), 15)++ def test_threads_sharing_a_tokenizer_have_separate_call_caches(self):+ tok = Tokenizer()+ barrier = threading.Barrier(2)++ @_reuse_question_tokens+ def run():+ first = build_sequence(tok, "state", QUESTION, state_ids=[])+ barrier.wait(timeout=10)+ second = build_sequence(tok, "state", QUESTION, state_ids=[])+ return first, second++ with ThreadPoolExecutor(max_workers=2) as pool:+ results = list(pool.map(lambda _: run(), range(2)))+ self.assertEqual(results[0], results[1])+ self.assertEqual(len(tok.calls), 6)+ self.assertEqual(len({tid for tid, _, _ in tok.calls}), 2)++ def test_tokenizer_identity_and_mutated_sequences_do_not_leak(self):+ first, second = Tokenizer(), Tokenizer(offset=100)+ expected = build_sequence(first, "state", QUESTION, state_ids=[])++ @_reuse_question_tokens+ def run():+ ids, markers = build_sequence(first, "state", QUESTION, state_ids=[])+ ids[:] = [999]+ markers[:] = []+ again = build_sequence(first, "state", QUESTION, state_ids=[])+ other = build_sequence(second, "state", QUESTION, state_ids=[])+ return again, other++ again, other = run()+ self.assertEqual(again, expected)+ self.assertNotEqual(other, expected)+ self.assertEqual(len(second.calls), 3)++ def test_copied_hook_context_cannot_keep_a_finished_call_cache(self):+ tok = Tokenizer()++ @_reuse_question_tokens+ def run():+ build_sequence(tok, "state", QUESTION, state_ids=[])+ return contextvars.copy_context()++ copied = run()+ copied.run(build_sequence, tok, "state", QUESTION, state_ids=[])+ self.assertEqual(len(tok.calls), 6)++ def test_copied_hook_context_does_not_share_tokens_with_a_worker(self):+ tok = Tokenizer()++ @_reuse_question_tokens+ def run():+ build_sequence(tok, "state", QUESTION, state_ids=[])+ copied = contextvars.copy_context()+ with ThreadPoolExecutor(max_workers=1) as pool:+ pool.submit(copied.run, build_sequence, tok, "state", QUESTION, state_ids=[]).result()+ build_sequence(tok, "state", QUESTION, state_ids=[])++ run()+ self.assertEqual(len(tok.calls), 6)+++if __name__ == "__main__":+ unittest.main()