Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

perf(agent): reuse question tokenization within prediction batches

周诚宇(Zhou Chengyu)d868c9f2c066
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 7a9cca5f..e3e78d3d 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -112,6 +112,7 @@ jobs:
python tests/test_doc_tables.py
python tests/test_tokenizer_cache.py
python tests/test_tokenizer_concurrency.py
+ python tests/test_question_token_reuse.py
python tests/test_lazy_import.py
python tests/test_runtime_fixes.py
python tests/test_lang_guess.py
@@ -254,6 +255,10 @@ jobs:
python tests/test_hooks_api.py
shell: bash
+ - name: Question token reuse tests
+ run: python tests/test_question_token_reuse.py
+ shell: bash
+
# Same five pytest suites as the Linux job. They are named explicitly rather than added to
# the loop above, because the loop runs each file as a script and that is exactly what does
# not execute a pytest suite (#374).
diff --git a/laya/agent.py b/laya/agent.py
index 1b58de6f..cdfd4c16 100644
--- a/laya/agent.py
+++ b/laya/agent.py
@@ -23,6 +23,8 @@ from .common import (
answer_confidence,
confidence_from_probs,
_resolve_noul_labels,
+ _reuse_question_tokens,
+ _disable_question_token_reuse,
encode_text,
render_options,
serialize_state,
@@ -814,6 +816,7 @@ class Agent(HookRegistry):
return answers
@torch.no_grad()
+ @_reuse_question_tokens
def predict_batch(self, states: List[Union[str, dict, list]], questions: Dict[str, Dict[str, Any]],
batch_size: Optional[int] = None, lang: Optional[str] = None,
hooks=None,
@@ -873,6 +876,8 @@ class Agent(HookRegistry):
"predict_batch expects a list of states; pass a single state to predict()/system_one()."
)
states = list(states)
+ if len(states) == 1:
+ _disable_question_token_reuse()
if not states:
ctx.results = []
else:
diff --git a/laya/common.py b/laya/common.py
index 91f702a5..3cf70571 100644
--- a/laya/common.py
+++ b/laya/common.py
@@ -4,6 +4,8 @@ import math
import os
import threading
from contextlib import nullcontext
+from contextvars import ContextVar
+from functools import wraps
from typing import Dict, List, Optional, Union
import numpy as np
@@ -22,6 +24,7 @@ _DEFAULT_NOUL_LABELS = {"false": "false", "true": "true"}
# `RuntimeError: Already borrowed`. Serialise encoding instead: it is a small fraction of a call
# next to the forward pass, and this keeps the cache's single parse.
_TOKENIZE_LOCK = threading.RLock()
+_QUESTION_TOKEN_CACHE = ContextVar("laya_question_token_cache", default=None)
def encode_text(tok, text, **kwargs):
@@ -30,6 +33,43 @@ def encode_text(tok, text, **kwargs):
return tok(text, **kwargs)
+def _reuse_question_tokens(fn):
+ @wraps(fn)
+ def wrapped(*args, **kwargs):
+ # One cache per prediction call: nested calls get their own scope, and exceptions
+ # restore the outer scope. Nothing is retained on an Agent or shared across threads.
+ scope = {"thread": threading.get_ident(), "tokens": {}}
+ token = _QUESTION_TOKEN_CACHE.set(scope)
+ try:
+ return fn(*args, **kwargs)
+ finally:
+ # A timed hook can copy this context to a worker that outlives the call.
+ scope["tokens"] = None
+ _QUESTION_TOKEN_CACHE.reset(token)
+ return wrapped
+
+
+def _disable_question_token_reuse():
+ scope = _QUESTION_TOKEN_CACHE.get()
+ if scope is not None:
+ scope["tokens"] = None
+
+
+def _encode_question_text(tok, text, **kwargs):
+ scope = _QUESTION_TOKEN_CACHE.get()
+ cache = scope["tokens"] if scope is not None and scope["thread"] == threading.get_ident() else None
+ if cache is None:
+ return encode_text(tok, text, **kwargs)["input_ids"]
+ # Key the rendered, mask-sanitized text and encoding settings, not a JSON question:
+ # option order, structured criteria and custom noul labels must keep their meaning.
+ key = (id(tok), text, tuple(kwargs.items()))
+ if key not in cache:
+ # Keep the tokenizer alive so its identity cannot be reused within this scope.
+ cache[key] = (tok, tuple(encode_text(tok, text, **kwargs)["input_ids"]))
+ # Sequence assembly must not mutate token lists retained for later states.
+ return list(cache[key][1])
+
+
def serialize_state(state: Union[str, dict, list]) -> str:
if isinstance(state, str):
return state
@@ -110,19 +150,19 @@ def build_sequence(
opts = render_options(q)
order = option_order if option_order is not None else list(range(len(opts)))
ins = str(q["ins"]).replace(mask_tok, " ")
- head_ids = encode_text(tok, "%s question: %s" % (q["t"], ins), add_special_tokens=False)["input_ids"]
+ head_ids = _encode_question_text(tok, "%s question: %s" % (q["t"], ins), add_special_tokens=False)
opt_ids = []
for i in order:
# Cap at the tokenizer, not after the fact: `[:48]` still makes the tokenizer process the
# whole (possibly long) description. truncation=True, max_length=48 keeps the first 48
# tokens, which is exactly what the previous slice produced.
- opt_tokens = encode_text(
+ opt_tokens = _encode_question_text(
tok,
" " + opts[i].replace(mask_tok, " "),
add_special_tokens=False,
truncation=True,
max_length=48,
- )["input_ids"]
+ )
opt_ids.append([tok.mask_token_id] + opt_tokens)
opt_budget = head_max_len - sum(len(o) for o in opt_ids)
if opt_budget < 16:
diff --git a/research/results/question_token_reuse_cuda.json b/research/results/question_token_reuse_cuda.json
new file mode 100644
index 00000000..3ce02b42
--- /dev/null
+++ b/research/results/question_token_reuse_cuda.json
@@ -0,0 +1,349 @@
+{
+ "base": "4066d5d5fbf08b66c6757ddeedbd797bd7655bc0",
+ "model": "../models/laya",
+ "model_revision": "55cf4c4ebb4ebe31b2550e8bdf3bd21b99753851",
+ "torch": "2.7.1+cu128",
+ "device": "cuda",
+ "gpu": "NVIDIA GeForce RTX 4060 Laptop GPU",
+ "threads": 4,
+ "repeats": 16,
+ "workloads": [
+ {
+ "name": "long_20_options_default",
+ "windows": 30,
+ "before_p50_ms": 770.7229144999985,
+ "after_p50_ms": 741.8036320000141,
+ "saving_percent": 3.752228194583475,
+ "paired_saving_p50_ms": 25.38752499998509,
+ "after_faster_pairs": 16,
+ "question_encode_calls": {
+ "before": 630,
+ "after": 21
+ },
+ "pairs_ms": [
+ {
+ "before": 755.3662249999888,
+ "after": 731.0198440000022
+ },
+ {
+ "after": 734.5862699999941,
+ "before": 756.0385290000227
+ },
+ {
+ "before": 763.4225589999915,
+ "after": 743.674795000004
+ },
+ {
+ "after": 749.1004630000191,
+ "before": 781.9324560000211
+ },
+ {
+ "before": 771.513586999987,
+ "after": 745.0849180000034
+ },
+ {
+ "after": 739.2042749999916,
+ "before": 775.3677320000065
+ },
+ {
+ "before": 775.0609379999958,
+ "after": 733.2346009999924
+ },
+ {
+ "after": 742.5324480000199,
+ "before": 773.6810950000006
+ },
+ {
+ "before": 741.9111699999803,
+ "after": 739.8703190000049
+ },
+ {
+ "after": 741.0748160000082,
+ "before": 769.993301999989
+ },
+ {
+ "before": 766.7506409999874,
+ "after": 766.6943709999998
+ },
+ {
+ "after": 739.4848539999828,
+ "before": 769.5560379999904
+ },
+ {
+ "before": 779.1554670000096,
+ "after": 750.8460010000135
+ },
+ {
+ "after": 766.0764169999936,
+ "before": 771.4525270000081
+ },
+ {
+ "before": 775.2085010000087,
+ "after": 755.3889690000233
+ },
+ {
+ "after": 711.069574000021,
+ "before": 725.6501040000103
+ }
+ ],
+ "all_input_tensors_equal": true,
+ "outputs_equal": true,
+ "decision_logits_and_action_probabilities_equal": true
+ },
+ {
+ "name": "long_20_options_batch8",
+ "windows": 30,
+ "before_p50_ms": 772.4074305000102,
+ "after_p50_ms": 742.8103704999955,
+ "saving_percent": 3.831793795776311,
+ "paired_saving_p50_ms": 25.337741500010225,
+ "after_faster_pairs": 16,
+ "question_encode_calls": {
+ "before": 630,
+ "after": 21
+ },
+ "pairs_ms": [
+ {
+ "before": 747.2378940000226,
+ "after": 722.010795000017
+ },
+ {
+ "after": 724.8769889999949,
+ "before": 749.1776330000448
+ },
+ {
+ "before": 763.6030759999812,
+ "after": 747.3942290000082
+ },
+ {
+ "after": 737.9716530000451,
+ "before": 762.8258909999772
+ },
+ {
+ "before": 773.0705460000422,
+ "after": 758.7429350000434
+ },
+ {
+ "after": 741.4936989999887,
+ "before": 774.9147319999565
+ },
+ {
+ "before": 763.6511570000266,
+ "after": 740.7228280000027
+ },
+ {
+ "after": 743.3846420000236,
+ "before": 772.8549529999782
+ },
+ {
+ "before": 767.094568999994,
+ "after": 743.7462590000337
+ },
+ {
+ "after": 745.6122489999757,
+ "before": 771.9599080000421
+ },
+ {
+ "before": 774.0217000000484,
+ "after": 742.2360989999675
+ },
+ {
+ "after": 750.0837059999981,
+ "before": 775.5320900000129
+ },
+ {
+ "before": 776.6750619999812,
+ "after": 741.2745449999534
+ },
+ {
+ "after": 742.0869170000515,
+ "before": 756.5714419999949
+ },
+ {
+ "before": 784.0836970000282,
+ "after": 758.568293000053
+ },
+ {
+ "after": 778.9746190000528,
+ "before": 816.0115910000059
+ }
+ ],
+ "all_input_tensors_equal": true,
+ "outputs_equal": true,
+ "decision_logits_and_action_probabilities_equal": true
+ },
+ {
+ "name": "long_4_options_default",
+ "windows": 30,
+ "before_p50_ms": 543.4064060000026,
+ "after_p50_ms": 539.2311045000042,
+ "saving_percent": 0.7683570627612957,
+ "paired_saving_p50_ms": 12.141736000018,
+ "after_faster_pairs": 13,
+ "question_encode_calls": {
+ "before": 150,
+ "after": 5
+ },
+ "pairs_ms": [
+ {
+ "before": 597.1686920000252,
+ "after": 570.3511920000324
+ },
+ {
+ "after": 548.8087200000109,
+ "before": 548.5951840000212
+ },
+ {
+ "before": 563.6884250000094,
+ "after": 541.0735230000228
+ },
+ {
+ "after": 545.9787620000043,
+ "before": 543.492614999991
+ },
+ {
+ "before": 555.453449999959,
+ "after": 549.3829319999577
+ },
+ {
+ "after": 567.2547649999728,
+ "before": 585.6271679999736
+ },
+ {
+ "before": 585.101796999993,
+ "after": 564.3718290000379
+ },
+ {
+ "after": 577.6956290000044,
+ "before": 599.3873820000317
+ },
+ {
+ "before": 540.1400929999909,
+ "after": 523.965282000006
+ },
+ {
+ "after": 525.9081159999823,
+ "before": 543.3201970000141
+ },
+ {
+ "before": 525.2991820000261,
+ "after": 524.15587400003
+ },
+ {
+ "after": 528.3915110000521,
+ "before": 533.8170489999925
+ },
+ {
+ "before": 534.4393409999952,
+ "after": 537.3886859999857
+ },
+ {
+ "after": 526.5865289999851,
+ "before": 541.0266940000383
+ },
+ {
+ "before": 538.9093260000095,
+ "after": 529.0660190000267
+ },
+ {
+ "after": 531.238535,
+ "before": 531.8594930000131
+ }
+ ],
+ "all_input_tensors_equal": true,
+ "outputs_equal": true,
+ "decision_logits_and_action_probabilities_equal": true
+ },
+ {
+ "name": "single_short_20_options",
+ "windows": 1,
+ "before_p50_ms": 27.625586999988627,
+ "after_p50_ms": 27.79482599999028,
+ "saving_percent": -0.6126168468446398,
+ "paired_saving_p50_ms": -0.2940649999914058,
+ "after_faster_pairs": 6,
+ "question_encode_calls": {
+ "before": 21,
+ "after": 21
+ },
+ "pairs_ms": [
+ {
+ "before": 24.87685800002737,
+ "after": 25.161067999988518
+ },
+ {
+ "after": 25.549729000033494,
+ "before": 32.6728030000254
+ },
+ {
+ "before": 24.855655999999726,
+ "after": 27.84138399999847
+ },
+ {
+ "after": 33.07467999997016,
+ "before": 24.60957400001007
+ },
+ {
+ "before": 26.697087000002284,
+ "after": 26.525146000039967
+ },
+ {
+ "after": 28.13503100003345,
+ "before": 33.05376600002319
+ },
+ {
+ "before": 28.00941899999998,
+ "after": 27.74826799998209
+ },
+ {
+ "after": 31.101750999994238,
+ "before": 27.477479999959087
+ },
+ {
+ "before": 28.22349100000565,
+ "after": 31.531492999988586
+ },
+ {
+ "after": 27.645273000018733,
+ "before": 25.38376799998332
+ },
+ {
+ "before": 27.79949600000009,
+ "after": 27.734253000005538
+ },
+ {
+ "after": 28.282738000029894,
+ "before": 27.978818000008232
+ },
+ {
+ "before": 29.842222000013408,
+ "after": 30.09515899998405
+ },
+ {
+ "after": 26.838087000044197,
+ "before": 26.429774000007455
+ },
+ {
+ "before": 27.773694000018168,
+ "after": 27.15556599997626
+ },
+ {
+ "after": 30.651570999964406,
+ "before": 25.120577999985017
+ }
+ ],
+ "all_input_tensors_equal": true,
+ "outputs_equal": true,
+ "decision_logits_and_action_probabilities_equal": true
+ }
+ ],
+ "python": "3.10.12",
+ "transformers": "4.51.3",
+ "os": "WSL2 Ubuntu-22.04",
+ "gpu_memory_gib": 8,
+ "amp": "bf16",
+ "weights": "fp32",
+ "warmup_per_mode": 3,
+ "approximate_filler_tokens": 4800,
+ "workload_note": "MASSIVE seed-13 English first case + bench_long_context filler is a latency workload, not an accuracy benchmark."
+}
diff --git a/research/scripts/bench_question_token_reuse.py b/research/scripts/bench_question_token_reuse.py
new file mode 100644
index 00000000..b5b3d557
--- /dev/null
+++ b/research/scripts/bench_question_token_reuse.py
@@ -0,0 +1,157 @@
+"""Compare the actual patch against the original functions at the checked-out base commit."""
+import argparse
+import ast
+import contextlib
+import importlib
+import json
+import os
+import pathlib
+import sys
+import statistics
+import subprocess
+import time
+from unittest.mock import patch
+
+ROOT = pathlib.Path(__file__).resolve().parents[2]
+sys.path.insert(0, str(ROOT))
+os.environ.setdefault("USE_TF", "0")
+os.environ.setdefault("TOKENIZERS_PARALLELISM", "false")
+import torch # noqa: E402
+import laya # noqa: E402
+from bench_long_context import FILLER, QUESTIONS # noqa: E402
+import laya.common as common # noqa: E402
+
+agent_module = importlib.import_module("laya.agent")
+after_batch = laya.Agent.predict_batch
+
+
+def original_function(base, file, name, namespace, cls=None):
+ # Execute only the original function from this checkout, with its original dependencies.
+ source = subprocess.check_output(["git", "show", base + ":" + file], cwd=ROOT, text=True)
+ tree = ast.parse(source)
+ nodes = tree.body
+ if cls:
+ nodes = next(node for node in nodes if isinstance(node, ast.ClassDef) and node.name == cls).body
+ fn = next(node for node in nodes if isinstance(node, ast.FunctionDef) and node.name == name)
+ module = ast.Module(body=[fn], type_ignores=[])
+ exec(compile(module, file + "@" + base, "exec"), namespace)
+ return namespace[name]
+
+
+def mode(name, before_batch):
+ with patch.object(laya.Agent, "predict_batch", before_batch if name == "before" else after_batch):
+ yield
+
+
+def timed(fn, device):
+ if device.type == "cuda":
+ torch.cuda.synchronize(device)
+ start = time.perf_counter()
+ result = fn()
+ if device.type == "cuda":
+ torch.cuda.synchronize(device)
+ return result, (time.perf_counter() - start) * 1000
+
+
+def capture(agent, fn):
+ batches, raw = [], []
+ forward = agent._forward
+
+ def saved(batch):
+ batches.append({key: val.clone() for key, val in batch.items() if isinstance(val, torch.Tensor)})
+ result = forward(batch)
+ raw.append(tuple(val.copy() for val in result))
+ return result
+
+ with patch.object(agent, "_forward", saved):
+ result = fn()
+ return result, batches, raw
+
+
+def main():
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument("--model", required=True, help="Checkpoint directory or Hugging Face repo")
+ parser.add_argument("--revision", default=None)
+ parser.add_argument("--baseline-ref", required=True, help="Trusted Git ref before question reuse")
+ parser.add_argument("--device", default="cuda")
+ parser.add_argument("--repeats", type=int, default=16)
+ parser.add_argument("--out", type=pathlib.Path, required=True)
+ args = parser.parse_args()
+ if args.repeats < 1:
+ parser.error("--repeats must be positive")
+ torch.set_num_threads(4)
+ before_build = original_function(args.baseline_ref, "laya/common.py", "build_sequence", dict(common.__dict__))
+ before_batch = original_function(args.baseline_ref, "laya/agent.py", "predict_batch",
+ dict(agent_module.__dict__, build_sequence=before_build), cls="Agent")
+ agent = laya.Agent(args.model, revision=args.revision, device=args.device, fast=False, compile=False)
+ # First seed-13 MASSIVE English regression case, combined with the long-context filler.
+ # This is a latency workload; no dataset accuracy is claimed for the padded input.
+ case = {"state": {"utterance": "wake me up at five am this week"},
+ "instructions": "What is the user asking for in `utterance`?",
+ "options": ["email_query", "takeaway_order", "qa_definition", "cooking_recipe", "qa_factoid",
+ "calendar_set", "music_settings", "social_query", "general_quirky", "datetime_query",
+ "qa_currency", "audio_volume_other", "transport_query", "alarm_remove", "general_greet",
+ "transport_traffic", "email_querycontact", "alarm_set", "transport_ticket", "takeaway_query"]}
+ questions20 = {"intent": {"type": "choice", "instructions": case["instructions"],
+ "criteria": {key: key.replace("_", " ") for key in case["options"]}}}
+ repeats = round(4800 / len(agent.tok(FILLER, add_special_tokens=False)["input_ids"]))
+ state = FILLER * repeats + "\n\nActual request: " + common.serialize_state(case["state"])
+ workloads = [("long_20_options_default", lambda: agent.predict_long(state, questions20)),
+ ("long_20_options_batch8", lambda: agent.predict_long(state, questions20, batch_size=8)),
+ ("long_4_options_default", lambda: agent.predict_long(state, QUESTIONS)),
+ ("single_short_20_options", lambda: agent.system_one(case["state"], questions20))]
+ report = {"base": args.baseline_ref, "model": args.model, "model_revision": args.revision,
+ "torch": torch.__version__, "device": str(agent.device), "gpu": torch.cuda.get_device_name(agent.device) if agent.device.type == "cuda" else None, "threads": 4,
+ "repeats": args.repeats, "workloads": []}
+ for name, fn in workloads:
+ for which in ("before", "after"):
+ with mode(which, before_batch):
+ for _ in range(3):
+ fn()
+ with mode("before", before_batch):
+ expected, before_tensors, before_raw = capture(agent, fn)
+ with mode("after", before_batch):
+ actual, after_tensors, after_raw = capture(agent, fn)
+ assert expected == actual
+ assert len(before_tensors) == len(after_tensors)
+ assert all(left[key].equal(right[key]) for left, right in zip(before_tensors, after_tensors) for key in left)
+ assert all((x == y).all() for left, right in zip(before_raw, after_raw) for x, y in zip(left, right))
+ counts = {}
+ for which in ("before", "after"):
+ seen = []
+ encode = common.encode_text
+
+ def counting(tok, text, **kwargs):
+ seen.append(text)
+ return encode(tok, text, **kwargs)
+
+ # The original build_sequence has its own global namespace.
+ with mode(which, before_batch), patch.object(common, "encode_text", counting), patch.dict(
+ before_build.__globals__, encode_text=counting):
+ fn()
+ counts[which] = len(seen)
+ pairs = []
+ for i in range(args.repeats):
+ pair = {}
+ for which in (("before", "after") if i % 2 == 0 else ("after", "before")):
+ with mode(which, before_batch):
+ result, pair[which] = timed(fn, agent.device)
+ assert result == expected
+ assert agent.device.type == args.device.split(":")[0]
+ pairs.append(pair)
+ before = statistics.median(pair["before"] for pair in pairs)
+ after = statistics.median(pair["after"] for pair in pairs)
+ row = {"name": name, "windows": expected["usage"].get("windows", 1),
+ "before_p50_ms": before, "after_p50_ms": after, "saving_percent": (before - after) / before * 100,
+ "paired_saving_p50_ms": statistics.median(pair["before"] - pair["after"] for pair in pairs),
+ "after_faster_pairs": sum(pair["after"] < pair["before"] for pair in pairs),
+ "question_encode_calls": counts, "pairs_ms": pairs,
+ "all_input_tensors_equal": True, "decision_logits_and_action_probabilities_equal": True, "outputs_equal": True}
+ report["workloads"].append(row)
+ args.out.write_text(json.dumps(report, indent=2))
+ print(json.dumps({key: value for key, value in row.items() if key != "pairs_ms"}), flush=True)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/tests/test_question_token_reuse.py b/tests/test_question_token_reuse.py
new file mode 100644
index 00000000..a68e48e9
--- /dev/null
+++ b/tests/test_question_token_reuse.py
@@ -0,0 +1,255 @@
+"""Question encoding reuse: run with python tests/test_question_token_reuse.py (no weights)."""
+import copy
+import contextvars
+import os
+import sys
+import threading
+import unittest
+from concurrent.futures import ThreadPoolExecutor
+
+import numpy as np
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from laya.agent import Agent # noqa: E402
+from laya.common import _reuse_question_tokens, build_sequence # noqa: E402
+
+
+class Tokenizer:
+ mask_token = "[MASK]"
+ mask_token_id = 1
+ cls_token_id = 2
+ sep_token_id = 3
+ pad_token_id = 0
+
+ def __init__(self, offset=0):
+ self.calls = []
+ self.offset = offset
+
+ def __call__(self, text, **kwargs):
+ self.calls.append((threading.get_ident(), text, kwargs))
+ ids = [ord(c) + 4 + self.offset for c in text]
+ if kwargs.get("truncation"):
+ ids = ids[:kwargs["max_length"]]
+ return {"input_ids": ids}
+
+ def decode(self, ids, **kwargs):
+ return "".join(chr(i - 4 - self.offset) for i in ids)
+
+
+QUESTION = {"t": "choice", "ins": "Which team?", "crit": {"billing": "charges", "tech": "errors"}}
+QUESTIONS = {"team": {"type": "choice", "instructions": QUESTION["ins"], "criteria": QUESTION["crit"]}}
+
+
+def agent_with(tok):
+ agent = Agent.__new__(Agent)
+ agent.tok = tok
+ agent.cfg = {"max_len": 512, "head_max_len": 192}
+ agent.batches = []
+
+ def forward(batch):
+ agent.batches.append({key: value.clone() for key, value in batch.items() if hasattr(value, "clone")})
+ shape = batch["marker_pos"].shape
+ return np.zeros(shape, dtype=np.float32), np.full((shape[0], 2), 0.5, dtype=np.float32)
+
+ def decode(logits, act, items, ids, internal, offset, **kwargs):
+ return {qid: {"ids": item["ids"], "markers": item["markers"], "answer_confidence": 0.5}
+ for qid, item in zip(ids, items)}
+
+ agent._forward = forward
+ agent._decode_answers = decode
+ return agent
+
+
+class QuestionTokenReuse(unittest.TestCase):
+ def test_reuse_across_chunks_preserves_all_collated_tensors(self):
+ states = ["state %d" % i for i in range(19)]
+ tok = Tokenizer()
+ agent = agent_with(tok)
+ for grouped in (False, True):
+ for batch_size in (None, 1, 3):
+ with self.subTest(grouped=grouped, batch_size=batch_size):
+ tok.calls.clear()
+ agent.batches.clear()
+ got = agent.predict_batch(states, QUESTIONS, batch_size=batch_size, sort_by_length=grouped)
+ self.assertEqual(len(tok.calls), len(states) + 3)
+ cached = agent.batches[:]
+ agent.batches.clear()
+ # The real undecorated method is the same orchestration without reuse.
+ plain = Agent.predict_batch.__wrapped__.__wrapped__(
+ agent, states, QUESTIONS, batch_size=batch_size, sort_by_length=grouped)
+ self.assertEqual(got, plain)
+ self.assertEqual(len(cached), len(agent.batches))
+ for left, right in zip(cached, agent.batches):
+ self.assertEqual(set(left), set(right))
+ for key in left:
+ self.assertTrue(left[key].equal(right[key]), key)
+
+ def test_predict_long_uses_the_same_cache_for_all_windows(self):
+ tok = Tokenizer()
+ agent = agent_with(tok)
+ agent.predict_long("x" * 1000, QUESTIONS, window=80, stride=50, batch_size=3)
+ head = "choice question: " + QUESTION["ins"]
+ self.assertEqual(sum(text == head for _, text, _ in tok.calls), 1)
+ self.assertEqual(sum(text == " billing: charges" for _, text, _ in tok.calls), 1)
+ self.assertGreater(len(agent.batches), 1)
+
+ def test_sequence_parity_for_rendering_order_and_budgets(self):
+ cases = [
+ QUESTION,
+ {"t": "choice", "ins": "a [MASK] b", "crit": {7: None, "zero": 0, "false": False}},
+ {"t": "choice", "ins": "中文", "crit": {"a": {"desc": [1, "二"]}, "b": "z" * 200}},
+ {"t": "score", "ins": "level", "crit": [{"desc": "one"}, ["two"], 0]},
+ {"t": "noul", "ins": "holds?", "crit": {"false": {"reason": "no"}, "true": ["yes"]},
+ "labels": {"false": "否", "true": "是"}},
+ ]
+ tok = Tokenizer()
+ for q in cases:
+ for max_len, head_len in ((512, 192), (128, 32), (16, 8)):
+ for left in (False, True):
+ for order in (None, list(reversed(range(len(q.get("crit", {})))))):
+ with self.subTest(q=q, max_len=max_len, left=left, order=order):
+ kwargs = dict(max_len=max_len, head_max_len=head_len,
+ option_order=order, truncate_left=left)
+ expected = build_sequence(tok, [{"text": "x" * 600}], q, **kwargs)
+
+ @_reuse_question_tokens
+ def repeated():
+ first = build_sequence(tok, [{"text": "x" * 600}], q, **kwargs)
+ second = build_sequence(tok, [{"text": "x" * 600}], q, **kwargs)
+ return first, second
+
+ self.assertEqual(repeated(), (expected, expected))
+
+ def test_state_encodings_are_not_cached(self):
+ tok = Tokenizer()
+ agent_with(tok).predict_batch(["same"] * 20, QUESTIONS, batch_size=2)
+ self.assertEqual(sum(text == "same" for _, text, _ in tok.calls), 20)
+ self.assertEqual(len(tok.calls), 23)
+
+ def test_single_state_skips_cache_but_a_hook_can_expand_it(self):
+ tok = Tokenizer()
+ agent = agent_with(tok)
+ duplicate = dict(QUESTIONS, another=QUESTIONS["team"])
+ agent.predict_batch(["state"], duplicate)
+ self.assertEqual(len(tok.calls), 7) # one state plus both uncached three-tokenizer-call prefixes
+ tok.calls.clear()
+
+ def expand(ctx):
+ ctx.states = ["state", "other"]
+
+ agent.predict_batch(["state"], duplicate, on_predict_start=expand)
+ self.assertEqual(len(tok.calls), 5) # two states plus one shared prefix
+
+ def test_new_call_reencodes_and_sees_question_changes(self):
+ tok = Tokenizer()
+ agent = agent_with(tok)
+ questions = copy.deepcopy(QUESTIONS)
+ before = agent.predict_batch(["state"] * 2, questions)
+ first_count = len(tok.calls)
+ self.assertEqual(agent.predict_batch(["state"] * 2, questions), before)
+ self.assertEqual(len(tok.calls), first_count * 2)
+ questions["team"]["criteria"]["billing"] = {"rubric": "refunds"}
+ after = agent.predict_batch(["state"] * 2, questions)
+ self.assertNotEqual(before, after)
+ self.assertTrue(any("refunds" in text for _, text, _ in tok.calls))
+
+ def test_start_hook_question_rewrite_is_encoded(self):
+ tok = Tokenizer()
+ agent = agent_with(tok)
+
+ def rewrite(ctx):
+ ctx.questions = copy.deepcopy(QUESTIONS)
+ ctx.questions["team"]["instructions"] = "Replacement"
+
+ agent.predict_batch(["state"] * 2, QUESTIONS, on_predict_start=rewrite)
+ self.assertEqual(sum(text == "choice question: Replacement" for _, text, _ in tok.calls), 1)
+ self.assertFalse(any(text == "choice question: Which team?" for _, text, _ in tok.calls))
+
+ def test_nested_call_restores_outer_cache_and_exception_discards_it(self):
+ tok = Tokenizer()
+
+ @_reuse_question_tokens
+ def inner():
+ build_sequence(tok, "state", QUESTION, state_ids=[])
+
+ @_reuse_question_tokens
+ def outer(fail=False):
+ build_sequence(tok, "state", QUESTION, state_ids=[])
+ inner()
+ build_sequence(tok, "state", QUESTION, state_ids=[])
+ if fail:
+ raise RuntimeError("failure")
+
+ outer()
+ self.assertEqual(len(tok.calls), 6)
+ with self.assertRaisesRegex(RuntimeError, "failure"):
+ outer(fail=True)
+ self.assertEqual(len(tok.calls), 12)
+ build_sequence(tok, "state", QUESTION, state_ids=[])
+ self.assertEqual(len(tok.calls), 15)
+
+ def test_threads_sharing_a_tokenizer_have_separate_call_caches(self):
+ tok = Tokenizer()
+ barrier = threading.Barrier(2)
+
+ @_reuse_question_tokens
+ def run():
+ first = build_sequence(tok, "state", QUESTION, state_ids=[])
+ barrier.wait(timeout=10)
+ second = build_sequence(tok, "state", QUESTION, state_ids=[])
+ return first, second
+
+ with ThreadPoolExecutor(max_workers=2) as pool:
+ results = list(pool.map(lambda _: run(), range(2)))
+ self.assertEqual(results[0], results[1])
+ self.assertEqual(len(tok.calls), 6)
+ self.assertEqual(len({tid for tid, _, _ in tok.calls}), 2)
+
+ def test_tokenizer_identity_and_mutated_sequences_do_not_leak(self):
+ first, second = Tokenizer(), Tokenizer(offset=100)
+ expected = build_sequence(first, "state", QUESTION, state_ids=[])
+
+ @_reuse_question_tokens
+ def run():
+ ids, markers = build_sequence(first, "state", QUESTION, state_ids=[])
+ ids[:] = [999]
+ markers[:] = []
+ again = build_sequence(first, "state", QUESTION, state_ids=[])
+ other = build_sequence(second, "state", QUESTION, state_ids=[])
+ return again, other
+
+ again, other = run()
+ self.assertEqual(again, expected)
+ self.assertNotEqual(other, expected)
+ self.assertEqual(len(second.calls), 3)
+
+ def test_copied_hook_context_cannot_keep_a_finished_call_cache(self):
+ tok = Tokenizer()
+
+ @_reuse_question_tokens
+ def run():
+ build_sequence(tok, "state", QUESTION, state_ids=[])
+ return contextvars.copy_context()
+
+ copied = run()
+ copied.run(build_sequence, tok, "state", QUESTION, state_ids=[])
+ self.assertEqual(len(tok.calls), 6)
+
+ def test_copied_hook_context_does_not_share_tokens_with_a_worker(self):
+ tok = Tokenizer()
+
+ @_reuse_question_tokens
+ def run():
+ build_sequence(tok, "state", QUESTION, state_ids=[])
+ copied = contextvars.copy_context()
+ with ThreadPoolExecutor(max_workers=1) as pool:
+ pool.submit(copied.run, build_sequence, tok, "state", QUESTION, state_ids=[]).result()
+ build_sequence(tok, "state", QUESTION, state_ids=[])
+
+ run()
+ self.assertEqual(len(tok.calls), 6)
+
+
+if __name__ == "__main__":
+ unittest.main()