Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

perf(agent): reuse compiled CUDA graphs across token lengths

周诚宇(Zhou Chengyu)fb4eb0c11260
diff --git a/README.md b/README.md
index 328a6194..77352ed0 100644
--- a/README.md
+++ b/README.md
@@ -776,6 +776,9 @@ Measured with `benchmarks/bench_compile.py --device cuda [--warmup]` (English ch
requests of changing shape): without it the first request took 51 s and the first single-question request,
the eighth, took another 41 s; after `warmup()` no request took more than 30 ms. It works the same with
`fast=True` (kernels and CUDA graphs for those buckets) and costs a few forward passes on the stock path.
+On CUDA, compiled inference pads the masked end of each sequence to a multiple of eight tokens. This
+avoids extra SDPA graph specialisations on PyTorch versions that distinguish lengths modulo eight;
+reported token usage still counts the original, unpadded sequence.
Inductor caches compiled graphs under `TORCHINDUCTOR_CACHE_DIR` (by default in `/tmp`); point it at a
persistent directory to keep them across restarts.
diff --git a/laya/agent.py b/laya/agent.py
index c98e6859..aeda5a68 100644
--- a/laya/agent.py
+++ b/laya/agent.py
@@ -208,6 +208,22 @@ def _amp_context(device, dtype, enabled: bool):
return torch.autocast(device_type=device.type, dtype=dtype)
+def _pad_cuda_compile_batch(batch: Dict, pad_token_id: int) -> Dict:
+ """Keep compiled CUDA attention on the same sequence-length guard family.
+
+ Some Inductor SDPA versions specialise on the token length modulo eight even with
+ dynamic shapes. Pad only the masked tail; marker positions and caller-visible token
+ accounting continue to refer to the original sequence.
+ """
+ missing = -batch["input_ids"].shape[1] % 8
+ if not missing:
+ return batch
+ padded = dict(batch)
+ padded["input_ids"] = torch.nn.functional.pad(batch["input_ids"], (0, missing), value=pad_token_id)
+ padded["attention_mask"] = torch.nn.functional.pad(batch["attention_mask"], (0, missing), value=0)
+ return padded
+
+
MPS_AMP_MIN_ROWS_DEFAULT = 5
# An unsupported autocast op fails the same way on every request. Retry that request in
# full precision, and only turn AMP off after this many failures in a row (#351).
@@ -930,6 +946,8 @@ class Agent(HookRegistry):
def _infer(self, b: Dict):
"""Run the forward pass under autocast, degrading gracefully on OOM or unsupported autocast."""
+ if self._compiled and self._fast is None and self.device.type == "cuda":
+ b = _pad_cuda_compile_batch(b, self.tok.pad_token_id)
use_amp = self._amp_enabled_for(b["input_ids"].shape[0])
if self._fast is not None and b["input_ids"].shape[1] > self._fast.max_len:
diff --git a/tests/test_compile.py b/tests/test_compile.py
index 1d833e89..edecb6bc 100644
--- a/tests/test_compile.py
+++ b/tests/test_compile.py
@@ -15,7 +15,7 @@ import torch
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from laya._compile import compile_model, independent_dims # noqa: E402
-from laya.agent import Agent # noqa: E402
+from laya.agent import Agent, _pad_cuda_compile_batch # noqa: E402
from laya.common import DecisionModel # noqa: E402
fx_config = torch.fx.experimental._config
@@ -32,7 +32,11 @@ def tiny_model():
num_attention_heads=2, global_attn_every_n_layers=2, local_attention=16,
max_position_embeddings=512, pad_token_id=0, bos_token_id=1, eos_token_id=2,
cls_token_id=1, sep_token_id=2)
- return DecisionModel(AutoModel.from_config(cfg, attn_implementation="sdpa"), 1, 2).eval()
+ encoder = AutoModel.from_config(cfg, attn_implementation="sdpa")
+ # The CPU fixture checks Laya's dynamic dimensions, not ModernBERT's own
+ # first-forward mutation of reference_compile ("auto" -> False on CPU).
+ encoder.config.reference_compile = False
+ return DecisionModel(encoder, 1, 2).eval()
def batch(rows, tokens, markers):
@@ -60,6 +64,22 @@ def graphs():
return counters["stats"]["unique_graphs"]
+def test_cuda_compile_padding_masks_only_the_new_tail():
+ b = batch(2, 49, 3)
+ padded = _pad_cuda_compile_batch(b, 7)
+ assert padded["input_ids"].shape == (2, 56)
+ assert padded["attention_mask"].shape == (2, 56)
+ assert torch.equal(padded["input_ids"][:, :49], b["input_ids"])
+ assert torch.equal(padded["attention_mask"][:, :49], b["attention_mask"])
+ assert torch.all(padded["input_ids"][:, 49:] == 7)
+ assert not padded["attention_mask"][:, 49:].any()
+ assert padded["marker_pos"] is b["marker_pos"]
+ assert padded["marker_mask"] is b["marker_mask"]
+ assert padded["qtype"] is b["qtype"]
+ assert b["input_ids"].shape == (2, 49)
+ assert _pad_cuda_compile_batch(padded, 0) is padded
+
+
def test_one_graph_across_shapes_and_same_outputs():
torch._dynamo.reset()
model = tiny_model()