Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
perf(agent): reuse compiled CUDA graphs across token lengths
周诚宇(Zhou Chengyu)fb4eb0c11260
diff --git a/README.md b/README.mdindex 328a6194..77352ed0 100644--- a/README.md+++ b/README.md@@ -776,6 +776,9 @@ Measured with `benchmarks/bench_compile.py --device cuda [--warmup]` (English chrequests of changing shape): without it the first request took 51 s and the first single-question request,the eighth, took another 41 s; after `warmup()` no request took more than 30 ms. It works the same with`fast=True` (kernels and CUDA graphs for those buckets) and costs a few forward passes on the stock path.+On CUDA, compiled inference pads the masked end of each sequence to a multiple of eight tokens. This+avoids extra SDPA graph specialisations on PyTorch versions that distinguish lengths modulo eight;+reported token usage still counts the original, unpadded sequence.Inductor caches compiled graphs under `TORCHINDUCTOR_CACHE_DIR` (by default in `/tmp`); point it at apersistent directory to keep them across restarts.diff --git a/laya/agent.py b/laya/agent.pyindex c98e6859..aeda5a68 100644--- a/laya/agent.py+++ b/laya/agent.py@@ -208,6 +208,22 @@ def _amp_context(device, dtype, enabled: bool):return torch.autocast(device_type=device.type, dtype=dtype)+def _pad_cuda_compile_batch(batch: Dict, pad_token_id: int) -> Dict:+ """Keep compiled CUDA attention on the same sequence-length guard family.++ Some Inductor SDPA versions specialise on the token length modulo eight even with+ dynamic shapes. Pad only the masked tail; marker positions and caller-visible token+ accounting continue to refer to the original sequence.+ """+ missing = -batch["input_ids"].shape[1] % 8+ if not missing:+ return batch+ padded = dict(batch)+ padded["input_ids"] = torch.nn.functional.pad(batch["input_ids"], (0, missing), value=pad_token_id)+ padded["attention_mask"] = torch.nn.functional.pad(batch["attention_mask"], (0, missing), value=0)+ return padded++MPS_AMP_MIN_ROWS_DEFAULT = 5# An unsupported autocast op fails the same way on every request. Retry that request in# full precision, and only turn AMP off after this many failures in a row (#351).@@ -930,6 +946,8 @@ class Agent(HookRegistry):def _infer(self, b: Dict):"""Run the forward pass under autocast, degrading gracefully on OOM or unsupported autocast."""+ if self._compiled and self._fast is None and self.device.type == "cuda":+ b = _pad_cuda_compile_batch(b, self.tok.pad_token_id)use_amp = self._amp_enabled_for(b["input_ids"].shape[0])if self._fast is not None and b["input_ids"].shape[1] > self._fast.max_len:diff --git a/tests/test_compile.py b/tests/test_compile.pyindex 1d833e89..edecb6bc 100644--- a/tests/test_compile.py+++ b/tests/test_compile.py@@ -15,7 +15,7 @@ import torchsys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))from laya._compile import compile_model, independent_dims # noqa: E402-from laya.agent import Agent # noqa: E402+from laya.agent import Agent, _pad_cuda_compile_batch # noqa: E402from laya.common import DecisionModel # noqa: E402fx_config = torch.fx.experimental._config@@ -32,7 +32,11 @@ def tiny_model():num_attention_heads=2, global_attn_every_n_layers=2, local_attention=16,max_position_embeddings=512, pad_token_id=0, bos_token_id=1, eos_token_id=2,cls_token_id=1, sep_token_id=2)- return DecisionModel(AutoModel.from_config(cfg, attn_implementation="sdpa"), 1, 2).eval()+ encoder = AutoModel.from_config(cfg, attn_implementation="sdpa")+ # The CPU fixture checks Laya's dynamic dimensions, not ModernBERT's own+ # first-forward mutation of reference_compile ("auto" -> False on CPU).+ encoder.config.reference_compile = False+ return DecisionModel(encoder, 1, 2).eval()def batch(rows, tokens, markers):@@ -60,6 +64,22 @@ def graphs():return counters["stats"]["unique_graphs"]+def test_cuda_compile_padding_masks_only_the_new_tail():+ b = batch(2, 49, 3)+ padded = _pad_cuda_compile_batch(b, 7)+ assert padded["input_ids"].shape == (2, 56)+ assert padded["attention_mask"].shape == (2, 56)+ assert torch.equal(padded["input_ids"][:, :49], b["input_ids"])+ assert torch.equal(padded["attention_mask"][:, :49], b["attention_mask"])+ assert torch.all(padded["input_ids"][:, 49:] == 7)+ assert not padded["attention_mask"][:, 49:].any()+ assert padded["marker_pos"] is b["marker_pos"]+ assert padded["marker_mask"] is b["marker_mask"]+ assert padded["qtype"] is b["qtype"]+ assert b["input_ids"].shape == (2, 49)+ assert _pad_cuda_compile_batch(padded, 0) is padded++def test_one_graph_across_shapes_and_same_outputs():torch._dynamo.reset()model = tiny_model()