Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(model): match action head dtype for AOTInductor packaging

cklxx036cb4817011
diff --git a/benchmarks/results/aoti_multilingual_rtx4070.json b/benchmarks/results/aoti_multilingual_rtx4070.json
new file mode 100644
index 00000000..47881e86
--- /dev/null
+++ b/benchmarks/results/aoti_multilingual_rtx4070.json
@@ -0,0 +1,161 @@
+{
+ "base_commit": "fa9a2a7",
+ "date": "2026-10-03",
+ "transformers": "5.17.0",
+ "python": "3.12.13",
+ "method": "Median of five synchronized wall-clock groups of 30 forwards after ten warmups; torch.compile(dynamic=True), default mode, no explicit CUDA graphs. Each run starts with its own empty Inductor cache; AMP compilation precedes packaging. Shared GPU and CPU, clocks not locked.",
+ "before": {
+ "torch": "2.11.0+cu130",
+ "model": "/home/ckl/.cache/huggingface/hub/models--convaiinnovations--laya/snapshots/1c5edc17a7acd8701df6fc341c0d179f1c62c982/multilingual",
+ "gpu": "NVIDIA GeForce RTX 4070 Ti SUPER",
+ "cache": "/tmp/laya-aoti-before/cache",
+ "gpu_state_before": "Sat Oct 3 18:42:26 2026 \n+-----------------------------------------------------------------------------------------+\n| NVIDIA-SMI 615.71.09 KMD Version: 615.71.09 CUDA UMD Version: 13.4 |\n+-----------------------------------------+------------------------+----------------------+\n| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC |\n| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. |\n| | | MIG M. |\n|=========================================+========================+======================|\n| 0 NVIDIA GeForce RTX 4070 ... On | 00000000:26:00.0 Off | N/A |\n| 0% 33C P8 12W / 295W | 2817MiB / 16376MiB | 28% Default |\n| | | N/A |\n+-----------------------------------------+------------------------+----------------------+\n\n+-----------------------------------------------------------------------------------------+\n| Processes: |\n| GPU GI CI PID Type Process name GPU Memory |\n| ID ID Usage |\n|=========================================================================================|\n| 0 N/A N/A 892 G Hyprland 156MiB |\n| 0 N/A N/A 952 G quickshell 38MiB |\n| 0 N/A N/A 995 G Xwayland 4MiB |\n| 0 N/A N/A 1168 G monocloud 3MiB |\n| 0 N/A N/A 3324 C+G /usr/bin/sunshine 215MiB |\n| 0 N/A N/A 12883 G /usr/bin/ghostty 100MiB |\n| 0 N/A N/A 3906709 C+G ...rack-uuid=3190708988185955192 35MiB |\n| 0 N/A N/A 4068228 C ...jects/S/laya/.venv/bin/python 2074MiB |\n+-----------------------------------------------------------------------------------------+\n",
+ "amp_eager_ms": [
+ 15.1173,
+ 15.4687,
+ 14.8452
+ ],
+ "amp_compiled_ms": [
+ 6.7117,
+ 6.7274,
+ 7.3872
+ ],
+ "export_s": 5.004336104961112,
+ "failure_stage": "package",
+ "failure_s": 16.594587769010104,
+ "error": "mat1 and mat2 must have the same dtype, but got Float and BFloat16",
+ "package_s": null,
+ "package_bytes": null,
+ "aoti_ms": null,
+ "gpu_state_after": "Sat Oct 3 18:43:47 2026 \n+-----------------------------------------------------------------------------------------+\n| NVIDIA-SMI 615.71.09 KMD Version: 615.71.09 CUDA UMD Version: 13.4 |\n+-----------------------------------------+------------------------+----------------------+\n| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC |\n| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. |\n| | | MIG M. |\n|=========================================+========================+======================|\n| 0 NVIDIA GeForce RTX 4070 ... On | 00000000:26:00.0 Off | N/A |\n| 30% 36C P3 23W / 295W | 8560MiB / 16376MiB | 10% Default |\n| | | N/A |\n+-----------------------------------------+------------------------+----------------------+\n\n+-----------------------------------------------------------------------------------------+\n| Processes: |\n| GPU GI CI PID Type Process name GPU Memory |\n| ID ID Usage |\n|=========================================================================================|\n| 0 N/A N/A 892 G Hyprland 156MiB |\n| 0 N/A N/A 952 G quickshell 38MiB |\n| 0 N/A N/A 995 G Xwayland 4MiB |\n| 0 N/A N/A 1168 G monocloud 3MiB |\n| 0 N/A N/A 3324 C+G /usr/bin/sunshine 215MiB |\n| 0 N/A N/A 12883 G /usr/bin/ghostty 100MiB |\n| 0 N/A N/A 3906709 C+G ...rack-uuid=3190708988185955192 35MiB |\n| 0 N/A N/A 4069106 C ...jects/S/laya/.venv/bin/python 3176MiB |\n| 0 N/A N/A 4070160 C ...jects/S/laya/.venv/bin/python 2174MiB |\n| 0 N/A N/A 4070256 C ...jects/S/laya/.venv/bin/python 1990MiB |\n| 0 N/A N/A 4070519 C ...jects/S/laya/.venv/bin/python 246MiB |\n| 0 N/A N/A 4070896 C python 210MiB |\n+-----------------------------------------------------------------------------------------+\n"
+ },
+ "after": {
+ "torch": "2.11.0+cu130",
+ "model": "/home/ckl/.cache/huggingface/hub/models--convaiinnovations--laya/snapshots/1c5edc17a7acd8701df6fc341c0d179f1c62c982/multilingual",
+ "gpu": "NVIDIA GeForce RTX 4070 Ti SUPER",
+ "cache": "/tmp/laya-aoti-after/cache",
+ "gpu_state_before": "Sat Oct 3 18:44:33 2026 \n+-----------------------------------------------------------------------------------------+\n| NVIDIA-SMI 615.71.09 KMD Version: 615.71.09 CUDA UMD Version: 13.4 |\n+-----------------------------------------+------------------------+----------------------+\n| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC |\n| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. |\n| | | MIG M. |\n|=========================================+========================+======================|\n| 0 NVIDIA GeForce RTX 4070 ... On | 00000000:26:00.0 Off | N/A |\n| 30% 33C P8 12W / 295W | 2634MiB / 16376MiB | 0% Default |\n| | | N/A |\n+-----------------------------------------+------------------------+----------------------+\n\n+-----------------------------------------------------------------------------------------+\n| Processes: |\n| GPU GI CI PID Type Process name GPU Memory |\n| ID ID Usage |\n|=========================================================================================|\n| 0 N/A N/A 892 G Hyprland 146MiB |\n| 0 N/A N/A 952 G quickshell 38MiB |\n| 0 N/A N/A 995 G Xwayland 4MiB |\n| 0 N/A N/A 1168 G monocloud 3MiB |\n| 0 N/A N/A 3324 C+G /usr/bin/sunshine 215MiB |\n| 0 N/A N/A 12883 G /usr/bin/ghostty 100MiB |\n| 0 N/A N/A 3906709 C+G ...rack-uuid=3190708988185955192 53MiB |\n| 0 N/A N/A 4070160 C ...jects/S/laya/.venv/bin/python 1632MiB |\n| 0 N/A N/A 4070519 C ...jects/S/laya/.venv/bin/python 246MiB |\n+-----------------------------------------------------------------------------------------+\n",
+ "existing_eager_max_delta": 0.0,
+ "amp_eager_ms": [
+ 14.12,
+ 14.4156,
+ 14.5384
+ ],
+ "amp_compiled_ms": [
+ 6.9076,
+ 5.4772,
+ 5.5934
+ ],
+ "export_s": 3.8901766380295157,
+ "package_s": 60.01539184397552,
+ "package_bytes": 645729621,
+ "load_s": 0.44709984108339995,
+ "cases": [
+ {
+ "shape": [
+ 2,
+ 128,
+ 3
+ ],
+ "aoti_vs_bf16_eager": {
+ "decision_logits": 0.1875,
+ "decision_probabilities": 3.672063542126125e-09,
+ "decision_answers_equal": true,
+ "action_logits": 0.0,
+ "action_probabilities": 0.0,
+ "action_answers_equal": true
+ },
+ "aoti_vs_amp_eager": {
+ "decision_logits": 0.25,
+ "decision_probabilities": 6.553003473186436e-09,
+ "decision_answers_equal": true,
+ "action_logits": 0.0,
+ "action_probabilities": 0.0,
+ "action_answers_equal": true
+ },
+ "bf16_eager_ms": 13.9635,
+ "aoti_ms": 2.2674,
+ "bf16_compiled_ms": 4.8835,
+ "compiled_vs_bf16_eager": {
+ "decision_logits": 0.1875,
+ "decision_probabilities": 4.09005274093488e-09,
+ "decision_answers_equal": true,
+ "action_logits": 0.0,
+ "action_probabilities": 0.0,
+ "action_answers_equal": true
+ }
+ },
+ {
+ "shape": [
+ 1,
+ 256,
+ 3
+ ],
+ "aoti_vs_bf16_eager": {
+ "decision_logits": 0.1875,
+ "decision_probabilities": 3.672063542126125e-09,
+ "decision_answers_equal": true,
+ "action_logits": 0.0,
+ "action_probabilities": 0.0,
+ "action_answers_equal": true
+ },
+ "aoti_vs_amp_eager": {
+ "decision_logits": 0.3125,
+ "decision_probabilities": 8.276904495119197e-09,
+ "decision_answers_equal": true,
+ "action_logits": 0.0,
+ "action_probabilities": 0.0,
+ "action_answers_equal": true
+ },
+ "bf16_eager_ms": 14.7994,
+ "aoti_ms": 2.4154,
+ "bf16_compiled_ms": 4.725,
+ "compiled_vs_bf16_eager": {
+ "decision_logits": 0.1875,
+ "decision_probabilities": 4.09005274093488e-09,
+ "decision_answers_equal": true,
+ "action_logits": 0.0,
+ "action_probabilities": 0.0,
+ "action_answers_equal": true
+ }
+ },
+ {
+ "shape": [
+ 4,
+ 160,
+ 5
+ ],
+ "aoti_vs_bf16_eager": {
+ "decision_logits": 0.5625,
+ "decision_probabilities": 0.0065985918045043945,
+ "decision_answers_equal": true,
+ "action_logits": 0.0,
+ "action_probabilities": 0.0,
+ "action_answers_equal": true
+ },
+ "aoti_vs_amp_eager": {
+ "decision_logits": 0.4375,
+ "decision_probabilities": 0.007699102163314819,
+ "decision_answers_equal": true,
+ "action_logits": 8.0,
+ "action_probabilities": 0.0,
+ "action_answers_equal": true
+ },
+ "bf16_eager_ms": 14.7115,
+ "aoti_ms": 3.6018,
+ "bf16_compiled_ms": 5.1393,
+ "compiled_vs_bf16_eager": {
+ "decision_logits": 0.5,
+ "decision_probabilities": 0.0065985918045043945,
+ "decision_answers_equal": true,
+ "action_logits": 8.0,
+ "action_probabilities": 0.0,
+ "action_answers_equal": true
+ }
+ }
+ ],
+ "gpu_state_after": "Sat Oct 3 18:48:36 2026 \n+-----------------------------------------------------------------------------------------+\n| NVIDIA-SMI 615.71.09 KMD Version: 615.71.09 CUDA UMD Version: 13.4 |\n+-----------------------------------------+------------------------+----------------------+\n| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC |\n| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. |\n| | | MIG M. |\n|=========================================+========================+======================|\n| 0 NVIDIA GeForce RTX 4070 ... On | 00000000:26:00.0 Off | N/A |\n| 0% 42C P2 160W / 295W | 6476MiB / 16376MiB | 73% Default |\n| | | N/A |\n+-----------------------------------------+------------------------+----------------------+\n\n+-----------------------------------------------------------------------------------------+\n| Processes: |\n| GPU GI CI PID Type Process name GPU Memory |\n| ID ID Usage |\n|=========================================================================================|\n| 0 N/A N/A 892 G Hyprland 156MiB |\n| 0 N/A N/A 952 G quickshell 38MiB |\n| 0 N/A N/A 995 G Xwayland 4MiB |\n| 0 N/A N/A 1168 G monocloud 3MiB |\n| 0 N/A N/A 3324 C+G /usr/bin/sunshine 215MiB |\n| 0 N/A N/A 12883 G /usr/bin/ghostty 100MiB |\n| 0 N/A N/A 3906709 C+G ...rack-uuid=3190708988185955192 36MiB |\n| 0 N/A N/A 4072238 C ...jects/S/laya/.venv/bin/python 3802MiB |\n| 0 N/A N/A 4077174 C ...jects/S/laya/.venv/bin/python 1926MiB |\n+-----------------------------------------------------------------------------------------+\n",
+ "fresh_process_load_s": 4.867020106990822,
+ "fresh_process_note": "Standalone package and saved inputs only, torch.compile and aoti_compile_and_package blocked; both heads agree on all seven rows. Empty cache acquired a CPU AVX capability probe (.cpp/.so/lock), no model graph or CUDA kernels."
+ }
+}
diff --git a/docs/fast-backends.md b/docs/fast-backends.md
new file mode 100644
index 00000000..cd0eb2c3
--- /dev/null
+++ b/docs/fast-backends.md
@@ -0,0 +1,120 @@
+# Fast inference: AOTInductor
+
+## AOTInductor
+
+`DecisionModel` can be exported, compiled into a `.pt2` package, and loaded with
+`torch._inductor.aoti_load_package`. The package returns both decision logits and action logits.
+Tokenization, padding, temperature calibration and answer formatting remain the caller's job;
+this does not add an `Agent` backend or change its default execution path.
+
+The [closed PR #472](https://github.com/NandhaKishorM/laya/pull/472) documented the earlier dtype
+blocker. The action head's pooled state and confidence features are computed in fp32, even
+when a model's weights are explicitly bf16. Without autocast, the first action linear therefore
+received an fp32 input and bf16 weights. The forward now casts the concatenated input to the
+head's weight dtype **only outside autocast**. Softmax, entropy and the returned decision logits
+retain their fp32 calculations. Existing fp32-parameter eager inference, including fp16/bf16
+AMP, keeps its numerics; mixed weight/autocast dtypes also retain autocast's original conversion.
+
+Export a separate evaluation copy converted to bf16, outside autocast. Declare the token
+axis as `16 * Dim("tokens16", ...)` to satisfy the attention alignment guards. Rows and marker
+counts can be dynamic independently. Do not assume an export captured under autocast can be
+packaged outside that context: use explicit weight dtypes for this recipe.
+
+## Reproduce offline
+
+The assert-based check uses a tiny randomly initialized ModernBERT by default, without network
+access. Pass a local checkpoint directory for a real-model measurement. Missing CUDA,
+AOTInductor APIs or a C++ compiler produces an explicit `SKIP`; compilation and parity failures
+on a supported installation fail the check.
+
+```bash
+python scripts/check_aoti.py --output-dir /tmp/laya-aoti-smoke
+HF_HUB_OFFLINE=1 TORCHINDUCTOR_CACHE_DIR=/tmp/laya-aoti/cache \
+ python scripts/check_aoti.py --model /path/to/local/multilingual \
+ --output-dir /tmp/laya-aoti
+```
+
+The output directory holds `decision.pt2`, `results.json`, `inputs.pt` and `eager.pt`. The JSON
+includes full `nvidia-smi` snapshots, export/package/load times, package bytes, latency and
+maximum absolute logit/probability deltas for both heads. Binary artifacts and compiler caches
+are deliberately kept out of the repository.
+
+## Measured on RTX 4070 Ti SUPER
+
+2026-10-03, Linux x86-64, Python 3.12.13, torch 2.11.0+cu130, transformers 5.17.0,
+NVIDIA driver 615.71.09, 16,376 MiB VRAM. Checkpoint: `convaiinnovations/laya`,
+`multilingual` subdirectory at revision `1c5edc17a7acd8701df6fc341c0d179f1c62c982`.
+The baseline is main `fa9a2a7`. Full machine snapshots and unrounded measurements are in
+[`aoti_multilingual_rtx4070.json`](https://github.com/NandhaKishorM/laya/blob/main/benchmarks/results/aoti_multilingual_rtx4070.json).
+
+| Measurement | Before | After |
+|---|---:|---:|
+| Export | 5.00 s | 3.89 s |
+| Packaging | failed after 16.59 s | 60.02 s |
+| Package size | no artifact | 645,729,621 bytes (615.82 MiB) |
+| Load in the already initialized process | unavailable | 0.447 s |
+| Load in a fresh process with an empty cache | unavailable | 4.867 s |
+
+The reproduced failure is at packaging, after successful export:
+`mat1 and mat2 must have the same dtype, but got Float and BFloat16`.
+Each run used a separate initially empty Inductor cache; AMP compilation ran before packaging,
+so the package timing is not a fresh-interpreter cold-start measurement.
+The fresh-process check loaded only the package and saved inputs (no checkpoint), with
+`torch.compile` and packaging entry points blocked. It reproduced both heads' answers.
+PyTorch still compiled a small CPU AVX capability probe into the initially empty cache;
+no model graph or CUDA kernel was compiled. A blanket "no compiler activity at load" claim
+would therefore be inaccurate on this runtime.
+
+The same saved inputs contain seven decision rows across three batches, with real tokenized
+billing/refund text, choice/score/noul questions, padding and unequal valid marker counts.
+Each latency is the median of five groups of 30 forwards after ten warmups, using synchronized
+wall-clock timing. `torch.compile(dynamic=True)` uses the default mode. No explicit CUDA graphs,
+tokenization, export or compilation are included in these latency numbers.
+
+| Rows × tokens × marker slots | AMP eager before → after (ms) | AMP compiled before → after (ms) | Explicit bf16 eager (ms) | Explicit bf16 compiled (ms) | AOTI bf16 (ms) |
+|---|---:|---:|---:|---:|---:|
+| 2 × 128 × 3 | 15.1173 → 14.1200 | 6.7117 → 6.9076 | 13.9635 | 4.8835 | 2.2674 |
+| 1 × 256 × 3 | 15.4687 → 14.4156 | 6.7274 → 5.4772 | 14.7994 | 4.7250 | 2.4154 |
+| 4 × 160 × 5 | 14.8452 → 14.5384 | 7.3872 → 5.5934 | 14.7115 | 5.1393 | 3.6018 |
+
+AMP here means fp32 parameters with bf16 autocast. Explicit bf16 means weights and the residual
+stream are bf16, without autocast; use those columns for the closest execution comparison to
+the package. Explicit bf16 eager/compiled and AOTI were unavailable before this fix.
+
+| Comparison, maximum over all seven rows | Decision logits | Decision probabilities | Action logits | Action probabilities |
+|---|---:|---:|---:|---:|
+| Existing eager before vs after, fp32 and fp16/bf16 AMP | 0 | 0 | 0 | 0 |
+| AOTI vs explicit bf16 eager | 0.5625 | 0.00659859 | 0 | 0 |
+| AOTI vs existing bf16 AMP eager | 0.4375 | 0.00769910 | 8.0 | 0 |
+
+Both decision and action argmax agree on **7/7 rows** for both AOTI comparisons. Probabilities
+are raw softmax outputs, without calibration. The action distribution is saturated on these
+inputs, so its zero probability delta does not imply identical underlying logits against AMP.
+The explicit bf16 compiled forward also differs from explicit bf16 eager (maximum decision
+logit delta 0.5, probability delta 0.00659859). Reduced-precision compilation is not bit-exact.
+
+The GPU was shared with desktop applications and other Python jobs; CPU compilation was also
+shared, and clocks were not locked. These are observed timings, not an isolated speedup claim.
+`nvidia-smi` snapshots bracketing the runs:
+
+| Run / snapshot | GPU utilization | Used VRAM | Power | Temperature / state |
+|---|---:|---:|---:|---|
+| Before / start | 28% | 2,817 MiB | 12 W | 33°C / P8 |
+| Before / end | 10% | 8,560 MiB | 23 W | 36°C / P3 |
+| After / start | 0% | 2,634 MiB | 12 W | 33°C / P8 |
+| After / end | 73% | 6,476 MiB | 160 W | 42°C / P2 |
+
+## Remaining limits
+
+- The artifact is specific to this checkpoint, precision, PyTorch/runtime stack and GPU target;
+ portability to other hardware or PyTorch versions was not tested.
+- This check exports rows 1–8, tokens 32–512 in multiples of 16, and marker slots 2–8. It exercises
+ three shapes, including shapes different from the export example, rather than every point
+ in those ranges. One valid option can use padded marker slots; a true one-slot tensor takes
+ the separate single-option branch and needs a separate export. Arbitrary token lengths and
+ a production bucketing/dispatch layer are outside this change.
+- Seven rows verify the packaging regression, not broad checkpoint accuracy or calibrated
+ confidence parity. Casting an export copy to bf16 differs from retaining fp32 weights under
+ autocast; the existing eager path itself remains unchanged.
+- The script requires a CUDA-capable PyTorch build and local compiler toolchain to create the
+ artifact. The `.pt2` embeds the model and CUDA kernels; no hosted service is involved.
diff --git a/laya/common.py b/laya/common.py
index 874b0450..1cf74f26 100644
--- a/laya/common.py
+++ b/laya/common.py
@@ -539,7 +539,12 @@ class DecisionModel(nn.Module):
top2 = torch.cat([top1, torch.zeros_like(top1)], dim=-1)
feats = torch.stack([top2[:, 0], top2[:, 0] - top2[:, 1], ent, k / 255.0], -1)
pooled = h[:, 0].float()
- act_logits = self.act_head(torch.cat([pooled, feats], -1))
+ act_input = torch.cat([pooled, feats], -1)
+ # Explicit-dtype exports run without autocast; keep the confidence maths in fp32,
+ # then match the head's weights. Autocast already chooses the linear's input dtype.
+ if not torch.is_autocast_enabled(h.device.type):
+ act_input = act_input.to(self.act_head[0].weight.dtype)
+ act_logits = self.act_head(act_input)
return logits, act_logits
diff --git a/scripts/check_aoti.py b/scripts/check_aoti.py
new file mode 100644
index 00000000..0865f021
--- /dev/null
+++ b/scripts/check_aoti.py
@@ -0,0 +1,189 @@
+"""Offline AOTInductor packaging/parity check and CUDA forward benchmark.
+
+Run with --model pointing at a local Laya checkpoint. Without it, use a tiny
+random ModernBERT (no downloads). Outputs and compiler caches belong outside git.
+"""
+import argparse
+import copy
+import json
+import os
+from pathlib import Path
+import shutil
+import statistics
+import subprocess
+import sys
+import time
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+
+
+def main():
+ ap = argparse.ArgumentParser(description=__doc__)
+ ap.add_argument("--model", help="Local checkpoint directory; never downloads weights")
+ ap.add_argument("--output-dir", required=True)
+ ap.add_argument("--expect-export-failure", action="store_true", help="Measure the unpatched baseline")
+ ap.add_argument("--compare-eager", help="Baseline eager.pt to check bit identity")
+ args = ap.parse_args()
+ try:
+ import torch
+ from torch.export import Dim
+ from torch._inductor import aoti_compile_and_package, aoti_load_package
+ except ImportError:
+ print("SKIP: torch.export / AOTInductor unavailable")
+ return
+ if not torch.cuda.is_available() or not shutil.which("g++"):
+ print("SKIP: CUDA and a C++ compiler are required")
+ return
+ from laya.common import DecisionModel, QTYPES, build_sequence
+
+ outdir = Path(args.output_dir)
+ outdir.mkdir(parents=True, exist_ok=True)
+ torch.manual_seed(0)
+ torch.set_num_threads(4)
+ report = {"torch": torch.__version__, "model": args.model or "tiny random ModernBERT",
+ "gpu": torch.cuda.get_device_name(), "cache": os.environ.get("TORCHINDUCTOR_CACHE_DIR"),
+ "gpu_state_before": subprocess.check_output(["nvidia-smi"], text=True)}
+ if args.model:
+ if not Path(args.model).is_dir():
+ ap.error("--model must be an existing local checkpoint directory")
+ from laya.agent import Agent
+ agent = Agent(args.model, device="cuda", compile=False)
+ model = agent.model.eval()
+ else:
+ from transformers import ModernBertConfig, ModernBertModel
+ cfg = ModernBertConfig(vocab_size=128, hidden_size=64, intermediate_size=128,
+ num_hidden_layers=1, num_attention_heads=4, max_position_embeddings=512,
+ pad_token_id=0, bos_token_id=1, eos_token_id=2, cls_token_id=1, sep_token_id=2,
+ reference_compile=False, attn_implementation="sdpa")
+ model = DecisionModel(ModernBertModel(cfg), head_layers=1, dropout=0).cuda().eval()
+
+ cases = []
+ for rows, tokens, markers in ((2, 128, 3), (1, 256, 3), (4, 160, 5)):
+ ids = torch.zeros((rows, tokens), dtype=torch.long)
+ att = torch.zeros_like(ids)
+ pos = torch.zeros((rows, markers), dtype=torch.long)
+ mask = torch.zeros_like(pos, dtype=torch.bool)
+ qt = torch.arange(rows) % 3
+ for row in range(rows):
+ count = markers if row == 0 else max(1, markers - row)
+ if args.model:
+ kind = ("choice", "score", "noul")[row % 3]
+ question = {"type": kind, "instructions": "Which team should handle this request?"}
+ if kind == "choice":
+ question["criteria"] = dict(list({"billing": "refunds", "tech": "bugs", "sales": "pricing",
+ "account": "login", "support": "other"}.items())[:count])
+ elif kind == "score":
+ question["criteria"] = ["low", "medium", "high"][:count]
+ else:
+ question["instructions"] = "Is the customer asking for a refund?"
+ qq = agent._to_internal(question)
+ seq, mp = build_sequence(agent.tok, {"body": "We were billed twice. Please refund the duplicate charge."},
+ qq, tokens, min(tokens, agent.cfg["head_max_len"]))
+ qt[row] = QTYPES[qq["t"]]
+ ids[row].fill_(agent.tok.pad_token_id or 0)
+ else:
+ seq = torch.randint(1, 128, (tokens - 16 * row,)).tolist()
+ mp = list(range(1, count + 1))
+ assert len(seq) <= tokens and len(mp) <= markers
+ ids[row, :len(seq)] = torch.tensor(seq)
+ att[row, :len(seq)] = 1
+ pos[row, :len(mp)] = torch.tensor(mp)
+ mask[row, :len(mp)] = True
+ cases.append(tuple(t.cuda() for t in (ids, att, pos, mask, qt)))
+ torch.save([[t.cpu() for t in case] for case in cases], outdir / "inputs.pt")
+
+ def latency(fn, inputs):
+ for _ in range(10):
+ fn(*inputs)
+ torch.cuda.synchronize()
+ samples = []
+ for _ in range(5):
+ start = time.perf_counter()
+ for _ in range(30):
+ fn(*inputs)
+ torch.cuda.synchronize()
+ samples.append((time.perf_counter() - start) * 1000 / 30)
+ return round(statistics.median(samples), 4)
+
+ def delta(ref, got):
+ result = {}
+ for name, a, b in zip(("decision", "action"), ref, got):
+ assert torch.isfinite(b).all()
+ result[name + "_logits"] = (a.float() - b.float()).abs().max().item()
+ result[name + "_probabilities"] = (a.float().softmax(-1) - b.float().softmax(-1)).abs().max().item()
+ result[name + "_answers_equal"] = bool(torch.equal(a.argmax(-1), b.argmax(-1)))
+ return result
+
+ with torch.no_grad():
+ # Pin the existing fp32-parameter path, both without AMP and with fp16/bf16 AMP.
+ eager = {}
+ for dtype in (torch.float32, torch.float16, torch.bfloat16):
+ with torch.autocast("cuda", dtype=dtype, enabled=dtype != torch.float32):
+ eager[str(dtype)] = [[x.cpu() for x in model(*case)] for case in cases]
+ torch.save(eager, outdir / "eager.pt")
+ if args.compare_eager:
+ baseline = torch.load(args.compare_eager, weights_only=True)
+ for key in eager:
+ for old, new in zip(baseline[key], eager[key]):
+ for a, b in zip(old, new):
+ torch.testing.assert_close(a, b, rtol=0, atol=0)
+ report["existing_eager_max_delta"] = 0.0
+
+ with torch.autocast("cuda", dtype=torch.bfloat16):
+ report["amp_eager_ms"] = [latency(model, case) for case in cases]
+ compiled = torch.compile(model, dynamic=True)
+ report["amp_compiled_ms"] = [latency(compiled, case) for case in cases]
+ print(json.dumps(report), flush=True)
+ del compiled
+ candidate = copy.deepcopy(model).to(torch.bfloat16)
+ rows, markers = Dim("rows", min=1, max=8), Dim("markers", min=2, max=8)
+ tokens = 16 * Dim("tokens16", min=2, max=32)
+ dynamic = ({0: rows, 1: tokens}, {0: rows, 1: tokens}, {0: rows, 1: markers},
+ {0: rows, 1: markers}, {0: rows})
+ start = time.perf_counter()
+ stage = "export"
+ try:
+ exported = torch.export.export(candidate, cases[0], dynamic_shapes=dynamic, strict=False)
+ report["export_s"] = time.perf_counter() - start
+ print("Export complete; packaging", flush=True)
+ stage = "package"
+ start = time.perf_counter()
+ path = aoti_compile_and_package(exported, package_path=str(outdir / "decision.pt2"))
+ report["package_s"] = time.perf_counter() - start
+ report["package_bytes"] = Path(path).stat().st_size
+ except Exception as exc:
+ if not args.expect_export_failure or not any(s in str(exc) for s in ("same dtype", "dtype mismatch")):
+ raise
+ report.update(failure_stage=stage, failure_s=time.perf_counter() - start, error=str(exc),
+ package_s=None, package_bytes=None, aoti_ms=None)
+ else:
+ assert not args.expect_export_failure, "Expected the baseline dtype failure"
+ start = time.perf_counter()
+ runner = aoti_load_package(path)
+ report["load_s"] = time.perf_counter() - start
+ report["cases"] = []
+ for i, case in enumerate(cases):
+ reference = candidate(*case)
+ actual = runner(*case)
+ diff = delta(reference, actual)
+ amp_diff = delta([x.cuda() for x in eager["torch.bfloat16"][i]], actual)
+ for comparison in (diff, amp_diff):
+ for name in ("decision", "action"):
+ assert comparison[name + "_probabilities"] < 0.02, comparison
+ assert comparison[name + "_answers_equal"], comparison
+ item = {"shape": [*case[0].shape, case[2].shape[1]], "aoti_vs_bf16_eager": diff,
+ "aoti_vs_amp_eager": amp_diff,
+ "bf16_eager_ms": latency(candidate, case), "aoti_ms": latency(runner, case)}
+ report["cases"].append(item)
+ compiled = torch.compile(candidate, dynamic=True)
+ for item, case in zip(report["cases"], cases):
+ item["bf16_compiled_ms"] = latency(compiled, case)
+ item["compiled_vs_bf16_eager"] = delta(candidate(*case), compiled(*case))
+ report["gpu_state_after"] = subprocess.check_output(["nvidia-smi"], text=True)
+ (outdir / "results.json").write_text(json.dumps(report, indent=2) + "\n")
+ print(json.dumps(report, indent=2), flush=True)
+ print("PASS: expected baseline failure" if args.expect_export_failure else "PASS: AOTI package, parity and latency")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/tests/test_decision_model.py b/tests/test_decision_model.py
index ecd784d9..720c3499 100644
--- a/tests/test_decision_model.py
+++ b/tests/test_decision_model.py
@@ -93,9 +93,34 @@ def test_multi_option_question_is_unaffected():
assert torch.isfinite(act_logits).all()
+def test_action_head_explicit_dtype_and_autocast():
+ # The old boundary always passed fp32 to the head. Preserve that under AMP,
+ # including bf16 weights with fp16 autocast (an extra bf16 cast would round twice).
+ device = "cuda" if torch.cuda.is_available() else "cpu"
+ for weights in (torch.float32, torch.bfloat16):
+ for amp in (None, torch.bfloat16, torch.float16):
+ if device == "cpu" and amp == torch.float16:
+ continue
+ model = _tiny_model().to(device=device, dtype=weights)
+ inputs = tuple(t.to(device) for t in _inputs(2, 10, 4))
+ captured = []
+ hook = model.act_head.register_forward_pre_hook(lambda module, args: captured.append(args[0]))
+ with torch.no_grad(), torch.autocast(device, dtype=amp or torch.bfloat16, enabled=amp is not None):
+ logits, act_logits = model(*inputs)
+ hook.remove()
+ expected_dtype = torch.float32 if amp is not None else weights
+ assert captured[0].dtype == expected_dtype
+ assert logits.dtype == torch.float32
+ assert torch.isfinite(act_logits).all()
+ if amp is not None or weights == torch.float32:
+ legacy = model.act_head(captured[0].float())
+ torch.testing.assert_close(act_logits, legacy, rtol=0, atol=0)
+
+
if __name__ == "__main__":
test_single_option_question_does_not_crash()
test_single_option_top1_minus_top2_is_exactly_one()
test_multi_option_question_is_unaffected()
+ test_action_head_explicit_dtype_and_autocast()
print("all decision model tests passed")