Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

feat(agent): warm compiled models during load by default

cklxxdcbda34d496d
diff --git a/README.md b/README.md
index af65dae7..aeb2b6c9 100644
--- a/README.md
+++ b/README.md
@@ -864,20 +864,17 @@ restores it. Kernels compile once per shape bucket on first use (a few seconds,
### Warm-up before serving: `agent.warmup()`
-`compile=True` compiles on the first request that needs a graph, and the first single-question request
-needs a second one (torch specialises a batch of 1). Both stall a live request. `agent.warmup()` runs the
-forward on a few synthetic shapes now and returns the seconds it took, so the compiles happen before
-traffic arrives:
+`compile=True` now runs `agent.warmup()` before returning from load. This moves the batch and
+single-row graph compiles into startup; startup can take tens of seconds. To defer the work:
```python
-agent = laya.load("convaiinnovations/laya", compile=True)
-agent.warmup() # ~46 s on an RTX 4070 Ti SUPER; every later request ~10-30 ms
+agent = laya.load("convaiinnovations/laya", compile=True, compile_warmup=False)
+seconds = agent.warmup() # manual entry point, also accepts shapes=[(rows, tokens, markers)]
```
-Measured with `benchmarks/bench_compile.py --device cuda [--warmup]` (English checkpoint, torch 2.11, ten
-requests of changing shape): without it the first request took 51 s and the first single-question request,
-the eighth, took another 41 s; after `warmup()` no request took more than 30 ms. It works the same with
-`fast=True` (kernels and CUDA graphs for those buckets) and costs a few forward passes on the stock path.
+Warm-up uses synthetic inputs without prediction hooks or prediction caches. It covers common batch
+and single-row shapes, not every possible guard or CUDA graph shape. `compile=False` and `fast=True`
+do not warm automatically. Manual warm-up still works on both paths.
On CUDA, compiled inference pads the masked end of each sequence to a multiple of eight tokens. This
avoids extra SDPA graph specialisations on PyTorch versions that distinguish lengths modulo eight;
reported token usage still counts the original, unpadded sequence.
diff --git a/benchmarks/bench_compile_defaults.py b/benchmarks/bench_compile_defaults.py
new file mode 100644
index 00000000..b178a237
--- /dev/null
+++ b/benchmarks/bench_compile_defaults.py
@@ -0,0 +1,70 @@
+"""Cold process measurements; use a fresh cache, then repeat the identical command.
+
+Example: TORCHINDUCTOR_CACHE_DIR=/tmp/laya-probe python benchmarks/bench_compile_defaults.py --model /path/to/checkpoint
+"""
+import argparse
+import json
+import os
+import subprocess
+import sys
+import time
+
+os.environ.setdefault("TOKENIZERS_PARALLELISM", "false")
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+from bench_compile import CALLS, WORDS, questions
+
+
+def main():
+ parser = argparse.ArgumentParser()
+ parser.add_argument("--model", required=True)
+ parser.add_argument("--no-warmup", action="store_true")
+ parser.add_argument("--cache", action="store_true")
+ parser.add_argument("--mode", default=None)
+ args = parser.parse_args()
+ import torch
+ from torch._dynamo.utils import counters
+ from laya import Agent
+
+ def smi():
+ print(subprocess.check_output(["nvidia-smi"], text=True), flush=True)
+
+ smi()
+ print("torch", torch.__version__, "model", args.model, flush=True)
+ kw = {}
+ if args.no_warmup:
+ kw["compile_warmup"] = False
+ if args.cache:
+ kw["compile_cache"] = True
+ if args.mode:
+ kw["compile_mode"] = args.mode
+ t = time.perf_counter()
+ agent = Agent(args.model, device="cuda", compile=True, **kw)
+ torch.cuda.synchronize()
+ load_s = time.perf_counter() - t
+ print("load_s", load_s, "cache", os.environ.get("TORCHINDUCTOR_CACHE_DIR"), flush=True)
+ times = []
+ # Repeat the sequence to expose CUDA graph recording/replay and retained-output issues.
+ answers = []
+ for i, (n, k, words) in enumerate(CALLS * 3):
+ state = " ".join(WORDS[j % len(WORDS)] for j in range(words))
+ t = time.perf_counter()
+ result = agent.predict(state, questions(n, k))
+ torch.cuda.synchronize()
+ elapsed = time.perf_counter() - t
+ times.append(elapsed)
+ if i < len(CALLS):
+ answers.append(result["answers"])
+ else:
+ assert result["answers"] == answers[i % len(CALLS)], i
+ print("request", i + 1, "seconds", elapsed, "graphs", counters["stats"]["unique_graphs"], flush=True)
+ print("RESULT", json.dumps({"load_s": load_s, "requests_s": times,
+ "allocated_mib": torch.cuda.memory_allocated() / 2**20,
+ "reserved_mib": torch.cuda.memory_reserved() / 2**20,
+ "peak_allocated_mib": torch.cuda.max_memory_allocated() / 2**20,
+ "peak_reserved_mib": torch.cuda.max_memory_reserved() / 2**20,
+ "counters": {k: dict(v) for k, v in counters.items() if k in ("stats", "inductor")}}), flush=True)
+ smi()
+
+
+if __name__ == "__main__":
+ main()
diff --git a/benchmarks/results/compile-defaults/README.md b/benchmarks/results/compile-defaults/README.md
new file mode 100644
index 00000000..76d7dbd6
--- /dev/null
+++ b/benchmarks/results/compile-defaults/README.md
@@ -0,0 +1,53 @@
+# Compile defaults measurements (2026-10-03)
+
+Source: PR bodies #718 and #576 (`gh pr view 718 --repo NandhaKishorM/laya --json body`
+and the same command for 576). The requested `docs/fast-backends.md` is now
+`docs/compile-and-fast-path.md`. Base: fa9a2a7, branch feat/compile-defaults.
+
+Hardware: RTX 4070 Ti SUPER 16 GiB, torch 2.11.0+cu130, driver 615.71.09.
+Interpreter: `/home/ckl/projects/S/laya/.venv/bin/python`.
+Checkpoint: locally cached English `convaiinnovations/laya` snapshot
+`55cf4c4ebb4ebe31b2550e8bdf3bd21b99753851`; no network downloads.
+
+Each `.command` contains the exact benchmark invocation; the matching `.log` contains
+stdout/stderr, all 30 request timings, Dynamo/Inductor cache counters, allocator memory,
+and full `nvidia-smi` snapshots before load and after requests. No caches were cleared:
+each cold experiment starts with a new directory, and its restart uses the same directory
+in a separate process. Three passes over the same ten varying request shapes assert
+identical decoded answers within each process. Request 8 is the first single-row request.
+Memory columns are PyTorch peak allocated/reserved MiB, not total device usage.
+
+This is a shared desktop GPU. Baseline began at 755 MiB occupied. Other worktrees started
+AOTInductor and GPU parity jobs during the automatic-warmup experiments; their processes
+are visible in the snapshots. Contended timings are observations, not isolated speedup
+estimates. The checkpoint's existing out-of-range temperature warning is retained in logs.
+Disk-cache reuse is verified by FX cache counters, independently of timing.
+
+## Item 1: automatic warm-up
+
+`compile=True` warms after runtime/device initialization. `compile_warmup=False` preserves
+lazy startup; `agent.warmup()` remains callable. Eager and TileLang paths do not auto-warm.
+The constructor regression uses a tiny CPU model with mocked checkpoint I/O; the existing
+Dynamo eager-backend check verifies the actual warm-up builds two graphs and later shapes
+add none. API defaults are pinned in `tests/test_hooks_api.py`.
+
+Validation commands and full output are in `warmup-validation.log` (all exit codes recorded).
+Documentation dependencies were checked with:
+
+```
+uv pip install --python /home/ckl/projects/S/laya/.venv/bin/python -r requirements-docs.txt
+```
+
+Output: `Using Python 3.12.13 environment at: /home/ckl/projects/S/laya/.venv`;
+`Checked 3 packages in 2ms`. Strict docs builds must have no `griffe:` lines.
+
+| Run | Load s | Request 1 ms | Request 8 ms | Peak alloc/reserved MiB | FX hits/misses |
+|---|---:|---:|---:|---:|---:|
+| baseline-cold | 3.078 | 50154.96 | 40468.15 | 1680.0 / 1820.0 | 0 / 2 |
+| baseline-restart | 3.124 | 23281.17 | 18366.73 | 1671.0 / 1794.0 | 2 / 0 |
+| warmup-cold | 108.498 | 26.61 | 10.91 | 1685.6 / 1820.0 | 0 / 2 |
+| warmup-restart | 61.314 | 27.42 | 10.69 | 1671.0 / 1796.0 | 2 / 0 |
+
+Baseline-cold ran before code edits; use `--no-warmup` to reproduce its lazy behavior on this tree.
+Warm-up shifts cost into load; it does not eliminate compilation or all possible guard specializations.
+The two warm-up runs were contended. Cross-process reuse yielded two FX hits and zero misses.
diff --git a/benchmarks/results/compile-defaults/baseline-cold.command b/benchmarks/results/compile-defaults/baseline-cold.command
new file mode 100644
index 00000000..c2c2581d
--- /dev/null
+++ b/benchmarks/results/compile-defaults/baseline-cold.command
@@ -0,0 +1 @@
+TORCHINDUCTOR_CACHE_DIR=/tmp/laya-compile-defaults-baseline-20261003 HF_HUB_OFFLINE=1 /home/ckl/projects/S/laya/.venv/bin/python benchmarks/bench_compile_defaults.py --model /home/ckl/.cache/huggingface/hub/models--convaiinnovations--laya/snapshots/$(cat /home/ckl/.cache/huggingface/hub/models--convaiinnovations--laya/refs/main)
diff --git a/benchmarks/results/compile-defaults/baseline-cold.log b/benchmarks/results/compile-defaults/baseline-cold.log
new file mode 100644
index 00000000..616ca40c
--- /dev/null
+++ b/benchmarks/results/compile-defaults/baseline-cold.log
@@ -0,0 +1,107 @@
+Sat Oct 3 18:38:20 2026
++-----------------------------------------------------------------------------------------+
+| NVIDIA-SMI 615.71.09 KMD Version: 615.71.09 CUDA UMD Version: 13.4 |
++-----------------------------------------+------------------------+----------------------+
+| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC |
+| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. |
+| | | MIG M. |
+|=========================================+========================+======================|
+| 0 NVIDIA GeForce RTX 4070 ... On | 00000000:26:00.0 Off | N/A |
+| 0% 33C P8 12W / 295W | 738MiB / 16376MiB | 0% Default |
+| | | N/A |
++-----------------------------------------+------------------------+----------------------+
+
++-----------------------------------------------------------------------------------------+
+| Processes: |
+| GPU GI CI PID Type Process name GPU Memory |
+| ID ID Usage |
+|=========================================================================================|
+| 0 N/A N/A 892 G Hyprland 156MiB |
+| 0 N/A N/A 952 G quickshell 38MiB |
+| 0 N/A N/A 995 G Xwayland 4MiB |
+| 0 N/A N/A 1168 G monocloud 3MiB |
+| 0 N/A N/A 3324 C+G /usr/bin/sunshine 215MiB |
+| 0 N/A N/A 12883 G /usr/bin/ghostty 100MiB |
+| 0 N/A N/A 3906709 C+G ...rack-uuid=3190708988185955192 35MiB |
++-----------------------------------------------------------------------------------------+
+
+torch 2.11.0+cu130 model /home/ckl/.cache/huggingface/hub/models--convaiinnovations--laya/snapshots/55cf4c4ebb4ebe31b2550e8bdf3bd21b99753851
+/home/ckl/projects/S/laya-wt/compile-defaults/benchmarks/bench_compile_defaults.py:41: RuntimeWarning: laya: this checkpoint ships invalid temperatures or values outside [0.5, 5]; using choice:11+=0.10058280825614929 -> 0.5. Treat confidence from the affected entries as uncalibrated.
+ agent = Agent(args.model, device="cuda", compile=True, **kw)
+load_s 3.0783330149715766 cache /tmp/laya-compile-defaults-baseline-20261003
+/home/ckl/projects/S/laya/.venv/lib/python3.12/site-packages/torch/_inductor/compile_fx.py:322: UserWarning: TensorFloat32 tensor cores for float32 matrix multiplication available but not enabled. Consider setting `torch.set_float32_matmul_precision('high')` for better performance.
+ warnings.warn(
+W1003 18:39:01.514000 4065271 torch/_inductor/utils.py:1731] [0/0_1] Not enough SMs to use max_autotune_gemm mode
+/home/ckl/projects/S/laya/.venv/lib/python3.12/site-packages/torch/_inductor/lowering.py:7836: UserWarning:
+Online softmax is disabled on the fly since Inductor decides to
+split the reduction. Cut an issue to PyTorch if this is an
+important use case and you want to speed it up with online
+softmax.
+
+ warnings.warn(
+request 1 seconds 50.15496456599794 graphs 1
+request 2 seconds 0.013748151017352939 graphs 1
+request 3 seconds 0.010274930973537266 graphs 1
+request 4 seconds 0.01512157404795289 graphs 1
+request 5 seconds 0.01144568098243326 graphs 1
+request 6 seconds 0.026219401974231005 graphs 1
+request 7 seconds 0.018255921080708504 graphs 1
+/home/ckl/projects/S/laya/.venv/lib/python3.12/site-packages/torch/_inductor/lowering.py:7836: UserWarning:
+Online softmax is disabled on the fly since Inductor decides to
+split the reduction. Cut an issue to PyTorch if this is an
+important use case and you want to speed it up with online
+softmax.
+
+ warnings.warn(
+request 8 seconds 40.46814876003191 graphs 2
+request 9 seconds 0.02900244400370866 graphs 2
+request 10 seconds 0.013732966035604477 graphs 2
+request 11 seconds 0.009560407022945583 graphs 2
+request 12 seconds 0.011857088073156774 graphs 2
+request 13 seconds 0.009506426053121686 graphs 2
+request 14 seconds 0.01309512893203646 graphs 2
+request 15 seconds 0.008509077015332878 graphs 2
+request 16 seconds 0.023166831000708044 graphs 2
+request 17 seconds 0.015763194067403674 graphs 2
+request 18 seconds 0.008592027937993407 graphs 2
+request 19 seconds 0.028354599024169147 graphs 2
+request 20 seconds 0.013804925954900682 graphs 2
+request 21 seconds 0.009410365018993616 graphs 2
+request 22 seconds 0.01175862702075392 graphs 2
+request 23 seconds 0.009463655995205045 graphs 2
+request 24 seconds 0.013094828929752111 graphs 2
+request 25 seconds 0.008573779021389782 graphs 2
+request 26 seconds 0.023287873016670346 graphs 2
+request 27 seconds 0.016199808102101088 graphs 2
+request 28 seconds 0.008353455923497677 graphs 2
+request 29 seconds 0.028438869980163872 graphs 2
+request 30 seconds 0.013581363949924707 graphs 2
+RESULT {"load_s": 3.0783330149715766, "requests_s": [50.15496456599794, 0.013748151017352939, 0.010274930973537266, 0.01512157404795289, 0.01144568098243326, 0.026219401974231005, 0.018255921080708504, 40.46814876003191, 0.02900244400370866, 0.013732966035604477, 0.009560407022945583, 0.011857088073156774, 0.009506426053121686, 0.01309512893203646, 0.008509077015332878, 0.023166831000708044, 0.015763194067403674, 0.008592027937993407, 0.028354599024169147, 0.013804925954900682, 0.009410365018993616, 0.01175862702075392, 0.009463655995205045, 0.013094828929752111, 0.008573779021389782, 0.023287873016670346, 0.016199808102101088, 0.008353455923497677, 0.028438869980163872, 0.013581363949924707], "allocated_mib": 1616.48291015625, "reserved_mib": 1820.0, "peak_allocated_mib": 1680.029296875, "peak_reserved_mib": 1820.0, "counters": {"stats": {"calls_captured": 6700, "unique_graphs": 2}, "inductor": {"pattern_matcher_count": 786, "pattern_matcher_nodes": 910, "benchmarking.InductorBenchmarker.benchmark_gpu": 102, "pad_mm_bench": 4, "fxgraph_cache_miss": 2, "async_compile_cache_miss": 162, "extern_calls": 384, "async_compile_cache_hit": 81, "triton_bundler_save_kernel": 1024, "triton_bundler_save_static_autotuner": 2, "benchmarking.InductorBenchmarker.benchmark": 94}}}
+Sat Oct 3 18:39:54 2026
++-----------------------------------------------------------------------------------------+
+| NVIDIA-SMI 615.71.09 KMD Version: 615.71.09 CUDA UMD Version: 13.4 |
++-----------------------------------------+------------------------+----------------------+
+| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC |
+| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. |
+| | | MIG M. |
+|=========================================+========================+======================|
+| 0 NVIDIA GeForce RTX 4070 ... On | 00000000:26:00.0 Off | N/A |
+| 0% 44C P2 153W / 295W | 2813MiB / 16376MiB | 83% Default |
+| | | N/A |
++-----------------------------------------+------------------------+----------------------+
+
++-----------------------------------------------------------------------------------------+
+| Processes: |
+| GPU GI CI PID Type Process name GPU Memory |
+| ID ID Usage |
+|=========================================================================================|
+| 0 N/A N/A 892 G Hyprland 146MiB |
+| 0 N/A N/A 952 G quickshell 38MiB |
+| 0 N/A N/A 995 G Xwayland 4MiB |
+| 0 N/A N/A 1168 G monocloud 3MiB |
+| 0 N/A N/A 3324 C+G /usr/bin/sunshine 215MiB |
+| 0 N/A N/A 12883 G /usr/bin/ghostty 100MiB |
+| 0 N/A N/A 3906709 C+G ...rack-uuid=3190708988185955192 36MiB |
+| 0 N/A N/A 4065271 C ...jects/S/laya/.venv/bin/python 2080MiB |
++-----------------------------------------------------------------------------------------+
+
diff --git a/benchmarks/results/compile-defaults/baseline-restart.command b/benchmarks/results/compile-defaults/baseline-restart.command
new file mode 100644
index 00000000..9c0ec043
--- /dev/null
+++ b/benchmarks/results/compile-defaults/baseline-restart.command
@@ -0,0 +1 @@
+TORCHINDUCTOR_CACHE_DIR=/tmp/laya-compile-defaults-baseline-20261003 HF_HUB_OFFLINE=1 /home/ckl/projects/S/laya/.venv/bin/python benchmarks/bench_compile_defaults.py --model /home/ckl/.cache/huggingface/hub/models--convaiinnovations--laya/snapshots/55cf4c4ebb4ebe31b2550e8bdf3bd21b99753851 --no-warmup
diff --git a/benchmarks/results/compile-defaults/baseline-restart.log b/benchmarks/results/compile-defaults/baseline-restart.log
new file mode 100644
index 00000000..51dd617a
--- /dev/null
+++ b/benchmarks/results/compile-defaults/baseline-restart.log
@@ -0,0 +1,92 @@
+Sat Oct 3 18:40:29 2026
++-----------------------------------------------------------------------------------------+
+| NVIDIA-SMI 615.71.09 KMD Version: 615.71.09 CUDA UMD Version: 13.4 |
++-----------------------------------------+------------------------+----------------------+
+| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC |
+| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. |
+| | | MIG M. |
+|=========================================+========================+======================|
+| 0 NVIDIA GeForce RTX 4070 ... On | 00000000:26:00.0 Off | N/A |
+| 0% 34C P8 11W / 295W | 2541MiB / 16376MiB | 0% Default |
+| | | N/A |
++-----------------------------------------+------------------------+----------------------+
+
++-----------------------------------------------------------------------------------------+
+| Processes: |
+| GPU GI CI PID Type Process name GPU Memory |
+| ID ID Usage |
+|=========================================================================================|
+| 0 N/A N/A 892 G Hyprland 146MiB |
+| 0 N/A N/A 952 G quickshell 38MiB |
+| 0 N/A N/A 995 G Xwayland 4MiB |
+| 0 N/A N/A 1168 G monocloud 3MiB |
+| 0 N/A N/A 3324 C+G /usr/bin/sunshine 215MiB |
+| 0 N/A N/A 12883 G /usr/bin/ghostty 100MiB |
+| 0 N/A N/A 3906709 C+G ...rack-uuid=3190708988185955192 36MiB |
+| 0 N/A N/A 4067080 C ...jects/S/laya/.venv/bin/python 1808MiB |
++-----------------------------------------------------------------------------------------+
+
+torch 2.11.0+cu130 model /home/ckl/.cache/huggingface/hub/models--convaiinnovations--laya/snapshots/55cf4c4ebb4ebe31b2550e8bdf3bd21b99753851
+/home/ckl/projects/S/laya-wt/compile-defaults/benchmarks/bench_compile_defaults.py:41: RuntimeWarning: laya: this checkpoint ships invalid temperatures or values outside [0.5, 5]; using choice:11+=0.10058280825614929 -> 0.5. Treat confidence from the affected entries as uncalibrated.
+ agent = Agent(args.model, device="cuda", compile=True, **kw)
+load_s 3.1235701739788055 cache /tmp/laya-compile-defaults-baseline-20261003
+request 1 seconds 23.28117333201226 graphs 1
+request 2 seconds 0.025100769009441137 graphs 1
+request 3 seconds 0.03008991596288979 graphs 1
+request 4 seconds 0.03646164701785892 graphs 1
+request 5 seconds 0.028050957014784217 graphs 1
+request 6 seconds 0.05628728598821908 graphs 1
+request 7 seconds 0.03729775606188923 graphs 1
+request 8 seconds 18.366730210953392 graphs 2
+request 9 seconds 0.22316628799308091 graphs 2
+request 10 seconds 0.015891883056610823 graphs 2
+request 11 seconds 0.010749883949756622 graphs 2
+request 12 seconds 0.013169366982765496 graphs 2
+request 13 seconds 0.01048382103908807 graphs 2
+request 14 seconds 0.014847902930341661 graphs 2
+request 15 seconds 0.009504331042990088 graphs 2
+request 16 seconds 0.02506487094797194 graphs 2
+request 17 seconds 0.017540149041451514 graphs 2
+request 18 seconds 0.009501251042820513 graphs 2
+request 19 seconds 0.030772635946050286 graphs 2
+request 20 seconds 0.015021344064734876 graphs 2
+request 21 seconds 0.010195098002441227 graphs 2
+request 22 seconds 0.012504570069722831 graphs 2
+request 23 seconds 0.010158507036976516 graphs 2
+request 24 seconds 0.014270768035203218 graphs 2
+request 25 seconds 0.009358069975860417 graphs 2
+request 26 seconds 0.023969601024873555 graphs 2
+request 27 seconds 0.016719870967790484 graphs 2
+request 28 seconds 0.009138458059169352 graphs 2
+request 29 seconds 0.02911479107569903 graphs 2
+request 30 seconds 0.015058215009048581 graphs 2
+RESULT {"load_s": 3.1235701739788055, "requests_s": [23.28117333201226, 0.025100769009441137, 0.03008991596288979, 0.03646164701785892, 0.028050957014784217, 0.05628728598821908, 0.03729775606188923, 18.366730210953392, 0.22316628799308091, 0.015891883056610823, 0.010749883949756622, 0.013169366982765496, 0.01048382103908807, 0.014847902930341661, 0.009504331042990088, 0.02506487094797194, 0.017540149041451514, 0.009501251042820513, 0.030772635946050286, 0.015021344064734876, 0.010195098002441227, 0.012504570069722831, 0.010158507036976516, 0.014270768035203218, 0.009358069975860417, 0.023969601024873555, 0.016719870967790484, 0.009138458059169352, 0.02911479107569903, 0.015058215009048581], "allocated_mib": 1616.48291015625, "reserved_mib": 1794.0, "peak_allocated_mib": 1671.017578125, "peak_reserved_mib": 1794.0, "counters": {"stats": {"calls_captured": 6700, "unique_graphs": 2}, "inductor": {"triton_bundler_load_static_autotuner": 68, "async_compile_cache_miss": 243, "async_compile_cache_hit": 149, "pattern_matcher_count": 20, "pattern_matcher_nodes": 26, "extern_calls": 384, "fxgraph_cache_hit": 2}}}
+Sat Oct 3 18:41:15 2026
++-----------------------------------------------------------------------------------------+
+| NVIDIA-SMI 615.71.09 KMD Version: 615.71.09 CUDA UMD Version: 13.4 |
++-----------------------------------------+------------------------+----------------------+
+| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC |
+| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. |
+| | | MIG M. |
+|=========================================+========================+======================|
+| 0 NVIDIA GeForce RTX 4070 ... On | 00000000:26:00.0 Off | N/A |
+| 30% 43C P2 49W / 295W | 4647MiB / 16376MiB | 100% Default |
+| | | N/A |
++-----------------------------------------+------------------------+----------------------+
+
++-----------------------------------------------------------------------------------------+
+| Processes: |
+| GPU GI CI PID Type Process name GPU Memory |
+| ID ID Usage |
+|=========================================================================================|
+| 0 N/A N/A 892 G Hyprland 146MiB |
+| 0 N/A N/A 952 G quickshell 38MiB |
+| 0 N/A N/A 995 G Xwayland 4MiB |
+| 0 N/A N/A 1168 G monocloud 3MiB |
+| 0 N/A N/A 3324 C+G /usr/bin/sunshine 215MiB |
+| 0 N/A N/A 12883 G /usr/bin/ghostty 100MiB |
+| 0 N/A N/A 3906709 C+G ...rack-uuid=3190708988185955192 34MiB |
+| 0 N/A N/A 4067080 C ...jects/S/laya/.venv/bin/python 1856MiB |
+| 0 N/A N/A 4067428 C ...jects/S/laya/.venv/bin/python 2054MiB |
++-----------------------------------------------------------------------------------------+
+
diff --git a/benchmarks/results/compile-defaults/docs-install.log b/benchmarks/results/compile-defaults/docs-install.log
new file mode 100644
index 00000000..10814c2c
--- /dev/null
+++ b/benchmarks/results/compile-defaults/docs-install.log
@@ -0,0 +1,2 @@
+Using Python 3.12.13 environment at: /home/ckl/projects/S/laya/.venv
+Checked 3 packages in 2ms
diff --git a/benchmarks/results/compile-defaults/warmup-api.log b/benchmarks/results/compile-defaults/warmup-api.log
new file mode 100644
index 00000000..8ee19915
--- /dev/null
+++ b/benchmarks/results/compile-defaults/warmup-api.log
@@ -0,0 +1,3 @@
+
+399 passed, 0 failed
+all hook API tests passed
diff --git a/benchmarks/results/compile-defaults/warmup-cold.command b/benchmarks/results/compile-defaults/warmup-cold.command
new file mode 100644
index 00000000..e1b93a3e
--- /dev/null
+++ b/benchmarks/results/compile-defaults/warmup-cold.command
@@ -0,0 +1 @@
+TORCHINDUCTOR_CACHE_DIR=/tmp/laya-compile-defaults-warmup-20261003 HF_HUB_OFFLINE=1 /home/ckl/projects/S/laya/.venv/bin/python benchmarks/bench_compile_defaults.py --model /home/ckl/.cache/huggingface/hub/models--convaiinnovations--laya/snapshots/55cf4c4ebb4ebe31b2550e8bdf3bd21b99753851
diff --git a/benchmarks/results/compile-defaults/warmup-cold.log b/benchmarks/results/compile-defaults/warmup-cold.log
new file mode 100644
index 00000000..601c7415
--- /dev/null
+++ b/benchmarks/results/compile-defaults/warmup-cold.log
@@ -0,0 +1,109 @@
+Sat Oct 3 18:41:18 2026
++-----------------------------------------------------------------------------------------+
+| NVIDIA-SMI 615.71.09 KMD Version: 615.71.09 CUDA UMD Version: 13.4 |
++-----------------------------------------+------------------------+----------------------+
+| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC |
+| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. |
+| | | MIG M. |
+|=========================================+========================+======================|
+| 0 NVIDIA GeForce RTX 4070 ... On | 00000000:26:00.0 Off | N/A |
+| 30% 35C P3 31W / 295W | 2587MiB / 16376MiB | 0% Default |
+| | | N/A |
++-----------------------------------------+------------------------+----------------------+
+
++-----------------------------------------------------------------------------------------+
+| Processes: |
+| GPU GI CI PID Type Process name GPU Memory |
+| ID ID Usage |
+|=========================================================================================|
+| 0 N/A N/A 892 G Hyprland 146MiB |
+| 0 N/A N/A 952 G quickshell 38MiB |
+| 0 N/A N/A 995 G Xwayland 4MiB |
+| 0 N/A N/A 1168 G monocloud 3MiB |
+| 0 N/A N/A 3324 C+G /usr/bin/sunshine 215MiB |
+| 0 N/A N/A 12883 G /usr/bin/ghostty 100MiB |
+| 0 N/A N/A 3906709 C+G ...rack-uuid=3190708988185955192 34MiB |
+| 0 N/A N/A 4067080 C ...jects/S/laya/.venv/bin/python 1856MiB |
++-----------------------------------------------------------------------------------------+
+
+torch 2.11.0+cu130 model /home/ckl/.cache/huggingface/hub/models--convaiinnovations--laya/snapshots/55cf4c4ebb4ebe31b2550e8bdf3bd21b99753851
+/home/ckl/projects/S/laya-wt/compile-defaults/benchmarks/bench_compile_defaults.py:41: RuntimeWarning: laya: this checkpoint ships invalid temperatures or values outside [0.5, 5]; using choice:11+=0.10058280825614929 -> 0.5. Treat confidence from the affected entries as uncalibrated.
+ agent = Agent(args.model, device="cuda", compile=True, **kw)
+/home/ckl/projects/S/laya/.venv/lib/python3.12/site-packages/torch/_inductor/compile_fx.py:322: UserWarning: TensorFloat32 tensor cores for float32 matrix multiplication available but not enabled. Consider setting `torch.set_float32_matmul_precision('high')` for better performance.
+ warnings.warn(
+W1003 18:41:55.568000 4068228 torch/_inductor/utils.py:1731] [0/0_1] Not enough SMs to use max_autotune_gemm mode
+/home/ckl/projects/S/laya/.venv/lib/python3.12/site-packages/torch/_inductor/lowering.py:7836: UserWarning:
+Online softmax is disabled on the fly since Inductor decides to
+split the reduction. Cut an issue to PyTorch if this is an
+important use case and you want to speed it up with online
+softmax.
+
+ warnings.warn(
+/home/ckl/projects/S/laya/.venv/lib/python3.12/site-packages/torch/_inductor/lowering.py:7836: UserWarning:
+Online softmax is disabled on the fly since Inductor decides to
+split the reduction. Cut an issue to PyTorch if this is an
+important use case and you want to speed it up with online
+softmax.
+
+ warnings.warn(
+load_s 108.49820890603587 cache /tmp/laya-compile-defaults-warmup-20261003
+request 1 seconds 0.02660586207639426 graphs 2
+request 2 seconds 0.015146274003200233 graphs 2
+request 3 seconds 0.010777569958008826 graphs 2
+request 4 seconds 0.015865221968851984 graphs 2
+request 5 seconds 0.012163253966718912 graphs 2
+request 6 seconds 0.029346649069339037 graphs 2
+request 7 seconds 0.019118595053441823 graphs 2
+request 8 seconds 0.010908621014095843 graphs 2
+request 9 seconds 0.032982866978272796 graphs 2
+request 10 seconds 0.016992122982628644 graphs 2
+request 11 seconds 0.010969631955958903 graphs 2
+request 12 seconds 0.015091703971847892 graphs 2
+request 13 seconds 0.012968681985512376 graphs 2
+request 14 seconds 0.016305777011439204 graphs 2
+request 15 seconds 0.01148064702283591 graphs 2
+request 16 seconds 0.02667000296059996 graphs 2
+request 17 seconds 0.01863549102563411 graphs 2
+request 18 seconds 0.011158164008520544 graphs 2
+request 19 seconds 0.03343016095459461 graphs 2
+request 20 seconds 0.01674849190749228 graphs 2
+request 21 seconds 0.012811930966563523 graphs 2
+request 22 seconds 0.015374476904980838 graphs 2
+request 23 seconds 0.012070533004589379 graphs 2
+request 24 seconds 0.015654639923013747 graphs 2
+request 25 seconds 0.010528837912715971 graphs 2
+request 26 seconds 0.027839143993332982 graphs 2
+request 27 seconds 0.018755842000246048 graphs 2
+request 28 seconds 0.010280744056217372 graphs 2
+request 29 seconds 0.033292329986579716 graphs 2
+request 30 seconds 0.01766330993268639 graphs 2
+RESULT {"load_s": 108.49820890603587, "requests_s": [0.02660586207639426, 0.015146274003200233, 0.010777569958008826, 0.015865221968851984, 0.012163253966718912, 0.029346649069339037, 0.019118595053441823, 0.010908621014095843, 0.032982866978272796, 0.016992122982628644, 0.010969631955958903, 0.015091703971847892, 0.012968681985512376, 0.016305777011439204, 0.01148064702283591, 0.02667000296059996, 0.01863549102563411, 0.011158164008520544, 0.03343016095459461, 0.01674849190749228, 0.012811930966563523, 0.015374476904980838, 0.012070533004589379, 0.015654639923013747, 0.010528837912715971, 0.027839143993332982, 0.018755842000246048, 0.010280744056217372, 0.033292329986579716, 0.01766330993268639], "allocated_mib": 1616.48291015625, "reserved_mib": 1820.0, "peak_allocated_mib": 1685.626953125, "peak_reserved_mib": 1820.0, "counters": {"stats": {"calls_captured": 6700, "unique_graphs": 2}, "inductor": {"pattern_matcher_count": 786, "pattern_matcher_nodes": 910, "benchmarking.InductorBenchmarker.benchmark_gpu": 105, "pad_mm_bench": 4, "fxgraph_cache_miss": 2, "async_compile_cache_miss": 162, "extern_calls": 384, "async_compile_cache_hit": 81, "triton_bundler_save_kernel": 1048, "triton_bundler_save_static_autotuner": 2, "benchmarking.InductorBenchmarker.benchmark": 97}}}
+Sat Oct 3 18:43:07 2026
++-----------------------------------------------------------------------------------------+
+| NVIDIA-SMI 615.71.09 KMD Version: 615.71.09 CUDA UMD Version: 13.4 |
++-----------------------------------------+------------------------+----------------------+
+| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC |
+| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. |
+| | | MIG M. |
+|=========================================+========================+======================|
+| 0 NVIDIA GeForce RTX 4070 ... On | 00000000:26:00.0 Off | N/A |
+| 0% 43C P2 152W / 295W | 4636MiB / 16376MiB | 75% Default |
+| | | N/A |
++-----------------------------------------+------------------------+----------------------+
+
++-----------------------------------------------------------------------------------------+
+| Processes: |
+| GPU GI CI PID Type Process name GPU Memory |
+| ID ID Usage |
+|=========================================================================================|
+| 0 N/A N/A 892 G Hyprland 156MiB |
+| 0 N/A N/A 952 G quickshell 38MiB |
+| 0 N/A N/A 995 G Xwayland 4MiB |
+| 0 N/A N/A 1168 G monocloud 3MiB |
+| 0 N/A N/A 3324 C+G /usr/bin/sunshine 215MiB |
+| 0 N/A N/A 12883 G /usr/bin/ghostty 100MiB |
+| 0 N/A N/A 3906709 C+G ...rack-uuid=3190708988185955192 35MiB |
+| 0 N/A N/A 4068228 C ...jects/S/laya/.venv/bin/python 2080MiB |
+| 0 N/A N/A 4069106 C ...jects/S/laya/.venv/bin/python 1808MiB |
++-----------------------------------------------------------------------------------------+
+
diff --git a/benchmarks/results/compile-defaults/warmup-docs.log b/benchmarks/results/compile-defaults/warmup-docs.log
new file mode 100644
index 00000000..98aaabd6
--- /dev/null
+++ b/benchmarks/results/compile-defaults/warmup-docs.log
@@ -0,0 +1,3 @@
+Build started
+No issues found
+Build finished in 2.17s
diff --git a/benchmarks/results/compile-defaults/warmup-restart.command b/benchmarks/results/compile-defaults/warmup-restart.command
new file mode 100644
index 00000000..e1b93a3e
--- /dev/null
+++ b/benchmarks/results/compile-defaults/warmup-restart.command
@@ -0,0 +1 @@
+TORCHINDUCTOR_CACHE_DIR=/tmp/laya-compile-defaults-warmup-20261003 HF_HUB_OFFLINE=1 /home/ckl/projects/S/laya/.venv/bin/python benchmarks/bench_compile_defaults.py --model /home/ckl/.cache/huggingface/hub/models--convaiinnovations--laya/snapshots/55cf4c4ebb4ebe31b2550e8bdf3bd21b99753851
diff --git a/benchmarks/results/compile-defaults/warmup-restart.log b/benchmarks/results/compile-defaults/warmup-restart.log
new file mode 100644
index 00000000..e95f2aea
--- /dev/null
+++ b/benchmarks/results/compile-defaults/warmup-restart.log
@@ -0,0 +1,93 @@
+Sat Oct 3 18:43:15 2026
++-----------------------------------------------------------------------------------------+
+| NVIDIA-SMI 615.71.09 KMD Version: 615.71.09 CUDA UMD Version: 13.4 |
++-----------------------------------------+------------------------+----------------------+
+| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC |
+| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. |
+| | | MIG M. |
+|=========================================+========================+======================|
+| 0 NVIDIA GeForce RTX 4070 ... On | 00000000:26:00.0 Off | N/A |
+| 0% 34C P8 12W / 295W | 2551MiB / 16376MiB | 0% Default |
+| | | N/A |
++-----------------------------------------+------------------------+----------------------+
+
++-----------------------------------------------------------------------------------------+
+| Processes: |
+| GPU GI CI PID Type Process name GPU Memory |
+| ID ID Usage |
+|=========================================================================================|
+| 0 N/A N/A 892 G Hyprland 156MiB |
+| 0 N/A N/A 952 G quickshell 38MiB |
+| 0 N/A N/A 995 G Xwayland 4MiB |
+| 0 N/A N/A 1168 G monocloud 3MiB |
+| 0 N/A N/A 3324 C+G /usr/bin/sunshine 215MiB |
+| 0 N/A N/A 12883 G /usr/bin/ghostty 100MiB |
+| 0 N/A N/A 3906709 C+G ...rack-uuid=3190708988185955192 35MiB |
+| 0 N/A N/A 4069106 C ...jects/S/laya/.venv/bin/python 1808MiB |
++-----------------------------------------------------------------------------------------+
+
+torch 2.11.0+cu130 model /home/ckl/.cache/huggingface/hub/models--convaiinnovations--laya/snapshots/55cf4c4ebb4ebe31b2550e8bdf3bd21b99753851
+/home/ckl/projects/S/laya-wt/compile-defaults/benchmarks/bench_compile_defaults.py:41: RuntimeWarning: laya: this checkpoint ships invalid temperatures or values outside [0.5, 5]; using choice:11+=0.10058280825614929 -> 0.5. Treat confidence from the affected entries as uncalibrated.
+ agent = Agent(args.model, device="cuda", compile=True, **kw)
+load_s 61.31409852707293 cache /tmp/laya-compile-defaults-warmup-20261003
+request 1 seconds 0.0274208290502429 graphs 2
+request 2 seconds 0.01559443399310112 graphs 2
+request 3 seconds 0.011225447058677673 graphs 2
+request 4 seconds 0.016729915980249643 graphs 2
+request 5 seconds 0.011728283017873764 graphs 2
+request 6 seconds 0.02990892401430756 graphs 2
+request 7 seconds 0.020536146010272205 graphs 2
+request 8 seconds 0.010691441944800317 graphs 2
+request 9 seconds 0.03413175803143531 graphs 2
+request 10 seconds 0.017582115018740296 graphs 2
+request 11 seconds 0.011792163946665823 graphs 2
+request 12 seconds 0.013707653968594968 graphs 2
+request 13 seconds 0.012671043979935348 graphs 2
+request 14 seconds 0.01701312803197652 graphs 2
+request 15 seconds 0.011880895006470382 graphs 2
+request 16 seconds 0.027319276938214898 graphs 2
+request 17 seconds 0.01926259300671518 graphs 2
+request 18 seconds 0.010391199961304665 graphs 2
+request 19 seconds 0.03368271398358047 graphs 2
+request 20 seconds 0.01724065199960023 graphs 2
+request 21 seconds 0.0130963179981336 graphs 2
+request 22 seconds 0.015875616925768554 graphs 2
+request 23 seconds 0.01276965404395014 graphs 2
+request 24 seconds 0.017264161026105285 graphs 2
+request 25 seconds 0.010888433898799121 graphs 2
+request 26 seconds 0.026585378916934133 graphs 2
+request 27 seconds 0.019762587966397405 graphs 2
+request 28 seconds 0.011369779938831925 graphs 2
+request 29 seconds 0.03422279004007578 graphs 2
+request 30 seconds 0.01810811005998403 graphs 2
+RESULT {"load_s": 61.31409852707293, "requests_s": [0.0274208290502429, 0.01559443399310112, 0.011225447058677673, 0.016729915980249643, 0.011728283017873764, 0.02990892401430756, 0.020536146010272205, 0.010691441944800317, 0.03413175803143531, 0.017582115018740296, 0.011792163946665823, 0.013707653968594968, 0.012671043979935348, 0.01701312803197652, 0.011880895006470382, 0.027319276938214898, 0.01926259300671518, 0.010391199961304665, 0.03368271398358047, 0.01724065199960023, 0.0130963179981336, 0.015875616925768554, 0.01276965404395014, 0.017264161026105285, 0.010888433898799121, 0.026585378916934133, 0.019762587966397405, 0.011369779938831925, 0.03422279004007578, 0.01810811005998403], "allocated_mib": 1616.48291015625, "reserved_mib": 1796.0, "peak_allocated_mib": 1671.017578125, "peak_reserved_mib": 1796.0, "counters": {"stats": {"calls_captured": 6700, "unique_graphs": 2}, "inductor": {"triton_bundler_load_static_autotuner": 68, "async_compile_cache_miss": 243, "async_compile_cache_hit": 149, "pattern_matcher_count": 20, "pattern_matcher_nodes": 26, "extern_calls": 384, "fxgraph_cache_hit": 2}}}
+Sat Oct 3 18:44:16 2026
++-----------------------------------------------------------------------------------------+
+| NVIDIA-SMI 615.71.09 KMD Version: 615.71.09 CUDA UMD Version: 13.4 |
++-----------------------------------------+------------------------+----------------------+
+| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC |
+| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. |
+| | | MIG M. |
+|=========================================+========================+======================|
+| 0 NVIDIA GeForce RTX 4070 ... On | 00000000:26:00.0 Off | N/A |
+| 30% 40C P2 84W / 295W | 5229MiB / 16376MiB | 78% Default |
+| | | N/A |
++-----------------------------------------+------------------------+----------------------+
+
++-----------------------------------------------------------------------------------------+
+| Processes: |
+| GPU GI CI PID Type Process name GPU Memory |
+| ID ID Usage |
+|=========================================================================================|
+| 0 N/A N/A 892 G Hyprland 156MiB |
+| 0 N/A N/A 952 G quickshell 38MiB |
+| 0 N/A N/A 995 G Xwayland 4MiB |
+| 0 N/A N/A 1168 G monocloud 3MiB |
+| 0 N/A N/A 3324 C+G /usr/bin/sunshine 215MiB |
+| 0 N/A N/A 12883 G /usr/bin/ghostty 100MiB |
+| 0 N/A N/A 3906709 C+G ...rack-uuid=3190708988185955192 35MiB |
+| 0 N/A N/A 4070160 C ...jects/S/laya/.venv/bin/python 2174MiB |
+| 0 N/A N/A 4070256 C ...jects/S/laya/.venv/bin/python 2056MiB |
+| 0 N/A N/A 4070519 C ...jects/S/laya/.venv/bin/python 246MiB |
++-----------------------------------------------------------------------------------------+
+
diff --git a/benchmarks/results/compile-defaults/warmup-tests.log b/benchmarks/results/compile-defaults/warmup-tests.log
new file mode 100644
index 00000000..eee4927b
--- /dev/null
+++ b/benchmarks/results/compile-defaults/warmup-tests.log
@@ -0,0 +1,8 @@
+ok test_cuda_compile_padding_masks_only_the_new_tail
+ok test_one_graph_across_shapes_and_same_outputs
+ok test_warmup_builds_every_graph_before_the_first_request
+ok test_duck_shape_restored_after_errors_and_nesting
+ok test_duck_shape_restored_across_threads
+ok test_eager_agent_leaves_the_setting_alone
+ok test_constructor_warms_only_the_active_compiled_path
+all compile tests passed
diff --git a/benchmarks/results/compile-defaults/warmup-validation.log b/benchmarks/results/compile-defaults/warmup-validation.log
new file mode 100644
index 00000000..8c9208ae
--- /dev/null
+++ b/benchmarks/results/compile-defaults/warmup-validation.log
@@ -0,0 +1,56 @@
+PATH=/home/ckl/projects/S/laya/.venv/bin:$PATH
+
+$ ruff check laya/ --select=E9,F63,F7,F82,F401,F811 --line-length=120
+All checks passed!
+
+exit=0
+
+$ python -m compileall -q laya/ tests/
+
+exit=0
+
+$ python tests/test_compile.py
+ok test_cuda_compile_padding_masks_only_the_new_tail
+ok test_one_graph_across_shapes_and_same_outputs
+ok test_warmup_builds_every_graph_before_the_first_request
+ok test_duck_shape_restored_after_errors_and_nesting
+ok test_duck_shape_restored_across_threads
+ok test_eager_agent_leaves_the_setting_alone
+ok test_constructor_warms_only_the_active_compiled_path
+all compile tests passed
+
+exit=0
+
+$ python tests/test_hooks_api.py
+
+399 passed, 0 failed
+all hook API tests passed
+
+exit=0
+
+$ python tests/test_router.py
+
+703 passed, 0 failed
+all routing tests passed
+
+exit=0
+
+$ python tests/test_criteria.py
+
+198 passed, 0 failed
+
+exit=0
+
+$ python tests/test_hooks.py
+
+240 passed, 0 failed
+all hook tests passed
+
+exit=0
+
+$ zensical build --strict --clean
+Build started
+No issues found
+Build finished in 2.41s
+
+exit=0
diff --git a/docs/compile-and-fast-path.md b/docs/compile-and-fast-path.md
index 57a9dfb9..c3ea2205 100644
--- a/docs/compile-and-fast-path.md
+++ b/docs/compile-and-fast-path.md
@@ -22,8 +22,10 @@ by sequence length, so it has no such buffer.
## Cold start
- **First compile.** It takes tens of seconds per graph. `compile=True` needs two graphs: one for
- batches and one for a single row, which torch specialises. `agent.warmup()` (#718) builds both
- before traffic arrives.
+ batches and one for a single row, which torch specialises. `compile=True` now calls `agent.warmup()`
+ during load. `compile_warmup=False` restores lazy compilation, and `agent.warmup(shapes=...)`
+ remains available manually. Eager and TileLang loads do not warm automatically. These shapes
+ cover common requests, not every possible shape guard.
- **Across restarts.** Inductor's FX-graph cache keeps compiled graphs under
`TORCHINDUCTOR_CACHE_DIR`. The default is under `/tmp`, which does not survive a reboot or a
container restart. Set it to a persistent directory, or a volume in a container, and a second
diff --git a/laya/agent.py b/laya/agent.py
index 83b18da4..e9a523d9 100644
--- a/laya/agent.py
+++ b/laya/agent.py
@@ -492,6 +492,7 @@ class Agent(HookRegistry):
hooks_concurrent: bool = True,
hooks_timeout: Optional[float] = None,
calibration: Optional[str] = None,
+ compile_warmup: bool = True,
):
"""Load a Laya checkpoint.
@@ -510,6 +511,8 @@ class Agent(HookRegistry):
new one on almost every request, so those graphs usually cost more than they return;
use it when the traffic is repetitive. `fast=True` takes precedence, because the
TileLang path replaces the forward that would be compiled.
+ Compiled agents run `warmup()` before returning; `compile_warmup=False` defers that
+ work to requests or a manual `warmup()` call. Eager and fast agents are unchanged.
`subfolder` selects one checkpoint from a repo that bundles several, e.g.
`Agent("convaiinnovations/laya", subfolder="multilingual")`. Only that subfolder is
@@ -755,6 +758,9 @@ class Agent(HookRegistry):
" See https://pytorch.org/get-started/locally/\n"
% (fell_back_from, fell_back_why), RuntimeWarning)
+ if self._compiled and compile_warmup:
+ self.warmup()
+
def accelerate(self, use_graphs: bool = True, strict: bool = False):
"""Replace the model forward with the TileLang fast path (fused GEMM/GEGLU/LayerNorm/RoPE kernels,
sliding-window flash attention, 16-bit resident weights, CUDA graphs per shape bucket).
@@ -798,10 +804,10 @@ class Agent(HookRegistry):
def warmup(self, shapes=None) -> float:
"""Run the forward on synthetic input of each shape now and return the seconds it took.
- `compile=True` traces and compiles on the first request that needs a graph (tens of
- seconds on a GPU), and `fast=True` builds its kernels and CUDA graphs per shape bucket on
- first use. Calling this after loading, before serving, moves that cost out of the first
- requests. With the stock forward it is a few ordinary forward passes. `shapes` is a list
+ `compile=True` calls this at load unless `compile_warmup=False`. Extra shapes can still
+ be warmed manually. `fast=True` builds its kernels and CUDA graphs per shape bucket on
+ first use; calling this before serving moves that cost out of the first requests.
+ With the stock forward it is a few ordinary forward passes. `shapes` is a list
of (rows, tokens, markers); tokens are capped at the agent's `max_len`. Nothing is
returned to or recorded for any caller, and hooks do not run.
"""
@@ -1907,7 +1913,7 @@ def load(model_id_or_path: str = "convaiinnovations/laya", device: Optional[str]
hooks=None, on_predict_start=None, on_predict_end=None,
hooks_raise: bool = True, hooks_concurrent: bool = True,
hooks_timeout: Optional[float] = None,
- calibration: Optional[str] = None) -> Agent:
+ calibration: Optional[str] = None, compile_warmup: bool = True) -> Agent:
"""Load a Laya agent.
`subfolder` picks one checkpoint out of a repo that bundles several:
@@ -1943,7 +1949,7 @@ def load(model_id_or_path: str = "convaiinnovations/laya", device: Optional[str]
if spec is not None:
model_id_or_path, subfolder = spec
return Agent(model_id_or_path, device=device, token=token, subfolder=subfolder, fast=fast,
- compile=compile,
+ compile=compile, compile_warmup=compile_warmup,
revision=revision, expected_sha256=expected_sha256,
lang_temperatures=lang_temperatures,
hooks=hooks, on_predict_start=on_predict_start, on_predict_end=on_predict_end,
diff --git a/tests/test_compile.py b/tests/test_compile.py
index edecb6bc..e8801162 100644
--- a/tests/test_compile.py
+++ b/tests/test_compile.py
@@ -173,6 +173,38 @@ def test_eager_agent_leaves_the_setting_alone():
assert seen == [fx_config.use_duck_shape]
+def test_constructor_warms_only_the_active_compiled_path():
+ import json
+ import tempfile
+ from pathlib import Path
+ from unittest.mock import patch
+ import laya.agent as module
+ from laya import load
+
+ model = tiny_model()
+ with tempfile.TemporaryDirectory() as directory:
+ root = Path(directory)
+ (root / "rl_agent_config.json").write_text(json.dumps({"encoder": "unused", "head_layers": 1}))
+ (root / "model.safetensors").touch()
+ with patch.object(module, "build_model", return_value=model), \
+ patch.object(module, "_load_tokenizer", return_value=SimpleNamespace(cls_token_id=1)), \
+ patch("safetensors.torch.load_file", return_value=model.state_dict()), \
+ patch.object(module, "compile_model", side_effect=lambda m: m) as compile_spy, \
+ patch.object(Agent, "accelerate"), \
+ patch.object(Agent, "warmup", autospec=True) as warm:
+ agent = load(directory, device="cpu", compile=True)
+ warm.assert_called_once_with(agent)
+ assert not agent.model.training and agent._compiled
+ warm.reset_mock()
+ load(directory, device="cpu", compile=True, compile_warmup=False)
+ load(directory, device="cpu", compile=False)
+ load(directory, device="cpu", compile=True, fast=True)
+ warm.assert_not_called()
+ assert compile_spy.call_count == 2
+ agent.warmup()
+ warm.assert_called_once_with(agent)
+
+
if __name__ == "__main__":
for name, fn in list(globals().items()):
if name.startswith("test_"):
diff --git a/tests/test_hooks_api.py b/tests/test_hooks_api.py
index a8aa4957..3e889433 100644
--- a/tests/test_hooks_api.py
+++ b/tests/test_hooks_api.py
@@ -75,6 +75,9 @@ for label, fn in (("Agent.__init__", Agent.__init__), ("load", load),
for param, default in HOOK_KEYS.items():
check_param(label, fn, param, default)
+for label, fn in (("Agent.__init__", Agent.__init__), ("load", load)):
+ check_param(label, fn, "compile_warmup", True)
+
# Router keeps lang_guess and explicit per-model revisions too
check_param("Router.__init__", Router.__init__, "lang_guess", None)
check_param("Router.__init__", Router.__init__, "revisions", None)