Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

feat(agent): opt-in warmup() so compile=True does not stall the first requests

cklxxecd3f9ef9ee6
diff --git a/README.md b/README.md
index 4aadb0c4..e2fb188d 100644
--- a/README.md
+++ b/README.md
@@ -683,6 +683,25 @@ The fast path runs in the agent's autocast dtype at the time `accelerate()` is c
Falls back to the stock forward on CPU/MPS or when `tilelang` is not installed; `agent.deaccelerate()`
restores it. Kernels compile once per shape bucket on first use (a few seconds, cached on disk).
+### Warm-up before serving: `agent.warmup()`
+
+`compile=True` compiles on the first request that needs a graph, and the first single-question request
+needs a second one (torch specialises a batch of 1). Both stall a live request. `agent.warmup()` runs the
+forward on a few synthetic shapes now and returns the seconds it took, so the compiles happen before
+traffic arrives:
+
+```python
+agent = laya.load("convaiinnovations/laya", compile=True)
+agent.warmup() # ~46 s on an RTX 4070 Ti SUPER; every later request ~10-30 ms
+```
+
+Measured with `benchmarks/bench_compile.py --device cuda [--warmup]` (English checkpoint, torch 2.11, ten
+requests of changing shape): without it the first request took 51 s and the first single-question request,
+the eighth, took another 41 s; after `warmup()` no request took more than 30 ms. It works the same with
+`fast=True` (kernels and CUDA graphs for those buckets) and costs a few forward passes on the stock path.
+Inductor caches compiled graphs under `TORCHINDUCTOR_CACHE_DIR` (by default in `/tmp`); point it at a
+persistent directory to keep them across restarts.
+
---
## Automated Confidence Gating
diff --git a/benchmarks/bench_compile.py b/benchmarks/bench_compile.py
index e8163404..ddd06560 100644
--- a/benchmarks/bench_compile.py
+++ b/benchmarks/bench_compile.py
@@ -1,6 +1,6 @@
"""Graphs built and per-call cost under `compile=True`: this tree vs the stock `torch.compile(model)`.
- python benchmarks/bench_compile.py [--subfolder multilingual] [--device cuda] [--stock]
+ python benchmarks/bench_compile.py [--subfolder multilingual] [--device cuda] [--stock] [--warmup]
python benchmarks/bench_compile.py --device cpu --dynamo-backend eager # graph count only, no inductor
Runs a fixed sequence of `predict()` calls whose questions, options and state length change on
@@ -8,6 +8,7 @@ every call, the way real traffic does, and prints each call's wall time and how
has built so far. A call that builds a graph is a (re)compile: tens of seconds on a GPU with
inductor. `--stock` compiles the way `compile=True` did before (`torch.compile(model)`, duck sizing
on); `--dynamo-backend eager` swaps inductor out so the graph count is cheap to check on CPU.
+`--warmup` calls `agent.warmup()` after loading and reports its time before the first request.
"""
import argparse, os, sys, time
from contextlib import nullcontext
@@ -31,6 +32,7 @@ def main():
ap.add_argument("--model", default="convaiinnovations/laya"); ap.add_argument("--subfolder", default=None)
ap.add_argument("--device", default=None); ap.add_argument("--stock", action="store_true")
ap.add_argument("--dynamo-backend", default=None, help="torch.compile backend (default inductor)")
+ ap.add_argument("--warmup", action="store_true", help="call agent.warmup() before the first request")
args = ap.parse_args()
import torch
import laya.agent as A
@@ -44,6 +46,8 @@ def main():
A.compile_model = lambda m: _compile.compile_model(m, **kw)
agent = A.Agent(args.model, subfolder=args.subfolder, device=args.device, compile=True)
print("%s compile, device %s, torch %s" % ("stock" if args.stock else "this tree", agent.device, torch.__version__))
+ if args.warmup:
+ print(" warmup(): %8.2f s, graphs so far %d" % (agent.warmup(), counters["stats"]["unique_graphs"]))
total = time.perf_counter()
for n, k, words in CALLS:
state = " ".join(WORDS[i % len(WORDS)] for i in range(words))
diff --git a/laya/agent.py b/laya/agent.py
index 5ca3861c..ae11ea94 100644
--- a/laya/agent.py
+++ b/laya/agent.py
@@ -585,6 +585,37 @@ class Agent(HookRegistry):
self.model.forward = self._fast.forward
return True
+ # (rows, tokens, markers) run by `warmup()`: one row, which torch specialises into a graph of
+ # its own, and a small batch. The three sizes differ within each shape, so none are tied.
+ WARMUP_SHAPES = ((1, 64, 3), (4, 128, 5))
+
+ @torch.no_grad()
+ def warmup(self, shapes=None) -> float:
+ """Run the forward on synthetic input of each shape now and return the seconds it took.
+
+ `compile=True` traces and compiles on the first request that needs a graph (tens of
+ seconds on a GPU), and `fast=True` builds its kernels and CUDA graphs per shape bucket on
+ first use. Calling this after loading, before serving, moves that cost out of the first
+ requests. With the stock forward it is a few ordinary forward passes. `shapes` is a list
+ of (rows, tokens, markers); tokens are capped at the agent's `max_len`. Nothing is
+ returned to or recorded for any caller, and hooks do not run.
+ """
+ max_len = int(self.cfg.get("max_len", 512))
+ fill = self.tok.cls_token_id or 0
+ t0 = time.perf_counter()
+ for rows, tokens, markers in (self.WARMUP_SHAPES if shapes is None else shapes):
+ tokens = max(markers + 2, min(int(tokens), max_len))
+ self._infer({
+ "input_ids": torch.full((rows, tokens), fill, dtype=torch.long),
+ "attention_mask": torch.ones((rows, tokens), dtype=torch.long),
+ "marker_pos": torch.arange(1, markers + 1, dtype=torch.long).repeat(rows, 1),
+ "marker_mask": torch.ones((rows, markers), dtype=torch.bool),
+ "qtype": torch.zeros(rows, dtype=torch.long),
+ })
+ if self.device.type == "cuda":
+ torch.cuda.synchronize(self.device)
+ return time.perf_counter() - t0
+
def deaccelerate(self):
"""Restore the stock forward."""
if self._fast is not None:
diff --git a/tests/test_compile.py b/tests/test_compile.py
index 8abc4983..1d833e89 100644
--- a/tests/test_compile.py
+++ b/tests/test_compile.py
@@ -8,6 +8,7 @@ compiler; the guards and recompiles being tested are dynamo's, the same under ev
import os
import sys
import threading
+from types import SimpleNamespace
import torch
@@ -78,6 +79,21 @@ def test_one_graph_across_shapes_and_same_outputs():
assert fx_config.use_duck_shape is before
+def test_warmup_builds_every_graph_before_the_first_request():
+ torch._dynamo.reset()
+ agent = compiled_agent(tiny_model())
+ agent.cfg = {"max_len": 512}
+ agent.tok = SimpleNamespace(cls_token_id=1)
+ start = graphs()
+ assert agent.warmup() >= 0.0
+ # the batch graph and torch's own single-row specialisation
+ assert graphs() - start == 2, graphs() - start
+ with torch.no_grad():
+ for shape in SHAPES + [(1, 50, 4), (1, 200, 2)]:
+ agent._infer(batch(*shape))
+ assert graphs() - start == 2, graphs() - start
+
+
def test_duck_shape_restored_after_errors_and_nesting():
before = fx_config.use_duck_shape
with independent_dims():