Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

perf(agent): stop compile=True from recompiling on every new request shape

cklxx0f0b3ef9dc42
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 7a9cca5f..77463bf2 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -149,13 +149,14 @@ jobs:
# test_onnx.py and test_fast.py are deliberately not here: both are skip-guarded on an extra
# this job does not install, so they would pass trivially, and their real execution belongs
# in a lane that installs `onnx` or has a GPU.
- - name: Pytest suites (serve, batch, system-one, audit regressions, truncation)
+ - name: Pytest suites (serve, batch, system-one, audit regressions, truncation, compile)
run: |
python -m pytest tests/test_serve.py \
tests/test_router_batch.py \
tests/test_system_one_lang.py \
tests/test_audit_regressions.py \
tests/test_truncation_direction.py \
+ tests/test_compile.py \
-q
# The eval harness needs numpy and laya only, so it stays a separate, weight-free job: metric
@@ -254,16 +255,17 @@ jobs:
python tests/test_hooks_api.py
shell: bash
- # Same five pytest suites as the Linux job. They are named explicitly rather than added to
+ # Same six pytest suites as the Linux job. They are named explicitly rather than added to
# the loop above, because the loop runs each file as a script and that is exactly what does
# not execute a pytest suite (#374).
- - name: Pytest suites (serve, batch, system-one, audit regressions, truncation)
+ - name: Pytest suites (serve, batch, system-one, audit regressions, truncation, compile)
run: |
python -m pytest tests/test_serve.py \
tests/test_router_batch.py \
tests/test_system_one_lang.py \
tests/test_audit_regressions.py \
tests/test_truncation_direction.py \
+ tests/test_compile.py \
-q
shell: bash
diff --git a/benchmarks/bench_compile.py b/benchmarks/bench_compile.py
new file mode 100644
index 00000000..e8163404
--- /dev/null
+++ b/benchmarks/bench_compile.py
@@ -0,0 +1,60 @@
+"""Graphs built and per-call cost under `compile=True`: this tree vs the stock `torch.compile(model)`.
+
+ python benchmarks/bench_compile.py [--subfolder multilingual] [--device cuda] [--stock]
+ python benchmarks/bench_compile.py --device cpu --dynamo-backend eager # graph count only, no inductor
+
+Runs a fixed sequence of `predict()` calls whose questions, options and state length change on
+every call, the way real traffic does, and prints each call's wall time and how many graphs dynamo
+has built so far. A call that builds a graph is a (re)compile: tens of seconds on a GPU with
+inductor. `--stock` compiles the way `compile=True` did before (`torch.compile(model)`, duck sizing
+on); `--dynamo-backend eager` swaps inductor out so the graph count is cheap to check on CPU.
+"""
+import argparse, os, sys, time
+from contextlib import nullcontext
+os.environ.setdefault("USE_TF", "0"); os.environ.setdefault("TOKENIZERS_PARALLELISM", "false")
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+# (questions, options per question, state words): rows, markers and tokens all move, and the first
+# call has rows == markers, the case duck sizing ties together
+CALLS = [(4, 4, 20), (4, 4, 60), (3, 4, 30), (5, 2, 80), (2, 3, 40), (8, 5, 120), (6, 6, 50),
+ (1, 3, 20), (7, 3, 200), (4, 4, 90)]
+WORDS = "the customer reports a duplicate charge on the March invoice and asks for a refund today".split()
+
+
+def questions(n, k):
+ return {"q%d" % i: {"type": "choice", "instructions": "Which team should handle this (%d)?" % i,
+ "criteria": {"team%d" % j: "handles case %d" % j for j in range(k)}} for i in range(n)}
+
+
+def main():
+ ap = argparse.ArgumentParser()
+ ap.add_argument("--model", default="convaiinnovations/laya"); ap.add_argument("--subfolder", default=None)
+ ap.add_argument("--device", default=None); ap.add_argument("--stock", action="store_true")
+ ap.add_argument("--dynamo-backend", default=None, help="torch.compile backend (default inductor)")
+ args = ap.parse_args()
+ import torch
+ import laya.agent as A
+ from laya import _compile
+ from torch._dynamo.utils import counters
+ kw = {"backend": args.dynamo_backend} if args.dynamo_backend else {}
+ if args.stock:
+ A.compile_model = lambda m: torch.compile(m, **kw)
+ A.independent_dims = nullcontext
+ elif kw:
+ A.compile_model = lambda m: _compile.compile_model(m, **kw)
+ agent = A.Agent(args.model, subfolder=args.subfolder, device=args.device, compile=True)
+ print("%s compile, device %s, torch %s" % ("stock" if args.stock else "this tree", agent.device, torch.__version__))
+ total = time.perf_counter()
+ for n, k, words in CALLS:
+ state = " ".join(WORDS[i % len(WORDS)] for i in range(words))
+ t = time.perf_counter()
+ agent.predict(state, questions(n, k))
+ if agent.device.type == "cuda":
+ torch.cuda.synchronize()
+ print(" %d questions x %d options, %3d words: %8.2f s, graphs so far %d"
+ % (n, k, words, time.perf_counter() - t, counters["stats"]["unique_graphs"]))
+ print("total %.1f s, %d graphs" % (time.perf_counter() - total, counters["stats"]["unique_graphs"]))
+
+
+if __name__ == "__main__":
+ main()
diff --git a/laya/_compile.py b/laya/_compile.py
new file mode 100644
index 00000000..a3ad5cd6
--- /dev/null
+++ b/laya/_compile.py
@@ -0,0 +1,69 @@
+"""The `compile=True` path: `torch.compile` over the whole decision model, without per-shape recompiles.
+
+Laya's forward sees a new (rows, tokens, markers) shape on almost every call. Under
+`torch.compile(model)` with its defaults that costs, on an RTX 4070 (measured in #472):
+
+* one recompile at the second distinct shape, because the first compile is static and only the
+ recompile switches to dynamic shapes ("automatic dynamic"): ~18 s;
+* one more for every request that breaks a "duck sizing" guard. Dynamo gives two dimensions
+ that happen to be equal at trace time one symbol and then guards on them staying equal, so a
+ first call with four questions of four options (rows == markers == 4) recompiles, ~25 s, as
+ soon as they differ.
+
+`compile_model` compiles with `dynamic=True`, which removes the first, and `independent_dims()`
+switches duck sizing off while a compiled forward runs, which removes the second: every dimension
+of Laya's inputs is independent, so each gets its own symbol. What is left is torch's own 0/1
+specialisation, one extra graph the first time a single-row batch arrives.
+
+`use_duck_shape` is a process-global torch setting read when a graph is traced, which happens lazily
+inside a call, so it cannot be set once at load and restored. It is set for the duration of each
+compiled forward instead and put back to whatever it was when the last one returns (refcounted, so
+concurrent calls on several threads do not restore it early). Nothing is left changed between calls.
+
+Mode and device are unchanged from what `compile=True` has always done: the default inductor mode,
+on whatever device the agent runs on, CPU included.
+"""
+import threading
+from contextlib import contextmanager
+
+import torch
+
+_lock = threading.Lock()
+_depth = 0
+_saved = None
+
+
+def compile_model(model, **kwargs):
+ """`torch.compile(model, dynamic=True)`; `kwargs` go to `torch.compile` (tests pass `backend=`)."""
+ return torch.compile(model, dynamic=True, **kwargs)
+
+
+def _fx_config():
+ try:
+ from torch.fx.experimental import _config
+ except ImportError: # a torch without symbolic shapes has nothing to switch
+ return None
+ return _config if hasattr(_config, "use_duck_shape") else None
+
+
+@contextmanager
+def independent_dims():
+ """Trace with `use_duck_shape = False` inside the block, restoring the prior value afterwards."""
+ global _depth, _saved
+ cfg = _fx_config()
+ if cfg is None:
+ yield
+ return
+ with _lock:
+ if _depth == 0:
+ _saved = cfg.use_duck_shape
+ cfg.use_duck_shape = False
+ _depth += 1
+ try:
+ yield
+ finally:
+ with _lock:
+ _depth -= 1
+ if _depth == 0:
+ cfg.use_duck_shape = _saved
+ _saved = None
diff --git a/laya/agent.py b/laya/agent.py
index 1b58de6f..0e325bdf 100644
--- a/laya/agent.py
+++ b/laya/agent.py
@@ -11,6 +11,7 @@ from typing import Any, Dict, List, Optional, Union
import numpy as np
import torch
+from ._compile import compile_model, independent_dims
from .common import (
QTYPES,
TEMP_MAX,
@@ -232,6 +233,8 @@ class Agent(HookRegistry):
mps_amp_min_rows = MPS_AMP_MIN_ROWS_DEFAULT
# The stock forward is also used by lightweight runtimes built with __new__ in tests.
_fast = None
+ # Set when `compile=True` wrapped the model in torch.compile.
+ _compiled = False
def __init__(
self,
@@ -384,8 +387,11 @@ class Agent(HookRegistry):
pass
# The TileLang fast path replaces the forward itself, so it takes precedence over compile.
+ # dynamic=True plus independent dimensions (laya/_compile.py) so a new request shape
+ # does not recompile.
if compile and not fast:
- self.model = torch.compile(self.model)
+ self.model = compile_model(self.model)
+ self._compiled = True
# Keep what the checkpoint shipped for inspection, but only ever apply clamped values:
# some buckets are fitted to sharpen rather than soften (see clamp_temperature).
@@ -704,7 +710,8 @@ class Agent(HookRegistry):
# Recomputed inside run() so a fallback that disables amp (or moves to CPU) takes
# effect on the retry. A disabled gate never enters torch.autocast at all.
enabled = self._amp_enabled_for(b["input_ids"].shape[0])
- with _amp_context(self.device, self.dtype, enabled):
+ dims = independent_dims() if self._compiled else nullcontext()
+ with _amp_context(self.device, self.dtype, enabled), dims:
return self.model(
b["input_ids"].to(self.device),
b["attention_mask"].to(self.device),
diff --git a/tests/test_compile.py b/tests/test_compile.py
new file mode 100644
index 00000000..fe48d3c5
--- /dev/null
+++ b/tests/test_compile.py
@@ -0,0 +1,142 @@
+"""`compile=True`: one graph across request shapes, and no torch setting left changed. No weights.
+
+Dynamo runs with `backend="eager"` so the graph count is checked on CPU without inductor or a C
+compiler; the guards and recompiles being tested are dynamo's, the same under every backend.
+
+ python tests/test_compile.py (or python -m pytest tests/test_compile.py)
+"""
+import os
+import sys
+import threading
+
+import torch
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from laya._compile import compile_model, independent_dims # noqa: E402
+from laya.agent import Agent # noqa: E402
+from laya.common import DecisionModel # noqa: E402
+
+fx_config = torch.fx.experimental._config
+
+# rows x tokens x markers; the first has rows == markers, which duck sizing would tie together
+SHAPES = [(4, 40, 4), (4, 57, 4), (3, 70, 4), (5, 33, 2), (2, 90, 3), (8, 130, 5), (6, 61, 6), (7, 45, 3)]
+
+
+def tiny_model():
+ from transformers import AutoModel, ModernBertConfig
+
+ torch.manual_seed(0)
+ cfg = ModernBertConfig(vocab_size=100, hidden_size=64, intermediate_size=128, num_hidden_layers=2,
+ num_attention_heads=2, global_attn_every_n_layers=2, local_attention=16,
+ max_position_embeddings=512, pad_token_id=0, bos_token_id=1, eos_token_id=2,
+ cls_token_id=1, sep_token_id=2)
+ return DecisionModel(AutoModel.from_config(cfg, attn_implementation="sdpa"), 1, 2).eval()
+
+
+def batch(rows, tokens, markers):
+ return {
+ "input_ids": torch.randint(3, 100, (rows, tokens)),
+ "attention_mask": torch.ones((rows, tokens), dtype=torch.long),
+ "marker_pos": torch.arange(1, markers + 1).repeat(rows, 1),
+ "marker_mask": torch.ones((rows, markers), dtype=torch.bool),
+ "qtype": torch.zeros((rows,), dtype=torch.long),
+ }
+
+
+def compiled_agent(model):
+ agent = Agent.__new__(Agent)
+ agent.device = torch.device("cpu")
+ agent.dtype = torch.float32
+ agent.amp_enabled = False
+ agent.model = compile_model(model, backend="eager")
+ agent._compiled = True
+ return agent
+
+
+def graphs():
+ from torch._dynamo.utils import counters
+ return counters["stats"]["unique_graphs"]
+
+
+def test_one_graph_across_shapes_and_same_outputs():
+ torch._dynamo.reset()
+ model = tiny_model()
+ agent = compiled_agent(model)
+ before = fx_config.use_duck_shape
+ start = graphs()
+ with torch.no_grad():
+ for shape in SHAPES:
+ b = batch(*shape)
+ logits, act = agent._infer(b)
+ ref_logits, ref_act = model(*b.values())
+ assert torch.allclose(logits, ref_logits, atol=1e-4), shape
+ assert torch.allclose(act, ref_act, atol=1e-4), shape
+ # stock torch.compile(model) builds 4 graphs for these shapes (static, then automatic dynamic,
+ # then two duck-sizing recompiles); dynamic=True with independent dimensions builds one
+ assert graphs() - start == 1, graphs() - start
+ assert fx_config.use_duck_shape is before
+
+
+def test_duck_shape_restored_after_errors_and_nesting():
+ before = fx_config.use_duck_shape
+ with independent_dims():
+ assert fx_config.use_duck_shape is False
+ with independent_dims():
+ assert fx_config.use_duck_shape is False
+ assert fx_config.use_duck_shape is False # an inner exit does not restore early
+ assert fx_config.use_duck_shape is before
+ try:
+ with independent_dims():
+ raise RuntimeError("forward failed")
+ except RuntimeError:
+ pass
+ assert fx_config.use_duck_shape is before
+
+
+def test_duck_shape_restored_across_threads():
+ before = fx_config.use_duck_shape
+ inside, release = threading.Barrier(4), threading.Event()
+ seen = []
+
+ def call():
+ with independent_dims():
+ inside.wait()
+ seen.append(fx_config.use_duck_shape)
+ release.wait()
+
+ threads = [threading.Thread(target=call) for _ in range(4)]
+ for t in threads:
+ t.start()
+ while len(seen) < 4:
+ pass
+ release.set()
+ for t in threads:
+ t.join()
+ assert seen == [False] * 4
+ assert fx_config.use_duck_shape is before
+
+
+def test_eager_agent_leaves_the_setting_alone():
+ agent = Agent.__new__(Agent)
+ agent.device = torch.device("cpu")
+ agent.dtype = torch.float32
+ agent.amp_enabled = False
+ seen = []
+
+ class Spy(torch.nn.Module):
+ def forward(self, ids, *rest):
+ seen.append(fx_config.use_duck_shape)
+ return torch.zeros(ids.shape[0], 2), torch.zeros(ids.shape[0], 2)
+
+ agent.model = Spy()
+ agent._infer(batch(2, 8, 2))
+ assert seen == [fx_config.use_duck_shape]
+
+
+if __name__ == "__main__":
+ for name, fn in list(globals().items()):
+ if name.startswith("test_"):
+ fn()
+ print("ok", name)
+ print("all compile tests passed")