Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
perf(agent): stop compile=True from recompiling on every new request shape
cklxx0f0b3ef9dc42
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.ymlindex 7a9cca5f..77463bf2 100644--- a/.github/workflows/ci.yml+++ b/.github/workflows/ci.yml@@ -149,13 +149,14 @@ jobs:# test_onnx.py and test_fast.py are deliberately not here: both are skip-guarded on an extra# this job does not install, so they would pass trivially, and their real execution belongs# in a lane that installs `onnx` or has a GPU.- - name: Pytest suites (serve, batch, system-one, audit regressions, truncation)+ - name: Pytest suites (serve, batch, system-one, audit regressions, truncation, compile)run: |python -m pytest tests/test_serve.py \tests/test_router_batch.py \tests/test_system_one_lang.py \tests/test_audit_regressions.py \tests/test_truncation_direction.py \+ tests/test_compile.py \-q# The eval harness needs numpy and laya only, so it stays a separate, weight-free job: metric@@ -254,16 +255,17 @@ jobs:python tests/test_hooks_api.pyshell: bash- # Same five pytest suites as the Linux job. They are named explicitly rather than added to+ # Same six pytest suites as the Linux job. They are named explicitly rather than added to# the loop above, because the loop runs each file as a script and that is exactly what does# not execute a pytest suite (#374).- - name: Pytest suites (serve, batch, system-one, audit regressions, truncation)+ - name: Pytest suites (serve, batch, system-one, audit regressions, truncation, compile)run: |python -m pytest tests/test_serve.py \tests/test_router_batch.py \tests/test_system_one_lang.py \tests/test_audit_regressions.py \tests/test_truncation_direction.py \+ tests/test_compile.py \-qshell: bashdiff --git a/benchmarks/bench_compile.py b/benchmarks/bench_compile.pynew file mode 100644index 00000000..e8163404--- /dev/null+++ b/benchmarks/bench_compile.py@@ -0,0 +1,60 @@+"""Graphs built and per-call cost under `compile=True`: this tree vs the stock `torch.compile(model)`.++ python benchmarks/bench_compile.py [--subfolder multilingual] [--device cuda] [--stock]+ python benchmarks/bench_compile.py --device cpu --dynamo-backend eager # graph count only, no inductor++Runs a fixed sequence of `predict()` calls whose questions, options and state length change on+every call, the way real traffic does, and prints each call's wall time and how many graphs dynamo+has built so far. A call that builds a graph is a (re)compile: tens of seconds on a GPU with+inductor. `--stock` compiles the way `compile=True` did before (`torch.compile(model)`, duck sizing+on); `--dynamo-backend eager` swaps inductor out so the graph count is cheap to check on CPU.+"""+import argparse, os, sys, time+from contextlib import nullcontext+os.environ.setdefault("USE_TF", "0"); os.environ.setdefault("TOKENIZERS_PARALLELISM", "false")+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))++# (questions, options per question, state words): rows, markers and tokens all move, and the first+# call has rows == markers, the case duck sizing ties together+CALLS = [(4, 4, 20), (4, 4, 60), (3, 4, 30), (5, 2, 80), (2, 3, 40), (8, 5, 120), (6, 6, 50),+ (1, 3, 20), (7, 3, 200), (4, 4, 90)]+WORDS = "the customer reports a duplicate charge on the March invoice and asks for a refund today".split()+++def questions(n, k):+ return {"q%d" % i: {"type": "choice", "instructions": "Which team should handle this (%d)?" % i,+ "criteria": {"team%d" % j: "handles case %d" % j for j in range(k)}} for i in range(n)}+++def main():+ ap = argparse.ArgumentParser()+ ap.add_argument("--model", default="convaiinnovations/laya"); ap.add_argument("--subfolder", default=None)+ ap.add_argument("--device", default=None); ap.add_argument("--stock", action="store_true")+ ap.add_argument("--dynamo-backend", default=None, help="torch.compile backend (default inductor)")+ args = ap.parse_args()+ import torch+ import laya.agent as A+ from laya import _compile+ from torch._dynamo.utils import counters+ kw = {"backend": args.dynamo_backend} if args.dynamo_backend else {}+ if args.stock:+ A.compile_model = lambda m: torch.compile(m, **kw)+ A.independent_dims = nullcontext+ elif kw:+ A.compile_model = lambda m: _compile.compile_model(m, **kw)+ agent = A.Agent(args.model, subfolder=args.subfolder, device=args.device, compile=True)+ print("%s compile, device %s, torch %s" % ("stock" if args.stock else "this tree", agent.device, torch.__version__))+ total = time.perf_counter()+ for n, k, words in CALLS:+ state = " ".join(WORDS[i % len(WORDS)] for i in range(words))+ t = time.perf_counter()+ agent.predict(state, questions(n, k))+ if agent.device.type == "cuda":+ torch.cuda.synchronize()+ print(" %d questions x %d options, %3d words: %8.2f s, graphs so far %d"+ % (n, k, words, time.perf_counter() - t, counters["stats"]["unique_graphs"]))+ print("total %.1f s, %d graphs" % (time.perf_counter() - total, counters["stats"]["unique_graphs"]))+++if __name__ == "__main__":+ main()diff --git a/laya/_compile.py b/laya/_compile.pynew file mode 100644index 00000000..a3ad5cd6--- /dev/null+++ b/laya/_compile.py@@ -0,0 +1,69 @@+"""The `compile=True` path: `torch.compile` over the whole decision model, without per-shape recompiles.++Laya's forward sees a new (rows, tokens, markers) shape on almost every call. Under+`torch.compile(model)` with its defaults that costs, on an RTX 4070 (measured in #472):++* one recompile at the second distinct shape, because the first compile is static and only the+ recompile switches to dynamic shapes ("automatic dynamic"): ~18 s;+* one more for every request that breaks a "duck sizing" guard. Dynamo gives two dimensions+ that happen to be equal at trace time one symbol and then guards on them staying equal, so a+ first call with four questions of four options (rows == markers == 4) recompiles, ~25 s, as+ soon as they differ.++`compile_model` compiles with `dynamic=True`, which removes the first, and `independent_dims()`+switches duck sizing off while a compiled forward runs, which removes the second: every dimension+of Laya's inputs is independent, so each gets its own symbol. What is left is torch's own 0/1+specialisation, one extra graph the first time a single-row batch arrives.++`use_duck_shape` is a process-global torch setting read when a graph is traced, which happens lazily+inside a call, so it cannot be set once at load and restored. It is set for the duration of each+compiled forward instead and put back to whatever it was when the last one returns (refcounted, so+concurrent calls on several threads do not restore it early). Nothing is left changed between calls.++Mode and device are unchanged from what `compile=True` has always done: the default inductor mode,+on whatever device the agent runs on, CPU included.+"""+import threading+from contextlib import contextmanager++import torch++_lock = threading.Lock()+_depth = 0+_saved = None+++def compile_model(model, **kwargs):+ """`torch.compile(model, dynamic=True)`; `kwargs` go to `torch.compile` (tests pass `backend=`)."""+ return torch.compile(model, dynamic=True, **kwargs)+++def _fx_config():+ try:+ from torch.fx.experimental import _config+ except ImportError: # a torch without symbolic shapes has nothing to switch+ return None+ return _config if hasattr(_config, "use_duck_shape") else None+++@contextmanager+def independent_dims():+ """Trace with `use_duck_shape = False` inside the block, restoring the prior value afterwards."""+ global _depth, _saved+ cfg = _fx_config()+ if cfg is None:+ yield+ return+ with _lock:+ if _depth == 0:+ _saved = cfg.use_duck_shape+ cfg.use_duck_shape = False+ _depth += 1+ try:+ yield+ finally:+ with _lock:+ _depth -= 1+ if _depth == 0:+ cfg.use_duck_shape = _saved+ _saved = Nonediff --git a/laya/agent.py b/laya/agent.pyindex 1b58de6f..0e325bdf 100644--- a/laya/agent.py+++ b/laya/agent.py@@ -11,6 +11,7 @@ from typing import Any, Dict, List, Optional, Unionimport numpy as npimport torch+from ._compile import compile_model, independent_dimsfrom .common import (QTYPES,TEMP_MAX,@@ -232,6 +233,8 @@ class Agent(HookRegistry):mps_amp_min_rows = MPS_AMP_MIN_ROWS_DEFAULT# The stock forward is also used by lightweight runtimes built with __new__ in tests._fast = None+ # Set when `compile=True` wrapped the model in torch.compile.+ _compiled = Falsedef __init__(self,@@ -384,8 +387,11 @@ class Agent(HookRegistry):pass# The TileLang fast path replaces the forward itself, so it takes precedence over compile.+ # dynamic=True plus independent dimensions (laya/_compile.py) so a new request shape+ # does not recompile.if compile and not fast:- self.model = torch.compile(self.model)+ self.model = compile_model(self.model)+ self._compiled = True# Keep what the checkpoint shipped for inspection, but only ever apply clamped values:# some buckets are fitted to sharpen rather than soften (see clamp_temperature).@@ -704,7 +710,8 @@ class Agent(HookRegistry):# Recomputed inside run() so a fallback that disables amp (or moves to CPU) takes# effect on the retry. A disabled gate never enters torch.autocast at all.enabled = self._amp_enabled_for(b["input_ids"].shape[0])- with _amp_context(self.device, self.dtype, enabled):+ dims = independent_dims() if self._compiled else nullcontext()+ with _amp_context(self.device, self.dtype, enabled), dims:return self.model(b["input_ids"].to(self.device),b["attention_mask"].to(self.device),diff --git a/tests/test_compile.py b/tests/test_compile.pynew file mode 100644index 00000000..fe48d3c5--- /dev/null+++ b/tests/test_compile.py@@ -0,0 +1,142 @@+"""`compile=True`: one graph across request shapes, and no torch setting left changed. No weights.++Dynamo runs with `backend="eager"` so the graph count is checked on CPU without inductor or a C+compiler; the guards and recompiles being tested are dynamo's, the same under every backend.++ python tests/test_compile.py (or python -m pytest tests/test_compile.py)+"""+import os+import sys+import threading++import torch++sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))++from laya._compile import compile_model, independent_dims # noqa: E402+from laya.agent import Agent # noqa: E402+from laya.common import DecisionModel # noqa: E402++fx_config = torch.fx.experimental._config++# rows x tokens x markers; the first has rows == markers, which duck sizing would tie together+SHAPES = [(4, 40, 4), (4, 57, 4), (3, 70, 4), (5, 33, 2), (2, 90, 3), (8, 130, 5), (6, 61, 6), (7, 45, 3)]+++def tiny_model():+ from transformers import AutoModel, ModernBertConfig++ torch.manual_seed(0)+ cfg = ModernBertConfig(vocab_size=100, hidden_size=64, intermediate_size=128, num_hidden_layers=2,+ num_attention_heads=2, global_attn_every_n_layers=2, local_attention=16,+ max_position_embeddings=512, pad_token_id=0, bos_token_id=1, eos_token_id=2,+ cls_token_id=1, sep_token_id=2)+ return DecisionModel(AutoModel.from_config(cfg, attn_implementation="sdpa"), 1, 2).eval()+++def batch(rows, tokens, markers):+ return {+ "input_ids": torch.randint(3, 100, (rows, tokens)),+ "attention_mask": torch.ones((rows, tokens), dtype=torch.long),+ "marker_pos": torch.arange(1, markers + 1).repeat(rows, 1),+ "marker_mask": torch.ones((rows, markers), dtype=torch.bool),+ "qtype": torch.zeros((rows,), dtype=torch.long),+ }+++def compiled_agent(model):+ agent = Agent.__new__(Agent)+ agent.device = torch.device("cpu")+ agent.dtype = torch.float32+ agent.amp_enabled = False+ agent.model = compile_model(model, backend="eager")+ agent._compiled = True+ return agent+++def graphs():+ from torch._dynamo.utils import counters+ return counters["stats"]["unique_graphs"]+++def test_one_graph_across_shapes_and_same_outputs():+ torch._dynamo.reset()+ model = tiny_model()+ agent = compiled_agent(model)+ before = fx_config.use_duck_shape+ start = graphs()+ with torch.no_grad():+ for shape in SHAPES:+ b = batch(*shape)+ logits, act = agent._infer(b)+ ref_logits, ref_act = model(*b.values())+ assert torch.allclose(logits, ref_logits, atol=1e-4), shape+ assert torch.allclose(act, ref_act, atol=1e-4), shape+ # stock torch.compile(model) builds 4 graphs for these shapes (static, then automatic dynamic,+ # then two duck-sizing recompiles); dynamic=True with independent dimensions builds one+ assert graphs() - start == 1, graphs() - start+ assert fx_config.use_duck_shape is before+++def test_duck_shape_restored_after_errors_and_nesting():+ before = fx_config.use_duck_shape+ with independent_dims():+ assert fx_config.use_duck_shape is False+ with independent_dims():+ assert fx_config.use_duck_shape is False+ assert fx_config.use_duck_shape is False # an inner exit does not restore early+ assert fx_config.use_duck_shape is before+ try:+ with independent_dims():+ raise RuntimeError("forward failed")+ except RuntimeError:+ pass+ assert fx_config.use_duck_shape is before+++def test_duck_shape_restored_across_threads():+ before = fx_config.use_duck_shape+ inside, release = threading.Barrier(4), threading.Event()+ seen = []++ def call():+ with independent_dims():+ inside.wait()+ seen.append(fx_config.use_duck_shape)+ release.wait()++ threads = [threading.Thread(target=call) for _ in range(4)]+ for t in threads:+ t.start()+ while len(seen) < 4:+ pass+ release.set()+ for t in threads:+ t.join()+ assert seen == [False] * 4+ assert fx_config.use_duck_shape is before+++def test_eager_agent_leaves_the_setting_alone():+ agent = Agent.__new__(Agent)+ agent.device = torch.device("cpu")+ agent.dtype = torch.float32+ agent.amp_enabled = False+ seen = []++ class Spy(torch.nn.Module):+ def forward(self, ids, *rest):+ seen.append(fx_config.use_duck_shape)+ return torch.zeros(ids.shape[0], 2), torch.zeros(ids.shape[0], 2)++ agent.model = Spy()+ agent._infer(batch(2, 8, 2))+ assert seen == [fx_config.use_duck_shape]+++if __name__ == "__main__":+ for name, fn in list(globals().items()):+ if name.startswith("test_"):+ fn()+ print("ok", name)+ print("all compile tests passed")