Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(agent): keep compile=True meaningful when automatic warmup cannot compile

cklxx0f716a15ea75
diff --git a/README.md b/README.md
index 129b0b60..a186aadc 100644
--- a/README.md
+++ b/README.md
@@ -872,6 +872,11 @@ agent = laya.load("convaiinnovations/laya", compile=True, compile_warmup=False)
seconds = agent.warmup() # manual entry point, also accepts shapes=[(rows, tokens, markers)]
```
+If automatic warm-up fails, load emits a `RuntimeWarning` naming the underlying error and returns
+with the `torch.compile` wrapper and compile settings intact. Later requests still use the compiled
+model and propagate compilation failures; there is no automatic eager fallback. Explicit
+`agent.warmup()` calls also propagate failures, even after automatic warm-up has failed.
+
Warm-up uses synthetic inputs without prediction hooks or prediction caches. It covers common batch
and single-row shapes, not every possible guard or CUDA graph shape. `compile=False` and `fast=True`
do not warm automatically. Manual warm-up still works on both paths.
diff --git a/docs/compile-and-fast-path.md b/docs/compile-and-fast-path.md
index d491652f..69462735 100644
--- a/docs/compile-and-fast-path.md
+++ b/docs/compile-and-fast-path.md
@@ -26,6 +26,13 @@ by sequence length, so it has no such buffer.
during load. `compile_warmup=False` restores lazy compilation, and `agent.warmup(shapes=...)`
remains available manually. Eager and TileLang loads do not warm automatically. These shapes
cover common requests, not every possible shape guard.
+- **Warm-up failure.** Automatic warm-up is best effort: a failure emits a `RuntimeWarning`
+ naming the error (including the underlying compiler error) and load returns with the
+ `torch.compile` wrapper and compile settings intact. For example, Windows without MSVC can
+ load with `compile=True` even though warm-up fails. Later requests still use the compiled
+ model and surface compilation failures; Laya does not switch them to eager execution.
+ Explicit `agent.warmup()` calls also propagate failures, including after a failed automatic
+ warm-up. A successful load therefore does not guarantee that compiled inference is ready.
- **Across restarts.** Inductor's FX-graph cache keeps compiled graphs under
`TORCHINDUCTOR_CACHE_DIR`. The default is under `/tmp`, which does not survive a reboot or a
container restart. Set it to a persistent directory, or a volume in a container, and a second
diff --git a/laya/agent.py b/laya/agent.py
index 751a5a5d..df5abb41 100644
--- a/laya/agent.py
+++ b/laya/agent.py
@@ -774,7 +774,14 @@ class Agent(HookRegistry):
% (fell_back_from, fell_back_why), RuntimeWarning)
if self._compiled and compile_warmup:
- self.warmup()
+ try:
+ self.warmup()
+ except Exception as e:
+ # Compilation is lazy: unavailable toolchains fail on the first forward.
+ # Only automatic warm-up is best effort; explicit warmup() still raises.
+ warnings.warn("Warning: laya compile warm-up failed (%s: %s); keeping the compiled model. "
+ "Later requests and explicit warmup() calls may still raise."
+ % (type(e).__name__, e), RuntimeWarning)
def accelerate(self, use_graphs: bool = True, strict: bool = False):
"""Replace the model forward with the TileLang fast path (fused GEMM/GEGLU/LayerNorm/RoPE kernels,
diff --git a/tests/test_compile.py b/tests/test_compile.py
index 0fa0a8d5..0a12b46f 100644
--- a/tests/test_compile.py
+++ b/tests/test_compile.py
@@ -232,6 +232,79 @@ def test_constructor_warms_only_the_active_compiled_path():
cache_spy.assert_not_called()
+def test_missing_compiler_preserves_wrapper_and_warns_only_during_automatic_warmup():
+ import json
+ import tempfile
+ import warnings
+ from pathlib import Path
+ from unittest.mock import patch
+ from torch._inductor.exc import InvalidCxxCompiler
+ from torch._dynamo.eval_frame import OptimizedModule
+ import laya.agent as module
+ from laya import load
+
+ attempts = []
+
+ def missing_compiler(graph, inputs, **kwargs):
+ attempts.append(True)
+ # Torch versions differ: some take the compiler, others read cpp.cxx.
+ with torch._inductor.config.patch({"cpp.cxx": ("laya-missing-cxx",)}):
+ try:
+ error = InvalidCxxCompiler()
+ except TypeError:
+ error = InvalidCxxCompiler("laya-missing-cxx")
+ raise error
+
+ def compile_without_toolchain(model, **kwargs):
+ return compile_model(model, backend=missing_compiler, **kwargs)
+
+ torch._dynamo.reset()
+ model = tiny_model()
+ with tempfile.TemporaryDirectory() as directory:
+ root = Path(directory)
+ (root / "rl_agent_config.json").write_text(json.dumps({"encoder": "unused", "head_layers": 1}))
+ (root / "model.safetensors").touch()
+ with patch.object(module, "build_model", return_value=model), \
+ patch.object(module, "_load_tokenizer", return_value=SimpleNamespace(cls_token_id=1)), \
+ patch("safetensors.torch.load_file", return_value=model.state_dict()), \
+ patch.object(module, "compile_model", side_effect=compile_without_toolchain):
+ with warnings.catch_warnings(record=True) as caught:
+ warnings.simplefilter("always")
+ agent = load(directory, device="cpu", compile=True, compile_mode="reduce-overhead")
+ assert attempts, "automatic warm-up never reached the compile backend"
+ assert any(issubclass(w.category, RuntimeWarning)
+ and "InvalidCxxCompiler" in str(w.message)
+ and "laya-missing-cxx" in str(w.message)
+ and "keeping the compiled model" in str(w.message) for w in caught)
+ wrapped = agent.model
+ assert isinstance(wrapped, OptimizedModule) and wrapped._orig_mod is model
+ assert agent._compiled and agent._reduce_overhead
+ assert model.encoder.config.reference_compile is True
+ for operation in (lambda: agent._infer(batch(2, 16, 3)), agent.warmup):
+ calls = len(attempts)
+ try:
+ operation()
+ except Exception as error:
+ assert "InvalidCxxCompiler" in str(error) and "laya-missing-cxx" in str(error)
+ else:
+ raise AssertionError("request or explicit warmup swallowed the compiler failure")
+ assert len(attempts) > calls
+ assert agent.model is wrapped and agent._compiled and agent._reduce_overhead
+ assert model.encoder.config.reference_compile is True
+ calls = len(attempts)
+
+ explicit = load(directory, device="cpu", compile=True, compile_warmup=False)
+ assert len(attempts) == calls
+ try:
+ explicit.warmup()
+ except Exception as error:
+ assert "InvalidCxxCompiler" in str(error) and "laya-missing-cxx" in str(error)
+ else:
+ raise AssertionError("explicit warmup swallowed the compiler failure")
+ assert len(attempts) > calls and explicit._compiled
+ torch._dynamo.reset()
+
+
def test_persistent_cache_is_opt_in_and_respects_the_environment():
import tempfile
from pathlib import Path