Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
fix(agent): keep compile=True meaningful when automatic warmup cannot compile
cklxx0f716a15ea75
diff --git a/README.md b/README.mdindex 129b0b60..a186aadc 100644--- a/README.md+++ b/README.md@@ -872,6 +872,11 @@ agent = laya.load("convaiinnovations/laya", compile=True, compile_warmup=False)seconds = agent.warmup() # manual entry point, also accepts shapes=[(rows, tokens, markers)]```+If automatic warm-up fails, load emits a `RuntimeWarning` naming the underlying error and returns+with the `torch.compile` wrapper and compile settings intact. Later requests still use the compiled+model and propagate compilation failures; there is no automatic eager fallback. Explicit+`agent.warmup()` calls also propagate failures, even after automatic warm-up has failed.+Warm-up uses synthetic inputs without prediction hooks or prediction caches. It covers common batchand single-row shapes, not every possible guard or CUDA graph shape. `compile=False` and `fast=True`do not warm automatically. Manual warm-up still works on both paths.diff --git a/docs/compile-and-fast-path.md b/docs/compile-and-fast-path.mdindex d491652f..69462735 100644--- a/docs/compile-and-fast-path.md+++ b/docs/compile-and-fast-path.md@@ -26,6 +26,13 @@ by sequence length, so it has no such buffer.during load. `compile_warmup=False` restores lazy compilation, and `agent.warmup(shapes=...)`remains available manually. Eager and TileLang loads do not warm automatically. These shapescover common requests, not every possible shape guard.+- **Warm-up failure.** Automatic warm-up is best effort: a failure emits a `RuntimeWarning`+ naming the error (including the underlying compiler error) and load returns with the+ `torch.compile` wrapper and compile settings intact. For example, Windows without MSVC can+ load with `compile=True` even though warm-up fails. Later requests still use the compiled+ model and surface compilation failures; Laya does not switch them to eager execution.+ Explicit `agent.warmup()` calls also propagate failures, including after a failed automatic+ warm-up. A successful load therefore does not guarantee that compiled inference is ready.- **Across restarts.** Inductor's FX-graph cache keeps compiled graphs under`TORCHINDUCTOR_CACHE_DIR`. The default is under `/tmp`, which does not survive a reboot or acontainer restart. Set it to a persistent directory, or a volume in a container, and a seconddiff --git a/laya/agent.py b/laya/agent.pyindex 751a5a5d..df5abb41 100644--- a/laya/agent.py+++ b/laya/agent.py@@ -774,7 +774,14 @@ class Agent(HookRegistry):% (fell_back_from, fell_back_why), RuntimeWarning)if self._compiled and compile_warmup:- self.warmup()+ try:+ self.warmup()+ except Exception as e:+ # Compilation is lazy: unavailable toolchains fail on the first forward.+ # Only automatic warm-up is best effort; explicit warmup() still raises.+ warnings.warn("Warning: laya compile warm-up failed (%s: %s); keeping the compiled model. "+ "Later requests and explicit warmup() calls may still raise."+ % (type(e).__name__, e), RuntimeWarning)def accelerate(self, use_graphs: bool = True, strict: bool = False):"""Replace the model forward with the TileLang fast path (fused GEMM/GEGLU/LayerNorm/RoPE kernels,diff --git a/tests/test_compile.py b/tests/test_compile.pyindex 0fa0a8d5..0a12b46f 100644--- a/tests/test_compile.py+++ b/tests/test_compile.py@@ -232,6 +232,79 @@ def test_constructor_warms_only_the_active_compiled_path():cache_spy.assert_not_called()+def test_missing_compiler_preserves_wrapper_and_warns_only_during_automatic_warmup():+ import json+ import tempfile+ import warnings+ from pathlib import Path+ from unittest.mock import patch+ from torch._inductor.exc import InvalidCxxCompiler+ from torch._dynamo.eval_frame import OptimizedModule+ import laya.agent as module+ from laya import load++ attempts = []++ def missing_compiler(graph, inputs, **kwargs):+ attempts.append(True)+ # Torch versions differ: some take the compiler, others read cpp.cxx.+ with torch._inductor.config.patch({"cpp.cxx": ("laya-missing-cxx",)}):+ try:+ error = InvalidCxxCompiler()+ except TypeError:+ error = InvalidCxxCompiler("laya-missing-cxx")+ raise error++ def compile_without_toolchain(model, **kwargs):+ return compile_model(model, backend=missing_compiler, **kwargs)++ torch._dynamo.reset()+ model = tiny_model()+ with tempfile.TemporaryDirectory() as directory:+ root = Path(directory)+ (root / "rl_agent_config.json").write_text(json.dumps({"encoder": "unused", "head_layers": 1}))+ (root / "model.safetensors").touch()+ with patch.object(module, "build_model", return_value=model), \+ patch.object(module, "_load_tokenizer", return_value=SimpleNamespace(cls_token_id=1)), \+ patch("safetensors.torch.load_file", return_value=model.state_dict()), \+ patch.object(module, "compile_model", side_effect=compile_without_toolchain):+ with warnings.catch_warnings(record=True) as caught:+ warnings.simplefilter("always")+ agent = load(directory, device="cpu", compile=True, compile_mode="reduce-overhead")+ assert attempts, "automatic warm-up never reached the compile backend"+ assert any(issubclass(w.category, RuntimeWarning)+ and "InvalidCxxCompiler" in str(w.message)+ and "laya-missing-cxx" in str(w.message)+ and "keeping the compiled model" in str(w.message) for w in caught)+ wrapped = agent.model+ assert isinstance(wrapped, OptimizedModule) and wrapped._orig_mod is model+ assert agent._compiled and agent._reduce_overhead+ assert model.encoder.config.reference_compile is True+ for operation in (lambda: agent._infer(batch(2, 16, 3)), agent.warmup):+ calls = len(attempts)+ try:+ operation()+ except Exception as error:+ assert "InvalidCxxCompiler" in str(error) and "laya-missing-cxx" in str(error)+ else:+ raise AssertionError("request or explicit warmup swallowed the compiler failure")+ assert len(attempts) > calls+ assert agent.model is wrapped and agent._compiled and agent._reduce_overhead+ assert model.encoder.config.reference_compile is True+ calls = len(attempts)++ explicit = load(directory, device="cpu", compile=True, compile_warmup=False)+ assert len(attempts) == calls+ try:+ explicit.warmup()+ except Exception as error:+ assert "InvalidCxxCompiler" in str(error) and "laya-missing-cxx" in str(error)+ else:+ raise AssertionError("explicit warmup swallowed the compiler failure")+ assert len(attempts) > calls and explicit._compiled+ torch._dynamo.reset()++def test_persistent_cache_is_opt_in_and_respects_the_environment():import tempfilefrom pathlib import Path