Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

Merge pull request #898 from cklxx/feat-compile-defaults

Nandakishor8a1cba1b0bdc
diff --cc laya/agent.py
index eeaf0d98,df5abb41..9ef1b0f3
--- a/laya/agent.py
+++ b/laya/agent.py
@@@ -13,13 -13,10 +13,13 @@@ from typing import Any, Dict, List, Opt
import numpy as np
import torch
- from ._compile import compile_model, independent_dims
+ from ._compile import compile_model, independent_dims, configure_cache, cuda_graph_step
from .calibrate import (
_install_temperatures,
+ apply_binning_map,
apply_calibration_payload,
+ MIN_BINNING_BUCKET_N,
+ fit_binning_map,
calibration_payload,
fit_temperature_map,
)
@@@ -496,7 -493,9 +497,10 @@@ class Agent(HookRegistry)
hooks_concurrent: bool = True,
hooks_timeout: Optional[float] = None,
calibration: Optional[str] = None,
+ backend: Optional[str] = None,
+ compile_warmup: bool = True,
+ compile_cache: bool = False,
+ compile_mode: str = "default",
):
"""Load a Laya checkpoint.
@@@ -533,13 -535,9 +544,16 @@@
`hooks_concurrent=False` serialises hooks that are not safe to run in parallel, and
`hooks_timeout` bounds each hook call in seconds (None means no limit).
"""
+ if backend is not None:
+ from .backends import normalise
+ backend = normalise(backend)
+ if backend == "onnx":
+ raise ValueError("use laya.load(..., backend='onnx') or ONNXAgent")
+ fast = compile = False
+
+ # Model loading can import Dynamo, which populates Inductor's default directory.
+ if compile and not fast and compile_cache:
+ configure_cache()
self.hooks = normalise_hooks(hooks, on_predict_start, on_predict_end)
self.hooks_raise = bool(hooks_raise)
self.hooks_concurrent = bool(hooks_concurrent)
@@@ -803,34 -773,16 +821,44 @@@
" See https://pytorch.org/get-started/locally/\n"
% (fell_back_from, fell_back_why), RuntimeWarning)
+ if self._compiled and compile_warmup:
+ try:
+ self.warmup()
+ except Exception as e:
+ # Compilation is lazy: unavailable toolchains fail on the first forward.
+ # Only automatic warm-up is best effort; explicit warmup() still raises.
+ warnings.warn("Warning: laya compile warm-up failed (%s: %s); keeping the compiled model. "
+ "Later requests and explicit warmup() calls may still raise."
+ % (type(e).__name__, e), RuntimeWarning)
+
+ @property
+ def backend(self) -> str:
+ """The active inference backend, including the legacy compile and fast flags."""
+ if self._backend is not None:
+ return self._backend.name
+ return "tilelang" if self._fast is not None else "compile" if self._compiled else "eager"
+
+ @property
+ def backend_object(self):
+ """The installed Backend object, or None for a legacy runtime."""
+ return self._backend
+
+ def set_backend(self, name: str = "auto", strict: bool = False, **options) -> str:
+ """Switch backends; unavailable backends warn and use eager unless `strict=True`.
+
+ Options go to the backend constructor, e.g. `warmup=False` for compile or
+ `use_graphs=False` for tilelang. Switching waits for in-flight inference.
+ """
+ from . import backends
+ name = backends.normalise(name)
+ with self._gate.write_lock():
+ self.deaccelerate()
+ if self._compiled:
+ self.model = self.model._orig_mod
+ self._compiled = False
+ self.model.encoder.config.reference_compile = False
+ return backends.install(self, name, strict=strict, **options).name
+
def accelerate(self, use_graphs: bool = True, strict: bool = False):
"""Replace the model forward with the TileLang fast path (fused GEMM/GEGLU/LayerNorm/RoPE kernels,
sliding-window flash attention, 16-bit resident weights, CUDA graphs per shape bucket).
@@@ -2023,8 -1939,8 +2055,9 @@@ def load(model_id_or_path: str = "conva
hooks=None, on_predict_start=None, on_predict_end=None,
hooks_raise: bool = True, hooks_concurrent: bool = True,
hooks_timeout: Optional[float] = None,
- calibration: Optional[str] = None, compile_warmup: bool = True,
+ calibration: Optional[str] = None, backend: Optional[str] = None,
- onnx_path: Optional[str] = None):
++ onnx_path: Optional[str] = None, compile_warmup: bool = True,
+ compile_cache: bool = False, compile_mode: str = "default") -> Agent:
"""Load a Laya agent.
`subfolder` picks one checkpoint out of a repo that bundles several:
@@@ -2063,19 -1975,9 +2096,20 @@@
spec = resolve_model_spec(model_id_or_path)
if spec is not None:
model_id_or_path, subfolder = spec
+ if backend is not None:
+ from .backends import normalise
+ backend = normalise(backend)
+ if backend == "onnx":
+ from .onnx_agent import ONNXAgent
+ return ONNXAgent(model_id_or_path, onnx_path=onnx_path or "laya.onnx",
+ token=token, subfolder=subfolder, revision=revision, expected_sha256=expected_sha256,
+ lang_temperatures=lang_temperatures, calibration=calibration,
+ hooks=hooks, on_predict_start=on_predict_start, on_predict_end=on_predict_end,
+ hooks_raise=hooks_raise, hooks_concurrent=hooks_concurrent, hooks_timeout=hooks_timeout)
+ options = {"backend": backend} if backend is not None else {}
return Agent(model_id_or_path, device=device, token=token, subfolder=subfolder, fast=fast,
- compile=compile,
+ compile=compile, compile_warmup=compile_warmup, compile_cache=compile_cache,
+ compile_mode=compile_mode,
revision=revision, expected_sha256=expected_sha256,
lang_temperatures=lang_temperatures,
hooks=hooks, on_predict_start=on_predict_start, on_predict_end=on_predict_end,
diff --cc tests/test_download.py
index c9c4c977,3e49b74d..ba776716
--- a/tests/test_download.py
+++ b/tests/test_download.py
@@@ -345,11 -327,12 +345,14 @@@ class DownloadTests(unittest.TestCase)
# copied across rather than silently diverging again.
# fast/compile -- the TileLang path and torch.compile, neither of which exists
# inside onnxruntime.
+ # backend -- selects a PyTorch forward; load() routes ONNX to ONNXAgent.
+ # compile_warmup/compile_cache/compile_mode -- configure torch.compile, which
+ # does not exist inside onnxruntime either.
# device -- ONNXAgent picks an execution provider from what onnxruntime
# reports and takes no override; a different asymmetry, with its
# own fix.
- not_for_onnxruntime = {"fast", "compile", "device", "backend"}
- not_for_onnxruntime = {"fast", "compile", "device", "compile_warmup", "compile_cache", "compile_mode"}
++ not_for_onnxruntime = {"fast", "compile", "device", "backend",
++ "compile_warmup", "compile_cache", "compile_mode"}
agent_side = (set(inspect.signature(Agent.__init__).parameters)
- {"self", "model_id_or_path"} - not_for_onnxruntime)
onnx_side = set(inspect.signature(ONNXAgent.__init__).parameters) - {"self"}
diff --cc tests/test_packaging.py
index 5a00271f,6d36aa97..602b0ec8
--- a/tests/test_packaging.py
+++ b/tests/test_packaging.py
@@@ -206,7 -206,7 +206,8 @@@ EXEMPT_TEST_SUITES =
"test_mcp_local_e2e.py": "Requires local checkpoints under ~/laya_models (AGENTS.md)",
"test_onnx.py": "Requires onnx extra; skip-guarded on lane without it (AGENTS.md)",
"test_fast.py": "Requires CUDA and tilelang extra (AGENTS.md)",
+ "test_fast_cpu.py": "Requires tilelang extra and a C++ compiler; optional CUDA parity",
+ "test_compile_cuda.py": "Requires CUDA; CI installs CPU-only torch",
"test_server_example.py": "Requires cached or downloaded weights for examples/server.py",
}