Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
Merge pull request #898 from cklxx/feat-compile-defaults
Nandakishor8a1cba1b0bdc
diff --cc laya/agent.pyindex eeaf0d98,df5abb41..9ef1b0f3--- a/laya/agent.py+++ b/laya/agent.py@@@ -13,13 -13,10 +13,13 @@@ from typing import Any, Dict, List, Optimport numpy as npimport torch- from ._compile import compile_model, independent_dims+ from ._compile import compile_model, independent_dims, configure_cache, cuda_graph_stepfrom .calibrate import (_install_temperatures,+ apply_binning_map,apply_calibration_payload,+ MIN_BINNING_BUCKET_N,+ fit_binning_map,calibration_payload,fit_temperature_map,)@@@ -496,7 -493,9 +497,10 @@@ class Agent(HookRegistry)hooks_concurrent: bool = True,hooks_timeout: Optional[float] = None,calibration: Optional[str] = None,+ backend: Optional[str] = None,+ compile_warmup: bool = True,+ compile_cache: bool = False,+ compile_mode: str = "default",):"""Load a Laya checkpoint.@@@ -533,13 -535,9 +544,16 @@@`hooks_concurrent=False` serialises hooks that are not safe to run in parallel, and`hooks_timeout` bounds each hook call in seconds (None means no limit)."""+ if backend is not None:+ from .backends import normalise+ backend = normalise(backend)+ if backend == "onnx":+ raise ValueError("use laya.load(..., backend='onnx') or ONNXAgent")+ fast = compile = False++ # Model loading can import Dynamo, which populates Inductor's default directory.+ if compile and not fast and compile_cache:+ configure_cache()self.hooks = normalise_hooks(hooks, on_predict_start, on_predict_end)self.hooks_raise = bool(hooks_raise)self.hooks_concurrent = bool(hooks_concurrent)@@@ -803,34 -773,16 +821,44 @@@" See https://pytorch.org/get-started/locally/\n"% (fell_back_from, fell_back_why), RuntimeWarning)+ if self._compiled and compile_warmup:+ try:+ self.warmup()+ except Exception as e:+ # Compilation is lazy: unavailable toolchains fail on the first forward.+ # Only automatic warm-up is best effort; explicit warmup() still raises.+ warnings.warn("Warning: laya compile warm-up failed (%s: %s); keeping the compiled model. "+ "Later requests and explicit warmup() calls may still raise."+ % (type(e).__name__, e), RuntimeWarning)++ @property+ def backend(self) -> str:+ """The active inference backend, including the legacy compile and fast flags."""+ if self._backend is not None:+ return self._backend.name+ return "tilelang" if self._fast is not None else "compile" if self._compiled else "eager"++ @property+ def backend_object(self):+ """The installed Backend object, or None for a legacy runtime."""+ return self._backend++ def set_backend(self, name: str = "auto", strict: bool = False, **options) -> str:+ """Switch backends; unavailable backends warn and use eager unless `strict=True`.++ Options go to the backend constructor, e.g. `warmup=False` for compile or+ `use_graphs=False` for tilelang. Switching waits for in-flight inference.+ """+ from . import backends+ name = backends.normalise(name)+ with self._gate.write_lock():+ self.deaccelerate()+ if self._compiled:+ self.model = self.model._orig_mod+ self._compiled = False+ self.model.encoder.config.reference_compile = False+ return backends.install(self, name, strict=strict, **options).name+def accelerate(self, use_graphs: bool = True, strict: bool = False):"""Replace the model forward with the TileLang fast path (fused GEMM/GEGLU/LayerNorm/RoPE kernels,sliding-window flash attention, 16-bit resident weights, CUDA graphs per shape bucket).@@@ -2023,8 -1939,8 +2055,9 @@@ def load(model_id_or_path: str = "convahooks=None, on_predict_start=None, on_predict_end=None,hooks_raise: bool = True, hooks_concurrent: bool = True,hooks_timeout: Optional[float] = None,- calibration: Optional[str] = None, compile_warmup: bool = True,+ calibration: Optional[str] = None, backend: Optional[str] = None,- onnx_path: Optional[str] = None):++ onnx_path: Optional[str] = None, compile_warmup: bool = True,+ compile_cache: bool = False, compile_mode: str = "default") -> Agent:"""Load a Laya agent.`subfolder` picks one checkpoint out of a repo that bundles several:@@@ -2063,19 -1975,9 +2096,20 @@@spec = resolve_model_spec(model_id_or_path)if spec is not None:model_id_or_path, subfolder = spec+ if backend is not None:+ from .backends import normalise+ backend = normalise(backend)+ if backend == "onnx":+ from .onnx_agent import ONNXAgent+ return ONNXAgent(model_id_or_path, onnx_path=onnx_path or "laya.onnx",+ token=token, subfolder=subfolder, revision=revision, expected_sha256=expected_sha256,+ lang_temperatures=lang_temperatures, calibration=calibration,+ hooks=hooks, on_predict_start=on_predict_start, on_predict_end=on_predict_end,+ hooks_raise=hooks_raise, hooks_concurrent=hooks_concurrent, hooks_timeout=hooks_timeout)+ options = {"backend": backend} if backend is not None else {}return Agent(model_id_or_path, device=device, token=token, subfolder=subfolder, fast=fast,- compile=compile,+ compile=compile, compile_warmup=compile_warmup, compile_cache=compile_cache,+ compile_mode=compile_mode,revision=revision, expected_sha256=expected_sha256,lang_temperatures=lang_temperatures,hooks=hooks, on_predict_start=on_predict_start, on_predict_end=on_predict_end,diff --cc tests/test_download.pyindex c9c4c977,3e49b74d..ba776716--- a/tests/test_download.py+++ b/tests/test_download.py@@@ -345,11 -327,12 +345,14 @@@ class DownloadTests(unittest.TestCase)# copied across rather than silently diverging again.# fast/compile -- the TileLang path and torch.compile, neither of which exists# inside onnxruntime.+ # backend -- selects a PyTorch forward; load() routes ONNX to ONNXAgent.+ # compile_warmup/compile_cache/compile_mode -- configure torch.compile, which+ # does not exist inside onnxruntime either.# device -- ONNXAgent picks an execution provider from what onnxruntime# reports and takes no override; a different asymmetry, with its# own fix.- not_for_onnxruntime = {"fast", "compile", "device", "backend"}- not_for_onnxruntime = {"fast", "compile", "device", "compile_warmup", "compile_cache", "compile_mode"}++ not_for_onnxruntime = {"fast", "compile", "device", "backend",++ "compile_warmup", "compile_cache", "compile_mode"}agent_side = (set(inspect.signature(Agent.__init__).parameters)- {"self", "model_id_or_path"} - not_for_onnxruntime)onnx_side = set(inspect.signature(ONNXAgent.__init__).parameters) - {"self"}diff --cc tests/test_packaging.pyindex 5a00271f,6d36aa97..602b0ec8--- a/tests/test_packaging.py+++ b/tests/test_packaging.py@@@ -206,7 -206,7 +206,8 @@@ EXEMPT_TEST_SUITES ="test_mcp_local_e2e.py": "Requires local checkpoints under ~/laya_models (AGENTS.md)","test_onnx.py": "Requires onnx extra; skip-guarded on lane without it (AGENTS.md)","test_fast.py": "Requires CUDA and tilelang extra (AGENTS.md)",+ "test_fast_cpu.py": "Requires tilelang extra and a C++ compiler; optional CUDA parity",+ "test_compile_cuda.py": "Requires CUDA; CI installs CPU-only torch","test_server_example.py": "Requires cached or downloaded weights for examples/server.py",}