Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
feat(agent): restore inference backend class layer
cklxxfb4b0371716d
diff --git a/docs/compile-and-fast-path.md b/docs/compile-and-fast-path.mdindex 57a9dfb9..17464797 100644--- a/docs/compile-and-fast-path.md+++ b/docs/compile-and-fast-path.md@@ -5,6 +5,36 @@ come from measurements taken while working on #472, #576 and #718, on an RTX 407torch 2.11 and tilelang 0.1.14. They are here so the next person does not have to measure themagain.+## Backend selection++`Agent(..., backend="auto")` and `laya.load(..., backend="auto")` opt into the backend+class layer. The default remains eager. Explicit `backend=` takes precedence over `compile`+and `fast`; omitting it preserves both flags' existing behaviour.++- `eager`: the stock PyTorch forward, on any supported device.+- `compile`: CUDA-only `torch.compile` with dynamic shapes and `reduce-overhead` mode,+ bucket padding, persistent inductor cache, and warmup at installation. It reuses the same+ independent-dimension scope as `compile=True`, which keeps its existing default mode and+ CPU support. Set `LAYA_COMPILE_WARMUP=0` to defer backend warmup and `LAYA_INDUCTOR_CACHE_DIR`+ to choose its cache directory (default `~/.cache/laya/inductor`).+- `tilelang`: an adapter around the current fast path, using the agent's bf16 or fp16 dtype.+- `auto`: TileLang on CUDA with a supported ModernBERT encoder and dtype when TileLang is+ installed, otherwise compile on CUDA; eager on other devices.+- `onnx`: `laya.load(..., backend="onnx", onnx_path="model.onnx")` returns the existing+ `ONNXAgent`. Without `onnx_path`, it uses `laya.onnx`.++An unavailable backend emits a `RuntimeWarning` naming the resolved backend and falls back to+eager. To require a backend, use `agent.set_backend("tilelang", strict=True)`. Switching waits+for active inference; `agent.backend` reports the active name and `agent.backend_object`+exposes the installed object. `agent.set_backend("compile", warmup=False)` defers compilation+until inference, so compilation errors then surface on the request. `agent.warmup()` remains+available. `agent.deaccelerate()` removes a backend installed through the class layer.++Routers forward an explicit selection through `Router(agent_kwargs={"backend": "auto"})`.+They pass no backend argument by default, preserving compatibility with existing Agent-like+constructors. Scoped CPU OOM retries detach the backend and restore it when the model returns+to its original device.+## `compile=True` materialises the attention maskEager SDPA takes ModernBERT's `(rows, 1, L, L)` attention mask as a broadcast view. Under thediff --git a/laya/agent.py b/laya/agent.pyindex 83b18da4..0f5c8647 100644--- a/laya/agent.py+++ b/laya/agent.py@@ -465,6 +465,7 @@ class Agent(HookRegistry):_amp_failures = 0# The stock forward is also used by lightweight runtimes built with __new__ in tests._fast = None+ _backend = None# Scoped CPU-fallback observability: how often _infer's per-request OOM fallback fired# and what the last failure was, so an operator sees a slow lane in /health instead of# discovering it by accident. Class defaults cover instances built without __init__.@@ -492,9 +493,14 @@ class Agent(HookRegistry):hooks_concurrent: bool = True,hooks_timeout: Optional[float] = None,calibration: Optional[str] = None,+ backend: Optional[str] = None,):"""Load a Laya checkpoint.+ `backend` selects "eager", "auto", "compile" or "tilelang"; see `laya.backends`.+ It takes precedence over `fast` and `compile`. Omit it to keep those legacy flags.+ For ONNX use `load(backend="onnx")` instead.+`revision` optionally pins the Hub download to an explicit commit SHA/branch/tag;when omitted, huggingface_hub's normal default and existing offline cache are used.`expected_sha256` ({path relative to the checkpoint dir: hexdigest})@@ -524,6 +530,13 @@ class Agent(HookRegistry):`hooks_concurrent=False` serialises hooks that are not safe to run in parallel, and`hooks_timeout` bounds each hook call in seconds (None means no limit)."""+ if backend is not None:+ from .backends import normalise+ backend = normalise(backend)+ if backend == "onnx":+ raise ValueError("use laya.load(..., backend='onnx') or ONNXAgent")+ fast = compile = False+self.hooks = normalise_hooks(hooks, on_predict_start, on_predict_end)self.hooks_raise = bool(hooks_raise)self.hooks_concurrent = bool(hooks_concurrent)@@ -741,7 +754,9 @@ class Agent(HookRegistry):else:raise e- if fast:+ if backend is not None:+ self.set_backend(backend)+ elif fast:self.accelerate()if fell_back_from is not None:@@ -755,6 +770,34 @@ class Agent(HookRegistry):" See https://pytorch.org/get-started/locally/\n"% (fell_back_from, fell_back_why), RuntimeWarning)+ @property+ def backend(self) -> str:+ """The active inference backend, including the legacy compile and fast flags."""+ if self._backend is not None:+ return self._backend.name+ return "tilelang" if self._fast is not None else "compile" if self._compiled else "eager"++ @property+ def backend_object(self):+ """The installed Backend object, or None for a legacy runtime."""+ return self._backend++ def set_backend(self, name: str = "auto", strict: bool = False, **options) -> str:+ """Switch backends; unavailable backends warn and use eager unless `strict=True`.++ Options go to the backend constructor, e.g. `warmup=False` for compile or+ `use_graphs=False` for tilelang. Switching waits for in-flight inference.+ """+ from . import backends+ name = backends.normalise(name)+ with self._gate.write_lock():+ self.deaccelerate()+ if self._compiled:+ self.model = self.model._orig_mod+ self._compiled = False+ self.model.encoder.config.reference_compile = False+ return backends.install(self, name, strict=strict, **options).name+def accelerate(self, use_graphs: bool = True, strict: bool = False):"""Replace the model forward with the TileLang fast path (fused GEMM/GEGLU/LayerNorm/RoPE kernels,sliding-window flash attention, 16-bit resident weights, CUDA graphs per shape bucket).@@ -770,6 +813,8 @@ class Agent(HookRegistry):if strict:raise RuntimeError("laya fast path needs a CUDA device")return False+ if self._backend is not None:+ self.deaccelerate()last = Nonefor _attempt in range(2): # tilelang's JIT cache has been seen to fail once, then succeedtry:@@ -823,7 +868,11 @@ class Agent(HookRegistry):def deaccelerate(self):"""Restore the stock forward."""- if self._fast is not None:+ if self._backend is not None:+ self._backend.uninstall()+ self._backend = None+ self._fast = None+ elif self._fast is not None:self.model.forward = self._stock_forwardself._fast = None@@ -1093,6 +1142,10 @@ class Agent(HookRegistry):b = _pad_cuda_compile_batch(b, self.tok.pad_token_id)use_amp = self._amp_enabled_for(b["input_ids"].shape[0])+ limit = getattr(self._backend, "max_len", None)+ if limit is not None and b["input_ids"].shape[1] > limit:+ raise ValueError("the %s backend was built for max_len=%d; this request needs %d tokens"+ % (self.backend, limit, b["input_ids"].shape[1]))if self._fast is not None and b["input_ids"].shape[1] > self._fast.max_len:raise ValueError("the CUDA fast path was built for max_len=%d; this request needs %d tokens. "@@ -1140,7 +1193,8 @@ class Agent(HookRegistry):if self.device.type == "cpu":return run()held_device, held_dtype, held_amp = self.device, self.dtype, self.amp_enabled- had_fast = self._fast is not None+ held_backend = self._backend+ had_fast = self._fast is not None and held_backend is None# Only our batch scope can retain BF16 copies after this failed forward.# Release them before moving the model and retrying on CPU.if self.device.type == "cuda" and _BATCH_AUTOCAST_CACHE.get():@@ -1159,6 +1213,15 @@ class Agent(HookRegistry):return run()finally:self._restore_runtime(held_device, held_dtype, held_amp, had_fast)+ if held_backend is not None and self.device == held_device:+ from .backends import BackendUnavailable, warn_fallback+ try:+ held_backend.install()+ except BackendUnavailable as exc:+ warn_fallback(held_backend.name, exc)+ else:+ self._backend = held_backend+ self._fast = getattr(held_backend, "fast", None)if use_amp and self.device.type in ("mps", "cpu"):# Not every MPS/CPU build implements autocast for every op. Retry this# request in full precision. One miss must not turn AMP off; a build that@@ -1352,7 +1415,7 @@ class Agent(HookRegistry):# caller already owns a CUDA autocast scope, its cache is sufficient.if (chunk < len(states) and getattr(self, "device", None) is not Noneand self.device.type == "cuda" and self.amp_enabled- and self._fast is None and not self._compiled+ and self._fast is None and not self._compiled and self.backend != "compile"and not torch.is_autocast_enabled()):amp_stack.enter_context(torch.autocast(device_type="cuda", dtype=self.dtype,enabled=False))@@ -1907,7 +1970,8 @@ def load(model_id_or_path: str = "convaiinnovations/laya", device: Optional[str]hooks=None, on_predict_start=None, on_predict_end=None,hooks_raise: bool = True, hooks_concurrent: bool = True,hooks_timeout: Optional[float] = None,- calibration: Optional[str] = None) -> Agent:+ calibration: Optional[str] = None, backend: Optional[str] = None,+ onnx_path: Optional[str] = None):"""Load a Laya agent.`subfolder` picks one checkpoint out of a repo that bundles several:@@ -1925,6 +1989,10 @@ def load(model_id_or_path: str = "convaiinnovations/laya", device: Optional[str]Anything else (a Hub repo id, a local directory) is passed to `Agent` unchanged.+ `backend` selects "auto", "eager", "compile", "tilelang" or "onnx". ONNX returns+ the existing `ONNXAgent`, with `onnx_path` (default "laya.onnx").+ Other backends use `Agent`; an explicit backend takes precedence over the legacy flags.+`revision`/`expected_sha256` pin and verify the downloaded artifacts; see `Agent`.`hooks` / `on_predict_start` / `on_predict_end` observe or shape every prediction; see`laya.hooks`. `calibration` is the same optional JSON path accepted by `Agent`.@@ -1942,10 +2010,21 @@ def load(model_id_or_path: str = "convaiinnovations/laya", device: Optional[str]spec = resolve_model_spec(model_id_or_path)if spec is not None:model_id_or_path, subfolder = spec+ if backend is not None:+ from .backends import normalise+ backend = normalise(backend)+ if backend == "onnx":+ from .onnx_agent import ONNXAgent+ return ONNXAgent(model_id_or_path, onnx_path=onnx_path or "laya.onnx",+ token=token, subfolder=subfolder, revision=revision, expected_sha256=expected_sha256,+ lang_temperatures=lang_temperatures, calibration=calibration,+ hooks=hooks, on_predict_start=on_predict_start, on_predict_end=on_predict_end,+ hooks_raise=hooks_raise, hooks_concurrent=hooks_concurrent, hooks_timeout=hooks_timeout)+ options = {"backend": backend} if backend is not None else {}return Agent(model_id_or_path, device=device, token=token, subfolder=subfolder, fast=fast,compile=compile,revision=revision, expected_sha256=expected_sha256,lang_temperatures=lang_temperatures,hooks=hooks, on_predict_start=on_predict_start, on_predict_end=on_predict_end,hooks_raise=hooks_raise, hooks_concurrent=hooks_concurrent,- hooks_timeout=hooks_timeout, calibration=calibration)+ hooks_timeout=hooks_timeout, calibration=calibration, **options)diff --git a/laya/backends/__init__.py b/laya/backends/__init__.pynew file mode 100644index 00000000..08f1b116--- /dev/null+++ b/laya/backends/__init__.py@@ -0,0 +1,109 @@+"""Inference backends: one contract, several ways to run `DecisionModel.forward`.++ laya.load(..., backend="auto" | "eager" | "compile" | "tilelang" | "onnx")++| backend | what it is | needs |+|------------|-------------------------------------------------------------------|--------------------------|+| `eager` | the stock PyTorch forward under autocast (the default) | nothing |+| `compile` | `torch.compile(dynamic=True, mode="reduce-overhead")` + warm-up | CUDA |+| `tilelang` | fused TileLang kernels, 16-bit weights, one CUDA graph per bucket | CUDA, `laya[fast]`, bf16/fp16 |+| `onnx` | ONNX Runtime via `laya.onnx_agent.ONNXAgent` (the explicit CPU option, its own class) | `laya[onnx]`, an exported model |++`auto` picks per device: CUDA -> `tilelang` when tilelang imports and the encoder is a+ModernBERT-family model (the shipped checkpoints), else `compile`; CPU and MPS -> `eager`.+Every backend honours `agent.dtype`, and one that cannot run falls back to `eager` with one+`RuntimeWarning`; `strict=True` raises instead. See `laya.backends.base` for the contract and+the padding policy.+"""+from .base import Backend, BackendUnavailable, bucket_shape, pad_batch, warn_fallback++NAMES = ("auto", "eager", "compile", "tilelang", "onnx")+AUTO = "auto"+++def normalise(name):+ """Accept the documented spellings; anything else is a ValueError naming the choices."""+ key = str(name).strip().lower().replace("_", "-") if name is not None else "eager"+ aliases = {"fast": "tilelang", "torch": "eager", "torch.compile": "compile", "inductor": "compile"}+ key = aliases.get(key, key)+ if key not in NAMES:+ raise ValueError("unknown backend %r; use one of %s" % (name, ", ".join(NAMES)))+ return key+++def tilelang_available() -> bool:+ try:+ import tilelang # noqa: F401+ return True+ except Exception:+ return False+++def is_modernbert(model) -> bool:+ cfg = getattr(getattr(model, "encoder", None), "config", None)+ return getattr(cfg, "model_type", None) == "modernbert"+++def auto_policy(agent) -> str:+ """The backend `auto` resolves to for this agent (device, dtype, encoder, installed extras)."""+ if agent.device.type == "cuda":+ import torch+ if tilelang_available() and is_modernbert(agent.model) and agent.dtype in (torch.bfloat16, torch.float16):+ return "tilelang"+ return "compile"+ return "eager"+++def make_backend(name: str, agent, **options) -> Backend:+ """Build (not install) the backend `name` for `agent`. `auto` is resolved here."""+ key = normalise(name)+ if key == AUTO:+ key = auto_policy(agent)+ if key == "eager":+ from .eager import EagerBackend+ return EagerBackend(agent, **options)+ if key == "compile":+ from .compile import CompileBackend+ return CompileBackend(agent, **options)+ if key == "tilelang":+ try:+ from .tilelang import TileLangBackend+ except ImportError as e:+ raise BackendUnavailable("tilelang is not installed (pip install laya[fast]): %s" % e)+ return TileLangBackend(agent, **options)+ raise BackendUnavailable("backend %r is not a forward replacement; use laya.load(..., backend='onnx') "+ "or laya.onnx_agent.ONNXAgent directly" % key)+++def install(agent, name: str, strict: bool = False, **options) -> Backend:+ """Install `name` on `agent`, falling back to eager with one warning unless `strict`.++ Returns the backend that ended up installed. The agent's previous backend is uninstalled+ first, so switching is always eager -> new and never stacks two replacements.+ """+ from .eager import EagerBackend++ key = normalise(name)+ if key == AUTO:+ key = auto_policy(agent)+ current = getattr(agent, "_backend", None)+ if current is not None:+ current.uninstall()+ agent._backend = None+ agent._fast = None+ try:+ backend = make_backend(key, agent, **options)+ backend.install()+ except BackendUnavailable as e:+ if strict:+ raise+ warn_fallback(key, e)+ backend = EagerBackend(agent)+ backend.install()+ agent._backend = backend+ agent._fast = getattr(backend, "fast", None)+ return backend+++__all__ = ["Backend", "BackendUnavailable", "NAMES", "auto_policy", "bucket_shape",+ "install", "make_backend", "normalise", "pad_batch", "warn_fallback"]diff --git a/laya/backends/base.py b/laya/backends/base.pynew file mode 100644index 00000000..5a875843--- /dev/null+++ b/laya/backends/base.py@@ -0,0 +1,149 @@+"""The inference backend contract and padding helpers.++A backend owns one thing: how `DecisionModel.forward` is executed. The Agent keeps tokenization,+collation, temperature scaling and decoding; a backend is handed the collated batch on the agent's+device and returns `(logits, act_logits)` with the same shapes and semantics the stock forward has.++ backend = make_backend("compile", agent) # laya.backends+ backend.install() # swaps agent.model.forward; may raise+ ...+ backend.uninstall() # restores the stock forward++Padding helpers bound the shapes seen by graph backends. Dtype and autocast policy remain+owned by Agent; a backend honours the dtype already selected there.+"""+import warnings+from typing import Any, Optional, Tuple++import torch++# ---------------------------------------------------------------------------------- padding policy+DYNAMIC_MAX_L = 256 # up to here a dynamic-shape kernel is as fast as a static one: 16-token buckets+LONG_BUCKET = 64 # beyond it, 64-token buckets+++def bucket_rows(n: int) -> int:+ """Rows are padded to the next power of two (1, 2, 4, ... questions per call)."""+ return 1 << (max(1, n) - 1).bit_length()+++def bucket_tokens(n_tokens: int, max_len: int) -> int:+ """Tokens are padded to a 16-token bucket up to `DYNAMIC_MAX_L`, 64-token buckets beyond, capped+ at `max_len` (the checkpoint's context, which the tokenizer never exceeds)."""+ g = 16 if n_tokens <= DYNAMIC_MAX_L else LONG_BUCKET+ return min(max_len, ((n_tokens + g - 1) // g) * g)+++def bucket_shape(n_rows: int, n_tokens: int, max_len: int) -> Tuple[int, int]:+ return bucket_rows(n_rows), bucket_tokens(n_tokens, max_len)+++def pad_batch(input_ids, attention_mask, marker_pos, marker_mask, qtype, max_len: int, pad_id: int = 0,+ keep_one_token: bool = False):+ """Pad a collated batch to its `bucket_shape`. Returns the padded tensors and `(N, L0)` to slice back.++ Padding rows carry `pad_id` tokens, an empty attention mask, markers at position 0 and qtype 0,+ so a backend that reads only the first `N` rows of its output sees exactly the request.+ `keep_one_token=True` marks token 0 of a padding row as valid instead: SDPA/eager attention+ turns an all-masked row into NaN, and although those rows are sliced away, a NaN in the batch+ is enough to poison a CUDA-graph replay under `torch.compile`; the TileLang attention kernel+ keeps fully masked rows finite by construction and does not need it.+ """+ N, L0 = input_ids.shape+ B, L = bucket_shape(N, L0, max_len)+ dev = input_ids.device+ ids = torch.full((B, L), pad_id, dtype=input_ids.dtype, device=dev)+ ids[:N, :L0] = input_ids+ att = torch.zeros((B, L), dtype=attention_mask.dtype, device=dev)+ att[:N, :L0] = attention_mask+ if keep_one_token and B > N:+ att[N:, 0] = 1+ mpos = torch.zeros((B, marker_pos.shape[1]), dtype=marker_pos.dtype, device=dev)+ mpos[:N] = marker_pos+ mmask = torch.zeros((B, marker_mask.shape[1]), dtype=marker_mask.dtype, device=dev)+ mmask[:N] = marker_mask+ # a padding row keeps one live marker so its softmax/entropy stay finite (the stock head+ # clamps k to 2 and masks the rest to -1e4; an all-masked row is a uniform softmax, fine too)+ if B > N:+ mmask[N:, 0] = True+ qt = torch.zeros((B,), dtype=qtype.dtype, device=dev)+ qt[:N] = qtype+ return ids, att, mpos, mmask, qt, N, L0+++# ------------------------------------------------------------------------------------- the contract+class BackendUnavailable(RuntimeError):+ """The backend cannot run here (no CUDA, dependency missing, dtype unsupported): fall back."""+++class Backend:+ """Base class. Subclasses set `name`, and override `_install`/`_uninstall`/`forward`.++ `max_len` is the longest sequence the backend accepts once installed, or None for no limit+ beyond the checkpoint's; `Agent._infer` reports a request past it with a clear message.+ """++ name = "eager"+ max_len: Optional[int] = None++ def __init__(self, agent):+ self.agent = agent+ self.model = agent.model+ self.installed = False+ self._stock_forward = None+ self._prior_forward = None++ # -- lifecycle+ def install(self) -> None:+ """Swap `model.forward` for this backend's. Raises `BackendUnavailable` when it cannot run+ here (the caller then falls back to eager with one warning) and leaves the model untouched."""+ if self.installed:+ return+ # `forward` normally comes from the class; another replacement (an instance attribute) is+ # restored as it was, so backends never stack and uninstall leaves no trace.+ self._prior_forward = self.model.__dict__.get("forward")+ self._stock_forward = self.model.forward+ self._install()+ self.model.forward = self.forward+ self.installed = True++ def uninstall(self) -> None:+ if not self.installed:+ return+ if self._prior_forward is None:+ self.model.__dict__.pop("forward", None)+ else:+ self.model.forward = self._prior_forward+ self._stock_forward = self._prior_forward = None+ self._uninstall()+ self.installed = False++ def _install(self) -> None:+ pass++ def _uninstall(self) -> None:+ pass++ def warmup(self, shapes=None) -> float:+ """Run representative shapes through the backend so the first real call is not the slow one.+ Returns seconds spent. The default backend has nothing to warm."""+ return 0.0++ # -- the forward itself+ def stock_forward(self, *args, **kwargs):+ """The model's own forward, whatever backend is installed."""+ fn = self._stock_forward if self._stock_forward is not None else type(self.model).forward.__get__(self.model)+ return fn(*args, **kwargs)++ def forward(self, input_ids, attention_mask, marker_pos, marker_mask, qtype, detach_encoder=False):+ return self.stock_forward(input_ids, attention_mask, marker_pos, marker_mask, qtype, detach_encoder)++ def __repr__(self):+ return "%s(installed=%s)" % (type(self).__name__, self.installed)+++def warn_fallback(name: str, why: Any) -> None:+ """The one warning a caller sees when a requested backend cannot be used."""+ warnings.warn(+ "laya: backend %r is unavailable (%s); using the eager forward instead." % (name, why),+ RuntimeWarning, stacklevel=3)diff --git a/laya/backends/compile.py b/laya/backends/compile.pynew file mode 100644index 00000000..7b45e336--- /dev/null+++ b/laya/backends/compile.py@@ -0,0 +1,138 @@+"""The `torch.compile` backend: inductor kernels + CUDA graphs, no extra dependency.++ agent = laya.load("convaiinnovations/laya", backend="compile")++`torch.compile(model.forward, dynamic=True, mode="reduce-overhead")` uses the independent+input dimensions from `laya._compile`. Torch still specialises 0/1 dimensions. Requests are+padded to the bucket policy (`laya.backends.base.bucket_shape`) to limit CUDA-graph shapes,+and outputs are copied out of graph buffers before the lock is released.++Cold start is the cost: the first compile takes tens of seconds. Two things cut it:++* the inductor FX-graph cache, persisted in a stable directory. Set `LAYA_INDUCTOR_CACHE_DIR`+ (default `~/.cache/laya/inductor`); it is applied to `TORCHINDUCTOR_CACHE_DIR` unless that is+ already set, so a `TORCHINDUCTOR_*` variable the operator exports still wins. A second process+ loads the compiled graphs from there instead of compiling them again.+* warm-up at load (`warmup=True`, or `LAYA_COMPILE_WARMUP=0` to skip): a few representative+ shapes are run at load time so the first request pays nothing. `agent.backend_object.warmup_s`+ reports what it cost.++`torch.compile` is only used on CUDA: on CPU the inductor C++ path needs a compiler and buys+little at Laya's batch sizes, and MPS is unsupported, so those raise `BackendUnavailable`.+"""+import os+import threading+import time+from typing import Optional++import torch++from .._compile import compile_model, independent_dims+from .base import Backend, BackendUnavailable, pad_batch++CACHE_DIR_ENV = "LAYA_INDUCTOR_CACHE_DIR"+WARMUP_ENV = "LAYA_COMPILE_WARMUP"+DEFAULT_CACHE_DIR = os.path.join("~", ".cache", "laya", "inductor")+# rows x tokens x markers run at load: one short call, a few questions at a medium length, and a+# bigger batch; the three sizes differ within each shape so no two dimensions get tied together+WARMUP_SHAPES = ((1, 64, 3), (4, 128, 5), (8, 256, 6))+++def configure_inductor_cache() -> str:+ """Point inductor's on-disk caches at a stable directory and switch the FX-graph cache on.++ Returns the directory in use. Inductor reads `TORCHINDUCTOR_CACHE_DIR` when it opens the+ cache, so this must run before the first compile; an explicit `TORCHINDUCTOR_CACHE_DIR` from+ the environment is left alone.+ """+ cache_dir = os.path.expanduser(os.environ.get(CACHE_DIR_ENV) or DEFAULT_CACHE_DIR)+ os.environ.setdefault("TORCHINDUCTOR_CACHE_DIR", cache_dir)+ os.environ.setdefault("TORCHINDUCTOR_FX_GRAPH_CACHE", "1")+ os.environ.setdefault("TORCHINDUCTOR_AUTOGRAD_CACHE", "1")+ try:+ import torch._inductor.config as icfg+ icfg.fx_graph_cache = True+ except Exception:+ pass+ return os.environ["TORCHINDUCTOR_CACHE_DIR"]+++def warmup_enabled(default: bool = True) -> bool:+ raw = os.environ.get(WARMUP_ENV)+ if raw is None:+ return default+ return raw.strip().lower() not in ("0", "false", "no", "off")+++class CompileBackend(Backend):+ name = "compile"++ def __init__(self, agent, mode: str = "reduce-overhead",+ warmup: Optional[bool] = None):+ super().__init__(agent)+ self.mode = mode+ self.do_warmup = warmup_enabled(True) if warmup is None else bool(warmup)+ self.compiled = None+ self.cache_dir = None+ self.warmup_s = 0.0+ # CUDA graphs replay into static buffers: keep one forward in flight at a time.+ self._lock = threading.RLock()++ def _install(self):+ agent = self.agent+ if agent.device.type != "cuda":+ raise BackendUnavailable("torch.compile is used on CUDA only here; %s runs eager" % agent.device.type)+ if not hasattr(torch, "compile"):+ raise BackendUnavailable("this torch has no torch.compile")+ self.cache_dir = configure_inductor_cache()+ self.max_len = int(agent.cfg.get("max_len", 512))+ try:+ self.compiled = compile_model(self._stock_forward, mode=self.mode)+ except Exception as e:+ raise BackendUnavailable(e)++ def _uninstall(self):+ self.compiled = None++ def install(self):+ if self.installed:+ return+ super().install()+ if self.do_warmup:+ try:+ self.warmup_s = self.warmup()+ except Exception as e:+ # A compile failure surfaces here rather than on the first request.+ self.uninstall()+ raise BackendUnavailable(e)++ @torch.no_grad()+ def forward(self, input_ids, attention_mask, marker_pos, marker_mask, qtype, detach_encoder=False):+ ids, att, mpos, mmask, qt, N, _ = pad_batch(+ input_ids, attention_mask, marker_pos, marker_mask, qtype, self.max_len,+ pad_id=self.agent.tok.pad_token_id or 0, keep_one_token=True)+ with self._lock, independent_dims():+ logits, act = self.compiled(ids, att, mpos, mmask, qt, detach_encoder)+ # copy out of the CUDA graph's static output buffers before the next replay+ return logits[:N].clone(), act[:N].clone()++ def warmup(self, shapes=None):+ agent = self.agent+ dev = agent.device+ t0 = time.perf_counter()+ with torch.no_grad(), torch.autocast(device_type="cuda", dtype=agent.dtype, enabled=agent.amp_enabled):+ for rows, tokens, markers in shapes or WARMUP_SHAPES:+ tokens = min(tokens, self.max_len)+ if tokens <= markers:+ raise ValueError("warmup needs more tokens than markers")+ ids = torch.full((rows, tokens), agent.tok.pad_token_id or 0, dtype=torch.long, device=dev)+ ids[:, 0] = agent.tok.cls_token_id if agent.tok.cls_token_id is not None else 0+ att = torch.ones((rows, tokens), dtype=torch.long, device=dev)+ mpos = torch.arange(1, markers + 1, device=dev).repeat(rows, 1)+ mmask = torch.ones((rows, markers), dtype=torch.bool, device=dev)+ qt = torch.zeros((rows,), dtype=torch.long, device=dev)+ # cudagraph trees record on the second call of a shape and replay from the third+ for _ in range(3):+ self.forward(ids, att, mpos, mmask, qt)+ torch.cuda.synchronize(dev)+ return time.perf_counter() - t0diff --git a/laya/backends/eager.py b/laya/backends/eager.pynew file mode 100644index 00000000..c8851384--- /dev/null+++ b/laya/backends/eager.py@@ -0,0 +1,14 @@+"""The stock PyTorch forward: no padding, no graphs, the reference every other backend is measured against."""+from .base import Backend+++class EagerBackend(Backend):+ name = "eager"++ def install(self) -> None:+ # Nothing to swap: `model.forward` is already the eager forward. Marking it installed keeps+ # the lifecycle uniform for the Agent.+ self.installed = True++ def uninstall(self) -> None:+ self.installed = Falsediff --git a/laya/backends/tilelang.py b/laya/backends/tilelang.pynew file mode 100644index 00000000..6191effb--- /dev/null+++ b/laya/backends/tilelang.py@@ -0,0 +1,45 @@+"""Backend adapter for the current TileLang fast path."""+import torch++from .base import Backend, BackendUnavailable+++class TileLangBackend(Backend):+ name = "tilelang"++ def __init__(self, agent, use_graphs=True, verbose=False):+ super().__init__(agent)+ self.use_graphs = use_graphs+ self.verbose = verbose+ self.fast = None++ def _install(self):+ agent = self.agent+ if agent.device.type != "cuda":+ raise BackendUnavailable("needs a CUDA device")+ if agent.dtype not in (torch.bfloat16, torch.float16):+ raise BackendUnavailable("the TileLang kernels need bf16 or fp16 and the agent runs %s" % (agent.dtype,))+ if getattr(getattr(self.model.encoder, "config", None), "model_type", None) != "modernbert":+ raise BackendUnavailable("the kernels implement a ModernBERT-family encoder, got %r"+ % getattr(self.model.encoder.config, "model_type", None))+ last = None+ for _attempt in range(2): # tilelang's JIT cache has been seen to fail once, then succeed+ try:+ from ..fast import FastLaya+ self.fast = FastLaya(self.model, max_len=agent.cfg.get("max_len", 512),+ use_graphs=self.use_graphs, verbose=self.verbose, dtype=agent.dtype)+ break+ except Exception as e: # tilelang missing / unsupported arch+ last = e+ if self.fast is None:+ raise BackendUnavailable(last)+ self.max_len = self.fast.max_len++ def _uninstall(self):+ self.fast = None++ def forward(self, input_ids, attention_mask, marker_pos, marker_mask, qtype, detach_encoder=False):+ return self.fast.forward(input_ids, attention_mask, marker_pos, marker_mask, qtype, detach_encoder)++ def warmup(self, shapes=None):+ return self.agent.warmup(shapes)diff --git a/laya/fast.py b/laya/fast.pyindex 6597c592..1285fe1e 100644--- a/laya/fast.py+++ b/laya/fast.py@@ -8,6 +8,7 @@ import sysimport threadingimport torchfrom . import tl_kernels as K+from .backends.base import bucket_rows_KERNEL_DTYPES = {torch.bfloat16: "bfloat16", torch.float16: "float16"}@@ -21,7 +22,7 @@ def _top_two(probs):def _bucket_n(n):- return 1 << max(0, (n - 1).bit_length())+ return bucket_rows(n)class FastLaya:diff --git a/scripts/test_suites.py b/scripts/test_suites.pyindex ffe1ab20..43b3efd3 100644--- a/scripts/test_suites.py+++ b/scripts/test_suites.py@@ -59,6 +59,7 @@ SCRIPT_SUITES = ["tests/test_question_token_reuse.py","tests/test_lazy_import.py","tests/test_runtime_fixes.py",+ "tests/test_backends.py","tests/test_fast_head_partition.py","tests/test_agent_gate.py","tests/test_option_collapse.py",diff --git a/tests/test_backends.py b/tests/test_backends.pynew file mode 100644index 00000000..b76442f6--- /dev/null+++ b/tests/test_backends.py@@ -0,0 +1,276 @@+"""The inference backend layer (`laya.backends`), checked without a GPU or a checkpoint.++What is pinned here is the contract every backend is held to and the parts the Agent relies on:+the names and aliases `load(backend=...)` accepts, the `auto` policy off CUDA, the shared padding+policy both graph backends use, and that a backend which cannot run here falls back to eager with+exactly one warning (or raises under `strict=True`) and leaves the model's forward untouched.++Run: python tests/test_backends.py (or under pytest)+"""+import os+import sys+import warnings+from types import SimpleNamespace++sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))++import torch # noqa: E402++from laya import backends # noqa: E402+from laya.backends import BackendUnavailable, bucket_shape, pad_batch # noqa: E402+from laya.backends.base import Backend, bucket_rows # noqa: E402+from laya.backends.eager import EagerBackend # noqa: E402++PASS, FAIL = [], []+++def check(name, got, want):+ (PASS if got == want else FAIL).append(name if got == want else "%s: got %r, want %r" % (name, got, want))+++def check_true(name, cond, detail=""):+ (PASS if cond else FAIL).append(name if cond else "%s: %s" % (name, detail))+++class TinyModel(torch.nn.Module):+ """A forward with the DecisionModel signature, so a backend can be installed on it."""++ def __init__(self):+ super().__init__()+ self.encoder = SimpleNamespace(config=SimpleNamespace(model_type="bert", reference_compile=None))+ self.lin = torch.nn.Linear(4, 4)++ def forward(self, input_ids, attention_mask, marker_pos, marker_mask, qtype, detach_encoder=False):+ n, k = marker_mask.shape+ return torch.zeros(n, k) + attention_mask.sum(1, keepdim=True).float(), torch.zeros(n, 2)+++def fake_agent(device="cpu", dtype=torch.float32):+ return SimpleNamespace(model=TinyModel(), device=torch.device(device), dtype=dtype, amp_enabled=False,+ cfg={"max_len": 64}, tok=SimpleNamespace(pad_token_id=0, cls_token_id=1), _backend=None)+++# ------------------------------------------------------------------ names and aliases+for raw, want in (("eager", "eager"), ("Eager", "eager"), ("fast", "tilelang"), ("TileLang", "tilelang"),+ ("torch.compile", "compile"), ("inductor", "compile"), ("auto", "auto"), ("onnx", "onnx"), (None, "eager")):+ check("normalise(%r)" % (raw,), backends.normalise(raw), want)+try:+ backends.normalise("cuda-graphs")+ FAIL.append("normalise/unknown name accepted")+except ValueError as e:+ check_true("normalise/unknown name names the choices", "eager" in str(e) and "tilelang" in str(e), str(e))++# ------------------------------------------------------------------ auto policy off CUDA+check("auto_policy/cpu -> eager", backends.auto_policy(fake_agent("cpu")), "eager")+check("make_backend(auto)/cpu is eager", backends.make_backend("auto", fake_agent("cpu")).name, "eager")++# ------------------------------------------------------------------ padding policy+check("bucket_rows/zero", bucket_rows(0), 1)+for (n, L, max_len), want in (((1, 5, 512), (1, 16)), ((3, 70, 512), (4, 80)), ((5, 256, 512), (8, 256)),+ ((2, 257, 512), (2, 320)), ((1, 1000, 512), (1, 512)), ((16, 16, 512), (16, 16))):+ check("bucket_shape(%d, %d, %d)" % (n, L, max_len), bucket_shape(n, L, max_len), want)++ids = torch.tensor([[7, 8, 9], [7, 8, 0]])+att = torch.tensor([[1, 1, 1], [1, 1, 0]])+mpos = torch.tensor([[1, 2], [1, 0]])+mmask = torch.tensor([[True, True], [True, False]])+qt = torch.tensor([0, 2])+p_ids, p_att, p_mpos, p_mmask, p_qt, N, L0 = pad_batch(ids, att, mpos, mmask, qt, max_len=64, pad_id=5)+check("pad_batch/shape", tuple(p_ids.shape), (2, 16))+check("pad_batch/(N, L0)", (N, L0), (2, 3))+check_true("pad_batch/request kept", torch.equal(p_ids[:2, :3], ids) and torch.equal(p_att[:2, :3], att)+ and torch.equal(p_qt[:2], qt) and torch.equal(p_mmask[:2], mmask))+check("pad_batch/pad id fills", int(p_ids[0, 3]), 5)+check("pad_batch/padding tokens masked", int(p_att[:, 3:].sum()), 0)+p3 = pad_batch(ids[:1], att[:1], mpos[:1], mmask[:1], qt[:1], max_len=64)+check("pad_batch/one row stays one row", tuple(p3[0].shape), (1, 16))+p5 = pad_batch(ids.repeat(3, 1)[:5], att.repeat(3, 1)[:5], mpos.repeat(3, 1)[:5], mmask.repeat(3, 1)[:5], qt.repeat(3)[:5], 64,+ keep_one_token=True)+check("pad_batch/five rows pad to eight", tuple(p5[0].shape), (8, 16))+check("pad_batch/padding rows have empty masks", int(p5[1][5:].sum()), 3) # keep_one_token: token 0 valid+check("pad_batch/padding rows keep one marker", int(p5[3][5:].sum()), 3)+check("pad_batch/padding rows have qtype 0", int(p5[4][5:].sum()), 0)++# ------------------------------------------------------------------ install / uninstall / fallback+agent = fake_agent()+++def untouched(model):+ """The class forward is in use and no instance attribute shadows it."""+ return "forward" not in model.__dict__ and model.forward.__func__ is TinyModel.forward+++be = backends.install(agent, "eager")+check("install(eager)/name", be.name, "eager")+check_true("install(eager)/forward untouched", untouched(agent.model))++with warnings.catch_warnings(record=True) as w:+ warnings.simplefilter("always")+ be = backends.install(agent, "compile") # CPU: compile declines+check("install(compile)/cpu falls back to eager", be.name, "eager")+rw = [x for x in w if issubclass(x.category, RuntimeWarning)]+check("install(compile)/cpu warns exactly once", len(rw), 1)+check_true("install(compile)/warning names the backend and the fallback",+ rw and "'compile'" in str(rw[0].message) and "eager" in str(rw[0].message), [str(x.message) for x in rw])+check_true("install(compile)/forward untouched after fallback", untouched(agent.model))++with warnings.catch_warnings(record=True) as w:+ warnings.simplefilter("always")+ be = backends.install(agent, "tilelang") # CPU and/or no tilelang: declines+check("install(tilelang)/cpu falls back to eager", be.name, "eager")+check("install(tilelang)/cpu warns exactly once", len([x for x in w if issubclass(x.category, RuntimeWarning)]), 1)++try:+ backends.install(agent, "compile", strict=True)+ FAIL.append("install(compile, strict)/cpu did not raise")+except BackendUnavailable:+ PASS.append("install(compile, strict)/cpu raises BackendUnavailable")+check_true("install(strict)/forward untouched after raise", untouched(agent.model))+++class Doubling(Backend):+ """A backend that runs the stock forward and doubles the logits: proves install swaps the forward."""+ name = "doubling"++ def forward(self, *args, **kw):+ logits, act = self.stock_forward(*args, **kw)+ return logits * 2, act+++b = {"input_ids": ids, "attention_mask": att, "marker_pos": mpos, "marker_mask": mmask, "qtype": qt}+ref = agent.model(**b)[0]+d = Doubling(agent)+d.install()+check_true("custom backend/installed forward is used", torch.equal(agent.model(**b)[0], ref * 2))+check_true("custom backend/stock_forward still reaches the model", torch.equal(d.stock_forward(**b)[0], ref))+d.uninstall()+check_true("custom backend/uninstall restores the forward", untouched(agent.model) and torch.equal(agent.model(**b)[0], ref))+d.install(); d.install()+check_true("custom backend/install is idempotent", torch.equal(agent.model(**b)[0], ref * 2))+d.uninstall(); d.uninstall()+check_true("custom backend/uninstall is idempotent", untouched(agent.model))++e = EagerBackend(agent)+check("eager/max_len is unbounded", e.max_len, None)+check("eager/warmup is free", e.warmup(), 0.0)++# ------------------------------------------------------------------ the Agent surface+import inspect # noqa: E402+from laya.agent import Agent, load # noqa: E402++check("Agent.__init__/backend default None", inspect.signature(Agent.__init__).parameters["backend"].default, None)+check("load/backend default None", inspect.signature(load).parameters["backend"].default, None)+check("Agent/backend property default", Agent.__new__(Agent).backend, "eager")+check_true("Agent/set_backend, accelerate, deaccelerate exist",+ all(callable(getattr(Agent, n, None)) for n in ("set_backend", "accelerate", "deaccelerate")))++# ------------------------------------------------------------------ resolved fallback / compile lifecycle+from unittest.mock import patch # noqa: E402+from laya.backends.compile import CompileBackend # noqa: E402++with patch.object(backends, "auto_policy", return_value="compile"), warnings.catch_warnings(record=True) as w:+ warnings.simplefilter("always")+ be = backends.install(fake_agent(), "auto")+check("auto fallback/eager", be.name, "eager")+check("auto fallback/one warning", len(w), 1)+check_true("auto fallback/names resolved compile, not auto",+ len(w) == 1 and "backend 'compile'" in str(w[0].message) and "backend 'auto'" not in str(w[0].message))++cuda_agent = fake_agent("cuda")+cfg = torch.fx.experimental._config+prior_duck = cfg.use_duck_shape++def compile_stub(fn, **kwargs):+ check("compile/reuses dynamic helper", kwargs, {"dynamic": True, "mode": "reduce-overhead"})+ def call(*args):+ check("compile/independent dimensions during forward", cfg.use_duck_shape, False)+ return fn(*args)+ return call++with patch.object(torch, "compile", side_effect=compile_stub):+ be = backends.install(cuda_agent, "compile", strict=True, warmup=False)+ actual = cuda_agent.model(**b)+ check_true("compile/padding sliced away", torch.equal(actual[0], ref))+ check("compile/duck shape restored after forward", cfg.use_duck_shape, prior_duck)+ be.uninstall()+ check_true("compile/uninstall restores stock", untouched(cuda_agent.model))+with patch.object(torch, "compile", side_effect=compile_stub), \+ patch.object(CompileBackend, "warmup", side_effect=RuntimeError("compile failed")), \+ warnings.catch_warnings(record=True) as w:+ warnings.simplefilter("always")+ be = backends.install(cuda_agent, "compile")+check("compile/warmup failure falls back", be.name, "eager")+check_true("compile/warmup failure restores stock", untouched(cuda_agent.model))+check("compile/warmup failure warns once", len(w), 1)++# ------------------------------------------------------------------ forwarding and legacy Agent substitutes+from laya.router import Router # noqa: E402++class LegacyAgent:+ def __init__(self, repo, device=None, token=None, subfolder=None):+ self.repo = repo++with patch("laya.agent.Agent", LegacyAgent):+ router = Router()+ legacy = router._build(next(iter(router.models)))+check_true("Router/default supports constructor without backend", isinstance(legacy, LegacyAgent))+router = Router(agent_kwargs={"backend": "eager"})+with patch("laya.agent.Agent") as ctor:+ router._build(next(iter(router.models)))+check("Router/explicit backend forwarded via agent_kwargs", ctor.call_args.kwargs["backend"], "eager")+with patch("laya.agent.Agent") as ctor:+ load("unused/model")+check_true("load/default does not inject backend", "backend" not in ctor.call_args.kwargs)+with patch("laya.agent.Agent") as ctor:+ load("unused/model", backend="torch")+check("load/explicit backend forwarded", ctor.call_args.kwargs["backend"], "eager")+with patch("laya.onnx_agent.ONNXAgent") as ctor:+ load("unused/model", backend="onnx", onnx_path="local.onnx", token="test", revision="rev",+ expected_sha256={"config.json": "digest"}, hooks_timeout=2, calibration="cal.json",+ lang_temperatures={"en": {"temperature": [1, 1, 1]}})+check("load/ONNX uses existing class", ctor.call_count, 1)+for key, value in (("onnx_path", "local.onnx"), ("token", "test"), ("revision", "rev"),+ ("expected_sha256", {"config.json": "digest"}), ("hooks_timeout", 2),+ ("calibration", "cal.json"), ("lang_temperatures", {"en": {"temperature": [1, 1, 1]}})):+ check("load/ONNX forwards " + key, ctor.call_args.kwargs[key], value)++# ------------------------------------------------------------------ scoped OOM detaches and restores backend+class DeviceModel(TinyModel):+ def to(self, device):+ return self++class FailOnce(Backend):+ name = "test-oom"+ def forward(self, *args, **kwargs):+ raise torch.cuda.OutOfMemoryError("test out of memory")++a = Agent.__new__(Agent)+a.__dict__.update(fake_agent("cuda").__dict__)+a.model = DeviceModel()+held = FailOnce(a)+held.install()+a._backend = held+original_to = torch.Tensor.to++def cpu_inputs(tensor, *args, **kwargs):+ if args and isinstance(args[0], torch.device):+ return tensor+ return original_to(tensor, *args, **kwargs)++with patch.object(torch.Tensor, "to", cpu_inputs), warnings.catch_warnings(record=True):+ actual = a._infer(b)+check_true("OOM/CPU retry reaches stock forward", torch.equal(actual[0], ref))+check("OOM/restores device", a.device.type, "cuda")+check_true("OOM/restores same backend", a.backend_object is held and held.installed)+a.deaccelerate()+check_true("deaccelerate/restores eager", a.backend == "eager" and untouched(a.model))++print("\n%d passed, %d failed" % (len(PASS), len(FAIL)))+for f in FAIL:+ print(" FAIL " + f)+if __name__ == "__main__":+ sys.exit(1 if FAIL else 0)+++def test_backends():+ assert not FAIL, FAILdiff --git a/tests/test_download.py b/tests/test_download.pyindex 3e163d47..c9c4c977 100644--- a/tests/test_download.py+++ b/tests/test_download.py@@ -320,6 +320,24 @@ class DownloadTests(unittest.TestCase):self.assertIsInstance(compiled.model, OptimizedModule)self.assertIsInstance(compiled.model._orig_mod, DecisionModel)+ def test_backend_selection_preserves_legacy_compile_and_can_replace_it(self):+ compiled = load(str(self.repo), device="cpu", compile=True)+ self.assertEqual(compiled.backend, "compile")+ self.assertIsNone(compiled.backend_object)+ self.assertEqual(compiled.set_backend("eager"), "eager")+ self.assertNotIsInstance(compiled.model, OptimizedModule)+ self.assertFalse(compiled.model.encoder.config.reference_compile)+ self.assertEqual(compiled.predict("hello", self.questions)["answers"], self.expected["answers"])++ def test_explicit_backend_takes_precedence_over_legacy_flags(self):+ agent = load(str(self.repo), device="cpu", backend="eager", fast=True, compile=True)+ self.assertEqual(agent.backend, "eager")+ self.assertIsNotNone(agent.backend_object)+ self.assertNotIsInstance(agent.model, OptimizedModule)+ self.assertFalse(agent.model.encoder.config.reference_compile)+ self.assertIsNone(agent._fast)+ self.assertEqual(agent.predict("hello", self.questions)["answers"], self.expected["answers"])+def test_onnx_agent_accepts_every_hub_option_the_agent_does(self):# Both runtimes download the same checkpoint from the same place, so an option that# selects *which* checkpoint, or *how* to authenticate for it, has to exist on both.@@ -327,10 +345,11 @@ class DownloadTests(unittest.TestCase):# copied across rather than silently diverging again.# fast/compile -- the TileLang path and torch.compile, neither of which exists# inside onnxruntime.+ # backend -- selects a PyTorch forward; load() routes ONNX to ONNXAgent.# device -- ONNXAgent picks an execution provider from what onnxruntime# reports and takes no override; a different asymmetry, with its# own fix.- not_for_onnxruntime = {"fast", "compile", "device"}+ not_for_onnxruntime = {"fast", "compile", "device", "backend"}agent_side = (set(inspect.signature(Agent.__init__).parameters)- {"self", "model_id_or_path"} - not_for_onnxruntime)onnx_side = set(inspect.signature(ONNXAgent.__init__).parameters) - {"self"}diff --git a/tests/test_hooks_api.py b/tests/test_hooks_api.pyindex a8aa4957..474d85f9 100644--- a/tests/test_hooks_api.py+++ b/tests/test_hooks_api.py@@ -100,6 +100,13 @@ except ValueError as exc:# load() forwards Agent's own construction options, so none of them is reachable only# through the class; tests/test_download.py asserts that against both signatures.check_param("load", load, "compile", False)+check_param("load", load, "backend", None)+check_param("load", load, "onnx_path", None)+check_param("Agent.__init__", Agent.__init__, "backend", None)+check_param("Agent.set_backend", Agent.set_backend, "name", "auto")+check_param("Agent.set_backend", Agent.set_backend, "strict", False)+check_true("Agent.backend is a property", isinstance(Agent.backend, property))+check_true("Agent.backend_object is a property", isinstance(Agent.backend_object, property))# ONNXAgent downloads the same Hub artifacts as Agent, so it authenticates the same waycheck_param("ONNXAgent.__init__", ONNXAgent.__init__, "token", None)