Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

feat(agent): restore inference backend class layer

cklxxfb4b0371716d
diff --git a/docs/compile-and-fast-path.md b/docs/compile-and-fast-path.md
index 57a9dfb9..17464797 100644
--- a/docs/compile-and-fast-path.md
+++ b/docs/compile-and-fast-path.md
@@ -5,6 +5,36 @@ come from measurements taken while working on #472, #576 and #718, on an RTX 407
torch 2.11 and tilelang 0.1.14. They are here so the next person does not have to measure them
again.
+## Backend selection
+
+`Agent(..., backend="auto")` and `laya.load(..., backend="auto")` opt into the backend
+class layer. The default remains eager. Explicit `backend=` takes precedence over `compile`
+and `fast`; omitting it preserves both flags' existing behaviour.
+
+- `eager`: the stock PyTorch forward, on any supported device.
+- `compile`: CUDA-only `torch.compile` with dynamic shapes and `reduce-overhead` mode,
+ bucket padding, persistent inductor cache, and warmup at installation. It reuses the same
+ independent-dimension scope as `compile=True`, which keeps its existing default mode and
+ CPU support. Set `LAYA_COMPILE_WARMUP=0` to defer backend warmup and `LAYA_INDUCTOR_CACHE_DIR`
+ to choose its cache directory (default `~/.cache/laya/inductor`).
+- `tilelang`: an adapter around the current fast path, using the agent's bf16 or fp16 dtype.
+- `auto`: TileLang on CUDA with a supported ModernBERT encoder and dtype when TileLang is
+ installed, otherwise compile on CUDA; eager on other devices.
+- `onnx`: `laya.load(..., backend="onnx", onnx_path="model.onnx")` returns the existing
+ `ONNXAgent`. Without `onnx_path`, it uses `laya.onnx`.
+
+An unavailable backend emits a `RuntimeWarning` naming the resolved backend and falls back to
+eager. To require a backend, use `agent.set_backend("tilelang", strict=True)`. Switching waits
+for active inference; `agent.backend` reports the active name and `agent.backend_object`
+exposes the installed object. `agent.set_backend("compile", warmup=False)` defers compilation
+until inference, so compilation errors then surface on the request. `agent.warmup()` remains
+available. `agent.deaccelerate()` removes a backend installed through the class layer.
+
+Routers forward an explicit selection through `Router(agent_kwargs={"backend": "auto"})`.
+They pass no backend argument by default, preserving compatibility with existing Agent-like
+constructors. Scoped CPU OOM retries detach the backend and restore it when the model returns
+to its original device.
+
## `compile=True` materialises the attention mask
Eager SDPA takes ModernBERT's `(rows, 1, L, L)` attention mask as a broadcast view. Under the
diff --git a/laya/agent.py b/laya/agent.py
index 83b18da4..0f5c8647 100644
--- a/laya/agent.py
+++ b/laya/agent.py
@@ -465,6 +465,7 @@ class Agent(HookRegistry):
_amp_failures = 0
# The stock forward is also used by lightweight runtimes built with __new__ in tests.
_fast = None
+ _backend = None
# Scoped CPU-fallback observability: how often _infer's per-request OOM fallback fired
# and what the last failure was, so an operator sees a slow lane in /health instead of
# discovering it by accident. Class defaults cover instances built without __init__.
@@ -492,9 +493,14 @@ class Agent(HookRegistry):
hooks_concurrent: bool = True,
hooks_timeout: Optional[float] = None,
calibration: Optional[str] = None,
+ backend: Optional[str] = None,
):
"""Load a Laya checkpoint.
+ `backend` selects "eager", "auto", "compile" or "tilelang"; see `laya.backends`.
+ It takes precedence over `fast` and `compile`. Omit it to keep those legacy flags.
+ For ONNX use `load(backend="onnx")` instead.
+
`revision` optionally pins the Hub download to an explicit commit SHA/branch/tag;
when omitted, huggingface_hub's normal default and existing offline cache are used.
`expected_sha256` ({path relative to the checkpoint dir: hexdigest})
@@ -524,6 +530,13 @@ class Agent(HookRegistry):
`hooks_concurrent=False` serialises hooks that are not safe to run in parallel, and
`hooks_timeout` bounds each hook call in seconds (None means no limit).
"""
+ if backend is not None:
+ from .backends import normalise
+ backend = normalise(backend)
+ if backend == "onnx":
+ raise ValueError("use laya.load(..., backend='onnx') or ONNXAgent")
+ fast = compile = False
+
self.hooks = normalise_hooks(hooks, on_predict_start, on_predict_end)
self.hooks_raise = bool(hooks_raise)
self.hooks_concurrent = bool(hooks_concurrent)
@@ -741,7 +754,9 @@ class Agent(HookRegistry):
else:
raise e
- if fast:
+ if backend is not None:
+ self.set_backend(backend)
+ elif fast:
self.accelerate()
if fell_back_from is not None:
@@ -755,6 +770,34 @@ class Agent(HookRegistry):
" See https://pytorch.org/get-started/locally/\n"
% (fell_back_from, fell_back_why), RuntimeWarning)
+ @property
+ def backend(self) -> str:
+ """The active inference backend, including the legacy compile and fast flags."""
+ if self._backend is not None:
+ return self._backend.name
+ return "tilelang" if self._fast is not None else "compile" if self._compiled else "eager"
+
+ @property
+ def backend_object(self):
+ """The installed Backend object, or None for a legacy runtime."""
+ return self._backend
+
+ def set_backend(self, name: str = "auto", strict: bool = False, **options) -> str:
+ """Switch backends; unavailable backends warn and use eager unless `strict=True`.
+
+ Options go to the backend constructor, e.g. `warmup=False` for compile or
+ `use_graphs=False` for tilelang. Switching waits for in-flight inference.
+ """
+ from . import backends
+ name = backends.normalise(name)
+ with self._gate.write_lock():
+ self.deaccelerate()
+ if self._compiled:
+ self.model = self.model._orig_mod
+ self._compiled = False
+ self.model.encoder.config.reference_compile = False
+ return backends.install(self, name, strict=strict, **options).name
+
def accelerate(self, use_graphs: bool = True, strict: bool = False):
"""Replace the model forward with the TileLang fast path (fused GEMM/GEGLU/LayerNorm/RoPE kernels,
sliding-window flash attention, 16-bit resident weights, CUDA graphs per shape bucket).
@@ -770,6 +813,8 @@ class Agent(HookRegistry):
if strict:
raise RuntimeError("laya fast path needs a CUDA device")
return False
+ if self._backend is not None:
+ self.deaccelerate()
last = None
for _attempt in range(2): # tilelang's JIT cache has been seen to fail once, then succeed
try:
@@ -823,7 +868,11 @@ class Agent(HookRegistry):
def deaccelerate(self):
"""Restore the stock forward."""
- if self._fast is not None:
+ if self._backend is not None:
+ self._backend.uninstall()
+ self._backend = None
+ self._fast = None
+ elif self._fast is not None:
self.model.forward = self._stock_forward
self._fast = None
@@ -1093,6 +1142,10 @@ class Agent(HookRegistry):
b = _pad_cuda_compile_batch(b, self.tok.pad_token_id)
use_amp = self._amp_enabled_for(b["input_ids"].shape[0])
+ limit = getattr(self._backend, "max_len", None)
+ if limit is not None and b["input_ids"].shape[1] > limit:
+ raise ValueError("the %s backend was built for max_len=%d; this request needs %d tokens"
+ % (self.backend, limit, b["input_ids"].shape[1]))
if self._fast is not None and b["input_ids"].shape[1] > self._fast.max_len:
raise ValueError(
"the CUDA fast path was built for max_len=%d; this request needs %d tokens. "
@@ -1140,7 +1193,8 @@ class Agent(HookRegistry):
if self.device.type == "cpu":
return run()
held_device, held_dtype, held_amp = self.device, self.dtype, self.amp_enabled
- had_fast = self._fast is not None
+ held_backend = self._backend
+ had_fast = self._fast is not None and held_backend is None
# Only our batch scope can retain BF16 copies after this failed forward.
# Release them before moving the model and retrying on CPU.
if self.device.type == "cuda" and _BATCH_AUTOCAST_CACHE.get():
@@ -1159,6 +1213,15 @@ class Agent(HookRegistry):
return run()
finally:
self._restore_runtime(held_device, held_dtype, held_amp, had_fast)
+ if held_backend is not None and self.device == held_device:
+ from .backends import BackendUnavailable, warn_fallback
+ try:
+ held_backend.install()
+ except BackendUnavailable as exc:
+ warn_fallback(held_backend.name, exc)
+ else:
+ self._backend = held_backend
+ self._fast = getattr(held_backend, "fast", None)
if use_amp and self.device.type in ("mps", "cpu"):
# Not every MPS/CPU build implements autocast for every op. Retry this
# request in full precision. One miss must not turn AMP off; a build that
@@ -1352,7 +1415,7 @@ class Agent(HookRegistry):
# caller already owns a CUDA autocast scope, its cache is sufficient.
if (chunk < len(states) and getattr(self, "device", None) is not None
and self.device.type == "cuda" and self.amp_enabled
- and self._fast is None and not self._compiled
+ and self._fast is None and not self._compiled and self.backend != "compile"
and not torch.is_autocast_enabled()):
amp_stack.enter_context(torch.autocast(device_type="cuda", dtype=self.dtype,
enabled=False))
@@ -1907,7 +1970,8 @@ def load(model_id_or_path: str = "convaiinnovations/laya", device: Optional[str]
hooks=None, on_predict_start=None, on_predict_end=None,
hooks_raise: bool = True, hooks_concurrent: bool = True,
hooks_timeout: Optional[float] = None,
- calibration: Optional[str] = None) -> Agent:
+ calibration: Optional[str] = None, backend: Optional[str] = None,
+ onnx_path: Optional[str] = None):
"""Load a Laya agent.
`subfolder` picks one checkpoint out of a repo that bundles several:
@@ -1925,6 +1989,10 @@ def load(model_id_or_path: str = "convaiinnovations/laya", device: Optional[str]
Anything else (a Hub repo id, a local directory) is passed to `Agent` unchanged.
+ `backend` selects "auto", "eager", "compile", "tilelang" or "onnx". ONNX returns
+ the existing `ONNXAgent`, with `onnx_path` (default "laya.onnx").
+ Other backends use `Agent`; an explicit backend takes precedence over the legacy flags.
+
`revision`/`expected_sha256` pin and verify the downloaded artifacts; see `Agent`.
`hooks` / `on_predict_start` / `on_predict_end` observe or shape every prediction; see
`laya.hooks`. `calibration` is the same optional JSON path accepted by `Agent`.
@@ -1942,10 +2010,21 @@ def load(model_id_or_path: str = "convaiinnovations/laya", device: Optional[str]
spec = resolve_model_spec(model_id_or_path)
if spec is not None:
model_id_or_path, subfolder = spec
+ if backend is not None:
+ from .backends import normalise
+ backend = normalise(backend)
+ if backend == "onnx":
+ from .onnx_agent import ONNXAgent
+ return ONNXAgent(model_id_or_path, onnx_path=onnx_path or "laya.onnx",
+ token=token, subfolder=subfolder, revision=revision, expected_sha256=expected_sha256,
+ lang_temperatures=lang_temperatures, calibration=calibration,
+ hooks=hooks, on_predict_start=on_predict_start, on_predict_end=on_predict_end,
+ hooks_raise=hooks_raise, hooks_concurrent=hooks_concurrent, hooks_timeout=hooks_timeout)
+ options = {"backend": backend} if backend is not None else {}
return Agent(model_id_or_path, device=device, token=token, subfolder=subfolder, fast=fast,
compile=compile,
revision=revision, expected_sha256=expected_sha256,
lang_temperatures=lang_temperatures,
hooks=hooks, on_predict_start=on_predict_start, on_predict_end=on_predict_end,
hooks_raise=hooks_raise, hooks_concurrent=hooks_concurrent,
- hooks_timeout=hooks_timeout, calibration=calibration)
+ hooks_timeout=hooks_timeout, calibration=calibration, **options)
diff --git a/laya/backends/__init__.py b/laya/backends/__init__.py
new file mode 100644
index 00000000..08f1b116
--- /dev/null
+++ b/laya/backends/__init__.py
@@ -0,0 +1,109 @@
+"""Inference backends: one contract, several ways to run `DecisionModel.forward`.
+
+ laya.load(..., backend="auto" | "eager" | "compile" | "tilelang" | "onnx")
+
+| backend | what it is | needs |
+|------------|-------------------------------------------------------------------|--------------------------|
+| `eager` | the stock PyTorch forward under autocast (the default) | nothing |
+| `compile` | `torch.compile(dynamic=True, mode="reduce-overhead")` + warm-up | CUDA |
+| `tilelang` | fused TileLang kernels, 16-bit weights, one CUDA graph per bucket | CUDA, `laya[fast]`, bf16/fp16 |
+| `onnx` | ONNX Runtime via `laya.onnx_agent.ONNXAgent` (the explicit CPU option, its own class) | `laya[onnx]`, an exported model |
+
+`auto` picks per device: CUDA -> `tilelang` when tilelang imports and the encoder is a
+ModernBERT-family model (the shipped checkpoints), else `compile`; CPU and MPS -> `eager`.
+Every backend honours `agent.dtype`, and one that cannot run falls back to `eager` with one
+`RuntimeWarning`; `strict=True` raises instead. See `laya.backends.base` for the contract and
+the padding policy.
+"""
+from .base import Backend, BackendUnavailable, bucket_shape, pad_batch, warn_fallback
+
+NAMES = ("auto", "eager", "compile", "tilelang", "onnx")
+AUTO = "auto"
+
+
+def normalise(name):
+ """Accept the documented spellings; anything else is a ValueError naming the choices."""
+ key = str(name).strip().lower().replace("_", "-") if name is not None else "eager"
+ aliases = {"fast": "tilelang", "torch": "eager", "torch.compile": "compile", "inductor": "compile"}
+ key = aliases.get(key, key)
+ if key not in NAMES:
+ raise ValueError("unknown backend %r; use one of %s" % (name, ", ".join(NAMES)))
+ return key
+
+
+def tilelang_available() -> bool:
+ try:
+ import tilelang # noqa: F401
+ return True
+ except Exception:
+ return False
+
+
+def is_modernbert(model) -> bool:
+ cfg = getattr(getattr(model, "encoder", None), "config", None)
+ return getattr(cfg, "model_type", None) == "modernbert"
+
+
+def auto_policy(agent) -> str:
+ """The backend `auto` resolves to for this agent (device, dtype, encoder, installed extras)."""
+ if agent.device.type == "cuda":
+ import torch
+ if tilelang_available() and is_modernbert(agent.model) and agent.dtype in (torch.bfloat16, torch.float16):
+ return "tilelang"
+ return "compile"
+ return "eager"
+
+
+def make_backend(name: str, agent, **options) -> Backend:
+ """Build (not install) the backend `name` for `agent`. `auto` is resolved here."""
+ key = normalise(name)
+ if key == AUTO:
+ key = auto_policy(agent)
+ if key == "eager":
+ from .eager import EagerBackend
+ return EagerBackend(agent, **options)
+ if key == "compile":
+ from .compile import CompileBackend
+ return CompileBackend(agent, **options)
+ if key == "tilelang":
+ try:
+ from .tilelang import TileLangBackend
+ except ImportError as e:
+ raise BackendUnavailable("tilelang is not installed (pip install laya[fast]): %s" % e)
+ return TileLangBackend(agent, **options)
+ raise BackendUnavailable("backend %r is not a forward replacement; use laya.load(..., backend='onnx') "
+ "or laya.onnx_agent.ONNXAgent directly" % key)
+
+
+def install(agent, name: str, strict: bool = False, **options) -> Backend:
+ """Install `name` on `agent`, falling back to eager with one warning unless `strict`.
+
+ Returns the backend that ended up installed. The agent's previous backend is uninstalled
+ first, so switching is always eager -> new and never stacks two replacements.
+ """
+ from .eager import EagerBackend
+
+ key = normalise(name)
+ if key == AUTO:
+ key = auto_policy(agent)
+ current = getattr(agent, "_backend", None)
+ if current is not None:
+ current.uninstall()
+ agent._backend = None
+ agent._fast = None
+ try:
+ backend = make_backend(key, agent, **options)
+ backend.install()
+ except BackendUnavailable as e:
+ if strict:
+ raise
+ warn_fallback(key, e)
+ backend = EagerBackend(agent)
+ backend.install()
+ agent._backend = backend
+ agent._fast = getattr(backend, "fast", None)
+ return backend
+
+
+__all__ = ["Backend", "BackendUnavailable", "NAMES", "auto_policy", "bucket_shape",
+ "install", "make_backend", "normalise", "pad_batch", "warn_fallback"]
diff --git a/laya/backends/base.py b/laya/backends/base.py
new file mode 100644
index 00000000..5a875843
--- /dev/null
+++ b/laya/backends/base.py
@@ -0,0 +1,149 @@
+"""The inference backend contract and padding helpers.
+
+A backend owns one thing: how `DecisionModel.forward` is executed. The Agent keeps tokenization,
+collation, temperature scaling and decoding; a backend is handed the collated batch on the agent's
+device and returns `(logits, act_logits)` with the same shapes and semantics the stock forward has.
+
+ backend = make_backend("compile", agent) # laya.backends
+ backend.install() # swaps agent.model.forward; may raise
+ ...
+ backend.uninstall() # restores the stock forward
+
+Padding helpers bound the shapes seen by graph backends. Dtype and autocast policy remain
+owned by Agent; a backend honours the dtype already selected there.
+"""
+import warnings
+from typing import Any, Optional, Tuple
+
+import torch
+
+# ---------------------------------------------------------------------------------- padding policy
+DYNAMIC_MAX_L = 256 # up to here a dynamic-shape kernel is as fast as a static one: 16-token buckets
+LONG_BUCKET = 64 # beyond it, 64-token buckets
+
+
+def bucket_rows(n: int) -> int:
+ """Rows are padded to the next power of two (1, 2, 4, ... questions per call)."""
+ return 1 << (max(1, n) - 1).bit_length()
+
+
+def bucket_tokens(n_tokens: int, max_len: int) -> int:
+ """Tokens are padded to a 16-token bucket up to `DYNAMIC_MAX_L`, 64-token buckets beyond, capped
+ at `max_len` (the checkpoint's context, which the tokenizer never exceeds)."""
+ g = 16 if n_tokens <= DYNAMIC_MAX_L else LONG_BUCKET
+ return min(max_len, ((n_tokens + g - 1) // g) * g)
+
+
+def bucket_shape(n_rows: int, n_tokens: int, max_len: int) -> Tuple[int, int]:
+ return bucket_rows(n_rows), bucket_tokens(n_tokens, max_len)
+
+
+def pad_batch(input_ids, attention_mask, marker_pos, marker_mask, qtype, max_len: int, pad_id: int = 0,
+ keep_one_token: bool = False):
+ """Pad a collated batch to its `bucket_shape`. Returns the padded tensors and `(N, L0)` to slice back.
+
+ Padding rows carry `pad_id` tokens, an empty attention mask, markers at position 0 and qtype 0,
+ so a backend that reads only the first `N` rows of its output sees exactly the request.
+ `keep_one_token=True` marks token 0 of a padding row as valid instead: SDPA/eager attention
+ turns an all-masked row into NaN, and although those rows are sliced away, a NaN in the batch
+ is enough to poison a CUDA-graph replay under `torch.compile`; the TileLang attention kernel
+ keeps fully masked rows finite by construction and does not need it.
+ """
+ N, L0 = input_ids.shape
+ B, L = bucket_shape(N, L0, max_len)
+ dev = input_ids.device
+ ids = torch.full((B, L), pad_id, dtype=input_ids.dtype, device=dev)
+ ids[:N, :L0] = input_ids
+ att = torch.zeros((B, L), dtype=attention_mask.dtype, device=dev)
+ att[:N, :L0] = attention_mask
+ if keep_one_token and B > N:
+ att[N:, 0] = 1
+ mpos = torch.zeros((B, marker_pos.shape[1]), dtype=marker_pos.dtype, device=dev)
+ mpos[:N] = marker_pos
+ mmask = torch.zeros((B, marker_mask.shape[1]), dtype=marker_mask.dtype, device=dev)
+ mmask[:N] = marker_mask
+ # a padding row keeps one live marker so its softmax/entropy stay finite (the stock head
+ # clamps k to 2 and masks the rest to -1e4; an all-masked row is a uniform softmax, fine too)
+ if B > N:
+ mmask[N:, 0] = True
+ qt = torch.zeros((B,), dtype=qtype.dtype, device=dev)
+ qt[:N] = qtype
+ return ids, att, mpos, mmask, qt, N, L0
+
+
+# ------------------------------------------------------------------------------------- the contract
+class BackendUnavailable(RuntimeError):
+ """The backend cannot run here (no CUDA, dependency missing, dtype unsupported): fall back."""
+
+
+class Backend:
+ """Base class. Subclasses set `name`, and override `_install`/`_uninstall`/`forward`.
+
+ `max_len` is the longest sequence the backend accepts once installed, or None for no limit
+ beyond the checkpoint's; `Agent._infer` reports a request past it with a clear message.
+ """
+
+ name = "eager"
+ max_len: Optional[int] = None
+
+ def __init__(self, agent):
+ self.agent = agent
+ self.model = agent.model
+ self.installed = False
+ self._stock_forward = None
+ self._prior_forward = None
+
+ # -- lifecycle
+ def install(self) -> None:
+ """Swap `model.forward` for this backend's. Raises `BackendUnavailable` when it cannot run
+ here (the caller then falls back to eager with one warning) and leaves the model untouched."""
+ if self.installed:
+ return
+ # `forward` normally comes from the class; another replacement (an instance attribute) is
+ # restored as it was, so backends never stack and uninstall leaves no trace.
+ self._prior_forward = self.model.__dict__.get("forward")
+ self._stock_forward = self.model.forward
+ self._install()
+ self.model.forward = self.forward
+ self.installed = True
+
+ def uninstall(self) -> None:
+ if not self.installed:
+ return
+ if self._prior_forward is None:
+ self.model.__dict__.pop("forward", None)
+ else:
+ self.model.forward = self._prior_forward
+ self._stock_forward = self._prior_forward = None
+ self._uninstall()
+ self.installed = False
+
+ def _install(self) -> None:
+ pass
+
+ def _uninstall(self) -> None:
+ pass
+
+ def warmup(self, shapes=None) -> float:
+ """Run representative shapes through the backend so the first real call is not the slow one.
+ Returns seconds spent. The default backend has nothing to warm."""
+ return 0.0
+
+ # -- the forward itself
+ def stock_forward(self, *args, **kwargs):
+ """The model's own forward, whatever backend is installed."""
+ fn = self._stock_forward if self._stock_forward is not None else type(self.model).forward.__get__(self.model)
+ return fn(*args, **kwargs)
+
+ def forward(self, input_ids, attention_mask, marker_pos, marker_mask, qtype, detach_encoder=False):
+ return self.stock_forward(input_ids, attention_mask, marker_pos, marker_mask, qtype, detach_encoder)
+
+ def __repr__(self):
+ return "%s(installed=%s)" % (type(self).__name__, self.installed)
+
+
+def warn_fallback(name: str, why: Any) -> None:
+ """The one warning a caller sees when a requested backend cannot be used."""
+ warnings.warn(
+ "laya: backend %r is unavailable (%s); using the eager forward instead." % (name, why),
+ RuntimeWarning, stacklevel=3)
diff --git a/laya/backends/compile.py b/laya/backends/compile.py
new file mode 100644
index 00000000..7b45e336
--- /dev/null
+++ b/laya/backends/compile.py
@@ -0,0 +1,138 @@
+"""The `torch.compile` backend: inductor kernels + CUDA graphs, no extra dependency.
+
+ agent = laya.load("convaiinnovations/laya", backend="compile")
+
+`torch.compile(model.forward, dynamic=True, mode="reduce-overhead")` uses the independent
+input dimensions from `laya._compile`. Torch still specialises 0/1 dimensions. Requests are
+padded to the bucket policy (`laya.backends.base.bucket_shape`) to limit CUDA-graph shapes,
+and outputs are copied out of graph buffers before the lock is released.
+
+Cold start is the cost: the first compile takes tens of seconds. Two things cut it:
+
+* the inductor FX-graph cache, persisted in a stable directory. Set `LAYA_INDUCTOR_CACHE_DIR`
+ (default `~/.cache/laya/inductor`); it is applied to `TORCHINDUCTOR_CACHE_DIR` unless that is
+ already set, so a `TORCHINDUCTOR_*` variable the operator exports still wins. A second process
+ loads the compiled graphs from there instead of compiling them again.
+* warm-up at load (`warmup=True`, or `LAYA_COMPILE_WARMUP=0` to skip): a few representative
+ shapes are run at load time so the first request pays nothing. `agent.backend_object.warmup_s`
+ reports what it cost.
+
+`torch.compile` is only used on CUDA: on CPU the inductor C++ path needs a compiler and buys
+little at Laya's batch sizes, and MPS is unsupported, so those raise `BackendUnavailable`.
+"""
+import os
+import threading
+import time
+from typing import Optional
+
+import torch
+
+from .._compile import compile_model, independent_dims
+from .base import Backend, BackendUnavailable, pad_batch
+
+CACHE_DIR_ENV = "LAYA_INDUCTOR_CACHE_DIR"
+WARMUP_ENV = "LAYA_COMPILE_WARMUP"
+DEFAULT_CACHE_DIR = os.path.join("~", ".cache", "laya", "inductor")
+# rows x tokens x markers run at load: one short call, a few questions at a medium length, and a
+# bigger batch; the three sizes differ within each shape so no two dimensions get tied together
+WARMUP_SHAPES = ((1, 64, 3), (4, 128, 5), (8, 256, 6))
+
+
+def configure_inductor_cache() -> str:
+ """Point inductor's on-disk caches at a stable directory and switch the FX-graph cache on.
+
+ Returns the directory in use. Inductor reads `TORCHINDUCTOR_CACHE_DIR` when it opens the
+ cache, so this must run before the first compile; an explicit `TORCHINDUCTOR_CACHE_DIR` from
+ the environment is left alone.
+ """
+ cache_dir = os.path.expanduser(os.environ.get(CACHE_DIR_ENV) or DEFAULT_CACHE_DIR)
+ os.environ.setdefault("TORCHINDUCTOR_CACHE_DIR", cache_dir)
+ os.environ.setdefault("TORCHINDUCTOR_FX_GRAPH_CACHE", "1")
+ os.environ.setdefault("TORCHINDUCTOR_AUTOGRAD_CACHE", "1")
+ try:
+ import torch._inductor.config as icfg
+ icfg.fx_graph_cache = True
+ except Exception:
+ pass
+ return os.environ["TORCHINDUCTOR_CACHE_DIR"]
+
+
+def warmup_enabled(default: bool = True) -> bool:
+ raw = os.environ.get(WARMUP_ENV)
+ if raw is None:
+ return default
+ return raw.strip().lower() not in ("0", "false", "no", "off")
+
+
+class CompileBackend(Backend):
+ name = "compile"
+
+ def __init__(self, agent, mode: str = "reduce-overhead",
+ warmup: Optional[bool] = None):
+ super().__init__(agent)
+ self.mode = mode
+ self.do_warmup = warmup_enabled(True) if warmup is None else bool(warmup)
+ self.compiled = None
+ self.cache_dir = None
+ self.warmup_s = 0.0
+ # CUDA graphs replay into static buffers: keep one forward in flight at a time.
+ self._lock = threading.RLock()
+
+ def _install(self):
+ agent = self.agent
+ if agent.device.type != "cuda":
+ raise BackendUnavailable("torch.compile is used on CUDA only here; %s runs eager" % agent.device.type)
+ if not hasattr(torch, "compile"):
+ raise BackendUnavailable("this torch has no torch.compile")
+ self.cache_dir = configure_inductor_cache()
+ self.max_len = int(agent.cfg.get("max_len", 512))
+ try:
+ self.compiled = compile_model(self._stock_forward, mode=self.mode)
+ except Exception as e:
+ raise BackendUnavailable(e)
+
+ def _uninstall(self):
+ self.compiled = None
+
+ def install(self):
+ if self.installed:
+ return
+ super().install()
+ if self.do_warmup:
+ try:
+ self.warmup_s = self.warmup()
+ except Exception as e:
+ # A compile failure surfaces here rather than on the first request.
+ self.uninstall()
+ raise BackendUnavailable(e)
+
+ @torch.no_grad()
+ def forward(self, input_ids, attention_mask, marker_pos, marker_mask, qtype, detach_encoder=False):
+ ids, att, mpos, mmask, qt, N, _ = pad_batch(
+ input_ids, attention_mask, marker_pos, marker_mask, qtype, self.max_len,
+ pad_id=self.agent.tok.pad_token_id or 0, keep_one_token=True)
+ with self._lock, independent_dims():
+ logits, act = self.compiled(ids, att, mpos, mmask, qt, detach_encoder)
+ # copy out of the CUDA graph's static output buffers before the next replay
+ return logits[:N].clone(), act[:N].clone()
+
+ def warmup(self, shapes=None):
+ agent = self.agent
+ dev = agent.device
+ t0 = time.perf_counter()
+ with torch.no_grad(), torch.autocast(device_type="cuda", dtype=agent.dtype, enabled=agent.amp_enabled):
+ for rows, tokens, markers in shapes or WARMUP_SHAPES:
+ tokens = min(tokens, self.max_len)
+ if tokens <= markers:
+ raise ValueError("warmup needs more tokens than markers")
+ ids = torch.full((rows, tokens), agent.tok.pad_token_id or 0, dtype=torch.long, device=dev)
+ ids[:, 0] = agent.tok.cls_token_id if agent.tok.cls_token_id is not None else 0
+ att = torch.ones((rows, tokens), dtype=torch.long, device=dev)
+ mpos = torch.arange(1, markers + 1, device=dev).repeat(rows, 1)
+ mmask = torch.ones((rows, markers), dtype=torch.bool, device=dev)
+ qt = torch.zeros((rows,), dtype=torch.long, device=dev)
+ # cudagraph trees record on the second call of a shape and replay from the third
+ for _ in range(3):
+ self.forward(ids, att, mpos, mmask, qt)
+ torch.cuda.synchronize(dev)
+ return time.perf_counter() - t0
diff --git a/laya/backends/eager.py b/laya/backends/eager.py
new file mode 100644
index 00000000..c8851384
--- /dev/null
+++ b/laya/backends/eager.py
@@ -0,0 +1,14 @@
+"""The stock PyTorch forward: no padding, no graphs, the reference every other backend is measured against."""
+from .base import Backend
+
+
+class EagerBackend(Backend):
+ name = "eager"
+
+ def install(self) -> None:
+ # Nothing to swap: `model.forward` is already the eager forward. Marking it installed keeps
+ # the lifecycle uniform for the Agent.
+ self.installed = True
+
+ def uninstall(self) -> None:
+ self.installed = False
diff --git a/laya/backends/tilelang.py b/laya/backends/tilelang.py
new file mode 100644
index 00000000..6191effb
--- /dev/null
+++ b/laya/backends/tilelang.py
@@ -0,0 +1,45 @@
+"""Backend adapter for the current TileLang fast path."""
+import torch
+
+from .base import Backend, BackendUnavailable
+
+
+class TileLangBackend(Backend):
+ name = "tilelang"
+
+ def __init__(self, agent, use_graphs=True, verbose=False):
+ super().__init__(agent)
+ self.use_graphs = use_graphs
+ self.verbose = verbose
+ self.fast = None
+
+ def _install(self):
+ agent = self.agent
+ if agent.device.type != "cuda":
+ raise BackendUnavailable("needs a CUDA device")
+ if agent.dtype not in (torch.bfloat16, torch.float16):
+ raise BackendUnavailable("the TileLang kernels need bf16 or fp16 and the agent runs %s" % (agent.dtype,))
+ if getattr(getattr(self.model.encoder, "config", None), "model_type", None) != "modernbert":
+ raise BackendUnavailable("the kernels implement a ModernBERT-family encoder, got %r"
+ % getattr(self.model.encoder.config, "model_type", None))
+ last = None
+ for _attempt in range(2): # tilelang's JIT cache has been seen to fail once, then succeed
+ try:
+ from ..fast import FastLaya
+ self.fast = FastLaya(self.model, max_len=agent.cfg.get("max_len", 512),
+ use_graphs=self.use_graphs, verbose=self.verbose, dtype=agent.dtype)
+ break
+ except Exception as e: # tilelang missing / unsupported arch
+ last = e
+ if self.fast is None:
+ raise BackendUnavailable(last)
+ self.max_len = self.fast.max_len
+
+ def _uninstall(self):
+ self.fast = None
+
+ def forward(self, input_ids, attention_mask, marker_pos, marker_mask, qtype, detach_encoder=False):
+ return self.fast.forward(input_ids, attention_mask, marker_pos, marker_mask, qtype, detach_encoder)
+
+ def warmup(self, shapes=None):
+ return self.agent.warmup(shapes)
diff --git a/laya/fast.py b/laya/fast.py
index 6597c592..1285fe1e 100644
--- a/laya/fast.py
+++ b/laya/fast.py
@@ -8,6 +8,7 @@ import sys
import threading
import torch
from . import tl_kernels as K
+from .backends.base import bucket_rows
_KERNEL_DTYPES = {torch.bfloat16: "bfloat16", torch.float16: "float16"}
@@ -21,7 +22,7 @@ def _top_two(probs):
def _bucket_n(n):
- return 1 << max(0, (n - 1).bit_length())
+ return bucket_rows(n)
class FastLaya:
diff --git a/scripts/test_suites.py b/scripts/test_suites.py
index ffe1ab20..43b3efd3 100644
--- a/scripts/test_suites.py
+++ b/scripts/test_suites.py
@@ -59,6 +59,7 @@ SCRIPT_SUITES = [
"tests/test_question_token_reuse.py",
"tests/test_lazy_import.py",
"tests/test_runtime_fixes.py",
+ "tests/test_backends.py",
"tests/test_fast_head_partition.py",
"tests/test_agent_gate.py",
"tests/test_option_collapse.py",
diff --git a/tests/test_backends.py b/tests/test_backends.py
new file mode 100644
index 00000000..b76442f6
--- /dev/null
+++ b/tests/test_backends.py
@@ -0,0 +1,276 @@
+"""The inference backend layer (`laya.backends`), checked without a GPU or a checkpoint.
+
+What is pinned here is the contract every backend is held to and the parts the Agent relies on:
+the names and aliases `load(backend=...)` accepts, the `auto` policy off CUDA, the shared padding
+policy both graph backends use, and that a backend which cannot run here falls back to eager with
+exactly one warning (or raises under `strict=True`) and leaves the model's forward untouched.
+
+Run: python tests/test_backends.py (or under pytest)
+"""
+import os
+import sys
+import warnings
+from types import SimpleNamespace
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+import torch # noqa: E402
+
+from laya import backends # noqa: E402
+from laya.backends import BackendUnavailable, bucket_shape, pad_batch # noqa: E402
+from laya.backends.base import Backend, bucket_rows # noqa: E402
+from laya.backends.eager import EagerBackend # noqa: E402
+
+PASS, FAIL = [], []
+
+
+def check(name, got, want):
+ (PASS if got == want else FAIL).append(name if got == want else "%s: got %r, want %r" % (name, got, want))
+
+
+def check_true(name, cond, detail=""):
+ (PASS if cond else FAIL).append(name if cond else "%s: %s" % (name, detail))
+
+
+class TinyModel(torch.nn.Module):
+ """A forward with the DecisionModel signature, so a backend can be installed on it."""
+
+ def __init__(self):
+ super().__init__()
+ self.encoder = SimpleNamespace(config=SimpleNamespace(model_type="bert", reference_compile=None))
+ self.lin = torch.nn.Linear(4, 4)
+
+ def forward(self, input_ids, attention_mask, marker_pos, marker_mask, qtype, detach_encoder=False):
+ n, k = marker_mask.shape
+ return torch.zeros(n, k) + attention_mask.sum(1, keepdim=True).float(), torch.zeros(n, 2)
+
+
+def fake_agent(device="cpu", dtype=torch.float32):
+ return SimpleNamespace(model=TinyModel(), device=torch.device(device), dtype=dtype, amp_enabled=False,
+ cfg={"max_len": 64}, tok=SimpleNamespace(pad_token_id=0, cls_token_id=1), _backend=None)
+
+
+# ------------------------------------------------------------------ names and aliases
+for raw, want in (("eager", "eager"), ("Eager", "eager"), ("fast", "tilelang"), ("TileLang", "tilelang"),
+ ("torch.compile", "compile"), ("inductor", "compile"), ("auto", "auto"), ("onnx", "onnx"), (None, "eager")):
+ check("normalise(%r)" % (raw,), backends.normalise(raw), want)
+try:
+ backends.normalise("cuda-graphs")
+ FAIL.append("normalise/unknown name accepted")
+except ValueError as e:
+ check_true("normalise/unknown name names the choices", "eager" in str(e) and "tilelang" in str(e), str(e))
+
+# ------------------------------------------------------------------ auto policy off CUDA
+check("auto_policy/cpu -> eager", backends.auto_policy(fake_agent("cpu")), "eager")
+check("make_backend(auto)/cpu is eager", backends.make_backend("auto", fake_agent("cpu")).name, "eager")
+
+# ------------------------------------------------------------------ padding policy
+check("bucket_rows/zero", bucket_rows(0), 1)
+for (n, L, max_len), want in (((1, 5, 512), (1, 16)), ((3, 70, 512), (4, 80)), ((5, 256, 512), (8, 256)),
+ ((2, 257, 512), (2, 320)), ((1, 1000, 512), (1, 512)), ((16, 16, 512), (16, 16))):
+ check("bucket_shape(%d, %d, %d)" % (n, L, max_len), bucket_shape(n, L, max_len), want)
+
+ids = torch.tensor([[7, 8, 9], [7, 8, 0]])
+att = torch.tensor([[1, 1, 1], [1, 1, 0]])
+mpos = torch.tensor([[1, 2], [1, 0]])
+mmask = torch.tensor([[True, True], [True, False]])
+qt = torch.tensor([0, 2])
+p_ids, p_att, p_mpos, p_mmask, p_qt, N, L0 = pad_batch(ids, att, mpos, mmask, qt, max_len=64, pad_id=5)
+check("pad_batch/shape", tuple(p_ids.shape), (2, 16))
+check("pad_batch/(N, L0)", (N, L0), (2, 3))
+check_true("pad_batch/request kept", torch.equal(p_ids[:2, :3], ids) and torch.equal(p_att[:2, :3], att)
+ and torch.equal(p_qt[:2], qt) and torch.equal(p_mmask[:2], mmask))
+check("pad_batch/pad id fills", int(p_ids[0, 3]), 5)
+check("pad_batch/padding tokens masked", int(p_att[:, 3:].sum()), 0)
+p3 = pad_batch(ids[:1], att[:1], mpos[:1], mmask[:1], qt[:1], max_len=64)
+check("pad_batch/one row stays one row", tuple(p3[0].shape), (1, 16))
+p5 = pad_batch(ids.repeat(3, 1)[:5], att.repeat(3, 1)[:5], mpos.repeat(3, 1)[:5], mmask.repeat(3, 1)[:5], qt.repeat(3)[:5], 64,
+ keep_one_token=True)
+check("pad_batch/five rows pad to eight", tuple(p5[0].shape), (8, 16))
+check("pad_batch/padding rows have empty masks", int(p5[1][5:].sum()), 3) # keep_one_token: token 0 valid
+check("pad_batch/padding rows keep one marker", int(p5[3][5:].sum()), 3)
+check("pad_batch/padding rows have qtype 0", int(p5[4][5:].sum()), 0)
+
+# ------------------------------------------------------------------ install / uninstall / fallback
+agent = fake_agent()
+
+
+def untouched(model):
+ """The class forward is in use and no instance attribute shadows it."""
+ return "forward" not in model.__dict__ and model.forward.__func__ is TinyModel.forward
+
+
+be = backends.install(agent, "eager")
+check("install(eager)/name", be.name, "eager")
+check_true("install(eager)/forward untouched", untouched(agent.model))
+
+with warnings.catch_warnings(record=True) as w:
+ warnings.simplefilter("always")
+ be = backends.install(agent, "compile") # CPU: compile declines
+check("install(compile)/cpu falls back to eager", be.name, "eager")
+rw = [x for x in w if issubclass(x.category, RuntimeWarning)]
+check("install(compile)/cpu warns exactly once", len(rw), 1)
+check_true("install(compile)/warning names the backend and the fallback",
+ rw and "'compile'" in str(rw[0].message) and "eager" in str(rw[0].message), [str(x.message) for x in rw])
+check_true("install(compile)/forward untouched after fallback", untouched(agent.model))
+
+with warnings.catch_warnings(record=True) as w:
+ warnings.simplefilter("always")
+ be = backends.install(agent, "tilelang") # CPU and/or no tilelang: declines
+check("install(tilelang)/cpu falls back to eager", be.name, "eager")
+check("install(tilelang)/cpu warns exactly once", len([x for x in w if issubclass(x.category, RuntimeWarning)]), 1)
+
+try:
+ backends.install(agent, "compile", strict=True)
+ FAIL.append("install(compile, strict)/cpu did not raise")
+except BackendUnavailable:
+ PASS.append("install(compile, strict)/cpu raises BackendUnavailable")
+check_true("install(strict)/forward untouched after raise", untouched(agent.model))
+
+
+class Doubling(Backend):
+ """A backend that runs the stock forward and doubles the logits: proves install swaps the forward."""
+ name = "doubling"
+
+ def forward(self, *args, **kw):
+ logits, act = self.stock_forward(*args, **kw)
+ return logits * 2, act
+
+
+b = {"input_ids": ids, "attention_mask": att, "marker_pos": mpos, "marker_mask": mmask, "qtype": qt}
+ref = agent.model(**b)[0]
+d = Doubling(agent)
+d.install()
+check_true("custom backend/installed forward is used", torch.equal(agent.model(**b)[0], ref * 2))
+check_true("custom backend/stock_forward still reaches the model", torch.equal(d.stock_forward(**b)[0], ref))
+d.uninstall()
+check_true("custom backend/uninstall restores the forward", untouched(agent.model) and torch.equal(agent.model(**b)[0], ref))
+d.install(); d.install()
+check_true("custom backend/install is idempotent", torch.equal(agent.model(**b)[0], ref * 2))
+d.uninstall(); d.uninstall()
+check_true("custom backend/uninstall is idempotent", untouched(agent.model))
+
+e = EagerBackend(agent)
+check("eager/max_len is unbounded", e.max_len, None)
+check("eager/warmup is free", e.warmup(), 0.0)
+
+# ------------------------------------------------------------------ the Agent surface
+import inspect # noqa: E402
+from laya.agent import Agent, load # noqa: E402
+
+check("Agent.__init__/backend default None", inspect.signature(Agent.__init__).parameters["backend"].default, None)
+check("load/backend default None", inspect.signature(load).parameters["backend"].default, None)
+check("Agent/backend property default", Agent.__new__(Agent).backend, "eager")
+check_true("Agent/set_backend, accelerate, deaccelerate exist",
+ all(callable(getattr(Agent, n, None)) for n in ("set_backend", "accelerate", "deaccelerate")))
+
+# ------------------------------------------------------------------ resolved fallback / compile lifecycle
+from unittest.mock import patch # noqa: E402
+from laya.backends.compile import CompileBackend # noqa: E402
+
+with patch.object(backends, "auto_policy", return_value="compile"), warnings.catch_warnings(record=True) as w:
+ warnings.simplefilter("always")
+ be = backends.install(fake_agent(), "auto")
+check("auto fallback/eager", be.name, "eager")
+check("auto fallback/one warning", len(w), 1)
+check_true("auto fallback/names resolved compile, not auto",
+ len(w) == 1 and "backend 'compile'" in str(w[0].message) and "backend 'auto'" not in str(w[0].message))
+
+cuda_agent = fake_agent("cuda")
+cfg = torch.fx.experimental._config
+prior_duck = cfg.use_duck_shape
+
+def compile_stub(fn, **kwargs):
+ check("compile/reuses dynamic helper", kwargs, {"dynamic": True, "mode": "reduce-overhead"})
+ def call(*args):
+ check("compile/independent dimensions during forward", cfg.use_duck_shape, False)
+ return fn(*args)
+ return call
+
+with patch.object(torch, "compile", side_effect=compile_stub):
+ be = backends.install(cuda_agent, "compile", strict=True, warmup=False)
+ actual = cuda_agent.model(**b)
+ check_true("compile/padding sliced away", torch.equal(actual[0], ref))
+ check("compile/duck shape restored after forward", cfg.use_duck_shape, prior_duck)
+ be.uninstall()
+ check_true("compile/uninstall restores stock", untouched(cuda_agent.model))
+with patch.object(torch, "compile", side_effect=compile_stub), \
+ patch.object(CompileBackend, "warmup", side_effect=RuntimeError("compile failed")), \
+ warnings.catch_warnings(record=True) as w:
+ warnings.simplefilter("always")
+ be = backends.install(cuda_agent, "compile")
+check("compile/warmup failure falls back", be.name, "eager")
+check_true("compile/warmup failure restores stock", untouched(cuda_agent.model))
+check("compile/warmup failure warns once", len(w), 1)
+
+# ------------------------------------------------------------------ forwarding and legacy Agent substitutes
+from laya.router import Router # noqa: E402
+
+class LegacyAgent:
+ def __init__(self, repo, device=None, token=None, subfolder=None):
+ self.repo = repo
+
+with patch("laya.agent.Agent", LegacyAgent):
+ router = Router()
+ legacy = router._build(next(iter(router.models)))
+check_true("Router/default supports constructor without backend", isinstance(legacy, LegacyAgent))
+router = Router(agent_kwargs={"backend": "eager"})
+with patch("laya.agent.Agent") as ctor:
+ router._build(next(iter(router.models)))
+check("Router/explicit backend forwarded via agent_kwargs", ctor.call_args.kwargs["backend"], "eager")
+with patch("laya.agent.Agent") as ctor:
+ load("unused/model")
+check_true("load/default does not inject backend", "backend" not in ctor.call_args.kwargs)
+with patch("laya.agent.Agent") as ctor:
+ load("unused/model", backend="torch")
+check("load/explicit backend forwarded", ctor.call_args.kwargs["backend"], "eager")
+with patch("laya.onnx_agent.ONNXAgent") as ctor:
+ load("unused/model", backend="onnx", onnx_path="local.onnx", token="test", revision="rev",
+ expected_sha256={"config.json": "digest"}, hooks_timeout=2, calibration="cal.json",
+ lang_temperatures={"en": {"temperature": [1, 1, 1]}})
+check("load/ONNX uses existing class", ctor.call_count, 1)
+for key, value in (("onnx_path", "local.onnx"), ("token", "test"), ("revision", "rev"),
+ ("expected_sha256", {"config.json": "digest"}), ("hooks_timeout", 2),
+ ("calibration", "cal.json"), ("lang_temperatures", {"en": {"temperature": [1, 1, 1]}})):
+ check("load/ONNX forwards " + key, ctor.call_args.kwargs[key], value)
+
+# ------------------------------------------------------------------ scoped OOM detaches and restores backend
+class DeviceModel(TinyModel):
+ def to(self, device):
+ return self
+
+class FailOnce(Backend):
+ name = "test-oom"
+ def forward(self, *args, **kwargs):
+ raise torch.cuda.OutOfMemoryError("test out of memory")
+
+a = Agent.__new__(Agent)
+a.__dict__.update(fake_agent("cuda").__dict__)
+a.model = DeviceModel()
+held = FailOnce(a)
+held.install()
+a._backend = held
+original_to = torch.Tensor.to
+
+def cpu_inputs(tensor, *args, **kwargs):
+ if args and isinstance(args[0], torch.device):
+ return tensor
+ return original_to(tensor, *args, **kwargs)
+
+with patch.object(torch.Tensor, "to", cpu_inputs), warnings.catch_warnings(record=True):
+ actual = a._infer(b)
+check_true("OOM/CPU retry reaches stock forward", torch.equal(actual[0], ref))
+check("OOM/restores device", a.device.type, "cuda")
+check_true("OOM/restores same backend", a.backend_object is held and held.installed)
+a.deaccelerate()
+check_true("deaccelerate/restores eager", a.backend == "eager" and untouched(a.model))
+
+print("\n%d passed, %d failed" % (len(PASS), len(FAIL)))
+for f in FAIL:
+ print(" FAIL " + f)
+if __name__ == "__main__":
+ sys.exit(1 if FAIL else 0)
+
+
+def test_backends():
+ assert not FAIL, FAIL
diff --git a/tests/test_download.py b/tests/test_download.py
index 3e163d47..c9c4c977 100644
--- a/tests/test_download.py
+++ b/tests/test_download.py
@@ -320,6 +320,24 @@ class DownloadTests(unittest.TestCase):
self.assertIsInstance(compiled.model, OptimizedModule)
self.assertIsInstance(compiled.model._orig_mod, DecisionModel)
+ def test_backend_selection_preserves_legacy_compile_and_can_replace_it(self):
+ compiled = load(str(self.repo), device="cpu", compile=True)
+ self.assertEqual(compiled.backend, "compile")
+ self.assertIsNone(compiled.backend_object)
+ self.assertEqual(compiled.set_backend("eager"), "eager")
+ self.assertNotIsInstance(compiled.model, OptimizedModule)
+ self.assertFalse(compiled.model.encoder.config.reference_compile)
+ self.assertEqual(compiled.predict("hello", self.questions)["answers"], self.expected["answers"])
+
+ def test_explicit_backend_takes_precedence_over_legacy_flags(self):
+ agent = load(str(self.repo), device="cpu", backend="eager", fast=True, compile=True)
+ self.assertEqual(agent.backend, "eager")
+ self.assertIsNotNone(agent.backend_object)
+ self.assertNotIsInstance(agent.model, OptimizedModule)
+ self.assertFalse(agent.model.encoder.config.reference_compile)
+ self.assertIsNone(agent._fast)
+ self.assertEqual(agent.predict("hello", self.questions)["answers"], self.expected["answers"])
+
def test_onnx_agent_accepts_every_hub_option_the_agent_does(self):
# Both runtimes download the same checkpoint from the same place, so an option that
# selects *which* checkpoint, or *how* to authenticate for it, has to exist on both.
@@ -327,10 +345,11 @@ class DownloadTests(unittest.TestCase):
# copied across rather than silently diverging again.
# fast/compile -- the TileLang path and torch.compile, neither of which exists
# inside onnxruntime.
+ # backend -- selects a PyTorch forward; load() routes ONNX to ONNXAgent.
# device -- ONNXAgent picks an execution provider from what onnxruntime
# reports and takes no override; a different asymmetry, with its
# own fix.
- not_for_onnxruntime = {"fast", "compile", "device"}
+ not_for_onnxruntime = {"fast", "compile", "device", "backend"}
agent_side = (set(inspect.signature(Agent.__init__).parameters)
- {"self", "model_id_or_path"} - not_for_onnxruntime)
onnx_side = set(inspect.signature(ONNXAgent.__init__).parameters) - {"self"}
diff --git a/tests/test_hooks_api.py b/tests/test_hooks_api.py
index a8aa4957..474d85f9 100644
--- a/tests/test_hooks_api.py
+++ b/tests/test_hooks_api.py
@@ -100,6 +100,13 @@ except ValueError as exc:
# load() forwards Agent's own construction options, so none of them is reachable only
# through the class; tests/test_download.py asserts that against both signatures.
check_param("load", load, "compile", False)
+check_param("load", load, "backend", None)
+check_param("load", load, "onnx_path", None)
+check_param("Agent.__init__", Agent.__init__, "backend", None)
+check_param("Agent.set_backend", Agent.set_backend, "name", "auto")
+check_param("Agent.set_backend", Agent.set_backend, "strict", False)
+check_true("Agent.backend is a property", isinstance(Agent.backend, property))
+check_true("Agent.backend_object is a property", isinstance(Agent.backend_object, property))
# ONNXAgent downloads the same Hub artifacts as Agent, so it authenticates the same way
check_param("ONNXAgent.__init__", ONNXAgent.__init__, "token", None)