Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Give the single-row decode harness the real _decode_graphs helpers (#15182)

Daniel Hanbf9eaf39ad37
diff --git a/tests/utils/test_single_row_cached_attention.py b/tests/utils/test_single_row_cached_attention.py
index 182f18c9925..72ed6ee22e0 100644
--- a/tests/utils/test_single_row_cached_attention.py
+++ b/tests/utils/test_single_row_cached_attention.py
@@ -16,6 +16,7 @@
import ast
import builtins
+import importlib.util
from math import sqrt
from pathlib import Path
from types import SimpleNamespace
@@ -42,6 +43,22 @@ class _IdentityRope:
return torch.ones(seq_len, 2), torch.zeros(seq_len, 2)
+def _decode_graphs_module(source):
+ """The real decode buffer helpers, by file path: importing unsloth needs an accelerator. Its
+ top level is torch and the standard library only. Graph replay is CUDA-only, so it is off."""
+ spec = importlib.util.spec_from_file_location(
+ "_unsloth_decode_graphs_cpu", source.with_name("_decode_graphs.py")
+ )
+ module = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(module)
+ module.ENABLED = False
+ return module
+
+
+def _captured_attention_only(*args, **kwargs):
+ raise AssertionError("a captured decode step ran on CPU; the harness keeps graph replay off")
+
+
@pytest.fixture(params = ["llama", "qwen3"])
def decoder(request):
name = f"{request.param.title()}Attention_fast_forward_inference"
@@ -65,6 +82,9 @@ def decoder(request):
# The fused decode kernels are CUDA-only; on CPU they decline and the unfused path runs.
"decode_rope_inplace": lambda *args, **kwargs: False,
"decode_qk_norm_rope": lambda *args, **kwargs: None,
+ # The cache and scratch buffers come from the real helpers; only the captured path is out.
+ "_decode_graphs": _decode_graphs_module(source),
+ "_decode_attention_core": _captured_attention_only,
}
# The decode paths share llama.py's grouped KV helpers (qwen3 imports them from there).
llama_source = source.with_name("llama.py")