Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"""Graph plan checked on CPU; CUDA renders must match the stock diffusers forward bit for bit."""
from __future__ import annotations
import sys
from pathlib import Path
import pytest
from core.inference import diffusion_capture_safe as cs
from core.inference import diffusion_cuda_graph as cg
from core.inference import diffusion_qwenimage21 as q
torch = pytest.importorskip("torch")
qmod = pytest.importorskip("diffusers.models.transformers.transformer_qwenimage21")
# pytest puts the rootdir on sys.path, not this directory
_TESTS_DIR = str(Path(__file__).resolve().parent)
if _TESTS_DIR not in sys.path:
sys.path.insert(0, _TESTS_DIR)
from test_diffusion_qwenimage21 import CASES, _inputs, _model # noqa: E402
_CUDA = torch.cuda.is_available()
@pytest.fixture(autouse = True)
def _stock_after(monkeypatch):
for env in (q.FAST_STEP_ENV, cs.CAPTURE_SAFE_ENV, cg.CUDA_GRAPH_DISABLE_ENV):
monkeypatch.delenv(env, raising = False)
monkeypatch.setenv(cg.SPEED_CHECK_ENV, "0") # tiny forwards on a shared card: timing is noise
q.uninstall()
yield
q.uninstall()
def _call_kwargs(inp, hs, timestep, kv, mode):
return dict(
hidden_states = hs,
timestep = timestep,
encoder_hidden_states = inp["encoder_hidden_states"],
encoder_hidden_states_mask = inp["encoder_hidden_states_mask"],
img_shapes = inp["img_shapes"],
img_mask = inp["img_mask"],
attention_kwargs = None,
kv_cache = kv,
kv_cache_mode = mode,
return_dict = False,
)
def _render(
m,
inp,
steps = 5,
call = None,
):
"""The pipeline's loop: step 0 prefills the cache, later steps decode from it."""
call = call or m
outs = []
kv = qmod.QwenImage21KVCache(len(m.transformer_blocks))
lat = inp["latents"]
with torch.inference_mode():
for i in range(steps):
hs = lat if inp["cond"] is None else torch.cat([inp["cond"], lat], 1)
timestep = torch.full((hs.shape[0],), 1.0 - i / steps, device = hs.device)
out = call(**_call_kwargs(inp, hs, timestep, kv, "extract" if i == 0 else "cached"))[0]
outs.append(out.clone())
lat = lat - 0.1 * out[:, -lat.shape[1] :]
return outs
def _prefilled(m, inp):
kv = qmod.QwenImage21KVCache(len(m.transformer_blocks))
hs = inp["latents"] if inp["cond"] is None else torch.cat([inp["cond"], inp["latents"]], 1)
t = torch.full((hs.shape[0],), 0.9, device = hs.device)
with torch.inference_mode():
m(**_call_kwargs(inp, hs, t, kv, "extract"))
return kv, hs
@pytest.mark.parametrize("case", CASES)
def test_planned_decode_step_is_bit_identical_to_the_eager_fast_step(case):
m = _model()
inp = _inputs(**case)
assert q.install()
kv, hs = _prefilled(m, inp)
t = torch.full((hs.shape[0],), 0.5)
with torch.inference_mode():
kwargs = _call_kwargs(inp, hs, t, kv, "cached")
want = m(**kwargs)[0]
plan = q.graph_plan(m, (), kwargs)
assert plan is not None
call, planned = plan
got = call(**planned)[0]
assert torch.equal(want, got)
key = cg.graph_key(((), planned))
assert not cg._uncapturable(key)
assert not cg._has_float(key)
assert len(planned["kv"]) == len(m.transformer_blocks)
def test_what_the_plan_leaves_eager(monkeypatch):
m = _model()
inp = _inputs(batch = 2, pad = 2)
assert q.install()
kv, hs = _prefilled(m, inp)
t = torch.full((2,), 0.5)
with torch.inference_mode():
cached = _call_kwargs(inp, hs, t, kv, "cached")
assert q.graph_plan(m, (), cached) is not None
# the prefill fills the cache from Python, so it stays eager
assert q.graph_plan(m, (), {**cached, "kv_cache_mode": "extract"}) is None
assert q.graph_plan(m, (), {**cached, "kv_cache": None, "kv_cache_mode": None}) is None
assert q.graph_plan(m, (), {**cached, "return_dict": True}) is None
assert q.graph_plan(m, (), {**cached, "attention_kwargs": {"scale": 0.5}}) is None
assert (
q.graph_plan(m, (hs,), {k: v for k, v in cached.items() if k != "hidden_states"})
is None
)
assert q.graph_plan(m, (), {**cached, "unknown": 1}) is None
empty = qmod.QwenImage21KVCache(len(m.transformer_blocks))
assert q.graph_plan(m, (), {**cached, "kv_cache": empty}) is None
assert q.graph_plan(m, (), {**cached, "img_mask": "not a tensor"}) is None
with torch.enable_grad():
assert q.graph_plan(m, (), cached) is None
monkeypatch.setattr(q, "GRAPH_KV_MAX_BYTES", 16)
with torch.inference_mode():
assert q.graph_plan(m, (), cached) is None
monkeypatch.setattr(q, "GRAPH_KV_MAX_BYTES", 1 << 30)
monkeypatch.setenv(q.FAST_STEP_ENV, "0")
with torch.inference_mode():
assert q.graph_plan(m, (), cached) is None
def test_resolve_follows_the_installed_fast_step(monkeypatch):
cls = qmod.QwenImage21Transformer2DModel
forward, why = cs.resolve(cls)
assert forward is None and "is not installed" in why
assert q.install()
forward, why = cs.resolve(cls)
assert why is None
assert forward is vars(cls)["forward"]
assert forward.__unsloth_graph_plan__ is q.graph_plan
monkeypatch.setenv(cs.CAPTURE_SAFE_ENV, "0")
forward, why = cs.resolve(cls)
assert forward is None and cs.CAPTURE_SAFE_ENV in why
monkeypatch.delenv(cs.CAPTURE_SAFE_ENV)
q.uninstall()
assert cs.resolve(cls)[0] is None
def test_graph_eligible_accepts_qwen_image_21_once_the_fast_step_is_installed(monkeypatch):
import types
monkeypatch.setattr(torch.cuda, "is_available", lambda: True)
def eligible():
return cg.graph_eligible(
types.SimpleNamespace(device = "cuda", backend = "cuda"),
family = types.SimpleNamespace(),
pipe = types.SimpleNamespace(transformer = _model()),
offload_active = False,
cache_active = False,
speed_mode = "default",
)
ok, why = eligible()
assert ok is False and why.startswith(
"QwenImage21Transformer2DModel forward is not capture-safe"
)
assert q.install()
assert eligible() == (True, "eligible")
def test_a_failed_capture_falls_back_to_the_callers_own_call():
"""A planned call whose capture fails (here: host tensors) must run the stock call, not the planned kwargs."""
m = _model()
inp = _inputs(batch = 2, pad = 3)
ref = _render(m, inp)
assert q.install()
handle = cg.GraphedForward(m).enable()
try:
got = _render(m, _inputs(batch = 2, pad = 3))
finally:
cg.uninstall_all([handle])
for a, b in zip(ref, got):
assert torch.equal(a, b)
assert handle.poisoned and handle.stats["refused_host_tensor"] == 1
assert cg._POOL_BOX[0] is None
def _cuda_inputs(**case):
return {k: (v.to("cuda") if torch.is_tensor(v) else v) for k, v in _inputs(**case).items()}
def _graphed(m, **kw):
handle = cg.GraphedForward(m, **kw).enable()
assert handle.capture_safe and handle.plan is q.graph_plan
return handle
@pytest.mark.skipif(not _CUDA, reason = "needs CUDA")
@pytest.mark.parametrize("case", CASES)
def test_graphed_render_is_bit_identical_to_the_stock_forward(case):
m = _model().to("cuda")
inp = _cuda_inputs(**case)
ref = _render(m, inp)
assert q.install()
handle = _graphed(m)
try:
got = _render(m, inp)
again = _render(m, _cuda_inputs(**case))
finally:
cg.uninstall_all([handle])
for a, b, c in zip(ref, got, again):
assert torch.equal(a, b)
assert torch.equal(a, c)
assert not handle.poisoned, handle.capture_error
# two prefills eager, one capture, every decode step replayed (the capturing one too)
assert handle.stats["planned_eager"] == 2
assert handle.stats["captures"] == 1
assert handle.stats["replays"] == 2 * (5 - 1)
assert not handle.poisoned, handle.capture_error
@pytest.mark.skipif(not _CUDA, reason = "needs CUDA")
def test_evicting_the_only_graph_does_not_reuse_its_dead_pool():
m = _model().to("cuda")
lengths = (9, 11, 13)
refs = [_render(m, _cuda_inputs(text = n, seed = n)) for n in lengths]
assert q.install()
handle = _graphed(m, max_graphs = 1)
try:
for n, ref in zip(lengths, refs):
for a, b in zip(ref, _render(m, _cuda_inputs(text = n, seed = n))):
assert torch.equal(a, b)
finally:
cg.uninstall_all([handle])
assert not handle.poisoned, handle.capture_error
assert handle.stats["captures"] == 3 and handle.stats["evictions"] == 2
@pytest.mark.skipif(not _CUDA, reason = "needs CUDA")
def test_a_new_prompt_length_recaptures_and_the_oldest_graph_is_evicted():
m = _model().to("cuda")
lengths = (9, 11, 9, 13)
refs = [_render(m, _cuda_inputs(text = n, seed = n)) for n in lengths]
assert q.install()
handle = _graphed(m, max_graphs = 2)
try:
for n, ref in zip(lengths, refs):
got = _render(m, _cuda_inputs(text = n, seed = n))
for a, b in zip(ref, got):
assert torch.equal(a, b)
finally:
cg.uninstall_all([handle])
# 9 and 11 capture, 9 replays, 13 evicts 11 (LRU); a new length warms up on its first decode step
assert handle.stats["captures"] == 3
assert handle.stats["shape_warmups"] == 2
assert handle.stats["evictions"] == 1
assert handle.stats["cap_skips"] == 0
assert handle.stats["replays"] == 4 * 4 - 2
assert not handle.poisoned
@pytest.mark.skipif(not _CUDA, reason = "needs CUDA")
def test_a_new_prompt_length_warms_on_tensors_laid_out_like_the_capture(monkeypatch):
"""Compiled blocks guard on strides and storage offsets. The warm-up step of a new length must run on fresh copies
(as the capture's statics are), else it compiles a variant for the caller's views and the capture that follows
compiles inside the recording, which fails it (seen on the real model under block offload)."""
m = _model().to("cuda")
assert q.install()
handle = _graphed(m)
stock = q._graph_step
seen: list = []
def spy(self, **kw):
if handle.stats["shape_warmups"] and not seen:
leaves: list = []
cg._flatten(kw, leaves)
seen.append(leaves)
return stock(self, **kw)
monkeypatch.setattr(q, "_graph_step", spy)
try:
for n in (9, 11):
_render(m, _cuda_inputs(text = n, seed = n))
finally:
cg.uninstall_all([handle])
assert handle.stats["shape_warmups"] == 1 and handle.stats["captures"] == 2
assert seen and all(t.storage_offset() == 0 for t in seen[0])
@pytest.mark.skipif(not _CUDA, reason = "needs CUDA")
def test_alternating_caches_of_one_shape_replay_their_own_prefix():
"""True CFG: cond and uncond share a graph key but not a KV cache; the sticky copy must follow identity."""
m = _model().to("cuda")
cond, uncond = _cuda_inputs(seed = 1), _cuda_inputs(seed = 2)
assert q.install()
def two_stream_render():
out = []
kvs = [qmod.QwenImage21KVCache(len(m.transformer_blocks)) for _ in range(2)]
lat = cond["latents"]
with torch.inference_mode():
for i in range(4):
t = torch.full((1,), 1.0 - i / 4, device = "cuda")
mode = "extract" if i == 0 else "cached"
a = m(**_call_kwargs(cond, lat, t, kvs[0], mode))[0][:, -lat.shape[1] :]
b = m(**_call_kwargs(uncond, lat, t, kvs[1], mode))[0][:, -lat.shape[1] :]
out += [a.clone(), b.clone()]
lat = lat - 0.1 * (a + b)
return out
ref = two_stream_render()
handle = _graphed(m)
try:
got = two_stream_render()
finally:
cg.uninstall_all([handle])
for a, b in zip(ref, got):
assert torch.equal(a, b)
assert handle.stats["captures"] == 1 and handle.stats["replays"] == 6
@pytest.mark.skipif(not _CUDA, reason = "needs CUDA")
def test_graph_statics_match_the_eager_inputs_strides_and_inference_mode():
"""A static that is not an inference tensor where eager passes one selects another compiled variant."""
m = _model().to("cuda")
inp = _cuda_inputs()
assert q.install()
handle = _graphed(m)
try:
_render(m, inp)
(entry,) = handle.cache.values()
kv, hs = _prefilled(m, inp)
with torch.inference_mode():
# decode latents come from the previous step, made in inference mode
hs = hs.clone()
t = torch.full((1,), 0.5, device = "cuda")
_, planned = q.graph_plan(m, (), _call_kwargs(inp, hs, t, kv, "cached"))
live: list = []
cg._flatten(((), planned), live)
assert len(live) == len(entry.static)
for src, dst in zip(live, entry.static):
assert dst.stride() == src.stride()
assert dst.is_inference() == src.is_inference()
finally:
cg.uninstall_all([handle])
@pytest.mark.skipif(not _CUDA, reason = "needs CUDA")
def test_planned_decode_steps_do_not_sync_the_host():
m = _model().to("cuda")
inp = _cuda_inputs(batch = 2, pad = 2)
assert q.install()
handle = _graphed(m)
kv = qmod.QwenImage21KVCache(len(m.transformer_blocks))
try:
with torch.inference_mode():
for i in range(5):
t = torch.full((2,), 1.0 - i / 5, device = "cuda")
# step 1 warm-up and capture synchronize by design; later replays must not
if i >= 2:
torch.cuda.set_sync_debug_mode("error")
try:
m(**_call_kwargs(inp, inp["latents"], t, kv, "extract" if i == 0 else "cached"))
finally:
torch.cuda.set_sync_debug_mode("default")
finally:
cg.uninstall_all([handle])
assert handle.stats["replays"] == 4
def _block_streamed(m):
import types
from core.inference import diffusion_memory as dm
pipe = types.SimpleNamespace(transformer = m, components = {"transformer": m})
assert dm._apply_group_offload(pipe, "cuda", None)
return pipe
@pytest.mark.skipif(not _CUDA, reason = "needs CUDA")
@pytest.mark.parametrize("case", [CASES[1], CASES[3]])
def test_block_streamed_decode_steps_replay_bit_identically(case):
"""Under block offload the planned step enters through the offload hooks (which onload the top-level weights)
and the copy-stream onloads are recorded with it: renders match the eager streamed and the resident forward."""
pytest.importorskip("diffusers.hooks")
from core.inference import diffusion_offload_prefetch as op
resident = _render(_model().to("cuda"), _cuda_inputs(**case))
assert q.install()
m = _model()
pipe = _block_streamed(m)
eager = _render(m, _cuda_inputs(**case))
handles, reason = cg.arm_after_placement(pipe)
assert len(handles) == 1 and handles[0].placement.mode == "group", reason
handle = handles[0]
assert handle.plan is q.graph_plan and handle.placed is q.placed_step
try:
got = _render(m, _cuda_inputs(**case))
again = _render(m, _cuda_inputs(**case))
kv = qmod.QwenImage21KVCache(len(m.transformer_blocks))
inp = _cuda_inputs(**case)
hs = inp["latents"] if inp["cond"] is None else torch.cat([inp["cond"], inp["latents"]], 1)
with torch.inference_mode():
for i in range(3):
t = torch.full((hs.shape[0],), 1.0 - i / 5, device = "cuda")
if i:
torch.cuda.set_sync_debug_mode("error")
try:
m(**_call_kwargs(inp, hs, t, kv, "extract" if i == 0 else "cached"))
finally:
torch.cuda.set_sync_debug_mode("default")
finally:
cg.uninstall_all(handles)
assert not handle.poisoned, handle.capture_error
for r, e, a, b in zip(resident, eager, got, again):
assert torch.equal(e, a) and torch.equal(e, b)
assert torch.equal(r, a)
assert handle.stats["captures"] == 1
assert handle.stats["replays"] == 2 * (5 - 1) + 2
assert next(m.transformer_blocks[-1].parameters()).device.type == "cpu"
assert op.module_prefetcher(m).stats["missed"] == 0
def test_offload_placement_names_a_forward_that_cannot_record_under_the_hooks(monkeypatch):
m = _model()
def planned(*_a, **_k):
return None
planned.__unsloth_graph_plan__ = q.graph_plan
monkeypatch.setattr(cs, "resolve", lambda cls: (planned, None))
assert "no entry through the offload hooks" in cg._forward_refusal(m, "group")
planned.__unsloth_graph_placed__ = q.placed_step
assert cg._forward_refusal(m, "group") is None
def rewrite(*_a, **_k):
return None
monkeypatch.setattr(cs, "resolve", lambda cls: (rewrite, None))
assert "capture-safe forward" in cg._forward_refusal(m, "group")
assert cg._forward_refusal(m, "model") is None
monkeypatch.setattr(cs, "resolve", lambda cls: (None, "X forward is not capture-safe (why)"))
assert cg._forward_refusal(m, "model") == "X forward is not capture-safe (why)"
@pytest.mark.skipif(not _CUDA, reason = "needs CUDA")
@pytest.mark.parametrize("every_length", [False, True])
def test_true_cfg_negative_of_another_length_records_on_the_first_render(monkeypatch, every_length):
if every_length:
monkeypatch.setenv(cg.EVERY_LENGTH_ENV, "1")
else:
monkeypatch.delenv(cg.EVERY_LENGTH_ENV, raising = False)
m = _model().to("cuda")
cond, uncond = _cuda_inputs(text = 9, seed = 1), _cuda_inputs(text = 13, seed = 2)
steps = 4
def cfg_render():
out = []
kvs = [qmod.QwenImage21KVCache(len(m.transformer_blocks)) for _ in range(2)]
lat = cond["latents"]
with torch.inference_mode():
for i in range(steps):
t = torch.full((1,), 1.0 - i / steps, device = "cuda")
mode = "extract" if i == 0 else "cached"
a = m(**_call_kwargs(cond, lat, t, kvs[0], mode))[0][:, -lat.shape[1] :]
b = m(**_call_kwargs(uncond, lat, t, kvs[1], mode))[0][:, -lat.shape[1] :]
out += [a.clone(), b.clone()]
lat = lat - 0.1 * (a - b)
return out
ref = cfg_render()
assert q.install()
handle = _graphed(m)
assert handle.length_varies is (not every_length)
try:
got = []
for _ in range(2):
got.append(cfg_render())
cg.seal_lengths([handle])
finally:
cg.uninstall_all([handle])
for render in got:
for a, b in zip(ref, render):
assert torch.equal(a, b)
assert not handle.poisoned, handle.capture_error
stats = {k: handle.stats[k] for k in ("captures", "replays", "shape_warmups", "length_eager")}
assert (stats["captures"], stats["shape_warmups"], stats["length_eager"]) == (2, 1, 0), stats
assert handle.stats["replays"] == 2 * (steps - 1) * 2 - 1