Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"""The deferred speed profile declines when the eager peak plus the fused QKV copies would not fit on the card."""
from __future__ import annotations
import types
import pytest
torch = pytest.importorskip("torch")
diffusers = pytest.importorskip("diffusers")
from core.inference import diffusion as dmod # noqa: E402
from core.inference import diffusion_speed as dspeed # noqa: E402
MIB = 1 << 20
HEADROOM_ENV = "UNSLOTH_DIFFUSION_DEFERRED_HEADROOM"
def _tiny_unet():
from diffusers import UNet2DConditionModel
torch.manual_seed(0)
return UNet2DConditionModel(
sample_size = 8,
in_channels = 4,
out_channels = 4,
layers_per_block = 1,
block_out_channels = (32, 64),
down_block_types = ("DownBlock2D", "CrossAttnDownBlock2D"),
up_block_types = ("CrossAttnUpBlock2D", "UpBlock2D"),
cross_attention_dim = 24,
attention_head_dim = 8,
norm_num_groups = 8,
)
def _bytes(module) -> int:
return sum(p.numel() * p.element_size() for p in module.parameters())
def _keep_copies(monkeypatch):
monkeypatch.setenv(dspeed.DROP_UNFUSED_QKV_ENV, "0")
def test_added_bytes_equal_what_diffusers_fuse_adds(monkeypatch):
_keep_copies(monkeypatch)
unet = _tiny_unet()
pipe = types.SimpleNamespace(unet = unet)
added = dspeed.fused_qkv_added_bytes(pipe)
before = _bytes(unet)
unet.fuse_qkv_projections()
assert added > 0
assert _bytes(unet) - before == added
# already fused: nothing more to add
assert dspeed.fused_qkv_added_bytes(pipe) == 0
def test_no_unet_adds_nothing():
assert dspeed.fused_qkv_added_bytes(types.SimpleNamespace(transformer = object())) == 0
@pytest.fixture
def card(monkeypatch):
"""A fake CUDA card: ``peak`` / ``free`` / ``reserved`` in MiB."""
reading = {"peak": 0, "free": 0, "reserved": 0}
monkeypatch.setattr(torch.cuda, "is_available", lambda: True)
monkeypatch.setattr(torch.cuda, "max_memory_allocated", lambda dev = None: reading["peak"] * MIB)
monkeypatch.setattr(
torch.cuda, "mem_get_info", lambda dev = None: (reading["free"] * MIB, 24576 * MIB)
)
monkeypatch.setattr(torch.cuda, "memory_reserved", lambda dev = None: reading["reserved"] * MIB)
monkeypatch.delenv(HEADROOM_ENV, raising = False)
# the fused copies stay beside the originals (the case the gate guards); test_drop_* cover the dropped case
monkeypatch.setenv(dspeed.DROP_UNFUSED_QKV_ENV, "0")
return reading
@pytest.fixture
def engage(monkeypatch):
ran = []
monkeypatch.setattr(
dmod.DiffusionBackend, "_engage_deferred_profile", lambda self, state: ran.append(state)
)
monkeypatch.setattr(dmod, "invalidate_cfg_replica", lambda *a, **k: None)
monkeypatch.setattr(dmod, "drop_displaced_cfg_router", lambda *a, **k: None)
backend = object.__new__(dmod.DiffusionBackend)
def make(offload_policy = dmod.OFFLOAD_MODEL):
return types.SimpleNamespace(
pipe = types.SimpleNamespace(unet = _tiny_unet()),
speed_deferred = True,
deferred_static_plan = {"every": 3, "min_steps": 20},
resolved = {"speed_mode": {"value": "deferred", "reason": "auto"}},
transformer_cache = None,
offload_policy = offload_policy,
device = "cuda:0",
)
def run(offload_policy = dmod.OFFLOAD_MODEL, state = None):
state = state if state is not None else make(offload_policy)
dmod.DiffusionBackend._engage_deferred_speed(backend, state)
return state, bool(ran and ran[-1] is state)
run.make = make
return run
def test_profile_declined_when_the_render_would_not_fit(card, engage):
# SDXL 2048^2 at 8 GiB on an L4: eager peak 6197 MiB, 544 MiB free beside a 7234 MiB cache
# (the tiny U-Net's fused copies are KiB: the peak sits 300 MiB inside capacity, short of the margin)
card.update(peak = 7778 - 300, free = 544, reserved = 7234)
state, engaged = engage()
assert not engaged
assert state.speed_deferred is False # never retried on this load
assert (
state.deferred_static_plan is None
) # no static skip either: renders stay as speed_mode=off
assert "declined" in state.resolved["speed_mode"]["reason"]
def test_profile_engages_when_the_render_fits(card, engage):
# SDXL 1024^2 at 8 GiB: eager peak 5232 MiB leaves room for the fused copies
card.update(peak = 5232, free = 1322, reserved = 6456)
_state, engaged = engage()
assert engaged
def test_header_names_the_kill_switch():
assert dmod.DEFERRED_HEADROOM_ENV == HEADROOM_ENV
def test_kill_switch_engages_regardless(card, engage, monkeypatch):
card.update(peak = 7700, free = 50, reserved = 7728)
monkeypatch.setenv(HEADROOM_ENV, "0")
_state, engaged = engage()
assert engaged
def test_grouped_offload_is_not_gated(card, engage):
card.update(peak = 7700, free = 50, reserved = 7728)
_state, engaged = engage(offload_policy = dmod.OFFLOAD_GROUP)
assert engaged
def test_unreadable_card_engages(card, engage, monkeypatch):
def boom(dev = None):
raise RuntimeError("no reading")
monkeypatch.setattr(torch.cuda, "mem_get_info", boom)
card.update(peak = 7700)
_state, engaged = engage()
assert engaged
def test_eager_peak_read_at_render_end_survives_a_counter_reset(card, engage):
# a caller resets the peak counters before the 3rd generate: the eager peak noted as render 2 ended still counts
state = engage.make()
card.update(peak = 7778 - 300, free = 544, reserved = 7234)
dmod._note_deferred_eager_peak(state)
card["peak"] = 10
_state, engaged = engage(state = state)
assert not engaged
def test_render_start_opens_a_fresh_peak_window_without_the_learned_probe(
card, engage, monkeypatch
):
# a previous model left a 23 GiB peak in the counters and no learned-peak probe reset them (LEARNED_PEAKS=0)
from core.inference import diffusion_stable_arena as arena
resets = []
monkeypatch.setattr(torch.cuda, "reset_peak_memory_stats", lambda dev = None: card.update(peak = 0))
monkeypatch.setattr(arena, "reset_peak_mapped", lambda index: resets.append(index))
state = engage.make()
card.update(peak = 23 * 1024, free = 16384, reserved = 1024)
dmod._open_deferred_peak_window(state)
assert card["peak"] == 0 and resets == [0]
card["peak"] = 5232 # this load's own eager render
dmod._note_deferred_eager_peak(state)
_state, engaged = engage(state = state)
assert engaged
def test_hires_refinement_peaks_are_noted_too():
# The refinement pass does not count as a generation, but its (larger) peak must still be noted before the next
# pass resets the counters, so the note may not sit under the generation-count branch.
import ast
import inspect
tree = ast.parse(inspect.getsource(dmod))
for node in ast.walk(tree):
if isinstance(node, ast.If) and "generation_count" in ast.unparse(node.body[0]):
assert "_note_deferred_eager_peak" not in ast.unparse(node), ast.unparse(node)[:200]
break
else:
pytest.fail("generation_count branch not found")
assert "_note_deferred_eager_peak(state)" in inspect.getsource(dmod)
def test_drop_makes_the_fuse_add_nothing(monkeypatch):
# with the unfused copies dropped after the fuse, the fused weights replace the originals
monkeypatch.delenv(dspeed.DROP_UNFUSED_QKV_ENV, raising = False)
pipe = types.SimpleNamespace(unet = _tiny_unet())
assert dspeed.unfused_projections_will_drop(pipe.unet)
assert dspeed.fused_qkv_added_bytes(pipe) == 0
def test_a_lora_keeps_the_copies_so_they_count(monkeypatch):
monkeypatch.delenv(dspeed.DROP_UNFUSED_QKV_ENV, raising = False)
unet = _tiny_unet()
attn = next(m for m in unet.modules() if callable(getattr(m, "fuse_projections", None)))
attn.to_q.lora_A = (
torch.nn.ModuleDict()
) # a PEFT adapter on the projections: the drop is skipped
pipe = types.SimpleNamespace(unet = unet)
assert not dspeed.unfused_projections_will_drop(unet)
assert dspeed.fused_qkv_added_bytes(pipe) > 0
def test_drop_lets_the_profile_engage_where_the_copies_would_not_fit(card, engage, monkeypatch):
# the reading test_profile_declined_when_the_render_would_not_fit declines with the copies kept engages once
# they are dropped: the fuse then adds nothing to the card
monkeypatch.delenv(dspeed.DROP_UNFUSED_QKV_ENV, raising = False)
card.update(peak = 7778 - 300, free = 544, reserved = 7234)
_state, engaged = engage()
assert engaged