Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"""A UNet too large for whole-module offload streams leaf by leaf; a streamed UNet skips the deferred speed profile."""
from __future__ import annotations
import types
import pytest
torch = pytest.importorskip("torch")
from core.inference import diffusion_memory as dm # noqa: E402
SDXL_UNET_MIB = 4897
SDXL_VAE_MIB = 159
def _pipe(denoiser = "unet", unet_mib = 4):
"""Diffusers-shaped pipe: a UNet (or transformer) of about ``unet_mib`` MiB of fp32 weights, a VAE, encoders."""
comps = {
denoiser: torch.nn.Linear(512, 512 * unet_mib, bias = False),
"vae": torch.nn.Linear(8, 8),
"text_encoder": torch.nn.Linear(8, 8),
}
calls = []
pipe = types.SimpleNamespace(components = comps, model_offload_calls = calls, transformer = None)
for name, module in comps.items():
setattr(pipe, name, module)
pipe.enable_model_cpu_offload = lambda device = None: calls.append(device)
pipe.to = lambda device = None: calls.append(("to", device))
return pipe
@pytest.fixture
def cuda_free(monkeypatch):
"""Fake one CUDA card with ``state['free']`` MiB free; record _apply_group_offload calls."""
state = {"free": 0, "group": []}
monkeypatch.setattr(torch.cuda, "is_available", lambda: True)
monkeypatch.setattr(
torch.cuda, "mem_get_info", lambda device = None: (state["free"] << 20, 24 << 30)
)
def fake_group(pipe, device, logger, **kw):
state["group"].append((device, kw))
return True
monkeypatch.setattr(dm, "_apply_group_offload", fake_group)
monkeypatch.setattr(dm, "keep_cpu_weights_on_offload", lambda pipe, logger = None: 0)
monkeypatch.delenv(dm.UNET_STREAM_ENV, raising = False)
return state
def _plan(policy = dm.OFFLOAD_MODEL):
return dm.MemoryPlan(
requested_mode = "auto",
offload_policy = policy,
vae_tiling = False,
vae_slicing = False,
device_memory = None,
estimates = {},
)
@pytest.mark.parametrize("budget_mib, free_mib", [(4096, 3684), (6144, 5700)])
def test_sdxl_streams_where_whole_module_offload_cannot_fit(budget_mib, free_mib):
room = dm.unet_stream_room_mib(free_mib, SDXL_UNET_MIB, SDXL_VAE_MIB, lend = False)
assert (
room is not None
), f"{budget_mib} MiB budget: whole-module offload of the 4.8 GiB UNet OOMs"
reserve = (
dm._measured_reserve_mib(dm._MEASURED_UNET_STREAMED_PEAK_MIB) + dm._UNET_STREAM_SLACK_MIB
)
assert room == max(0, free_mib - SDXL_VAE_MIB - reserve)
assert reserve >= dm._MEASURED_UNET_STREAMED_PEAK_MIB * dm._MEASURED_PEAK_MARGIN
@pytest.mark.parametrize("free_mib", [7700, 15000, 22000])
def test_sdxl_keeps_whole_module_offload_when_it_fits(free_mib):
assert dm.unet_stream_room_mib(free_mib, SDXL_UNET_MIB, SDXL_VAE_MIB) is None
def test_more_free_vram_keeps_more_leaves_resident():
rooms = [
dm.unet_stream_room_mib(f, SDXL_UNET_MIB, SDXL_VAE_MIB) for f in (3684, 4500, 5000, 5700)
]
assert rooms == sorted(rooms) and rooms[-1] > rooms[0]
def test_fp32_decode_of_an_fp16_vae_keeps_fewer_leaves_resident():
for free_mib in (5700, 8000 - 1):
r16 = dm.unet_stream_room_mib(free_mib, 7000, SDXL_VAE_MIB, lend = False)
r32 = dm.unet_stream_room_mib(free_mib, 7000, SDXL_VAE_MIB, fp32_decode = True, lend = False)
assert r16 is not None and r32 is not None and r32 < r16
assert dm._MEASURED_UNET_STREAMED_FP32_DECODE_PEAK_MIB > dm._MEASURED_UNET_STREAMED_PEAK_MIB
@pytest.mark.parametrize(
"dtype, force_upcast, fp32",
[(torch.float16, True, True), (torch.float16, False, False), (torch.bfloat16, True, False)],
)
def test_vae_decode_dtype_follows_the_pipeline_upcast(dtype, force_upcast, fp32):
vae = torch.nn.Linear(4, 4).to(dtype)
vae.config = types.SimpleNamespace(force_upcast = force_upcast)
assert dm._vae_decodes_in_fp32(vae) is fp32
def test_streamed_room_is_passed_to_group_offload(cuda_free):
pipe = _pipe(unet_mib = 4)
cuda_free["free"] = 8000
pipe.unet = pipe.components["unet"] = torch.nn.Linear(
512, 512 * 8000, bias = False, device = "meta"
)
policy, _ = dm.apply_memory_plan(pipe, _plan(), device = "cuda")
assert policy == dm.OFFLOAD_GROUP
((_, kw),) = cuda_free["group"]
expected = dm.unet_stream_room_mib(8000, 8000, 0)
assert expected and kw["resident_transformer_mib"] == expected
def test_vram_budget_reserve_is_kept_out_of_the_resident_leaves(cuda_free, monkeypatch):
pipe = _pipe(unet_mib = 4)
pipe.unet = pipe.components["unet"] = torch.nn.Linear(
512, 512 * 8000, bias = False, device = "meta"
)
cuda_free["free"] = 12000
monkeypatch.setattr(dm, "_vram_budget_fraction", lambda: 0.75)
reserve_mib = dm.user_budget_reserve_bytes(torch.device("cuda"), 24 << 30) >> 20
assert reserve_mib == 6144
policy, _ = dm.apply_memory_plan(pipe, _plan(), device = "cuda")
assert policy == dm.OFFLOAD_GROUP
((_, kw),) = cuda_free["group"]
assert kw["resident_transformer_mib"] == dm.unet_stream_room_mib(12000 - reserve_mib, 8000, 0)
def test_model_offload_plan_streams_a_unet_that_cannot_fit(cuda_free):
pipe = _pipe(unet_mib = 4)
cuda_free["free"] = 2
policy, _ = dm.apply_memory_plan(pipe, _plan(), device = "cuda")
assert policy == dm.OFFLOAD_GROUP
assert pipe.model_offload_calls == []
((device, kw),) = cuda_free["group"]
assert device == "cuda" and kw["denoiser"] == "unet" and kw["stream_text_encoders"] is True
assert pipe._unsloth_unet_streamed is True
assert pipe._unsloth_applied_plan.offload_policy == dm.OFFLOAD_GROUP
def test_model_offload_plan_unchanged_when_the_unet_fits(cuda_free):
pipe = _pipe(unet_mib = 4)
cuda_free["free"] = 20_000
policy, _ = dm.apply_memory_plan(pipe, _plan(), device = "cuda")
assert policy == dm.OFFLOAD_MODEL
assert pipe.model_offload_calls == ["cuda"] and cuda_free["group"] == []
assert not getattr(pipe, "_unsloth_unet_streamed", False)
def test_kill_switch_restores_whole_module_offload(cuda_free, monkeypatch):
monkeypatch.setenv(dm.UNET_STREAM_ENV, "0")
pipe = _pipe(unet_mib = 4)
cuda_free["free"] = 2
policy, _ = dm.apply_memory_plan(pipe, _plan(), device = "cuda")
assert policy == dm.OFFLOAD_MODEL
assert pipe.model_offload_calls == ["cuda"] and cuda_free["group"] == []
def test_transformer_pipelines_and_resident_plans_never_stream(cuda_free):
cuda_free["free"] = 2
dit = _pipe(denoiser = "transformer", unet_mib = 4)
assert dm._stream_unet_instead_of_model_offload(dit, "cuda") is False
resident = _pipe(unet_mib = 4)
policy, _ = dm.apply_memory_plan(resident, _plan(dm.OFFLOAD_NONE), device = "cuda")
assert policy == dm.OFFLOAD_NONE
assert cuda_free["group"] == [] and resident.model_offload_calls == [("to", "cuda")]
def test_unet_streaming_failure_falls_back_to_whole_module_offload(cuda_free, monkeypatch):
monkeypatch.setattr(dm, "_apply_group_offload", lambda pipe, device, logger, **kw: False)
pipe = _pipe(unet_mib = 4)
cuda_free["free"] = 2
policy, _ = dm.apply_memory_plan(pipe, _plan(), device = "cuda")
assert policy == dm.OFFLOAD_MODEL
assert pipe.model_offload_calls == ["cuda"]
assert not getattr(pipe, "_unsloth_unet_streamed", False)
def test_cpu_placement_never_streams(cuda_free):
cuda_free["free"] = 2
assert dm._stream_unet_instead_of_model_offload(_pipe(unet_mib = 4), "cpu") is False
assert cuda_free["group"] == []
def test_deferred_speed_profile_skipped_only_for_a_streamed_unet(monkeypatch):
from core.inference import diffusion as dmod
ran = []
monkeypatch.setattr(
dmod.DiffusionBackend, "_engage_deferred_profile", lambda self, state: ran.append(state)
)
monkeypatch.setattr(dmod, "invalidate_cfg_replica", lambda *a, **k: None)
monkeypatch.setattr(dmod, "drop_displaced_cfg_router", lambda *a, **k: None)
backend = object.__new__(dmod.DiffusionBackend)
states = {}
for streamed in (True, False):
pipe = types.SimpleNamespace()
if streamed:
pipe._unsloth_unet_streamed = True
states[streamed] = types.SimpleNamespace(
pipe = pipe,
speed_deferred = True,
deferred_static_plan = None,
resolved = {},
transformer_cache = None,
)
dmod.DiffusionBackend._engage_deferred_speed(backend, states[streamed])
assert states[streamed].speed_deferred is False
assert ran == [states[False]]
# ---- the decode borrows the resident leaves (UNSLOTH_DIFFUSION_UNET_STREAM_LEND)
@pytest.mark.parametrize("free_mib", [3684, 5732])
def test_lending_sizes_the_leaves_from_the_denoise_peak(free_mib, monkeypatch):
monkeypatch.delenv(dm.UNET_STREAM_LEND_ENV, raising = False)
lent = dm.unet_stream_room_mib(free_mib, SDXL_UNET_MIB, SDXL_VAE_MIB)
kept = dm.unet_stream_room_mib(free_mib, SDXL_UNET_MIB, SDXL_VAE_MIB, lend = False)
reserve = (
dm._measured_reserve_mib(dm._MEASURED_UNET_STREAMED_DENOISE_MIB) + dm._UNET_STREAM_SLACK_MIB
)
assert lent == free_mib - SDXL_VAE_MIB - reserve
assert lent > kept
# the decode's own peak no longer has to fit beside every leaf; the 2048^2 denoise still does
assert dm._MEASURED_UNET_STREAMED_DENOISE_MIB < dm._MEASURED_UNET_STREAMED_PEAK_MIB
def test_lend_kill_switch_restores_the_decode_sized_reserve(monkeypatch):
monkeypatch.setenv(dm.UNET_STREAM_LEND_ENV, "0")
assert dm.unet_stream_room_mib(3684, SDXL_UNET_MIB, SDXL_VAE_MIB) == dm.unet_stream_room_mib(
3684, SDXL_UNET_MIB, SDXL_VAE_MIB, lend = False
)
@pytest.mark.parametrize("env, lent", [(None, True), ("0", False)])
def test_streamed_unet_lends_its_leaves_to_the_decode(cuda_free, monkeypatch, env, lent):
if env is None:
monkeypatch.delenv(dm.UNET_STREAM_LEND_ENV, raising = False)
else:
monkeypatch.setenv(dm.UNET_STREAM_LEND_ENV, env)
pipe = _pipe(unet_mib = 4)
pipe.unet = pipe.components["unet"] = torch.nn.Linear(
512, 512 * 8000, bias = False, device = "meta"
)
cuda_free["free"] = 8000
policy, _ = dm.apply_memory_plan(pipe, _plan(), device = "cuda")
assert policy == dm.OFFLOAD_GROUP
lend = getattr(pipe, dm.DECODE_LEND_ATTR, None)
if not lent:
assert lend is None
return
assert lend == dm.unet_decode_lend(False)
# a 1024^2 decode needs the measured decode peak x margin; 4x the pixels needs 4x (tiled: an over-release)
need = dm.decode_need_mib(lend, width = 1024, height = 1024)
assert need >= dm._MEASURED_UNET_STREAMED_PEAK_MIB * dm._MEASURED_PEAK_MARGIN
assert dm.decode_need_mib(lend, width = 2048, height = 2048) >= 4 * need - 4
def test_fp32_decode_lends_the_fp32_peak():
assert dm.unet_decode_lend(True)[1] == dm._MEASURED_UNET_STREAMED_FP32_DECODE_PEAK_MIB
assert dm.unet_decode_lend(False)[1] == dm._MEASURED_UNET_STREAMED_PEAK_MIB
def test_release_for_decode_streams_the_unet_shortfall(monkeypatch):
"""At decode start with 1500 MiB free, the 1024^2 decode (2435 x 1.15 + slack) borrows the shortfall."""
pipe = types.SimpleNamespace(components = {})
setattr(pipe, dm.DECODE_LEND_ATTR, dm.unet_decode_lend(False))
monkeypatch.setattr(torch.cuda, "mem_get_info", lambda device = None: (1500 << 20, 24 << 30))
monkeypatch.setattr(torch.cuda, "memory_reserved", lambda device = None: 0)
monkeypatch.setattr(torch.cuda, "memory_allocated", lambda device = None: 0)
monkeypatch.setattr(torch.cuda, "current_device", lambda: 0)
monkeypatch.setattr(dm, "_graph_held_cache_bytes", lambda index: 0)
monkeypatch.setattr(dm, "user_budget_reserve_bytes", lambda dev, total: 0)
asked = []
monkeypatch.setattr(
dm,
"release_resident_groups",
lambda p, need, logger = None, **kw: asked.append(need) or (lambda: None),
)
restore = dm.release_for_decode(pipe, width = 1024, height = 1024, device = "cuda")
assert callable(restore)
need = dm.decode_need_mib(dm.unet_decode_lend(False), width = 1024, height = 1024)
assert asked == [need + dm._DECODE_SLACK_MIB - 1500]