Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"""Text encoders on unified memory (#11792): the int8 ConvRot encoder on Apple Silicon, the load-time refusal it
lifts, and releasing the encoders between prompts. The release tests run on whatever accelerator the runner has
(MPS on Apple Silicon runners, CUDA, else CPU), never skipped for the device."""
from __future__ import annotations
import os
import types
import pytest
torch = pytest.importorskip("torch")
from core.inference import diffusion_te_prequant as te_prequant
from core.inference import diffusion_te_release as te_release
from core.inference.diffusion_device import DiffusionDeviceTarget
from core.inference.diffusion_memory import (
DeviceMemory,
plan_diffusion_memory,
unified_memory_shortfall_message,
)
GIB = 1024
# Qwen-Image-2.1 on a 48 GB M5 Max, from the report: macOS said 27 GB free.
Q21_DIT_Q4_K_M_MIB = 4_199_565_024 >> 20
Q21_TE_BF16_MIB = 17_534_339_488 >> 20
Q21_TE_INT8_CONVROT_MIB = 9_349_769_248 >> 20
Q21_VAE_MIB = 650
def _target(device: str, dtype = None) -> DiffusionDeviceTarget:
return DiffusionDeviceTarget(
device = device,
dtype = dtype if dtype is not None else torch.bfloat16,
backend = device,
vendor = None,
supports_model_cpu_offload = device == "cuda",
supports_default_torch_compile = device == "cuda",
supports_pinned_transfer = device == "cuda",
)
def _q21_family():
from core.inference.diffusion_families import detect_family
fam = detect_family("Qwen/Qwen-Image-2.1")
assert fam is not None and fam.name == "qwen-image-2.1"
return fam
def _plan(
memory: DeviceMemory,
weights: int,
encoders: int,
headroom: int = 8 * GIB,
):
return plan_diffusion_memory(
target = _target("mps" if memory.memory_kind == "unified_memory" else "cuda"),
device_memory = memory,
model_dense_mib = weights,
runtime_headroom_mib = headroom,
companion_dense_mib = encoders + Q21_VAE_MIB,
text_encoder_dense_mib = encoders,
)
M5_MAX = DeviceMemory("mps", "mps", "unified_memory", free_mib = 27 * GIB, total_mib = 48 * GIB)
def test_mps_takes_the_int8_convrot_encoder_without_the_fp8_fallback():
sources = te_prequant.te_prequant_sources(
_q21_family(), te_quant_mode = "int8", target = _target("mps")
)
source = sources["text_encoder"]
assert source.filename == "Qwen-Image-2.1-text_encoder-INT8-ConvRot.safetensors"
assert source.fallback_filenames == ()
def test_mps_fp8_request_still_takes_no_precast_encoder():
assert (
te_prequant.te_prequant_sources(_q21_family(), te_quant_mode = "fp8", target = _target("mps"))
== {}
)
def test_cpu_takes_no_precast_encoder():
assert (
te_prequant.te_prequant_sources(
_q21_family(), te_quant_mode = "int8", target = _target("cpu", torch.float32)
)
== {}
)
def test_cuda_keeps_the_fp8_fallback(monkeypatch):
from core.inference import diffusion_precision
monkeypatch.setattr(diffusion_precision, "te_quant_supported", lambda target, mode: True)
source = te_prequant.te_prequant_sources(
_q21_family(), te_quant_mode = "int8", target = _target("cuda")
)["text_encoder"]
assert source.fallback_filenames, "the CUDA path must keep its fp8 fallback names"
assert (
te_prequant.te_prequant_budget_scale(
_q21_family(), te_quant_mode = "int8", target = _target("cuda"), base = "Qwen/Qwen-Image-2.1"
)
== te_prequant.TE_PREQUANT_BUDGET_SCALE
)
def test_mps_budgets_the_int8_encoder_at_its_own_size():
scale = te_prequant.te_prequant_budget_scale(
_q21_family(), te_quant_mode = "int8", target = _target("mps"), base = "Qwen/Qwen-Image-2.1"
)
assert scale == te_prequant.TE_INT8_CONVROT_BUDGET_SCALE
assert Q21_TE_BF16_MIB * scale >= Q21_TE_INT8_CONVROT_MIB
def test_int8_convrot_linear_runs_on_this_accelerator():
"""The int8 ConvRot projection the Apple Silicon path now loads: plain tensors, so it runs on MPS too."""
from core.inference.video_minimax_h3_te import _int8_convrot_linear_class
device = _device()
torch.manual_seed(0)
dense = torch.nn.Linear(512, 256, bias = True)
codes, scale = te_prequant.quantize_int8_convrot_weight(dense.weight.detach(), group_size = 256)
layer = _int8_convrot_linear_class()(512, 256, bias = True, group_size = 256)
layer.load_state_dict(
{"weight": codes, "weight_scale": scale, "bias": dense.bias.detach().to(torch.bfloat16)},
assign = True,
)
layer = layer.to(device)
x = torch.randn(4, 512)
with torch.no_grad():
got = layer(x.to(device)).float().cpu()
want = dense(x)
rel = (got - want).norm() / want.norm()
assert got.shape == (4, 256)
assert rel < 0.02, float(rel)
def test_m5_max_refuses_the_bf16_encoder():
weights = Q21_DIT_Q4_K_M_MIB + Q21_TE_BF16_MIB + Q21_VAE_MIB
assert unified_memory_shortfall_message(_plan(M5_MAX, weights, Q21_TE_BF16_MIB)) is not None
def test_m5_max_accepts_the_int8_convrot_encoder():
encoders = int(Q21_TE_BF16_MIB * te_prequant.TE_INT8_CONVROT_BUDGET_SCALE)
weights = Q21_DIT_Q4_K_M_MIB + encoders + Q21_VAE_MIB
assert unified_memory_shortfall_message(_plan(M5_MAX, weights, encoders)) is None
def test_a_genuinely_oversized_load_is_still_refused():
# The dense bf16 Qwen-Image-2.1 transformer with the int8 encoder does not fit 27 GB free either.
encoders = int(Q21_TE_BF16_MIB * te_prequant.TE_INT8_CONVROT_BUDGET_SCALE)
weights = (14_230_284_408 >> 20) + encoders + Q21_VAE_MIB
assert unified_memory_shortfall_message(_plan(M5_MAX, weights, encoders)) is not None
def test_discrete_vram_never_refuses_here():
memory = DeviceMemory("cuda", "cuda", "discrete_vram", free_mib = 8 * GIB, total_mib = 24 * GIB)
weights = Q21_DIT_Q4_K_M_MIB + Q21_TE_BF16_MIB + Q21_VAE_MIB
assert unified_memory_shortfall_message(_plan(memory, weights, Q21_TE_BF16_MIB)) is None
def test_release_engages_on_unified_memory_when_the_resident_set_does_not_fit(monkeypatch):
monkeypatch.delenv(te_release.RELEASE_ENV, raising = False)
encoders = int(Q21_TE_BF16_MIB * te_prequant.TE_INT8_CONVROT_BUDGET_SCALE)
weights = Q21_DIT_Q4_K_M_MIB + encoders + Q21_VAE_MIB
wanted, reason = te_release.release_wanted(_plan(M5_MAX, weights, encoders))
assert wanted, reason
def test_release_stays_off_when_everything_fits(monkeypatch):
monkeypatch.delenv(te_release.RELEASE_ENV, raising = False)
roomy = DeviceMemory("mps", "mps", "unified_memory", free_mib = 100 * GIB, total_mib = 128 * GIB)
weights = Q21_DIT_Q4_K_M_MIB + Q21_TE_BF16_MIB + Q21_VAE_MIB
assert te_release.release_wanted(_plan(roomy, weights, Q21_TE_BF16_MIB))[0] is False
def test_release_stays_off_on_discrete_vram(monkeypatch):
monkeypatch.delenv(te_release.RELEASE_ENV, raising = False)
memory = DeviceMemory("cuda", "cuda", "discrete_vram", free_mib = 8 * GIB, total_mib = 24 * GIB)
weights = Q21_DIT_Q4_K_M_MIB + Q21_TE_BF16_MIB + Q21_VAE_MIB
assert te_release.release_wanted(_plan(memory, weights, Q21_TE_BF16_MIB))[0] is False
@pytest.mark.parametrize("value, expected", [("1", True), ("0", False)])
def test_release_env_forces_either_way(monkeypatch, value, expected):
monkeypatch.setenv(te_release.RELEASE_ENV, value)
memory = DeviceMemory("cuda", "cuda", "discrete_vram", free_mib = 8 * GIB, total_mib = 24 * GIB)
assert te_release.release_wanted(_plan(memory, 1, 1))[0] is expected
@pytest.mark.parametrize("device", ["mps", "cuda"])
def test_a_failed_int8_encoder_on_mps_never_falls_back_to_dense(monkeypatch, device):
# The MPS plan priced the int8 file, not the dense shards beside it (#11792).
from core.inference import diffusion_precision, media_locality
monkeypatch.setattr(diffusion_precision, "te_quant_supported", lambda t, m: device == "cuda")
def _fail(
*a,
failures = None,
**k,
):
if failures is not None:
failures.append("bad header")
return None
monkeypatch.setattr(te_prequant, "load_prequant_text_encoder", _fail)
monkeypatch.setattr(te_prequant, "_held_locally", lambda *a, **k: True)
monkeypatch.setattr(media_locality, "_hosted_component_cached", lambda *a, **k: True)
call = lambda: te_prequant.te_prequant_pipe_kwargs( # noqa: E731
_q21_family(),
"Qwen/Qwen-Image-2.1",
te_quant_mode = "int8",
target = _target(device),
dtype = torch.bfloat16,
dense_source = "Qwen/Qwen-Image-2.1",
)
if device == "mps":
with pytest.raises(RuntimeError, match = r"bad header.*Dense \(bf16\)"):
call()
else:
assert call() == {}
def _device() -> str:
if getattr(torch.backends, "mps", None) is not None and torch.backends.mps.is_available():
return "mps"
if torch.cuda.is_available():
return "cuda"
return "cpu"
class _Encoder(torch.nn.Module):
"""A stand-in encoder: a tied embedding / head pair, int8 buffers like the ConvRot encoder, a small norm."""
def __init__(self):
super().__init__()
self.embed = torch.nn.Embedding(512, 1024)
self.proj = torch.nn.Linear(1024, 1024)
self.register_buffer("codes", torch.randint(-127, 127, (1024, 1024), dtype = torch.int8))
self.register_buffer("rotary", torch.arange(16, dtype = torch.float32), persistent = False)
self.norm = torch.nn.LayerNorm(1024)
self.head = torch.nn.Linear(1024, 512, bias = False)
self.head.weight = self.embed.weight
def forward(self, ids):
h = self.norm(
self.proj(self.embed(ids))
+ self.codes[: ids.shape[-1]].to(self.proj.weight.dtype).sum(-1, keepdim = True)
)
return self.head(h) + self.rotary.sum()
@pytest.fixture
def snapshot_root(tmp_path, monkeypatch):
monkeypatch.setattr(te_release, "_snapshot_root", lambda: tmp_path / "snap")
return tmp_path / "snap"
def _pipe(device: str):
torch.manual_seed(0)
encoder = _Encoder().to(device)
return types.SimpleNamespace(text_encoder = encoder, transformer = torch.nn.Linear(4, 4).to(device))
def test_release_frees_and_reload_restores_bit_for_bit(snapshot_root):
device = _device()
pipe = _pipe(device)
ids = torch.arange(8, device = device)
with torch.no_grad():
before = pipe.text_encoder(ids).cpu()
releaser = te_release.TextEncoderReleaser([("text_encoder", pipe.text_encoder)])
total = releaser.releasable_bytes
assert total > 2 << 20
assert releaser.release() == total
assert releaser.released
assert pipe.text_encoder.proj.weight.numel() == 0
assert pipe.text_encoder.codes.numel() == 0
# Device and dtype survive, so pipelines that read them before encoding still work.
assert pipe.text_encoder.proj.weight.device.type == device
assert pipe.text_encoder.proj.weight.dtype == torch.float32
assert pipe.text_encoder.rotary.numel() == 16
with torch.no_grad():
after = pipe.text_encoder(ids).cpu()
assert not releaser.released and releaser.reloads == 1
assert torch.equal(before, after)
# The tie survives: one tensor, released and restored once.
assert pipe.text_encoder.head.weight is pipe.text_encoder.embed.weight
# A second release reuses the snapshot; a submodule called directly also reloads.
shards = list(snapshot_root.rglob("*.safetensors"))
assert releaser.release() == total
assert sorted(snapshot_root.rglob("*.safetensors")) == sorted(shards)
with torch.no_grad():
pipe.text_encoder.proj(torch.zeros(1, 1024, device = device))
assert releaser.reloads == 2
releaser.close()
assert not any(snapshot_root.rglob("*.safetensors"))
def test_release_refuses_offload_hooked_encoders(snapshot_root):
pipe = _pipe(_device())
pipe.text_encoder.proj._hf_hook = object()
releaser = te_release.TextEncoderReleaser([("text_encoder", pipe.text_encoder)])
assert releaser.release() == 0
assert pipe.text_encoder.proj.weight.numel() > 0
def test_release_refuses_when_the_disk_is_full(snapshot_root, monkeypatch):
pipe = _pipe(_device())
usage = types.SimpleNamespace(total = 1, used = 1, free = 1)
monkeypatch.setattr(te_release.shutil, "disk_usage", lambda path: usage)
releaser = te_release.TextEncoderReleaser([("text_encoder", pipe.text_encoder)])
assert releaser.release() == 0
assert pipe.text_encoder.proj.weight.numel() > 0
def test_maybe_install_only_when_wanted(snapshot_root, monkeypatch):
pipe = _pipe(_device())
monkeypatch.setenv(te_release.RELEASE_ENV, "0")
assert te_release.maybe_install(pipe, None) is None
monkeypatch.setenv(te_release.RELEASE_ENV, "1")
releaser = te_release.maybe_install(pipe, None)
assert releaser is not None
releaser.close()
def _save_source(
encoder,
path,
prefix = "",
):
"""The encoder's weights as a checkpoint file, each tied tensor once."""
from safetensors.torch import save_file
tensors = {}
seen = set()
for name, tensor in list(encoder.named_parameters()) + list(encoder.named_buffers()):
if id(tensor) in seen or name == "rotary":
continue
seen.add(id(tensor))
tensors[prefix + name] = tensor.detach().to("cpu").contiguous().clone()
save_file(tensors, str(path))
return path
def _snapshot_bytes(root):
return sum(f.stat().st_size for f in root.rglob("*.safetensors")) if root.exists() else 0
def test_release_reloads_unchanged_weights_from_the_source_file(
snapshot_root, tmp_path, monkeypatch
):
monkeypatch.delenv(te_release.SOURCE_REUSE_ENV, raising = False)
device = _device()
pipe = _pipe(device)
source = _save_source(pipe.text_encoder, tmp_path / "te.safetensors")
setattr(pipe.text_encoder, te_release.WEIGHT_FILES_ATTR, (str(source),))
ids = torch.arange(8, device = device)
with torch.no_grad():
before = pipe.text_encoder(ids).cpu()
releaser = te_release.TextEncoderReleaser([("text_encoder", pipe.text_encoder)])
total = releaser.releasable_bytes
assert releaser.release() == total
assert len(releaser._from_source) == len(releaser._slots) == 3
assert _snapshot_bytes(snapshot_root) == 0
if not os.name == "nt":
# The open handle keeps the bytes even when the file is replaced or deleted meanwhile.
source.unlink()
with torch.no_grad():
after = pipe.text_encoder(ids).cpu()
assert releaser.reloads == 1
assert torch.equal(before, after)
assert pipe.text_encoder.head.weight is pipe.text_encoder.embed.weight
assert pipe.text_encoder.proj.weight.device.type == device
assert releaser.release() == total
with torch.no_grad():
assert torch.equal(before, pipe.text_encoder(ids).cpu())
releaser.close()
def test_release_snapshots_only_the_tensors_that_differ_from_the_source(
snapshot_root, tmp_path, monkeypatch
):
"""A merged / cast tensor no longer matches its file: it alone goes to the snapshot, and the reload is exact."""
monkeypatch.delenv(te_release.SOURCE_REUSE_ENV, raising = False)
device = _device()
pipe = _pipe(device)
# The checkpoint nests one level shallower than the module, as transformers 5 remaps some encoders.
_save_source(pipe.text_encoder, tmp_path / "model.safetensors")
with torch.no_grad():
pipe.text_encoder.proj.weight.add_(1e-3)
wrapper = torch.nn.Module()
wrapper.inner = pipe.text_encoder
wrapper.name_or_path = str(tmp_path)
ids = torch.arange(8, device = device)
with torch.no_grad():
before = wrapper.inner(ids).cpu()
releaser = te_release.TextEncoderReleaser([("text_encoder", wrapper)])
total = releaser.releasable_bytes
assert releaser.release() == total
assert sorted(releaser._from_source) == [
"text_encoder.inner.codes",
"text_encoder.inner.embed.weight",
]
proj = pipe.text_encoder.proj.weight
assert 0 < _snapshot_bytes(snapshot_root) < total
assert _snapshot_bytes(snapshot_root) >= 1024 * 1024 * proj.element_size()
with torch.no_grad():
after = wrapper.inner(ids).cpu()
assert torch.equal(before, after)
releaser.close()
def test_release_source_reuse_kill_switch_snapshots_everything(
snapshot_root, tmp_path, monkeypatch
):
monkeypatch.setenv(te_release.SOURCE_REUSE_ENV, "0")
pipe = _pipe(_device())
source = _save_source(pipe.text_encoder, tmp_path / "te.safetensors")
setattr(pipe.text_encoder, te_release.WEIGHT_FILES_ATTR, (str(source),))
releaser = te_release.TextEncoderReleaser([("text_encoder", pipe.text_encoder)])
total = releaser.releasable_bytes
assert releaser.release() == total
assert releaser._from_source == {}
assert _snapshot_bytes(snapshot_root) >= total
releaser.ensure_loaded()
releaser.close()
def test_release_without_a_known_source_still_snapshots(snapshot_root, tmp_path, monkeypatch):
monkeypatch.delenv(te_release.SOURCE_REUSE_ENV, raising = False)
pipe = _pipe(_device())
pipe.text_encoder.name_or_path = "some/repo-id"
releaser = te_release.TextEncoderReleaser([("text_encoder", pipe.text_encoder)])
total = releaser.releasable_bytes
assert releaser.release() == total
assert releaser._from_source == {}
assert _snapshot_bytes(snapshot_root) >= total
releaser.ensure_loaded()
releaser.close()
@pytest.mark.allow_network
def test_tiny_qwen_image_21_pipeline_renders_the_same_with_release(snapshot_root, monkeypatch):
"""The real QwenImage21Pipeline, tiny: release at denoise entry, reload for a new prompt, same images."""
diffusers = pytest.importorskip("diffusers")
pipeline_cls = getattr(diffusers, "QwenImage21Pipeline", None)
if pipeline_cls is None:
pytest.skip("this diffusers has no QwenImage21Pipeline")
from huggingface_hub import snapshot_download
try:
path = snapshot_download("hf-internal-testing/tiny-qwenimage21-pipe")
except Exception as exc: # noqa: BLE001
pytest.skip(f"tiny pipeline not reachable: {exc}")
from core.inference.media_decode_phase import denoise_phase
device = _device()
dtype = torch.float32
def render(
pipe,
prompt,
releaser = None,
):
def on_denoise():
if releaser is not None:
releaser.release()
with denoise_phase(pipe, on_denoise):
out = pipe(
prompt = prompt,
height = 64,
width = 64,
num_inference_steps = 2,
generator = torch.Generator("cpu").manual_seed(0),
output_type = "pt",
)
return out.images.cpu()
pipe = pipeline_cls.from_pretrained(path, torch_dtype = dtype).to(device)
pipe.set_progress_bar_config(disable = True)
reference = [render(pipe, "a red cube"), render(pipe, "a blue sphere")]
releaser = te_release.TextEncoderReleaser([("text_encoder", pipe.text_encoder)])
first = render(pipe, "a red cube", releaser)
assert releaser.released and releaser.releases == 1
second = render(pipe, "a blue sphere", releaser)
assert releaser.reloads == 1 and releaser.releases == 2
assert torch.equal(first, reference[0])
assert torch.equal(second, reference[1])
releaser.close()