Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio diffusion: load Qwen-Image-2.1's hosted int8 text encoder on float16 cards (T4 Turbo INT8 82.7 -> 31.9 s/image) (#17379)
Daniel Hanaa97d20989eb
diff --git a/studio/backend/core/inference/diffusion_te_prequant.py b/studio/backend/core/inference/diffusion_te_prequant.pyindex 5a59fa8c8c5..eefaf59eef6 100644--- a/studio/backend/core/inference/diffusion_te_prequant.py+++ b/studio/backend/core/inference/diffusion_te_prequant.py@@ -741,6 +741,27 @@ def int8_convrot_te_runs_on(target: Any) -> bool:return getattr(target, "device", None) == "mps"+TE_INT8_FP16_ENV = "UNSLOTH_TE_INT8_FP16"+# Families whose hosted int8 ConvRot encoder was validated in float16 compute.+_TE_INT8_FP16_FAMILIES = frozenset({"qwen-image-2.1"})+++def int8_convrot_te_runs_fp16(fam: Any, target: Any) -> bool:+ """float16 CUDA (no bf16/fp8) runs the hosted int8 ConvRot encoder; ``UNSLOTH_TE_INT8_FP16=0`` keeps the dense one."""+ import os++ if os.environ.get(TE_INT8_FP16_ENV, "1").strip().lower() in ("0", "false", "no", "off"):+ return False+ name = str(getattr(fam, "name", fam) or "").strip().lower()+ if name not in _TE_INT8_FP16_FAMILIES or getattr(target, "device", None) != "cuda":+ return False+ try:+ import torch+ return getattr(target, "dtype", None) is torch.float16+ except Exception: # noqa: BLE001+ return False++def hosted_int8_convrot_te_runs_on(fam: Any, target: Any) -> bool:"""Whether an explicit int8 encoder request on ``fam`` loads the hosted int8 ConvRot file on ``target`` without thefp8 fallback: the family hosts the file (and the lm_head trim it needs is on) and the device runs it as plain@@ -959,7 +980,10 @@ def te_prequant_sources(return {}# int8 falls back to the fp8 file, so it needs fp8; MPS runs the int8 file alone and drops the fp8 names.fp8_runs = te_quant_supported(target, TE_QUANT_FP8)- if not fp8_runs and not (mode == "int8" and int8_convrot_te_runs_on(target)):+ if not fp8_runs and not (+ mode == "int8"+ and (int8_convrot_te_runs_on(target) or int8_convrot_te_runs_fp16(fam, target))+ ):return {}sources: dict[str, TePrequantSource] = {}for component in components:@@ -1779,6 +1803,10 @@ def _build_int8_convrot_encoder()_fill_tied_aliases(encoder, state_dict)encoder.load_state_dict(state_dict, strict = True, assign = True)+ if dtype is torch.float16:+ for module in encoder.modules():+ if isinstance(module, linear_cls):+ module.scale_weight_first = True# assign=True gives a tied pair separate tensors; re-tie a headless encoder (T5 / UMT5's encoder.embed_tokens -># shared), as the fp8 path does. One with a (trimmed) head is left as loaded: its head is a stand-in with no weight# to tie.diff --git a/studio/backend/core/inference/video_minimax_h3_te.py b/studio/backend/core/inference/video_minimax_h3_te.pyindex 7f2de2c3b99..3e977fe1a40 100644--- a/studio/backend/core/inference/video_minimax_h3_te.py+++ b/studio/backend/core/inference/video_minimax_h3_te.py@@ -130,10 +130,18 @@ def _int8_convrot_linear_class() -> Any:)else:self.bias = None+ # float16: scale the weight before the matmul; the unscaled int8 sum overflows 65504.+ self.scale_weight_first = Falsedef forward(self, x: Any) -> Any: # noqa: D102h = build_convrot_hadamard(self.group_size, device = x.device, dtype = x.dtype)rotated = rotate_convrot_activation(x, h, self.group_size)+ if getattr(self, "scale_weight_first", False):+ weight = (self.weight.to(torch.float32) * self.weight_scale).to(x.dtype)+ out = torch.nn.functional.linear(rotated, weight)+ if self.bias is not None:+ out = out + self.bias.to(out.dtype)+ return outout = torch.nn.functional.linear(rotated, self.weight.to(x.dtype))out = out * self.weight_scale.reshape(1, -1).to(dtype = out.dtype, device = out.device)if self.bias is not None:diff --git a/studio/backend/tests/test_diffusion_te_int8_fp16.py b/studio/backend/tests/test_diffusion_te_int8_fp16.pynew file mode 100644index 00000000000..7420ebbfca5--- /dev/null+++ b/studio/backend/tests/test_diffusion_te_int8_fp16.py@@ -0,0 +1,90 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""float16 CUDA cards (T4) take Qwen-Image-2.1's hosted int8 ConvRot text encoder instead of the dense bf16 one."""++from __future__ import annotations++import types++import pytest++torch = pytest.importorskip("torch")+pytest.importorskip("transformers.models.qwen3_vl")+pytest.importorskip("accelerate")++import core.inference.diffusion_te_prequant as tpq # noqa: E402+from core.inference.video_minimax_h3_te import _int8_convrot_linear_class # noqa: E402++from .test_diffusion_te_int8_convrot import ( # noqa: E402, F401+ BASE,+ INT8_NAME,+ REPO,+ get_family,+ int8_file,+)++CUDA_FP16 = types.SimpleNamespace(device = "cuda", dtype = torch.float16)+CUDA_BF16 = types.SimpleNamespace(device = "cuda", dtype = torch.bfloat16)+KILL_SWITCH = "UNSLOTH_TE_INT8_FP16"++[email protected](autouse = True)+def _default_env(monkeypatch):+ monkeypatch.delenv(KILL_SWITCH, raising = False)+++def test_fp16_card_resolves_only_the_int8_file():+ fam = get_family("qwen-image-2.1")+ src = tpq.te_prequant_sources_for_base(fam, BASE, te_quant_mode = "int8", target = CUDA_FP16)[+ "text_encoder"+ ]+ assert (src.kind, src.location, src.filename) == ("repo", REPO, INT8_NAME)+ assert tpq.te_candidate_filenames(src) == (INT8_NAME,)+++def test_fp16_kill_switch_and_other_cases_unchanged(monkeypatch):+ fam = get_family("qwen-image-2.1")+ assert tpq.te_prequant_sources(fam, te_quant_mode = "fp8", target = CUDA_FP16) == {}+ assert tpq.te_prequant_sources(get_family("sdxl"), te_quant_mode = "int8", target = CUDA_FP16) == {}+ bf16 = tpq.te_prequant_sources(fam, te_quant_mode = "int8", target = CUDA_BF16)["text_encoder"]+ assert len(tpq.te_candidate_filenames(bf16)) > 1+ monkeypatch.setenv(KILL_SWITCH, "0")+ assert tpq.te_prequant_sources(fam, te_quant_mode = "int8", target = CUDA_FP16) == {}+++def test_fp16_scales_the_weight_before_the_matmul():+ from core.inference.diffusion_convrot import build_convrot_hadamard++ lin = _int8_convrot_linear_class()(256, 4, bias = False, group_size = 256)+ lin.weight = torch.full((4, 256), 127, dtype = torch.int8)+ lin.weight_scale = torch.full((4, 1), 1e-4, dtype = torch.float32)+ # Unscaled sum 640 * 127 = 81280 overflows float16; true output is 8.13.+ x = torch.full((1, 256), 40.0, dtype = torch.float16)+ h = build_convrot_hadamard(256, dtype = torch.float32)+ expected = torch.nn.functional.linear(x.float(), (lin.weight.float() * lin.weight_scale) @ h)+ assert not torch.isfinite(lin(x)).all()+ lin.scale_weight_first = True+ out = lin(x)+ assert out.dtype == torch.float16+ torch.testing.assert_close(out.float(), expected, rtol = 2e-3, atol = 2e-3)+++def test_fp16_load_flags_every_convrot_linear(int8_file):+ _, _, path = int8_file+ cls = _int8_convrot_linear_class()++ def load(dtype):+ enc = tpq.load_prequant_text_encoder(+ BASE,+ "text_encoder",+ tpq.TePrequantSource(kind = "path", location = str(path)),+ dtype = dtype,+ scheme = "int8",+ local_files_only = True,+ trim_lm_head = True,+ )+ return [getattr(m, "scale_weight_first", None) for m in enc.modules() if isinstance(m, cls)]++ assert set(load(torch.float16)) == {True}+ assert set(load(torch.bfloat16)) == {False}