Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio diffusion: load Qwen-Image-2.1's hosted int8 text encoder on float16 cards (T4 Turbo INT8 82.7 -> 31.9 s/image) (#17379)

Daniel Hanaa97d20989eb
diff --git a/studio/backend/core/inference/diffusion_te_prequant.py b/studio/backend/core/inference/diffusion_te_prequant.py
index 5a59fa8c8c5..eefaf59eef6 100644
--- a/studio/backend/core/inference/diffusion_te_prequant.py
+++ b/studio/backend/core/inference/diffusion_te_prequant.py
@@ -741,6 +741,27 @@ def int8_convrot_te_runs_on(target: Any) -> bool:
return getattr(target, "device", None) == "mps"
+TE_INT8_FP16_ENV = "UNSLOTH_TE_INT8_FP16"
+# Families whose hosted int8 ConvRot encoder was validated in float16 compute.
+_TE_INT8_FP16_FAMILIES = frozenset({"qwen-image-2.1"})
+
+
+def int8_convrot_te_runs_fp16(fam: Any, target: Any) -> bool:
+ """float16 CUDA (no bf16/fp8) runs the hosted int8 ConvRot encoder; ``UNSLOTH_TE_INT8_FP16=0`` keeps the dense one."""
+ import os
+
+ if os.environ.get(TE_INT8_FP16_ENV, "1").strip().lower() in ("0", "false", "no", "off"):
+ return False
+ name = str(getattr(fam, "name", fam) or "").strip().lower()
+ if name not in _TE_INT8_FP16_FAMILIES or getattr(target, "device", None) != "cuda":
+ return False
+ try:
+ import torch
+ return getattr(target, "dtype", None) is torch.float16
+ except Exception: # noqa: BLE001
+ return False
+
+
def hosted_int8_convrot_te_runs_on(fam: Any, target: Any) -> bool:
"""Whether an explicit int8 encoder request on ``fam`` loads the hosted int8 ConvRot file on ``target`` without the
fp8 fallback: the family hosts the file (and the lm_head trim it needs is on) and the device runs it as plain
@@ -959,7 +980,10 @@ def te_prequant_sources(
return {}
# int8 falls back to the fp8 file, so it needs fp8; MPS runs the int8 file alone and drops the fp8 names.
fp8_runs = te_quant_supported(target, TE_QUANT_FP8)
- if not fp8_runs and not (mode == "int8" and int8_convrot_te_runs_on(target)):
+ if not fp8_runs and not (
+ mode == "int8"
+ and (int8_convrot_te_runs_on(target) or int8_convrot_te_runs_fp16(fam, target))
+ ):
return {}
sources: dict[str, TePrequantSource] = {}
for component in components:
@@ -1779,6 +1803,10 @@ def _build_int8_convrot_encoder(
)
_fill_tied_aliases(encoder, state_dict)
encoder.load_state_dict(state_dict, strict = True, assign = True)
+ if dtype is torch.float16:
+ for module in encoder.modules():
+ if isinstance(module, linear_cls):
+ module.scale_weight_first = True
# assign=True gives a tied pair separate tensors; re-tie a headless encoder (T5 / UMT5's encoder.embed_tokens ->
# shared), as the fp8 path does. One with a (trimmed) head is left as loaded: its head is a stand-in with no weight
# to tie.
diff --git a/studio/backend/core/inference/video_minimax_h3_te.py b/studio/backend/core/inference/video_minimax_h3_te.py
index 7f2de2c3b99..3e977fe1a40 100644
--- a/studio/backend/core/inference/video_minimax_h3_te.py
+++ b/studio/backend/core/inference/video_minimax_h3_te.py
@@ -130,10 +130,18 @@ def _int8_convrot_linear_class() -> Any:
)
else:
self.bias = None
+ # float16: scale the weight before the matmul; the unscaled int8 sum overflows 65504.
+ self.scale_weight_first = False
def forward(self, x: Any) -> Any: # noqa: D102
h = build_convrot_hadamard(self.group_size, device = x.device, dtype = x.dtype)
rotated = rotate_convrot_activation(x, h, self.group_size)
+ if getattr(self, "scale_weight_first", False):
+ weight = (self.weight.to(torch.float32) * self.weight_scale).to(x.dtype)
+ out = torch.nn.functional.linear(rotated, weight)
+ if self.bias is not None:
+ out = out + self.bias.to(out.dtype)
+ return out
out = torch.nn.functional.linear(rotated, self.weight.to(x.dtype))
out = out * self.weight_scale.reshape(1, -1).to(dtype = out.dtype, device = out.device)
if self.bias is not None:
diff --git a/studio/backend/tests/test_diffusion_te_int8_fp16.py b/studio/backend/tests/test_diffusion_te_int8_fp16.py
new file mode 100644
index 00000000000..7420ebbfca5
--- /dev/null
+++ b/studio/backend/tests/test_diffusion_te_int8_fp16.py
@@ -0,0 +1,90 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""float16 CUDA cards (T4) take Qwen-Image-2.1's hosted int8 ConvRot text encoder instead of the dense bf16 one."""
+
+from __future__ import annotations
+
+import types
+
+import pytest
+
+torch = pytest.importorskip("torch")
+pytest.importorskip("transformers.models.qwen3_vl")
+pytest.importorskip("accelerate")
+
+import core.inference.diffusion_te_prequant as tpq # noqa: E402
+from core.inference.video_minimax_h3_te import _int8_convrot_linear_class # noqa: E402
+
+from .test_diffusion_te_int8_convrot import ( # noqa: E402, F401
+ BASE,
+ INT8_NAME,
+ REPO,
+ get_family,
+ int8_file,
+)
+
+CUDA_FP16 = types.SimpleNamespace(device = "cuda", dtype = torch.float16)
+CUDA_BF16 = types.SimpleNamespace(device = "cuda", dtype = torch.bfloat16)
+KILL_SWITCH = "UNSLOTH_TE_INT8_FP16"
+
+
[email protected](autouse = True)
+def _default_env(monkeypatch):
+ monkeypatch.delenv(KILL_SWITCH, raising = False)
+
+
+def test_fp16_card_resolves_only_the_int8_file():
+ fam = get_family("qwen-image-2.1")
+ src = tpq.te_prequant_sources_for_base(fam, BASE, te_quant_mode = "int8", target = CUDA_FP16)[
+ "text_encoder"
+ ]
+ assert (src.kind, src.location, src.filename) == ("repo", REPO, INT8_NAME)
+ assert tpq.te_candidate_filenames(src) == (INT8_NAME,)
+
+
+def test_fp16_kill_switch_and_other_cases_unchanged(monkeypatch):
+ fam = get_family("qwen-image-2.1")
+ assert tpq.te_prequant_sources(fam, te_quant_mode = "fp8", target = CUDA_FP16) == {}
+ assert tpq.te_prequant_sources(get_family("sdxl"), te_quant_mode = "int8", target = CUDA_FP16) == {}
+ bf16 = tpq.te_prequant_sources(fam, te_quant_mode = "int8", target = CUDA_BF16)["text_encoder"]
+ assert len(tpq.te_candidate_filenames(bf16)) > 1
+ monkeypatch.setenv(KILL_SWITCH, "0")
+ assert tpq.te_prequant_sources(fam, te_quant_mode = "int8", target = CUDA_FP16) == {}
+
+
+def test_fp16_scales_the_weight_before_the_matmul():
+ from core.inference.diffusion_convrot import build_convrot_hadamard
+
+ lin = _int8_convrot_linear_class()(256, 4, bias = False, group_size = 256)
+ lin.weight = torch.full((4, 256), 127, dtype = torch.int8)
+ lin.weight_scale = torch.full((4, 1), 1e-4, dtype = torch.float32)
+ # Unscaled sum 640 * 127 = 81280 overflows float16; true output is 8.13.
+ x = torch.full((1, 256), 40.0, dtype = torch.float16)
+ h = build_convrot_hadamard(256, dtype = torch.float32)
+ expected = torch.nn.functional.linear(x.float(), (lin.weight.float() * lin.weight_scale) @ h)
+ assert not torch.isfinite(lin(x)).all()
+ lin.scale_weight_first = True
+ out = lin(x)
+ assert out.dtype == torch.float16
+ torch.testing.assert_close(out.float(), expected, rtol = 2e-3, atol = 2e-3)
+
+
+def test_fp16_load_flags_every_convrot_linear(int8_file):
+ _, _, path = int8_file
+ cls = _int8_convrot_linear_class()
+
+ def load(dtype):
+ enc = tpq.load_prequant_text_encoder(
+ BASE,
+ "text_encoder",
+ tpq.TePrequantSource(kind = "path", location = str(path)),
+ dtype = dtype,
+ scheme = "int8",
+ local_files_only = True,
+ trim_lm_head = True,
+ )
+ return [getattr(m, "scale_weight_first", None) for m in enc.modules() if isinstance(m, cls)]
+
+ assert set(load(torch.float16)) == {True}
+ assert set(load(torch.bfloat16)) == {False}