Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio diffusion: keep single-file int8 weights as int8 on T4 / pre-Ampere NVIDIA and Intel XPU under offload (#17079)

Daniel Han3967e8156ecd
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.py
index e0239e14d6c..9e5dfe7978e 100644
--- a/studio/backend/core/inference/diffusion.py
+++ b/studio/backend/core/inference/diffusion.py
@@ -9961,6 +9961,7 @@ class DiffusionBackend:
fam.name,
base,
offload = _comfy_offload,
+ lora = _has_active_lora(loras),
),
fp8_backend = comfy_fp8_backend(
target,
@@ -11629,7 +11630,8 @@ class DiffusionBackend:
return comfy_resident_mib(
single_file_path,
scan,
- keep_int8 = comfy_int8_backend(target, name, base, offload = offload) is not None,
+ keep_int8 = comfy_int8_backend(target, name, base, offload = offload, lora = lora)
+ is not None,
keep_fp8 = comfy_fp8_backend(target, name, base, offload = offload) is not None,
fp8_stored = fp8_storage and comfy_fp8_storage(target, lora = lora),
target = target,
diff --git a/studio/backend/core/inference/diffusion_comfy_quant.py b/studio/backend/core/inference/diffusion_comfy_quant.py
index 54fce04c8e0..a70a90762ff 100644
--- a/studio/backend/core/inference/diffusion_comfy_quant.py
+++ b/studio/backend/core/inference/diffusion_comfy_quant.py
@@ -969,16 +969,20 @@ def comfy_int8_backend(
base_repo: Optional[str] = None,
*,
offload: bool = False,
+ lora: bool = False,
) -> Optional[str]:
- """int8 runtime: torchao when resident, native under offload, None to dequantise to bf16."""
+ """int8 runtime: torchao when resident, native under offload (or as storage, ``int8_storage_scheme``, where neither
+ runs; never under a LoRA), None to dequantise to bf16."""
if (os.environ.get(COMFY_INT8_ENV) or "").strip().lower() in ("0", "off", "false", "no"):
return None
try:
from .diffusion_transformer_quant import (
TQ_INT8,
+ int8_storage_scheme,
native_quant_scheme,
select_transformer_quant_scheme,
)
+
if native_quant_scheme(target, TQ_INT8, family = family, offload = offload) == TQ_INT8:
return "native"
if not offload and (
@@ -986,6 +990,8 @@ def comfy_int8_backend(
== TQ_INT8
):
return "torchao"
+ if not lora and int8_storage_scheme(target, family, offload = offload) == TQ_INT8:
+ return "native"
except Exception: # noqa: BLE001 -- an unanswerable probe takes the exact dense path
pass
return None
diff --git a/studio/backend/core/inference/diffusion_transformer_quant.py b/studio/backend/core/inference/diffusion_transformer_quant.py
index 1da8410756f..913bb4745e6 100644
--- a/studio/backend/core/inference/diffusion_transformer_quant.py
+++ b/studio/backend/core/inference/diffusion_transformer_quant.py
@@ -1216,10 +1216,53 @@ def native_quant_scheme(
return scheme
+# Checkpoint int8 codes on a GPU with no Studio int8 runtime (pre-Ampere fp16, XPU) stay int8, widened per layer.
+# Unset: offloaded plans only (resident, the expanded DiT is faster); 1: resident too; 0: always expand.
+INT8_STORAGE_ENV = "UNSLOTH_DIFFUSION_INT8_STORAGE"
+
+
+def _int8_storage_setting() -> str:
+ return (_os.environ.get(INT8_STORAGE_ENV) or "").strip().lower()
+
+
+def int8_storage_host(target: Any) -> bool:
+ """Whether ``target`` keeps checkpoint int8 codes as weight-only storage (no int8 GEMM)."""
+ if _int8_storage_setting() in ("0", "off", "false", "no"):
+ return False
+ device = getattr(target, "device", None)
+ try:
+ import torch
+
+ dtype = getattr(target, "dtype", None)
+ if device == "xpu":
+ return dtype in (torch.bfloat16, torch.float16)
+ if device != "cuda" or torch_is_rocm() or is_stubbed("torchao"):
+ return False
+ return dtype is torch.float16
+ except Exception:
+ return False
+
+
+def int8_storage_scheme(
+ target: Any,
+ family: Optional[str] = None,
+ *,
+ offload: bool = False,
+) -> Optional[str]:
+ if not (offload or _int8_storage_setting() in ("1", "on", "true", "yes")):
+ return None
+ if not int8_storage_host(target) or _family_denied(family, TQ_INT8):
+ return None
+ return TQ_INT8
+
+
def native_int8_act(target: Any) -> bool:
# MPS has no ``_int_mm``: W8A8 would only add the activation quant.
if _mps_bf16(target):
return False
+ # storage only (int8_storage_scheme): never an int8 GEMM
+ if int8_storage_host(target):
+ return False
if native_quant_host(target):
return int8_act_requested()
return not int8_act_disabled()
diff --git a/studio/backend/tests/test_diffusion_comfy_quant.py b/studio/backend/tests/test_diffusion_comfy_quant.py
index 16acd69a93c..fc6067193a1 100644
--- a/studio/backend/tests/test_diffusion_comfy_quant.py
+++ b/studio/backend/tests/test_diffusion_comfy_quant.py
@@ -1507,3 +1507,185 @@ def test_planner_prices_keep_fp32_layers_at_fp32_only_on_fp16(monkeypatch):
assert bf16["keep_fp32_bytes"] is None
flux = types.SimpleNamespace(transformer_class = "FluxTransformer2DModel")
assert d._comfy_keep_fp32_pricing(flux, types.SimpleNamespace(dtype = torch.float16)) == {}
+
+
+# ------------------------------------------------------------------------------------- int8 storage
+import core.inference.diffusion_transformer_quant as tq # noqa: E402
+
+_STORAGE_ENV = "UNSLOTH_DIFFUSION_INT8_STORAGE"
+
+
+def _target(device, dtype):
+ return types.SimpleNamespace(device = device, dtype = dtype, backend = device)
+
+
+def _nvidia(monkeypatch):
+ monkeypatch.setattr(tq, "torch_is_rocm", lambda: False)
+ monkeypatch.setattr(tq, "is_stubbed", lambda pkg: False)
+ # no torchao smoke probe here
+ monkeypatch.setattr(tq, "select_transformer_quant_scheme", lambda *a, **k: None)
+
+
+ "device, dtype, offload, env, want",
+ [
+ # pre-Ampere NVIDIA (fp16 compute) and XPU: stored under offload, or everywhere when forced
+ ("cuda", torch.float16, True, None, "native"),
+ ("cuda", torch.float16, False, None, None),
+ ("cuda", torch.float16, False, "1", "native"),
+ ("cuda", torch.float16, True, "0", None),
+ ("xpu", torch.bfloat16, True, None, "native"),
+ ("xpu", torch.float16, True, None, "native"),
+ ("xpu", torch.bfloat16, False, None, None),
+ ("xpu", torch.bfloat16, False, "1", "native"),
+ ("xpu", torch.bfloat16, True, "0", None),
+ # untouched: Ampere+ keeps its W8A8 offload route, CPU and an fp32 XPU expand
+ ("cuda", torch.bfloat16, True, None, "native"),
+ ("cuda", torch.bfloat16, True, "0", "native"),
+ ("cuda", torch.bfloat16, False, "1", None),
+ ("cpu", torch.float32, True, "1", None),
+ ("xpu", torch.float32, True, None, None),
+ ],
+)
+def test_int8_storage_backend_choice(monkeypatch, device, dtype, offload, env, want):
+ _nvidia(monkeypatch)
+ if env is None:
+ monkeypatch.delenv(_STORAGE_ENV, raising = False)
+ else:
+ monkeypatch.setenv(_STORAGE_ENV, env)
+ assert cq.comfy_int8_backend(_target(device, dtype), "z-image", offload = offload) == want
+
+
+def test_int8_storage_never_on_rocm_or_the_stub(monkeypatch):
+ monkeypatch.setenv(_STORAGE_ENV, "1")
+ monkeypatch.setattr(tq, "torch_is_rocm", lambda: True)
+ assert not tq.int8_storage_host(_target("cuda", torch.float16))
+ monkeypatch.setattr(tq, "torch_is_rocm", lambda: False)
+ monkeypatch.setattr(tq, "is_stubbed", lambda pkg: True)
+ assert not tq.int8_storage_host(_target("cuda", torch.float16))
+
+
+def test_int8_storage_respects_the_family_deny(monkeypatch):
+ _nvidia(monkeypatch)
+ monkeypatch.setattr(tq, "_family_denied", lambda family, scheme, *a: scheme == "int8")
+ assert cq.comfy_int8_backend(_target("cuda", torch.float16), "z-image", offload = True) is None
+
+
+def test_int8_storage_hosts_never_take_the_int8_gemm(monkeypatch):
+ _nvidia(monkeypatch)
+ monkeypatch.delenv(_STORAGE_ENV, raising = False)
+ assert tq.native_int8_act(_target("cuda", torch.float16)) is False
+ assert tq.native_int8_act(_target("xpu", torch.bfloat16)) is False
+ assert tq.native_int8_act(_target("cuda", torch.bfloat16)) is (not tq.int8_act_disabled())
+ monkeypatch.setenv(_STORAGE_ENV, "0")
+ assert tq.native_int8_act(_target("cuda", torch.float16)) is (not tq.int8_act_disabled())
+
+
+def plain_int8_file(tmp_path, monkeypatch):
+ """A ComfyUI int8_tensorwise file (no ConvRot) of ``_Tiny``."""
+ monkeypatch.setattr(cq, "_mapping", lambda cls: (_convert, _sfm()))
+ torch.manual_seed(1)
+ dense = _Tiny()
+ tensors = {}
+ for b, block in enumerate(dense.blocks):
+ qkv = torch.cat([block.to_q.weight, block.to_k.weight, block.to_v.weight]).detach()
+ bias = torch.cat([block.to_q.bias, block.to_k.bias, block.to_v.bias])
+ for name, (weight, b_) in {
+ f"blocks.{b}.qkv": (qkv, bias),
+ f"blocks.{b}.out": (block.to_out.weight.detach(), block.to_out.bias),
+ }.items():
+ q, s = _int8(weight)
+ tensors[f"{name}.weight"] = q
+ tensors[f"{name}.weight_scale"] = s
+ tensors[f"{name}.bias"] = b_.detach().clone()
+ tensors[f"{name}.comfy_quant"] = _conf(format = "int8_tensorwise")
+ tensors[f"blocks.{b}.adaLN_modulation.0.weight"] = (
+ block.adaLN_modulation[0].weight.detach().clone()
+ )
+ tensors[f"blocks.{b}.adaLN_modulation.0.bias"] = (
+ block.adaLN_modulation[0].bias.detach().clone()
+ )
+ tensors["norm.weight"] = dense.norm.weight.detach().clone()
+ tensors["norm.bias"] = dense.norm.bias.detach().clone()
+ return _save(tmp_path / "tiny_int8.safetensors", tensors)
+
+
+def _load_as(path, target, backend):
+ return cq.load_comfy_quant_transformer(
+ _Tiny,
+ path,
+ cq.refuse_comfy_quant(path),
+ {"torch_dtype": target.dtype, "config": "base/repo", "subfolder": "transformer"},
+ int8_backend = backend,
+ family = "z-image",
+ target = target,
+ )
+
+
+def _run(model, x):
+ outs = []
+ for block in model.blocks:
+ for name in ("to_q", "to_k", "to_v", "to_out"):
+ outs.append(getattr(block, name)(x))
+ return outs
+
+
+def _held(model) -> int:
+ return sum(t.nbytes for t in (*model.parameters(), *model.buffers()))
+
+
[email protected]("device, dtype", [("cuda", torch.float16), ("xpu", torch.bfloat16)])
+def test_int8_storage_keeps_codes_and_matches_the_expanded_dit_bitwise(
+ monkeypatch, plain_int8_file, device, dtype
+):
+ from core.inference.diffusion_native_quant import is_native_linear
+
+ _nvidia(monkeypatch)
+ monkeypatch.delenv(_STORAGE_ENV, raising = False)
+ target = _target(device, dtype)
+ backend = cq.comfy_int8_backend(target, "z-image", offload = True)
+ assert backend == "native"
+ expanded = _load_as(plain_int8_file, target, None)
+ stored = _load_as(plain_int8_file, target, backend)
+ assert (
+ stored._unsloth_comfy_quant["int8"] == 8
+ and stored._unsloth_comfy_quant["backend"] == "native"
+ )
+ for block in stored.blocks:
+ for name in ("to_q", "to_k", "to_v", "to_out"):
+ layer = getattr(block, name)
+ assert is_native_linear(layer) and not layer.act_int8 and not layer.rot_group
+ assert layer.weight_q.dtype == torch.int8
+ # 1 byte + a 4-byte row scale against itemsize per weight
+ assert _held(expanded) - _held(stored) == 8 * (DIM * DIM * (dtype.itemsize - 1) - 4 * DIM)
+ x = torch.randn(3, DIM).to(dtype)
+ for got, want in zip(_run(stored, x), _run(expanded, x)):
+ assert torch.equal(got, want)
+
+
+def test_int8_storage_convrot_rotates_activations_like_the_native_route(monkeypatch, comfy_file):
+ _nvidia(monkeypatch)
+ monkeypatch.delenv(_STORAGE_ENV, raising = False)
+ path, dense, _ = comfy_file
+ target = _target("cuda", torch.float16)
+ stored = _load_as(path, target, cq.comfy_int8_backend(target, "z-image", offload = True))
+ expanded = _load_as(path, target, None)
+ assert stored._unsloth_comfy_quant["convrot"] == 8
+ layer = stored.blocks[0].to_k
+ assert layer.rot_group == GROUP and not layer.act_int8
+ x = torch.randn(5, DIM).to(torch.float16)
+ for got, want in zip(_run(stored, x), _run(expanded, x)):
+ assert _cos(got, want) > 0.9999
+
+
+def test_int8_storage_stays_off_under_a_lora(monkeypatch):
+ _nvidia(monkeypatch)
+ monkeypatch.delenv(_STORAGE_ENV, raising = False)
+ t4 = _target("cuda", torch.float16)
+ assert cq.comfy_int8_backend(t4, "z-image", offload = True) == "native"
+ assert cq.comfy_int8_backend(t4, "z-image", offload = True, lora = True) is None
+ assert (
+ cq.comfy_int8_backend(_target("cuda", torch.bfloat16), "z-image", offload = True, lora = True)
+ == "native"
+ )