Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio diffusion: keep single-file int8 weights as int8 on T4 / pre-Ampere NVIDIA and Intel XPU under offload (#17079)
Daniel Han3967e8156ecd
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.pyindex e0239e14d6c..9e5dfe7978e 100644--- a/studio/backend/core/inference/diffusion.py+++ b/studio/backend/core/inference/diffusion.py@@ -9961,6 +9961,7 @@ class DiffusionBackend:fam.name,base,offload = _comfy_offload,+ lora = _has_active_lora(loras),),fp8_backend = comfy_fp8_backend(target,@@ -11629,7 +11630,8 @@ class DiffusionBackend:return comfy_resident_mib(single_file_path,scan,- keep_int8 = comfy_int8_backend(target, name, base, offload = offload) is not None,+ keep_int8 = comfy_int8_backend(target, name, base, offload = offload, lora = lora)+ is not None,keep_fp8 = comfy_fp8_backend(target, name, base, offload = offload) is not None,fp8_stored = fp8_storage and comfy_fp8_storage(target, lora = lora),target = target,diff --git a/studio/backend/core/inference/diffusion_comfy_quant.py b/studio/backend/core/inference/diffusion_comfy_quant.pyindex 54fce04c8e0..a70a90762ff 100644--- a/studio/backend/core/inference/diffusion_comfy_quant.py+++ b/studio/backend/core/inference/diffusion_comfy_quant.py@@ -969,16 +969,20 @@ def comfy_int8_backend(base_repo: Optional[str] = None,*,offload: bool = False,+ lora: bool = False,) -> Optional[str]:- """int8 runtime: torchao when resident, native under offload, None to dequantise to bf16."""+ """int8 runtime: torchao when resident, native under offload (or as storage, ``int8_storage_scheme``, where neither+ runs; never under a LoRA), None to dequantise to bf16."""if (os.environ.get(COMFY_INT8_ENV) or "").strip().lower() in ("0", "off", "false", "no"):return Nonetry:from .diffusion_transformer_quant import (TQ_INT8,+ int8_storage_scheme,native_quant_scheme,select_transformer_quant_scheme,)+if native_quant_scheme(target, TQ_INT8, family = family, offload = offload) == TQ_INT8:return "native"if not offload and (@@ -986,6 +990,8 @@ def comfy_int8_backend(== TQ_INT8):return "torchao"+ if not lora and int8_storage_scheme(target, family, offload = offload) == TQ_INT8:+ return "native"except Exception: # noqa: BLE001 -- an unanswerable probe takes the exact dense pathpassreturn Nonediff --git a/studio/backend/core/inference/diffusion_transformer_quant.py b/studio/backend/core/inference/diffusion_transformer_quant.pyindex 1da8410756f..913bb4745e6 100644--- a/studio/backend/core/inference/diffusion_transformer_quant.py+++ b/studio/backend/core/inference/diffusion_transformer_quant.py@@ -1216,10 +1216,53 @@ def native_quant_scheme(return scheme+# Checkpoint int8 codes on a GPU with no Studio int8 runtime (pre-Ampere fp16, XPU) stay int8, widened per layer.+# Unset: offloaded plans only (resident, the expanded DiT is faster); 1: resident too; 0: always expand.+INT8_STORAGE_ENV = "UNSLOTH_DIFFUSION_INT8_STORAGE"+++def _int8_storage_setting() -> str:+ return (_os.environ.get(INT8_STORAGE_ENV) or "").strip().lower()+++def int8_storage_host(target: Any) -> bool:+ """Whether ``target`` keeps checkpoint int8 codes as weight-only storage (no int8 GEMM)."""+ if _int8_storage_setting() in ("0", "off", "false", "no"):+ return False+ device = getattr(target, "device", None)+ try:+ import torch++ dtype = getattr(target, "dtype", None)+ if device == "xpu":+ return dtype in (torch.bfloat16, torch.float16)+ if device != "cuda" or torch_is_rocm() or is_stubbed("torchao"):+ return False+ return dtype is torch.float16+ except Exception:+ return False+++def int8_storage_scheme(+ target: Any,+ family: Optional[str] = None,+ *,+ offload: bool = False,+) -> Optional[str]:+ if not (offload or _int8_storage_setting() in ("1", "on", "true", "yes")):+ return None+ if not int8_storage_host(target) or _family_denied(family, TQ_INT8):+ return None+ return TQ_INT8++def native_int8_act(target: Any) -> bool:# MPS has no ``_int_mm``: W8A8 would only add the activation quant.if _mps_bf16(target):return False+ # storage only (int8_storage_scheme): never an int8 GEMM+ if int8_storage_host(target):+ return Falseif native_quant_host(target):return int8_act_requested()return not int8_act_disabled()diff --git a/studio/backend/tests/test_diffusion_comfy_quant.py b/studio/backend/tests/test_diffusion_comfy_quant.pyindex 16acd69a93c..fc6067193a1 100644--- a/studio/backend/tests/test_diffusion_comfy_quant.py+++ b/studio/backend/tests/test_diffusion_comfy_quant.py@@ -1507,3 +1507,185 @@ def test_planner_prices_keep_fp32_layers_at_fp32_only_on_fp16(monkeypatch):assert bf16["keep_fp32_bytes"] is Noneflux = types.SimpleNamespace(transformer_class = "FluxTransformer2DModel")assert d._comfy_keep_fp32_pricing(flux, types.SimpleNamespace(dtype = torch.float16)) == {}+++# ------------------------------------------------------------------------------------- int8 storage+import core.inference.diffusion_transformer_quant as tq # noqa: E402++_STORAGE_ENV = "UNSLOTH_DIFFUSION_INT8_STORAGE"+++def _target(device, dtype):+ return types.SimpleNamespace(device = device, dtype = dtype, backend = device)+++def _nvidia(monkeypatch):+ monkeypatch.setattr(tq, "torch_is_rocm", lambda: False)+ monkeypatch.setattr(tq, "is_stubbed", lambda pkg: False)+ # no torchao smoke probe here+ monkeypatch.setattr(tq, "select_transformer_quant_scheme", lambda *a, **k: None)+++ "device, dtype, offload, env, want",+ [+ # pre-Ampere NVIDIA (fp16 compute) and XPU: stored under offload, or everywhere when forced+ ("cuda", torch.float16, True, None, "native"),+ ("cuda", torch.float16, False, None, None),+ ("cuda", torch.float16, False, "1", "native"),+ ("cuda", torch.float16, True, "0", None),+ ("xpu", torch.bfloat16, True, None, "native"),+ ("xpu", torch.float16, True, None, "native"),+ ("xpu", torch.bfloat16, False, None, None),+ ("xpu", torch.bfloat16, False, "1", "native"),+ ("xpu", torch.bfloat16, True, "0", None),+ # untouched: Ampere+ keeps its W8A8 offload route, CPU and an fp32 XPU expand+ ("cuda", torch.bfloat16, True, None, "native"),+ ("cuda", torch.bfloat16, True, "0", "native"),+ ("cuda", torch.bfloat16, False, "1", None),+ ("cpu", torch.float32, True, "1", None),+ ("xpu", torch.float32, True, None, None),+ ],+)+def test_int8_storage_backend_choice(monkeypatch, device, dtype, offload, env, want):+ _nvidia(monkeypatch)+ if env is None:+ monkeypatch.delenv(_STORAGE_ENV, raising = False)+ else:+ monkeypatch.setenv(_STORAGE_ENV, env)+ assert cq.comfy_int8_backend(_target(device, dtype), "z-image", offload = offload) == want+++def test_int8_storage_never_on_rocm_or_the_stub(monkeypatch):+ monkeypatch.setenv(_STORAGE_ENV, "1")+ monkeypatch.setattr(tq, "torch_is_rocm", lambda: True)+ assert not tq.int8_storage_host(_target("cuda", torch.float16))+ monkeypatch.setattr(tq, "torch_is_rocm", lambda: False)+ monkeypatch.setattr(tq, "is_stubbed", lambda pkg: True)+ assert not tq.int8_storage_host(_target("cuda", torch.float16))+++def test_int8_storage_respects_the_family_deny(monkeypatch):+ _nvidia(monkeypatch)+ monkeypatch.setattr(tq, "_family_denied", lambda family, scheme, *a: scheme == "int8")+ assert cq.comfy_int8_backend(_target("cuda", torch.float16), "z-image", offload = True) is None+++def test_int8_storage_hosts_never_take_the_int8_gemm(monkeypatch):+ _nvidia(monkeypatch)+ monkeypatch.delenv(_STORAGE_ENV, raising = False)+ assert tq.native_int8_act(_target("cuda", torch.float16)) is False+ assert tq.native_int8_act(_target("xpu", torch.bfloat16)) is False+ assert tq.native_int8_act(_target("cuda", torch.bfloat16)) is (not tq.int8_act_disabled())+ monkeypatch.setenv(_STORAGE_ENV, "0")+ assert tq.native_int8_act(_target("cuda", torch.float16)) is (not tq.int8_act_disabled())+++def plain_int8_file(tmp_path, monkeypatch):+ """A ComfyUI int8_tensorwise file (no ConvRot) of ``_Tiny``."""+ monkeypatch.setattr(cq, "_mapping", lambda cls: (_convert, _sfm()))+ torch.manual_seed(1)+ dense = _Tiny()+ tensors = {}+ for b, block in enumerate(dense.blocks):+ qkv = torch.cat([block.to_q.weight, block.to_k.weight, block.to_v.weight]).detach()+ bias = torch.cat([block.to_q.bias, block.to_k.bias, block.to_v.bias])+ for name, (weight, b_) in {+ f"blocks.{b}.qkv": (qkv, bias),+ f"blocks.{b}.out": (block.to_out.weight.detach(), block.to_out.bias),+ }.items():+ q, s = _int8(weight)+ tensors[f"{name}.weight"] = q+ tensors[f"{name}.weight_scale"] = s+ tensors[f"{name}.bias"] = b_.detach().clone()+ tensors[f"{name}.comfy_quant"] = _conf(format = "int8_tensorwise")+ tensors[f"blocks.{b}.adaLN_modulation.0.weight"] = (+ block.adaLN_modulation[0].weight.detach().clone()+ )+ tensors[f"blocks.{b}.adaLN_modulation.0.bias"] = (+ block.adaLN_modulation[0].bias.detach().clone()+ )+ tensors["norm.weight"] = dense.norm.weight.detach().clone()+ tensors["norm.bias"] = dense.norm.bias.detach().clone()+ return _save(tmp_path / "tiny_int8.safetensors", tensors)+++def _load_as(path, target, backend):+ return cq.load_comfy_quant_transformer(+ _Tiny,+ path,+ cq.refuse_comfy_quant(path),+ {"torch_dtype": target.dtype, "config": "base/repo", "subfolder": "transformer"},+ int8_backend = backend,+ family = "z-image",+ target = target,+ )+++def _run(model, x):+ outs = []+ for block in model.blocks:+ for name in ("to_q", "to_k", "to_v", "to_out"):+ outs.append(getattr(block, name)(x))+ return outs+++def _held(model) -> int:+ return sum(t.nbytes for t in (*model.parameters(), *model.buffers()))++[email protected]("device, dtype", [("cuda", torch.float16), ("xpu", torch.bfloat16)])+def test_int8_storage_keeps_codes_and_matches_the_expanded_dit_bitwise(+ monkeypatch, plain_int8_file, device, dtype+):+ from core.inference.diffusion_native_quant import is_native_linear++ _nvidia(monkeypatch)+ monkeypatch.delenv(_STORAGE_ENV, raising = False)+ target = _target(device, dtype)+ backend = cq.comfy_int8_backend(target, "z-image", offload = True)+ assert backend == "native"+ expanded = _load_as(plain_int8_file, target, None)+ stored = _load_as(plain_int8_file, target, backend)+ assert (+ stored._unsloth_comfy_quant["int8"] == 8+ and stored._unsloth_comfy_quant["backend"] == "native"+ )+ for block in stored.blocks:+ for name in ("to_q", "to_k", "to_v", "to_out"):+ layer = getattr(block, name)+ assert is_native_linear(layer) and not layer.act_int8 and not layer.rot_group+ assert layer.weight_q.dtype == torch.int8+ # 1 byte + a 4-byte row scale against itemsize per weight+ assert _held(expanded) - _held(stored) == 8 * (DIM * DIM * (dtype.itemsize - 1) - 4 * DIM)+ x = torch.randn(3, DIM).to(dtype)+ for got, want in zip(_run(stored, x), _run(expanded, x)):+ assert torch.equal(got, want)+++def test_int8_storage_convrot_rotates_activations_like_the_native_route(monkeypatch, comfy_file):+ _nvidia(monkeypatch)+ monkeypatch.delenv(_STORAGE_ENV, raising = False)+ path, dense, _ = comfy_file+ target = _target("cuda", torch.float16)+ stored = _load_as(path, target, cq.comfy_int8_backend(target, "z-image", offload = True))+ expanded = _load_as(path, target, None)+ assert stored._unsloth_comfy_quant["convrot"] == 8+ layer = stored.blocks[0].to_k+ assert layer.rot_group == GROUP and not layer.act_int8+ x = torch.randn(5, DIM).to(torch.float16)+ for got, want in zip(_run(stored, x), _run(expanded, x)):+ assert _cos(got, want) > 0.9999+++def test_int8_storage_stays_off_under_a_lora(monkeypatch):+ _nvidia(monkeypatch)+ monkeypatch.delenv(_STORAGE_ENV, raising = False)+ t4 = _target("cuda", torch.float16)+ assert cq.comfy_int8_backend(t4, "z-image", offload = True) == "native"+ assert cq.comfy_int8_backend(t4, "z-image", offload = True, lora = True) is None+ assert (+ cq.comfy_int8_backend(_target("cuda", torch.bfloat16), "z-image", offload = True, lora = True)+ == "native"+ )