Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: keep single-file int8 denoiser weights as int8 on CPU instead of expanding them to float32 (#17178)

Daniel Han14896930e30b
diff --git a/studio/backend/core/inference/diffusion_native_quant.py b/studio/backend/core/inference/diffusion_native_quant.py
index 9d1eb55cfda..84911aa618d 100644
--- a/studio/backend/core/inference/diffusion_native_quant.py
+++ b/studio/backend/core/inference/diffusion_native_quant.py
@@ -10,6 +10,7 @@ import contextlib
import contextvars
import os
import re
+import threading
import time
from functools import lru_cache
from typing import Any, Callable, Optional
@@ -471,6 +472,22 @@ def fused_w8a8_state() -> dict:
}
+# Per-thread fp32 widen buffer: a fresh one per call costs ~4x the widen in page faults.
+_CPU_WIDEN = threading.local()
+
+
+def _cpu_widen_scratch(numel: int) -> Any:
+ import torch
+
+ buf = getattr(_CPU_WIDEN, "buf", None)
+ if buf is None or buf.numel() < numel:
+ _CPU_WIDEN.buf = None
+ # Not an inference tensor: those are read-only outside inference_mode.
+ with torch.inference_mode(False):
+ buf = _CPU_WIDEN.buf = torch.empty(numel, dtype = torch.float32)
+ return buf[:numel]
+
+
@lru_cache(maxsize = 1)
def native_linear_class():
"""``NativeWeightOnlyLinear``, defined on first use so this module imports torch-free."""
@@ -612,6 +629,21 @@ def native_linear_class():
wide = self.weight_q.view(getattr(torch, self.fp8_format)).to(torch.float32)
return (wide * scale[:, None]).to(dtype)
+ def _forward_weight(self, x: Any) -> Any:
+ # Not under autograd: a graph would save the buffer the next layer overwrites.
+ if (
+ self.scheme == NATIVE_INT8
+ and x.dtype is torch.float32
+ and x.device.type == "cpu"
+ and self.weight_q.device.type == "cpu"
+ and not torch.compiler.is_compiling()
+ and not (torch.is_grad_enabled() and x.requires_grad)
+ ):
+ out = _cpu_widen_scratch(self.weight_q.numel()).view(self.weight_q.shape)
+ out.copy_(self.weight_q)
+ return out.mul_(self.weight_scale.view(torch.float32)[:, None])
+ return self._stored_weight(x.dtype)
+
def _rotate(self, x: Any) -> Any:
g = self.rot_group
# Leading dims, not -1: a zero-row input has no inferable -1 on older torch.
@@ -813,7 +845,7 @@ def native_linear_class():
and self._int_mm_ok(x.numel() // max(1, self.in_features))
):
return self._forward_int_mm(x)
- return F.linear(x, self._stored_weight(x.dtype), self.bias)
+ return F.linear(x, self._forward_weight(x), self.bias)
def extra_repr(self) -> str:
return (
diff --git a/studio/backend/core/inference/diffusion_transformer_quant.py b/studio/backend/core/inference/diffusion_transformer_quant.py
index 6b661bbc6f4..6f4747ba0f3 100644
--- a/studio/backend/core/inference/diffusion_transformer_quant.py
+++ b/studio/backend/core/inference/diffusion_transformer_quant.py
@@ -1322,6 +1322,7 @@ def fp16_card_int8_act(target: Any) -> bool:
# Checkpoint int8 codes on a GPU with no Studio int8 runtime (pre-Ampere fp16, XPU) stay int8, widened per layer.
# Unset: offloaded plans only (resident, the expanded DiT is faster); 1: resident too; 0: always expand.
+# A float32 CPU target stores them unless 0: the expanded copy is 4 bytes a weight, the per-layer widen inside step noise.
INT8_STORAGE_ENV = "UNSLOTH_DIFFUSION_INT8_STORAGE"
@@ -1338,6 +1339,8 @@ def int8_storage_host(target: Any) -> bool:
import torch
dtype = getattr(target, "dtype", None)
+ if device == "cpu":
+ return dtype is torch.float32
if device == "xpu":
return dtype in (torch.bfloat16, torch.float16)
if device != "cuda" or torch_is_rocm() or is_stubbed("torchao"):
@@ -1353,7 +1356,8 @@ def int8_storage_scheme(
*,
offload: bool = False,
) -> Optional[str]:
- if not (offload or _int8_storage_setting() in ("1", "on", "true", "yes")):
+ cpu = getattr(target, "device", None) == "cpu"
+ if not (offload or cpu or _int8_storage_setting() in ("1", "on", "true", "yes")):
return None
if not int8_storage_host(target) or _family_denied(family, TQ_INT8):
return None
@@ -1364,8 +1368,8 @@ def native_int8_act(target: Any) -> bool:
# MPS has no ``_int_mm``: W8A8 would only add the activation quant.
if _mps_bf16(target):
return False
- # storage only (int8_storage_scheme): never an int8 GEMM
- if int8_storage_host(target):
+ # storage only (int8_storage_scheme): never an int8 GEMM, unless the host also runs a native W8A8 path
+ if int8_storage_host(target) and not (native_quant_host(target) or native_offload_host(target)):
return False
if native_quant_host(target):
return int8_act_requested()
diff --git a/studio/backend/tests/test_diffusion_comfy_quant.py b/studio/backend/tests/test_diffusion_comfy_quant.py
index fc6067193a1..c07a3d13263 100644
--- a/studio/backend/tests/test_diffusion_comfy_quant.py
+++ b/studio/backend/tests/test_diffusion_comfy_quant.py
@@ -1539,12 +1539,18 @@ def _nvidia(monkeypatch):
("xpu", torch.bfloat16, False, None, None),
("xpu", torch.bfloat16, False, "1", "native"),
("xpu", torch.bfloat16, True, "0", None),
- # untouched: Ampere+ keeps its W8A8 offload route, CPU and an fp32 XPU expand
+ # untouched: Ampere+ keeps its W8A8 offload route, an fp32 XPU and MPS outside bf16 expand
("cuda", torch.bfloat16, True, None, "native"),
("cuda", torch.bfloat16, True, "0", "native"),
("cuda", torch.bfloat16, False, "1", None),
- ("cpu", torch.float32, True, "1", None),
("xpu", torch.float32, True, None, None),
+ ("mps", torch.float16, False, None, None),
+ ("mps", torch.float32, False, "1", None),
+ # CPU (never offloaded): stored by default, expanded only under the kill switch
+ ("cpu", torch.float32, False, None, "native"),
+ ("cpu", torch.float32, False, "1", "native"),
+ ("cpu", torch.float32, False, "0", None),
+ ("cpu", torch.bfloat16, False, None, None),
],
)
def test_int8_storage_backend_choice(monkeypatch, device, dtype, offload, env, want):
@@ -1577,6 +1583,7 @@ def test_int8_storage_hosts_never_take_the_int8_gemm(monkeypatch):
assert tq.native_int8_act(_target("cuda", torch.float16)) is False
assert tq.native_int8_act(_target("xpu", torch.bfloat16)) is False
assert tq.native_int8_act(_target("cuda", torch.bfloat16)) is (not tq.int8_act_disabled())
+ assert tq.native_int8_act(_target("cpu", torch.float32)) is False
monkeypatch.setenv(_STORAGE_ENV, "0")
assert tq.native_int8_act(_target("cuda", torch.float16)) is (not tq.int8_act_disabled())
@@ -1664,6 +1671,113 @@ def test_int8_storage_keeps_codes_and_matches_the_expanded_dit_bitwise(
assert torch.equal(got, want)
+def test_cpu_keeps_int8_codes_resident_and_matches_the_expanded_dit_bitwise(
+ monkeypatch, plain_int8_file
+):
+ from core.inference.diffusion_native_quant import is_native_linear
+
+ _nvidia(monkeypatch)
+ monkeypatch.delenv(_STORAGE_ENV, raising = False)
+ dtype = torch.float32
+ target = _target("cpu", dtype)
+ backend = cq.comfy_int8_backend(target, "z-image")
+ assert backend == "native"
+ stored = _load_as(plain_int8_file, target, backend)
+ monkeypatch.setenv(_STORAGE_ENV, "0")
+ assert cq.comfy_int8_backend(target, "z-image") is None
+ expanded = _load_as(plain_int8_file, target, None)
+ assert stored._unsloth_comfy_quant["int8"] == 8
+ assert expanded._unsloth_comfy_quant["int8"] == 0
+ for block in stored.blocks:
+ for name in ("to_q", "to_k", "to_v", "to_out"):
+ layer = getattr(block, name)
+ assert is_native_linear(layer) and not layer.act_int8 and not layer.rot_group
+ assert layer.weight_q.dtype == torch.int8
+ for block in expanded.blocks:
+ assert not any(is_native_linear(m) for m in block.modules())
+ assert _held(expanded) - _held(stored) == 8 * (DIM * DIM * (dtype.itemsize - 1) - 4 * DIM)
+ for rows in (1, 3, 17):
+ x = torch.randn(rows, DIM).to(dtype)
+ for got, want in zip(_run(stored, x), _run(expanded, x)):
+ assert torch.equal(got, want)
+
+
+def _cpu_native(
+ out_f,
+ in_f,
+ seed,
+ bias = True,
+):
+ from core.inference.diffusion_native_quant import native_linear_class
+
+ torch.manual_seed(seed)
+ linear = torch.nn.Linear(in_f, out_f, bias = bias)
+ codes, scale = _int8(linear.weight.detach())
+ return native_linear_class()(linear, "int8", codes = codes, scale = scale)
+
+
+def test_cpu_forward_widens_into_a_reused_scratch_bitwise():
+ import core.inference.diffusion_native_quant as nq
+
+ nq._CPU_WIDEN.buf = None
+ small, big = _cpu_native(48, 64, 0), _cpu_native(96, 128, 1, bias = False)
+ x_small, x_big = torch.randn(5, 64), torch.randn(7, 128)
+ with torch.no_grad():
+ want_small = torch.nn.functional.linear(
+ x_small, small._stored_weight(torch.float32), small.bias
+ )
+ want_big = torch.nn.functional.linear(x_big, big._stored_weight(torch.float32), None)
+ got_small = small(x_small)
+ buf = nq._CPU_WIDEN.buf
+ got_big = big(x_big)
+ grown = nq._CPU_WIDEN.buf
+ again = small(x_small)
+ assert torch.equal(got_small, want_small) and torch.equal(got_big, want_big)
+ assert torch.equal(again, want_small)
+ assert buf.numel() == 48 * 64 and grown.numel() == 96 * 128
+ assert nq._CPU_WIDEN.buf is grown
+ assert small.weight.data_ptr() != grown.data_ptr()
+
+
+def test_cpu_scratch_made_under_inference_mode_still_serves_no_grad():
+ import core.inference.diffusion_native_quant as nq
+
+ nq._CPU_WIDEN.buf = None
+ layer = _cpu_native(40, 64, 5)
+ x = torch.randn(3, 64)
+ with torch.inference_mode():
+ first = layer(x)
+ with torch.no_grad():
+ second = layer(x)
+ second_grad_on = layer(x)
+ assert not nq._CPU_WIDEN.buf.is_inference()
+ assert torch.equal(first, second) and torch.equal(second, second_grad_on)
+
+
+def test_cpu_forward_under_autograd_keeps_its_own_weight():
+ layer = _cpu_native(32, 64, 2)
+ other = _cpu_native(32, 64, 3)
+ x = torch.randn(4, 64, requires_grad = True)
+ out = layer(x)
+ other(torch.randn(4, 64)) # would overwrite a shared scratch before backward
+ out.sum().backward()
+ ref = x.detach().clone().requires_grad_()
+ torch.nn.functional.linear(
+ ref, layer._stored_weight(torch.float32), layer.bias
+ ).sum().backward()
+ assert torch.equal(x.grad, ref.grad)
+
+
+def test_cpu_native_linear_compiles_without_a_graph_break():
+ torch._dynamo.reset()
+ layer = _cpu_native(64, 64, 4)
+ x = torch.randn(9, 64)
+ with torch.no_grad():
+ want = layer(x)
+ got = torch.compile(layer, backend = "eager", fullgraph = True, dynamic = True)(x)
+ assert torch.equal(got, want)
+
+
def test_int8_storage_convrot_rotates_activations_like_the_native_route(monkeypatch, comfy_file):
_nvidia(monkeypatch)
monkeypatch.delenv(_STORAGE_ENV, raising = False)