Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: keep single-file int8 denoiser weights as int8 on CPU instead of expanding them to float32 (#17178)
Daniel Han14896930e30b
diff --git a/studio/backend/core/inference/diffusion_native_quant.py b/studio/backend/core/inference/diffusion_native_quant.pyindex 9d1eb55cfda..84911aa618d 100644--- a/studio/backend/core/inference/diffusion_native_quant.py+++ b/studio/backend/core/inference/diffusion_native_quant.py@@ -10,6 +10,7 @@ import contextlibimport contextvarsimport osimport re+import threadingimport timefrom functools import lru_cachefrom typing import Any, Callable, Optional@@ -471,6 +472,22 @@ def fused_w8a8_state() -> dict:}+# Per-thread fp32 widen buffer: a fresh one per call costs ~4x the widen in page faults.+_CPU_WIDEN = threading.local()+++def _cpu_widen_scratch(numel: int) -> Any:+ import torch++ buf = getattr(_CPU_WIDEN, "buf", None)+ if buf is None or buf.numel() < numel:+ _CPU_WIDEN.buf = None+ # Not an inference tensor: those are read-only outside inference_mode.+ with torch.inference_mode(False):+ buf = _CPU_WIDEN.buf = torch.empty(numel, dtype = torch.float32)+ return buf[:numel]++@lru_cache(maxsize = 1)def native_linear_class():"""``NativeWeightOnlyLinear``, defined on first use so this module imports torch-free."""@@ -612,6 +629,21 @@ def native_linear_class():wide = self.weight_q.view(getattr(torch, self.fp8_format)).to(torch.float32)return (wide * scale[:, None]).to(dtype)+ def _forward_weight(self, x: Any) -> Any:+ # Not under autograd: a graph would save the buffer the next layer overwrites.+ if (+ self.scheme == NATIVE_INT8+ and x.dtype is torch.float32+ and x.device.type == "cpu"+ and self.weight_q.device.type == "cpu"+ and not torch.compiler.is_compiling()+ and not (torch.is_grad_enabled() and x.requires_grad)+ ):+ out = _cpu_widen_scratch(self.weight_q.numel()).view(self.weight_q.shape)+ out.copy_(self.weight_q)+ return out.mul_(self.weight_scale.view(torch.float32)[:, None])+ return self._stored_weight(x.dtype)+def _rotate(self, x: Any) -> Any:g = self.rot_group# Leading dims, not -1: a zero-row input has no inferable -1 on older torch.@@ -813,7 +845,7 @@ def native_linear_class():and self._int_mm_ok(x.numel() // max(1, self.in_features))):return self._forward_int_mm(x)- return F.linear(x, self._stored_weight(x.dtype), self.bias)+ return F.linear(x, self._forward_weight(x), self.bias)def extra_repr(self) -> str:return (diff --git a/studio/backend/core/inference/diffusion_transformer_quant.py b/studio/backend/core/inference/diffusion_transformer_quant.pyindex 6b661bbc6f4..6f4747ba0f3 100644--- a/studio/backend/core/inference/diffusion_transformer_quant.py+++ b/studio/backend/core/inference/diffusion_transformer_quant.py@@ -1322,6 +1322,7 @@ def fp16_card_int8_act(target: Any) -> bool:# Checkpoint int8 codes on a GPU with no Studio int8 runtime (pre-Ampere fp16, XPU) stay int8, widened per layer.# Unset: offloaded plans only (resident, the expanded DiT is faster); 1: resident too; 0: always expand.+# A float32 CPU target stores them unless 0: the expanded copy is 4 bytes a weight, the per-layer widen inside step noise.INT8_STORAGE_ENV = "UNSLOTH_DIFFUSION_INT8_STORAGE"@@ -1338,6 +1339,8 @@ def int8_storage_host(target: Any) -> bool:import torchdtype = getattr(target, "dtype", None)+ if device == "cpu":+ return dtype is torch.float32if device == "xpu":return dtype in (torch.bfloat16, torch.float16)if device != "cuda" or torch_is_rocm() or is_stubbed("torchao"):@@ -1353,7 +1356,8 @@ def int8_storage_scheme(*,offload: bool = False,) -> Optional[str]:- if not (offload or _int8_storage_setting() in ("1", "on", "true", "yes")):+ cpu = getattr(target, "device", None) == "cpu"+ if not (offload or cpu or _int8_storage_setting() in ("1", "on", "true", "yes")):return Noneif not int8_storage_host(target) or _family_denied(family, TQ_INT8):return None@@ -1364,8 +1368,8 @@ def native_int8_act(target: Any) -> bool:# MPS has no ``_int_mm``: W8A8 would only add the activation quant.if _mps_bf16(target):return False- # storage only (int8_storage_scheme): never an int8 GEMM- if int8_storage_host(target):+ # storage only (int8_storage_scheme): never an int8 GEMM, unless the host also runs a native W8A8 path+ if int8_storage_host(target) and not (native_quant_host(target) or native_offload_host(target)):return Falseif native_quant_host(target):return int8_act_requested()diff --git a/studio/backend/tests/test_diffusion_comfy_quant.py b/studio/backend/tests/test_diffusion_comfy_quant.pyindex fc6067193a1..c07a3d13263 100644--- a/studio/backend/tests/test_diffusion_comfy_quant.py+++ b/studio/backend/tests/test_diffusion_comfy_quant.py@@ -1539,12 +1539,18 @@ def _nvidia(monkeypatch):("xpu", torch.bfloat16, False, None, None),("xpu", torch.bfloat16, False, "1", "native"),("xpu", torch.bfloat16, True, "0", None),- # untouched: Ampere+ keeps its W8A8 offload route, CPU and an fp32 XPU expand+ # untouched: Ampere+ keeps its W8A8 offload route, an fp32 XPU and MPS outside bf16 expand("cuda", torch.bfloat16, True, None, "native"),("cuda", torch.bfloat16, True, "0", "native"),("cuda", torch.bfloat16, False, "1", None),- ("cpu", torch.float32, True, "1", None),("xpu", torch.float32, True, None, None),+ ("mps", torch.float16, False, None, None),+ ("mps", torch.float32, False, "1", None),+ # CPU (never offloaded): stored by default, expanded only under the kill switch+ ("cpu", torch.float32, False, None, "native"),+ ("cpu", torch.float32, False, "1", "native"),+ ("cpu", torch.float32, False, "0", None),+ ("cpu", torch.bfloat16, False, None, None),],)def test_int8_storage_backend_choice(monkeypatch, device, dtype, offload, env, want):@@ -1577,6 +1583,7 @@ def test_int8_storage_hosts_never_take_the_int8_gemm(monkeypatch):assert tq.native_int8_act(_target("cuda", torch.float16)) is Falseassert tq.native_int8_act(_target("xpu", torch.bfloat16)) is Falseassert tq.native_int8_act(_target("cuda", torch.bfloat16)) is (not tq.int8_act_disabled())+ assert tq.native_int8_act(_target("cpu", torch.float32)) is Falsemonkeypatch.setenv(_STORAGE_ENV, "0")assert tq.native_int8_act(_target("cuda", torch.float16)) is (not tq.int8_act_disabled())@@ -1664,6 +1671,113 @@ def test_int8_storage_keeps_codes_and_matches_the_expanded_dit_bitwise(assert torch.equal(got, want)+def test_cpu_keeps_int8_codes_resident_and_matches_the_expanded_dit_bitwise(+ monkeypatch, plain_int8_file+):+ from core.inference.diffusion_native_quant import is_native_linear++ _nvidia(monkeypatch)+ monkeypatch.delenv(_STORAGE_ENV, raising = False)+ dtype = torch.float32+ target = _target("cpu", dtype)+ backend = cq.comfy_int8_backend(target, "z-image")+ assert backend == "native"+ stored = _load_as(plain_int8_file, target, backend)+ monkeypatch.setenv(_STORAGE_ENV, "0")+ assert cq.comfy_int8_backend(target, "z-image") is None+ expanded = _load_as(plain_int8_file, target, None)+ assert stored._unsloth_comfy_quant["int8"] == 8+ assert expanded._unsloth_comfy_quant["int8"] == 0+ for block in stored.blocks:+ for name in ("to_q", "to_k", "to_v", "to_out"):+ layer = getattr(block, name)+ assert is_native_linear(layer) and not layer.act_int8 and not layer.rot_group+ assert layer.weight_q.dtype == torch.int8+ for block in expanded.blocks:+ assert not any(is_native_linear(m) for m in block.modules())+ assert _held(expanded) - _held(stored) == 8 * (DIM * DIM * (dtype.itemsize - 1) - 4 * DIM)+ for rows in (1, 3, 17):+ x = torch.randn(rows, DIM).to(dtype)+ for got, want in zip(_run(stored, x), _run(expanded, x)):+ assert torch.equal(got, want)+++def _cpu_native(+ out_f,+ in_f,+ seed,+ bias = True,+):+ from core.inference.diffusion_native_quant import native_linear_class++ torch.manual_seed(seed)+ linear = torch.nn.Linear(in_f, out_f, bias = bias)+ codes, scale = _int8(linear.weight.detach())+ return native_linear_class()(linear, "int8", codes = codes, scale = scale)+++def test_cpu_forward_widens_into_a_reused_scratch_bitwise():+ import core.inference.diffusion_native_quant as nq++ nq._CPU_WIDEN.buf = None+ small, big = _cpu_native(48, 64, 0), _cpu_native(96, 128, 1, bias = False)+ x_small, x_big = torch.randn(5, 64), torch.randn(7, 128)+ with torch.no_grad():+ want_small = torch.nn.functional.linear(+ x_small, small._stored_weight(torch.float32), small.bias+ )+ want_big = torch.nn.functional.linear(x_big, big._stored_weight(torch.float32), None)+ got_small = small(x_small)+ buf = nq._CPU_WIDEN.buf+ got_big = big(x_big)+ grown = nq._CPU_WIDEN.buf+ again = small(x_small)+ assert torch.equal(got_small, want_small) and torch.equal(got_big, want_big)+ assert torch.equal(again, want_small)+ assert buf.numel() == 48 * 64 and grown.numel() == 96 * 128+ assert nq._CPU_WIDEN.buf is grown+ assert small.weight.data_ptr() != grown.data_ptr()+++def test_cpu_scratch_made_under_inference_mode_still_serves_no_grad():+ import core.inference.diffusion_native_quant as nq++ nq._CPU_WIDEN.buf = None+ layer = _cpu_native(40, 64, 5)+ x = torch.randn(3, 64)+ with torch.inference_mode():+ first = layer(x)+ with torch.no_grad():+ second = layer(x)+ second_grad_on = layer(x)+ assert not nq._CPU_WIDEN.buf.is_inference()+ assert torch.equal(first, second) and torch.equal(second, second_grad_on)+++def test_cpu_forward_under_autograd_keeps_its_own_weight():+ layer = _cpu_native(32, 64, 2)+ other = _cpu_native(32, 64, 3)+ x = torch.randn(4, 64, requires_grad = True)+ out = layer(x)+ other(torch.randn(4, 64)) # would overwrite a shared scratch before backward+ out.sum().backward()+ ref = x.detach().clone().requires_grad_()+ torch.nn.functional.linear(+ ref, layer._stored_weight(torch.float32), layer.bias+ ).sum().backward()+ assert torch.equal(x.grad, ref.grad)+++def test_cpu_native_linear_compiles_without_a_graph_break():+ torch._dynamo.reset()+ layer = _cpu_native(64, 64, 4)+ x = torch.randn(9, 64)+ with torch.no_grad():+ want = layer(x)+ got = torch.compile(layer, backend = "eager", fullgraph = True, dynamic = True)(x)+ assert torch.equal(got, want)++def test_int8_storage_convrot_rotates_activations_like_the_native_route(monkeypatch, comfy_file):_nvidia(monkeypatch)monkeypatch.delenv(_STORAGE_ENV, raising = False)