Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"""ComfyUI quantised single-file loader; ConvRot must be installed on exactly the recorded layers."""
from __future__ import annotations
import json
import sys
import types
import pytest
torch = pytest.importorskip("torch")
safetensors_torch = pytest.importorskip("safetensors.torch")
nn = torch.nn
import core.inference.diffusion_comfy_quant as cq # noqa: E402
from core.inference.diffusion_convrot import build_convrot_hadamard, is_rotated_linear # noqa: E402
DIM = 512 # the int8 runtime's minimum Linear width
GROUP = 256
def _conf(**conf) -> torch.Tensor:
return torch.tensor(list(json.dumps(conf).encode("utf-8")), dtype = torch.uint8)
def _int8(weight: torch.Tensor, group: int = 0) -> tuple:
"""ComfyUI's int8_tensorwise layout: optional ConvRot, then symmetric per-row int8."""
w = weight.float()
if group:
h = build_convrot_hadamard(group)
w = (w.reshape(w.shape[0], -1, group) @ h.T).reshape(w.shape)
scale = w.abs().amax(dim = 1, keepdim = True).clamp_min(1e-8) / 127.0
return torch.round(w / scale).clamp(-127, 127).to(torch.int8), scale
def _save(
path,
tensors,
metadata = None,
) -> str:
tensors = {k: v.contiguous().clone() for k, v in tensors.items()}
safetensors_torch.save_file(tensors, str(path), metadata = metadata)
return str(path)
# ------------------------------------------------------------------------------------- detection
def test_plain_and_unscaled_fp8_files_are_not_comfy_quant(tmp_path):
plain = _save(tmp_path / "a.safetensors", {"x.weight": torch.zeros(4, 4, dtype = torch.bfloat16)})
fp8 = _save(tmp_path / "b.safetensors", {"x.weight": torch.zeros(4, 4).to(torch.float8_e4m3fn)})
assert cq.scan_comfy_quant(plain) is None
assert cq.scan_comfy_quant(fp8) is None
assert cq.scan_comfy_quant(str(tmp_path / "missing.safetensors")) is None
assert cq.scan_comfy_quant(None) is None
def test_per_layer_int8_convrot_is_detected(tmp_path):
q, s = _int8(torch.randn(8, GROUP), GROUP)
path = _save(
tmp_path / "m.safetensors",
{
"a.weight": q,
"a.weight_scale": s,
"a.comfy_quant": _conf(format = "int8_tensorwise", convrot = True, convrot_groupsize = GROUP),
"b.weight": q,
"b.weight_scale": s,
"b.comfy_quant": _conf(format = "int8_tensorwise", params = {"convrot": True}),
"c.weight": torch.zeros(8, 8),
},
)
scan = cq.scan_comfy_quant(path)
assert scan is not None and not scan.problems
assert scan.counts() == {"int8_tensorwise+convrot": 2}
assert scan.layers["b"].group == 256 # ComfyUI's default group, read through "params"
assert cq.comfy_quant_error(scan) is None
@pytest.mark.parametrize("fmt", ["convrot_w4a4", "asym_w4a8_int8", "w6a8_int8", "brand_new"])
def test_unsupported_formats_are_refused_by_name(tmp_path, fmt):
path = _save(
tmp_path / "m.safetensors",
{
"a.weight": torch.zeros(8, 8, dtype = torch.uint8),
"a.weight_scale": torch.ones(8, 1),
"a.comfy_quant": _conf(format = fmt),
},
)
with pytest.raises(ValueError, match = fmt):
cq.refuse_comfy_quant(path)
def test_header_quantization_metadata_is_read_and_refused(tmp_path):
"""Comfy-Org files declare formats only in the header, with no per-layer tensors."""
meta = {cq.QUANT_METADATA_KEY: json.dumps({"layers": {"a": {"format": "asym_w4a8_int8"}}})}
path = _save(
tmp_path / "m.safetensors",
{
"a.weight": torch.zeros(8, 4, dtype = torch.uint8),
"a.weight_scale": torch.zeros(8, 1, dtype = torch.uint8),
"a.weight_scale_2": torch.ones(()),
"a.input_scale": torch.ones(()),
},
meta,
)
scan = cq.scan_comfy_quant(path)
assert scan is not None and not scan.layers
with pytest.raises(ValueError, match = "'asym_w4a8_int8'"):
cq.refuse_comfy_quant(path)
def test_header_fp8_table_and_legacy_scaled_fp8_are_supported(tmp_path):
w8 = torch.randn(8, 8).to(torch.float8_e4m3fn)
meta = {cq.QUANT_METADATA_KEY: json.dumps({"layers": {"a": {"format": "float8_e4m3fn"}}})}
table = _save(
tmp_path / "t.safetensors",
{"a.weight": w8, "a.weight_scale": torch.tensor(0.5), "a.input_scale": torch.tensor(1.0)},
meta,
)
legacy = _save(
tmp_path / "l.safetensors",
{
"scaled_fp8": torch.zeros(2, dtype = torch.float8_e4m3fn),
"a.weight": w8,
"a.scale_weight": torch.tensor([0.5]),
"a.scale_input": torch.tensor([1.0]),
"a.bias": torch.zeros(8),
},
)
for path in (table, legacy):
scan = cq.refuse_comfy_quant(path)
assert scan.counts() == {"float8_e4m3fn": 1}
assert cq.scan_comfy_quant(legacy).legacy_scaled_fp8
def test_undeclared_scales(tmp_path):
"""An fp8 weight with a plain weight_scale has one meaning; scaled int8 with no format does not."""
fp8 = _save(
tmp_path / "f.safetensors",
{
"a.weight": torch.zeros(8, 8).to(torch.float8_e4m3fn),
"a.weight_scale": torch.tensor(2.0),
},
)
assert cq.refuse_comfy_quant(fp8).counts() == {"float8_e4m3fn": 1}
int8 = _save(
tmp_path / "i.safetensors",
{"a.weight": torch.zeros(8, 8, dtype = torch.int8), "a.weight_scale": torch.ones(8, 1)},
)
with pytest.raises(ValueError, match = "no quantization format declared"):
cq.refuse_comfy_quant(int8)
@pytest.mark.parametrize(
"tensors, why",
[
(
{
"a.weight": torch.zeros(8, GROUP, dtype = torch.int8),
"a.weight_scale": torch.ones(8, 4),
},
"block scales",
),
({"a.weight": torch.zeros(8, GROUP, dtype = torch.int8)}, "weight_scale is missing"),
(
{
"a.weight": torch.zeros(8, GROUP, dtype = torch.uint8),
"a.weight_scale": torch.ones(8, 1),
},
"stored as U8",
),
({"a.weight_scale": torch.ones(8, 1)}, "no a.weight"),
(
{
"a.weight": torch.zeros(2, 8, GROUP, dtype = torch.int8),
"a.weight_scale": torch.ones(2, 8, 1),
},
"2-D",
),
],
)
def test_malformed_int8_layers_are_refused(tmp_path, tensors, why):
tensors = dict(tensors, **{"a.comfy_quant": _conf(format = "int8_tensorwise")})
with pytest.raises(ValueError, match = why):
cq.refuse_comfy_quant(_save(tmp_path / "m.safetensors", tensors))
def test_bad_convrot_group_is_refused(tmp_path):
path = _save(
tmp_path / "m.safetensors",
{
"a.weight": torch.zeros(8, 384, dtype = torch.int8),
"a.weight_scale": torch.ones(8, 1),
"a.comfy_quant": _conf(format = "int8_tensorwise", convrot = True, convrot_groupsize = 128),
},
)
with pytest.raises(ValueError, match = "power of 4"):
cq.refuse_comfy_quant(path)
# ------------------------------------------------------------------------------------- loading
class _Block(nn.Module):
def __init__(self, dim: int) -> None:
super().__init__()
self.to_q = nn.Linear(dim, dim)
self.to_k = nn.Linear(dim, dim)
self.to_v = nn.Linear(dim, dim)
self.to_out = nn.Linear(dim, dim)
self.adaLN_modulation = nn.Sequential(nn.Linear(dim, 2 * dim))
class _Tiny(nn.Module):
"""Stand-in diffusers transformer: diffusers names, a ComfyUI checkpoint with a fused qkv."""
_keep_in_fp32_modules = None
_keys_to_ignore_on_load_unexpected = None
def __init__(
self,
dim: int = DIM,
blocks: int = 2,
) -> None:
super().__init__()
self.blocks = nn.ModuleList(_Block(dim) for _ in range(blocks))
self.norm = nn.LayerNorm(dim)
@classmethod
def load_config(cls, repo, **_kwargs):
assert repo == "base/repo"
return {"dim": DIM, "blocks": 2}
@staticmethod
def _get_signature_keys(_cls):
return {"dim", "blocks"}, set()
@classmethod
def from_config(cls, config):
return cls(**config)
def _convert(
checkpoint = None,
config = None,
mix_columns = False,
**_kwargs,
):
out = {}
for key, value in checkpoint.items():
if key.endswith(".qkv.weight") or key.endswith(".qkv.bias"):
stem, leaf = key.rsplit(".qkv.", 1)
chunks = list(value.chunk(3, dim = 0))
if mix_columns and leaf == "weight":
# q's left half spliced onto k's right half: rows of two layers in one weight
half = chunks[0].shape[1] // 2
chunks[0] = torch.cat([chunks[0][:, :half], chunks[1][:, half:]], dim = 1)
for part, chunk in zip(("to_q", "to_k", "to_v"), chunks):
out[f"{stem}.{part}.{leaf}"] = chunk
else:
out[key.replace(".out.", ".to_out.")] = value
return out
def _sfm():
return types.SimpleNamespace(
_should_convert_state_dict_to_diffusers = lambda model_sd, ckpt: set(model_sd) != set(ckpt),
_get_mapping_function_kwargs = lambda fn, **kw: {},
)
@pytest.fixture
def comfy_file(tmp_path, monkeypatch):
"""A ComfyUI int8_convrot file of ``_Tiny`` plus the dense weights it was quantized from."""
monkeypatch.setattr(cq, "_mapping", lambda cls: (_convert, _sfm()))
torch.manual_seed(0)
dense = _Tiny()
tensors = {}
for b, block in enumerate(dense.blocks):
qkv = torch.cat([block.to_q.weight, block.to_k.weight, block.to_v.weight]).detach()
layers = {
f"blocks.{b}.qkv": (
qkv,
torch.cat([block.to_q.bias, block.to_k.bias, block.to_v.bias]),
),
f"blocks.{b}.out": (block.to_out.weight.detach(), block.to_out.bias),
f"blocks.{b}.adaLN_modulation.0": (
block.adaLN_modulation[0].weight.detach(),
block.adaLN_modulation[0].bias,
),
}
for name, (weight, bias) in layers.items():
q, s = _int8(weight, GROUP)
tensors[f"{name}.weight"] = q
tensors[f"{name}.weight_scale"] = s
tensors[f"{name}.bias"] = bias.detach().clone()
tensors[f"{name}.comfy_quant"] = _conf(
format = "int8_tensorwise", convrot = True, convrot_groupsize = GROUP
)
tensors["norm.weight"] = dense.norm.weight.detach().clone()
tensors["norm.bias"] = dense.norm.bias.detach().clone()
path = _save(tmp_path / "tiny_int8_convrot.safetensors", tensors)
return path, dense, tensors
def _load(path, **kwargs):
kwargs.setdefault("int8_backend", None)
return cq.load_comfy_quant_transformer(
_Tiny,
path,
cq.refuse_comfy_quant(path),
{"torch_dtype": torch.float32, "config": "base/repo", "subfolder": "transformer"},
family = "z-image",
**kwargs,
)
def _cos(a, b) -> float:
return torch.nn.functional.cosine_similarity(
a.flatten().float(), b.flatten().float(), dim = 0
).item()
def test_dequant_path_reproduces_the_dense_weights(comfy_file):
path, dense, _ = comfy_file
model = _load(path)
assert model._unsloth_comfy_quant == {
"backend": None,
"int8": 0,
"convrot": 0,
"fp8_backend": None,
"fp8": 0,
"dequantized": 6,
}
assert not any(is_rotated_linear(m) for m in model.modules())
ref = dense.state_dict()
for key, value in model.state_dict().items():
assert value.dtype == torch.float32
if not ref[key].any():
assert not value.any(), key
continue
# int8 rounding only: the rotation is undone, so every weight lines up with its source
assert _cos(value, ref[key]) > 0.9999, key
assert not model.training
def test_int8_runtime_keeps_codes_and_scales_unchanged(comfy_file):
pytest.importorskip("torchao")
path, dense, tensors = comfy_file
model = _load(path, int8_backend = "torchao")
if not model._unsloth_comfy_quant["int8"]:
pytest.skip("this torchao has no Int8Tensor")
# 2 blocks x (q, k, v, out) as int8; adaLN is a modulation Linear Studio keeps dense
assert model._unsloth_comfy_quant == {
"backend": "torchao",
"int8": 8,
"convrot": 8,
"fp8_backend": None,
"fp8": 0,
"dequantized": 2,
}
assert model._unsloth_runtime_quant == "int8"
for b, block in enumerate(model.blocks):
q, s = tensors[f"blocks.{b}.qkv.weight"], tensors[f"blocks.{b}.qkv.weight_scale"]
for i, part in enumerate(("to_q", "to_k", "to_v")):
linear = getattr(block, part)
assert is_rotated_linear(linear) and linear.convrot_groupsize == GROUP
assert torch.equal(linear.weight.qdata, q[i * DIM : (i + 1) * DIM])
assert torch.equal(linear.weight.scale.reshape(-1, 1), s[i * DIM : (i + 1) * DIM])
assert torch.equal(block.to_out.weight.qdata, tensors[f"blocks.{b}.out.weight"])
adaln = block.adaLN_modulation[0]
assert type(adaln.weight.data) is torch.Tensor and not is_rotated_linear(adaln)
assert _cos(adaln.weight, dense.blocks[b].adaLN_modulation[0].weight) > 0.9999
def test_native_backend_keeps_codes_and_computes_the_dense_product(comfy_file):
"""Under offload the codes go to the torchao-free twin (plain buffers the hooks can move)."""
from core.inference.diffusion_native_quant import is_native_linear
path, dense, tensors = comfy_file
model = _load(path, int8_backend = "native")
assert model._unsloth_comfy_quant == {
"backend": "native",
"int8": 8,
"convrot": 8,
"fp8_backend": None,
"fp8": 0,
"dequantized": 2,
}
block = model.blocks[1]
linear = block.to_k
assert is_native_linear(linear) and linear.rot_group == GROUP
q, s = tensors["blocks.1.qkv.weight"], tensors["blocks.1.qkv.weight_scale"]
assert torch.equal(linear.weight_q, q[DIM : 2 * DIM])
assert torch.equal(linear.weight_scale.view(torch.float32), s[DIM : 2 * DIM, 0])
assert torch.equal(linear.bias, dense.blocks[1].to_k.bias)
# weight-only on CPU: rotate(x) @ dequant(W_rot).T is x @ W.T up to the int8 rounding
x = torch.randn(4, DIM)
want = torch.nn.functional.linear(x, dense.blocks[1].to_k.weight, dense.blocks[1].to_k.bias)
assert _cos(linear(x), want) > 0.9999
assert type(block.adaLN_modulation[0]) is nn.Linear
def test_fp16_keeps_fp32_modules_by_their_converted_names(comfy_file):
# Wan: ComfyUI's time_embedding.0 becomes diffusers' time_embedder only after conversion
path, dense, _ = comfy_file
class _KeepOut(_Tiny):
_keep_in_fp32_modules = ["to_out"]
model = cq.load_comfy_quant_transformer(
_KeepOut,
path,
cq.refuse_comfy_quant(path),
{"torch_dtype": torch.float16, "config": "base/repo", "subfolder": "transformer"},
int8_backend = None,
family = "z-image",
)
for b, block in enumerate(model.blocks):
assert block.to_out.weight.dtype == torch.float32
assert block.to_out.bias.dtype == torch.float32
assert torch.equal(block.to_out.bias, dense.blocks[b].to_out.bias)
assert _cos(block.to_out.weight, dense.blocks[b].to_out.weight) > 0.9999
assert block.to_q.weight.dtype == torch.float16
assert block.adaLN_modulation[0].bias.dtype == torch.float16
assert model.norm.weight.dtype == torch.float16
def test_fp8_layers_dequantize_with_their_scale(tmp_path, monkeypatch):
monkeypatch.setattr(cq, "_mapping", lambda cls: (_convert, _sfm()))
torch.manual_seed(1)
ref = _Tiny()
tensors = {k: v.detach().clone() for k, v in ref.state_dict().items()}
layer = "blocks.0.to_out"
w = tensors.pop(f"{layer}.weight")
scale = w.abs().max() / 448.0
codes = (w / scale).to(torch.float8_e4m3fn)
tensors.update({f"{layer}.weight": codes, f"{layer}.scale_weight": scale.reshape(1)})
tensors["scaled_fp8"] = torch.zeros(2, dtype = torch.float8_e4m3fn)
path = _save(tmp_path / "legacy_fp8_scaled.safetensors", tensors)
model = _load(path, fp8_storage = False)
assert torch.equal(model.blocks[0].to_out.weight, codes.float() * scale)
assert model._unsloth_comfy_quant["dequantized"] == 1
stored = _load(path, fp8_storage = True)
assert stored._unsloth_comfy_quant["dequantized"] == 0
assert stored._unsloth_comfy_quant["fp8_stored"] == 1
assert torch.equal(stored.blocks[0].to_out.weight, codes.float() * scale)
assert torch.equal(stored.blocks[0].to_out.weight_q, codes.view(torch.uint8))
def test_a_converter_that_mixes_columns_is_refused(comfy_file, monkeypatch):
pytest.importorskip("torchao")
path, _, _ = comfy_file
monkeypatch.setattr(
cq, "_mapping", lambda cls: (lambda **kw: _convert(mix_columns = True, **kw), _sfm())
)
with pytest.raises(ValueError, match = "mixed columns"):
_load(path, int8_backend = "torchao")
def test_missing_weights_after_conversion_are_refused(comfy_file, monkeypatch):
path, _, _ = comfy_file
monkeypatch.setattr(
cq,
"_mapping",
lambda cls: (
lambda **kw: {k: v for k, v in _convert(**kw).items() if "norm" not in k},
_sfm(),
),
)
with pytest.raises(ValueError, match = "missing 2 weight"):
_load(path)
def test_the_loader_refuses_what_the_scan_refuses(tmp_path):
path = _save(
tmp_path / "m.safetensors",
{
"a.weight": torch.zeros(8, 8, dtype = torch.uint8),
"a.weight_scale": torch.ones(8, 1),
"a.comfy_quant": _conf(format = "mxfp8"),
},
)
scan = cq.scan_comfy_quant(path)
with pytest.raises(ValueError, match = "mxfp8"):
cq.load_comfy_quant_transformer(
_Tiny, path, scan, {"config": "base/repo"}, int8_backend = None
)
# ------------------------------------------------------------------------------------- fp8 runtime
def _fp8_file(
tmp_path,
*,
fmt = "float8_e4m3fn",
per_row = False,
record = None,
):
"""A ComfyUI fp8 file of ``_Tiny`` (fused qkv, per-tensor or per-row ``weight_scale``) and its dense source."""
torch.manual_seed(2)
dense = _Tiny()
tensors = {}
fp8 = getattr(torch, fmt)
top = torch.finfo(fp8).max
for b, block in enumerate(dense.blocks):
layers = {
f"blocks.{b}.qkv": (
torch.cat([block.to_q.weight, block.to_k.weight, block.to_v.weight]).detach(),
torch.cat([block.to_q.bias, block.to_k.bias, block.to_v.bias]),
),
f"blocks.{b}.out": (block.to_out.weight.detach(), block.to_out.bias),
f"blocks.{b}.adaLN_modulation.0": (
block.adaLN_modulation[0].weight.detach(),
block.adaLN_modulation[0].bias,
),
}
for name, (weight, bias) in layers.items():
w = weight.float()
scale = (w.abs().amax(dim = 1, keepdim = True) if per_row else w.abs().max()) / top
tensors[f"{name}.weight"] = (w / scale).to(fp8)
tensors[f"{name}.weight_scale"] = scale.float()
tensors[f"{name}.input_scale"] = torch.tensor(1.0)
tensors[f"{name}.bias"] = bias.detach().clone()
tensors[f"{name}.comfy_quant"] = _conf(format = fmt, full_precision_matrix_mult = True)
tensors["norm.weight"] = dense.norm.weight.detach().clone()
tensors["norm.bias"] = dense.norm.bias.detach().clone()
meta = {cq.CONVERSION_RECORD_KEY: json.dumps(record)} if record else None
return _save(tmp_path / f"tiny_{fmt}.safetensors", tensors, metadata = meta), dense, tensors
def _load_bf16(path, **kwargs):
kwargs.setdefault("int8_backend", None)
return cq.load_comfy_quant_transformer(
_Tiny,
path,
cq.refuse_comfy_quant(path),
{"torch_dtype": torch.bfloat16, "config": "base/repo", "subfolder": "transformer"},
family = "z-image",
**kwargs,
)
@pytest.mark.parametrize("per_row", [False, True])
def test_fp8_runtime_keeps_codes_and_repeats_a_tensor_scale_per_row(tmp_path, monkeypatch, per_row):
pytest.importorskip("torchao")
monkeypatch.setattr(cq, "_mapping", lambda cls: (_convert, _sfm()))
path, _dense, tensors = _fp8_file(tmp_path, per_row = per_row)
model = _load_bf16(path, fp8_backend = "torchao")
assert model._unsloth_comfy_quant == {
"backend": None,
"int8": 0,
"convrot": 0,
"fp8_backend": "torchao",
"fp8": 10,
"dequantized": 0,
}
assert model._unsloth_runtime_quant == "fp8"
for b, block in enumerate(model.blocks):
codes = tensors[f"blocks.{b}.qkv.weight"]
scale = tensors[f"blocks.{b}.qkv.weight_scale"].reshape(-1, 1).expand(3 * DIM, 1)
for i, leaf in enumerate(("to_q", "to_k", "to_v")):
weight = getattr(block, leaf).weight
assert type(weight).__name__ == "Float8Tensor"
assert torch.equal(
weight.qdata.view(torch.uint8), codes[i * DIM : (i + 1) * DIM].view(torch.uint8)
)
assert torch.equal(weight.scale.float().reshape(-1, 1), scale[i * DIM : (i + 1) * DIM])
# the same per-row dynamic-activation layout as Studio's own FP8 checkpoints
assert weight.act_quant_kwargs.granularity.__class__.__name__ == "PerRow"
assert type(block.adaLN_modulation[0].weight).__name__ == "Float8Tensor"
assert not any(is_rotated_linear(m) for m in model.modules())
def test_fp8_native_backend_keeps_codes_and_computes_the_dense_product(tmp_path, monkeypatch):
monkeypatch.setattr(cq, "_mapping", lambda cls: (_convert, _sfm()))
path, dense, tensors = _fp8_file(tmp_path)
model = _load_bf16(path, fp8_backend = "native")
assert model._unsloth_comfy_quant["fp8_backend"] == "native"
assert model._unsloth_comfy_quant["fp8"] == 10
linear = model.blocks[1].to_k
assert type(linear).__name__ == "NativeWeightOnlyLinear" and linear.scheme == "fp8"
codes = tensors["blocks.1.qkv.weight"][DIM : 2 * DIM]
assert torch.equal(linear.weight_q, codes.view(torch.uint8))
x = torch.randn(4, DIM, dtype = torch.bfloat16)
want = torch.nn.functional.linear(
x.float(), dense.blocks[1].to_k.weight, dense.blocks[1].to_k.bias
)
assert _cos(linear(x), want) > 0.999
@pytest.mark.skipif(
not (getattr(torch.backends, "mps", None) and torch.backends.mps.is_available()),
reason = "needs Apple MPS",
)
def test_mps_fp8_storage_matches_the_load_time_dequant_bitwise(tmp_path, monkeypatch):
"""MPS has no fp8 cast before torch 2.15: stored codes widen through a lookup table to the same bf16 weights."""
monkeypatch.setattr(cq, "_mapping", lambda cls: (_convert, _sfm()))
path, _dense, _tensors = _fp8_file(tmp_path)
stored = _load_bf16(path, fp8_backend = None, fp8_storage = True).to("mps")
dense = _load_bf16(path, fp8_backend = None, fp8_storage = False).to("mps")
assert cq.fp8_stored_layers(stored) == 10
x = torch.randn(64, DIM, dtype = torch.bfloat16, device = "mps")
for name, layer in dense.named_modules():
if isinstance(layer, nn.Linear):
assert torch.equal(stored.get_submodule(name)(x), layer(x)), name
def test_fp8_without_a_runtime_dequantizes(tmp_path, monkeypatch):
monkeypatch.setattr(cq, "_mapping", lambda cls: (_convert, _sfm()))
path, _dense, tensors = _fp8_file(tmp_path)
model = _load_bf16(path, fp8_backend = None, fp8_storage = False)
assert model._unsloth_comfy_quant["fp8"] == 0
assert model._unsloth_comfy_quant["dequantized"] == 6
w = tensors["blocks.0.out.weight"].float() * tensors["blocks.0.out.weight_scale"]
assert torch.equal(model.blocks[0].to_out.weight, w.to(torch.bfloat16))
def test_e5m2_and_non_bf16_pipelines_dequantize(tmp_path, monkeypatch):
monkeypatch.setattr(cq, "_mapping", lambda cls: (_convert, _sfm()))
path, _dense, _tensors = _fp8_file(tmp_path, fmt = "float8_e5m2")
assert (
_load_bf16(path, fp8_backend = "torchao", fp8_storage = False)._unsloth_comfy_quant["fp8"] == 0
)
path, _dense, _tensors = _fp8_file(tmp_path)
model = _load(path, fp8_backend = "torchao", fp8_storage = False) # float32 pipeline
assert model._unsloth_comfy_quant["fp8"] == 0
assert model._unsloth_comfy_quant["dequantized"] == 6
def _linear_bytes(model) -> int:
"""Bytes the model's Linear-shaped weights hold (dense ``weight`` or stored ``weight_q`` + ``weight_scale``)."""
total = 0
for name, tensor in model.state_dict().items():
if name.endswith((".weight", ".weight_q", ".weight_scale")) and not name.startswith(
"norm."
):
total += tensor.numel() * tensor.element_size()
return total
@pytest.mark.parametrize(
"fmt, dtype",
[
("float8_e4m3fn", torch.bfloat16),
("float8_e4m3fn", torch.float16),
("float8_e4m3fn", torch.float32),
("float8_e5m2", torch.bfloat16),
],
)
@pytest.mark.parametrize("per_row", [False, True])
def test_fp8_without_a_runtime_stays_fp8_and_matches_the_dequant_exactly(
tmp_path, monkeypatch, fmt, dtype, per_row
):
"""No fp8 GEMM: codes stay 1 byte per weight and outputs are bit-identical to the load-time dequant."""
monkeypatch.setattr(cq, "_mapping", lambda cls: (_convert, _sfm()))
path, _dense, tensors = _fp8_file(tmp_path, fmt = fmt, per_row = per_row)
def load(**kwargs):
return cq.load_comfy_quant_transformer(
_Tiny,
path,
cq.refuse_comfy_quant(path),
{"torch_dtype": dtype, "config": "base/repo", "subfolder": "transformer"},
family = "z-image",
int8_backend = None,
fp8_backend = None,
**kwargs,
)
widened = load(fp8_storage = False)
stored = load(fp8_storage = True)
assert widened._unsloth_comfy_quant["dequantized"] == 6
assert stored._unsloth_comfy_quant["dequantized"] == 0
assert stored._unsloth_comfy_quant["fp8_stored"] == 10
assert stored._unsloth_comfy_quant["fp8"] == 0 # storage is not an fp8 runtime
assert getattr(stored, "_unsloth_runtime_quant", None) is None
for b in range(2):
for leaf in ("to_q", "to_k", "to_v", "to_out"):
layer = getattr(stored.blocks[b], leaf)
assert layer.weight_q.dtype is torch.uint8 and layer.fp8_format == fmt
assert layer.weight_scale.view(torch.float32).shape == (DIM,)
assert torch.equal(layer.weight, getattr(widened.blocks[b], leaf).weight)
# half the bytes of a 16-bit dequant (a quarter of fp32), plus a 4-byte scale per row
rows = 2 * (4 * DIM + 2 * DIM)
assert _linear_bytes(stored) == rows * DIM + rows * 4
assert _linear_bytes(widened) == rows * DIM * torch.finfo(dtype).bits // 8
x = torch.randn(3, 7, DIM).to(dtype)
for b in range(2):
for leaf in ("to_q", "to_out"):
got = getattr(stored.blocks[b], leaf)(x)
want = getattr(widened.blocks[b], leaf)(x)
assert got.dtype is dtype and torch.equal(got, want)
got = stored.blocks[b].adaLN_modulation(x)
assert torch.equal(got, widened.blocks[b].adaLN_modulation(x))
# no plan to judge by (None): widen, unless the env opts in; the kill switch beats an explicit request
monkeypatch.delenv(cq.FP8_STORAGE_ENV, raising = False)
assert load()._unsloth_comfy_quant["dequantized"] == 6
monkeypatch.setenv(cq.FP8_STORAGE_ENV, "1")
assert load()._unsloth_comfy_quant["fp8_stored"] == 10
monkeypatch.setenv(cq.FP8_STORAGE_ENV, "0")
assert cq.fp8_storage_wanted(None, offload = True) is False
def test_fp8_storage_follows_the_device_lora_and_kill_switch(monkeypatch):
monkeypatch.delenv(cq.FP8_STORAGE_ENV, raising = False)
cuda = types.SimpleNamespace(device = "cuda", dtype = torch.bfloat16)
assert cq.comfy_fp8_storage(cuda) is True
assert cq.comfy_fp8_storage(types.SimpleNamespace(device = "cpu")) is True
assert cq.comfy_fp8_storage(None) is True
# MPS widens stored codes through a lookup table; a LoRA needs dense Linears to attach to
assert cq.comfy_fp8_storage(types.SimpleNamespace(device = "mps")) is True
assert cq.comfy_fp8_storage(cuda, lora = True) is False
for off in ("0", "off", "false"):
monkeypatch.setenv(cq.FP8_STORAGE_ENV, off)
assert cq.comfy_fp8_storage(cuda) is False
def test_fp8_storage_keeps_fp32_modules_and_the_torchao_layers_apart(tmp_path, monkeypatch):
"""Only layers the fp8 GEMM runtime skips are stored; fp32-kept Linears still widen to fp32."""
monkeypatch.setattr(cq, "_mapping", lambda cls: (_convert, _sfm()))
path, _dense, _tensors = _fp8_file(tmp_path)
class _Keep(_Tiny):
_keep_in_fp32_modules = ["adaLN_modulation"]
model = cq.load_comfy_quant_transformer(
_Keep,
path,
cq.refuse_comfy_quant(path),
{"torch_dtype": torch.float16, "config": "base/repo", "subfolder": "transformer"},
family = "z-image",
int8_backend = None,
fp8_backend = None,
fp8_storage = True,
)
assert model._unsloth_comfy_quant["fp8_stored"] == 8
assert model._unsloth_comfy_quant["dequantized"] == 2
assert type(model.blocks[0].adaLN_modulation[0]) is nn.Linear
assert model.blocks[0].adaLN_modulation[0].weight.dtype is torch.float32
@pytest.mark.parametrize(
"native, supported, offload, streams, dtype, env, want",
[
(None, "fp8", False, False, "bfloat16", None, "torchao"),
(
None,
"fp8",
True,
False,
"bfloat16",
None,
None,
), # offloaded torchao fp8: dequantize, as before
(None, "fp8", True, True, "bfloat16", None, "torchao"), # streamed through group offload
(None, None, True, True, "bfloat16", None, None), # streamed, but no fp8 GEMM (A100)
(None, "fp8", True, True, "float16", None, None),
(None, "fp8", True, True, "bfloat16", "0", None),
(None, None, False, False, "bfloat16", None, None), # no fp8 GEMM (older GPU, CPU, MPS)
("fp8", None, False, False, "bfloat16", None, "native"), # ROCm / stubbed torchao
(None, "fp8", False, False, "float16", None, None),
(None, "fp8", False, False, "bfloat16", "0", None), # kill switch
],
)
def test_fp8_backend_follows_studios_own_fp8_rule(
monkeypatch, native, supported, offload, streams, dtype, env, want
):
import core.inference.diffusion_transformer_quant as tq
monkeypatch.setattr(tq, "native_quant_scheme", lambda *a, **k: native)
monkeypatch.setattr(tq, "select_transformer_quant_scheme", lambda *a, **k: supported)
if env is None:
monkeypatch.delenv(cq.COMFY_FP8_ENV, raising = False)
else:
monkeypatch.setenv(cq.COMFY_FP8_ENV, env)
target = types.SimpleNamespace(device = "cuda", dtype = getattr(torch, dtype))
assert cq.comfy_fp8_backend(target, "z-image", offload = offload, torchao_offload = streams) == want
@pytest.mark.parametrize(
"policy, stream, placed, want",
[
("group", True, "same", "same"),
("streaming", True, "same", "same"),
# whole-module offload whose quantised parts do not fit: streamed, as Studio's own fp8 is
("model", True, "streaming", "streaming"),
("model", True, "same", None), # fits whole: whole-module offload keeps fp8 storage
(
"group",
True,
None,
None,
), # torchao or diffusers too old, or the pin does not fit host RAM
("sequential", True, None, None),
("group", False, "same", None), # resident denoiser: the resident rule already applies
("none", True, "same", None),
],
)
def test_single_file_fp8_takes_torchao_only_when_offload_streams_it(
monkeypatch, policy, stream, placed, want
):
import core.inference.diffusion as d
plan = types.SimpleNamespace(offload_policy = policy, stream_transformer = stream)
streamed = types.SimpleNamespace(offload_policy = "streaming", stream_transformer = True)
seen = []
def offload_plan(p, scheme):
seen.append(scheme)
return {"same": p, "streaming": streamed, None: None}[placed]
monkeypatch.setattr(d, "torchao_offload_plan", offload_plan)
got = d.DiffusionBackend._comfy_fp8_stream_plan(plan)
assert got is {"same": plan, "streaming": streamed, None: None}[want]
assert seen in ([], ["fp8"])
# ------------------------------------------------------------------------------------- hosted prequant route
def test_comfy_prequant_scheme_is_read_from_the_header(tmp_path, comfy_file):
path, _, _ = comfy_file
assert cq.comfy_prequant_scheme(cq.scan_comfy_quant(path)) == "int8"
fp8_path, _, _ = _fp8_file(tmp_path)
assert cq.comfy_prequant_scheme(cq.scan_comfy_quant(fp8_path)) == "fp8"
from core.inference.diffusion_prequant import local_prequant_scheme
assert local_prequant_scheme(fp8_path) == "fp8"
assert local_prequant_scheme(path) == "int8"
def test_comfy_prequant_refuses_another_scheme_or_base(tmp_path, monkeypatch):
monkeypatch.setattr(cq, "_mapping", lambda cls: (_convert, _sfm()))
path, _, _ = _fp8_file(tmp_path, record = {"base_model_id": "org/other-model", "scheme": "fp8"})
kw = dict(base = "base/repo", family = "z-image", dtype = torch.bfloat16)
with pytest.raises(ValueError, match = "not a int8 checkpoint"):
cq.load_comfy_prequant(_Tiny, path, scheme = "int8", **kw)
with pytest.raises(ValueError, match = "converted from org/other-model"):
cq.load_comfy_prequant(_Tiny, path, scheme = "fp8", **kw)
with pytest.raises(ValueError, match = "needs a bfloat16 pipeline"):
cq.load_comfy_prequant(
_Tiny, path, scheme = "fp8", **dict(kw, dtype = torch.float32, base = "org/other-model")
)
def test_hosted_loader_rebuilds_a_comfy_file_into_studios_int8_weights(
comfy_file, monkeypatch, tmp_path
):
"""``load_prequantized_transformer`` on a ComfyUI file: Int8Tensor weights under ConvRot, the loaded file
recorded, the same placement tail as Studio's own checkpoints."""
pytest.importorskip("torchao")
import core.inference.diffusion_prequant as pq
path, _dense, tensors = comfy_file
monkeypatch.setenv(pq.ALLOW_LOCAL_PREQUANT_PATH_ENV, str(tmp_path))
source = pq.PrequantSource(kind = "path", location = path)
model = pq.load_prequantized_transformer(
_Tiny,
"base/repo",
source,
device = "cpu",
dtype = torch.float32,
scheme = "int8",
family = "z-image",
)
assert model is not None, pq.last_prequant_failure()
assert model._unsloth_prequant_path == path
assert model._unsloth_runtime_quant == "int8"
weight = model.blocks[0].to_out.weight
assert type(getattr(weight, "data", weight)).__name__ == "Int8Tensor"
assert torch.equal(weight.qdata, tensors["blocks.0.out.weight"])
assert is_rotated_linear(model.blocks[0].to_out)
# Studio's int8 filter keeps the modulation projection dense, as its own INT8 checkpoints do
assert type(model.blocks[0].adaLN_modulation[0].weight).__name__ in ("Parameter", "Tensor")
# asked for the wrong scheme: refused, with the reason recorded, so the caller quantizes dense instead
assert (
pq.load_prequantized_transformer(
_Tiny,
"base/repo",
source,
device = "cpu",
dtype = torch.float32,
scheme = "fp8",
family = "z-image",
)
is None
)
assert "not a fp8 checkpoint" in (pq.last_prequant_failure() or "")
def test_resident_size_is_priced_from_the_header_not_the_name(tmp_path, comfy_file):
"""Kept int8 / fp8 layers cost their stored bytes, dequantized ones twice that; the file name is not read."""
int8_path, _, tensors = comfy_file
scan = cq.scan_comfy_quant(int8_path)
quant = sum(tensors[f"{n}.weight"].numel() for n in scan.layers)
plain = sum(
v.numel()
for k, v in tensors.items()
if v.is_floating_point() and k.endswith((".bias", "norm.weight", "norm.bias"))
)
kept = cq.comfy_resident_mib(int8_path, scan, keep_int8 = True, keep_fp8 = True)
dequant = cq.comfy_resident_mib(int8_path, scan, keep_int8 = False, keep_fp8 = True)
assert dequant - kept == pytest.approx(quant / 2**20, abs = 1)
assert kept >= (quant + 2 * plain) / 2**20
fp8_path, _, _ = _fp8_file(tmp_path)
fp8_scan = cq.scan_comfy_quant(fp8_path)
assert cq.comfy_resident_mib(
fp8_path, fp8_scan, keep_int8 = False, keep_fp8 = True
) < cq.comfy_resident_mib(fp8_path, fp8_scan, keep_int8 = False, keep_fp8 = False)
assert (
cq.comfy_resident_mib(str(tmp_path / "missing.safetensors"), keep_int8 = True, keep_fp8 = True)
is None
)
def test_the_row_map_is_traced_on_narrow_tags_and_falls_back_to_full_width(comfy_file, monkeypatch):
"""The fast pass hands the converter 4-column tags; a converter that needs the real width (here: one that
reshapes by it) gets a second, full-width pass and the same result; one that drops columns is refused."""
pytest.importorskip("torchao")
path, _, tensors = comfy_file
widths = []
def _spy(
checkpoint = None,
config = None,
**kw,
):
widths.append(checkpoint["blocks.0.qkv.weight"].shape[1])
return _convert(checkpoint = checkpoint, config = config, **kw)
monkeypatch.setattr(cq, "_mapping", lambda cls: (_spy, _sfm()))
fast = _load(path, int8_backend = "torchao")
assert widths == [cq._NARROW_TAG_COLUMNS]
def _needs_width(
checkpoint = None,
config = None,
**kw,
):
w = checkpoint["blocks.0.qkv.weight"]
checkpoint["blocks.0.qkv.weight"] = w.reshape(-1, DIM) # only valid at the real width
return _convert(checkpoint = checkpoint, config = config, **kw)
monkeypatch.setattr(cq, "_mapping", lambda cls: (_needs_width, _sfm()))
full = _load(path, int8_backend = "torchao")
for a, b in zip(fast.state_dict().values(), full.state_dict().values()):
a, b = getattr(a, "qdata", a), getattr(b, "qdata", b)
assert torch.equal(a, b)
def _drops_columns(
checkpoint = None,
config = None,
**kw,
):
out = _convert(checkpoint = checkpoint, config = config, **kw)
out["blocks.0.to_out.weight"] = out["blocks.0.to_out.weight"].chunk(2, dim = 1)[0]
return out
monkeypatch.setattr(cq, "_mapping", lambda cls: (_drops_columns, _sfm()))
with pytest.raises(ValueError, match = "column count"):
_load(path, int8_backend = "torchao")
def test_the_hosted_route_registers_studios_single_file_converters(monkeypatch):
"""Qwen-Image-2.1's converter is Studio's own, registered by the single-file branch right before its call. The
hosted prequant route never passes through there, so the row map must register it itself (it used to fail with
"has no single-file converter" and fall back to quantizing the dense weights)."""
diffusers = pytest.importorskip("diffusers")
cls = getattr(diffusers, "QwenImage21Transformer2DModel", None)
if cls is None:
pytest.skip("this diffusers has no QwenImage21Transformer2DModel")
from diffusers.loaders import single_file_model as sfm
monkeypatch.setattr(
sfm,
"SINGLE_FILE_LOADABLE_CLASSES",
{k: v for k, v in sfm.SINGLE_FILE_LOADABLE_CLASSES.items() if k != cls.__name__},
)
mapping_fn, _ = cq._mapping(cls)
assert callable(mapping_fn)
def test_a_local_comfy_override_is_usable_only_for_an_image_family(tmp_path, monkeypatch):
"""A local ComfyUI-format override reads its scheme from the header for an image family; a video family's loader
cannot read the layout, so planning must not count on it there."""
import core.inference.diffusion_prequant as pq
from core.inference.diffusion_families import detect_family
from core.inference.video_families import _FAMILIES as VIDEO_FAMILIES
path, _, _ = _fp8_file(tmp_path)
monkeypatch.setenv(pq.ALLOW_LOCAL_PREQUANT_PATH_ENV, str(tmp_path))
monkeypatch.setattr(pq, "restricted_prequant_load_supported", lambda *a, **k: True)
image = detect_family("Tongyi-MAI/Z-Image-Turbo")
assert pq.usable_prequant_source(image, "fp8", path_override = path) is not None
assert pq.usable_prequant_source(image, "int8", path_override = path) is None
video = next(f for f in VIDEO_FAMILIES if any(s == "fp8" for s, _r in (f.prequant_repos or ())))
assert pq.usable_prequant_source(video, "fp8", path_override = path) is None
def test_only_torchao_comfy_loads_ask_for_compile(comfy_file):
path, _, _ = comfy_file
assert not cq.comfy_torchao_quantized(_load(path))
assert not cq.comfy_torchao_quantized(_load(path, int8_backend = "native"))
model = nn.Module()
model._unsloth_comfy_quant = {"backend": "torchao", "int8": 8}
assert cq.comfy_torchao_quantized(model)
assert not cq.comfy_torchao_quantized(nn.Linear(2, 2))
def test_buffers_built_in_init_follow_the_compute_dtype(comfy_file):
# Wan's rope tables are float64 non-persistent buffers; from_single_file casts them with model.to(dtype)
path, _, _ = comfy_file
class _Rope(_Tiny):
def __init__(self, **kwargs) -> None:
super().__init__(**kwargs)
self.register_buffer("freqs", torch.ones(4, dtype = torch.float64), persistent = False)
model = cq.load_comfy_quant_transformer(
_Rope,
path,
cq.refuse_comfy_quant(path),
{"torch_dtype": torch.bfloat16, "config": "base/repo", "subfolder": "transformer"},
int8_backend = "native",
family = "z-image",
)
assert model.freqs.dtype == torch.bfloat16
assert model.blocks[0].to_q.weight_q.dtype == torch.int8
def test_a_timestep_table_built_in_fp32_stays_fp32(comfy_file):
"""fp32 tables built in __init__ (Qwen-Image-2.1's timesteps, MiniMax-H3's RoPE) stay fp32, as from_pretrained
leaves them; a float64 one still follows the compute dtype."""
path, _, _ = comfy_file
class QwenImage21TemporalTimesteps(nn.Module):
def __init__(self) -> None:
super().__init__()
freqs = torch.exp(-torch.arange(0, 128, dtype = torch.float32) / 128 * 9.21)
self.register_buffer("freqs", freqs, persistent = False)
class _Timed(_Tiny):
def __init__(self, **kwargs) -> None:
super().__init__(**kwargs)
self.time_proj = QwenImage21TemporalTimesteps()
self.register_buffer("rope", torch.ones(4, dtype = torch.float32), persistent = False)
self.register_buffer("wide", torch.ones(4, dtype = torch.float64), persistent = False)
model = cq.load_comfy_quant_transformer(
_Timed,
path,
cq.refuse_comfy_quant(path),
{"torch_dtype": torch.bfloat16, "config": "base/repo", "subfolder": "transformer"},
int8_backend = "native",
family = "qwen-image-2.1",
)
assert model.time_proj.freqs.dtype == torch.float32
assert torch.equal(model.time_proj.freqs, QwenImage21TemporalTimesteps().freqs)
assert model.rope.dtype == torch.float32
assert model.wide.dtype == torch.bfloat16
def test_a_lora_on_stored_fp8_layers_is_refused_with_the_way_out(tmp_path, monkeypatch):
"""Stored fp8 Linears are not nn.Linear: a generation-time LoRA is refused by name."""
import threading
from core.inference.diffusion import DiffusionBackend
monkeypatch.setattr(cq, "_mapping", lambda cls: (_convert, _sfm()))
path, _dense, _tensors = _fp8_file(tmp_path)
model = _load_bf16(path, fp8_backend = None, fp8_storage = True)
assert cq.fp8_stored_layers(model) == 10
assert cq.fp8_stored_layers(_load_bf16(path, fp8_backend = None, fp8_storage = False)) == 0
assert cq.fp8_stored_layers(None) == 0
pipe = types.SimpleNamespace(transformer = model, _unsloth_loras = ())
state = types.SimpleNamespace(pipe = pipe, transformer_quant = None)
backend = DiffusionBackend.__new__(DiffusionBackend)
with pytest.raises(ValueError, match = "stored as fp8"):
DiffusionBackend._apply_loras(backend, state, [("some/lora", 1.0)], threading.Event())
DiffusionBackend._apply_loras(backend, state, [], threading.Event())
def test_fp8_storage_default_rule_is_offload_or_host_ram(monkeypatch):
"""Unset env stores only when widening would offload or crowd host RAM; ``1`` always; LoRA never."""
monkeypatch.delenv(cq.FP8_STORAGE_ENV, raising = False)
cuda = types.SimpleNamespace(device = "cuda", dtype = torch.bfloat16)
want = cq.fp8_storage_wanted
assert want(cuda) is False
assert want(cuda, offload = True) is True
reserve = cq.FP8_STORAGE_HOST_RESERVE_MIB
assert want(cuda, widened_mib = 24000, host_available_mib = 24000 + reserve + 1) is False
assert want(cuda, widened_mib = 24000, host_available_mib = 24000 + reserve - 1) is True
assert want(cuda, offload = True, lora = True) is False
assert want(types.SimpleNamespace(device = "mps"), offload = True) is True
monkeypatch.setenv(cq.FP8_STORAGE_ENV, "1")
assert want(cuda) is True
def test_fp8_upcast_is_read_from_the_header_not_the_basename(tmp_path):
"""A plain fp8 file named without 'fp8' still widens ~2x (and so can be stored); a bf16 one named 'fp8' does not."""
from safetensors.torch import save_file
from core.inference import diffusion as d
fam = types.SimpleNamespace(single_file_is_pipeline = False)
plain = tmp_path / "krea2.safetensors"
save_file({"blocks.0.weight": torch.zeros(4, 4, dtype = torch.float8_e4m3fn)}, str(plain))
dense = tmp_path / "krea2_fp8.safetensors"
save_file({"blocks.0.weight": torch.zeros(4, 4, dtype = torch.bfloat16)}, str(dense))
assert cq.header_has_fp8_weights(str(plain)) is True
assert cq.header_has_fp8_weights(str(dense)) is False
assert d._single_file_fp8_upcast(str(plain), fam) is True
assert d._single_file_fp8_upcast(str(dense), fam) is False
broken = tmp_path / "broken_fp8.safetensors"
broken.write_bytes(b"\x00" * 4)
assert cq.header_has_fp8_weights(str(broken)) is None
assert d._single_file_fp8_upcast(str(broken), fam) is True
assert (
d._single_file_fp8_upcast(str(plain), types.SimpleNamespace(single_file_is_pipeline = True))
is False
)
def test_fnuz_fp8_headers_are_priced_widened_and_never_stored(tmp_path):
"""FNUZ fp8 is seen by the header predicate and always priced widened."""
from safetensors.torch import save_file
from core.inference import diffusion as d
fam = types.SimpleNamespace(single_file_is_pipeline = False)
fnuz = tmp_path / "krea2.safetensors"
save_file({"blocks.0.weight": torch.zeros(4, 4, dtype = torch.float8_e4m3fnuz)}, str(fnuz))
ocp = tmp_path / "krea2_ocp.safetensors"
save_file({"blocks.0.weight": torch.zeros(4, 4, dtype = torch.float8_e4m3fn)}, str(ocp))
assert cq.header_has_fp8_weights(str(fnuz)) is True
assert d._single_file_fp8_upcast(str(fnuz), fam) is True
assert cq.header_has_fp8_weights(str(fnuz), fnuz_only = True) is True
assert cq.header_has_fp8_weights(str(ocp), fnuz_only = True) is False
def _plan_harness(
monkeypatch,
tmp_path,
*,
fam,
resident_at,
fits_mib,
comfy_mib = None,
mps_budget_mib = None,
):
"""Run ``_plan_memory`` for a single file with a fake planner (resident iff the DiT fits ``fits_mib``; with
``mps_budget_mib`` an MPS plan that stays resident against that budget)."""
from core.inference import diffusion as d
from core.inference import diffusion_memory as dm
path = tmp_path / "dit.safetensors"
if not path.exists():
path.write_bytes(b"\x00" * 16)
seen: list = []
def plan(**kwargs):
mib = kwargs["model_dense_mib"]
seen.append(mib)
if mps_budget_mib is not None:
return types.SimpleNamespace(
resident = True,
device_memory = types.SimpleNamespace(device = "mps", is_unified = True),
estimates = {
"resident_required_mib": mib,
"safe_device_budget_mib": mps_budget_mib,
},
)
return types.SimpleNamespace(resident = mib is not None and mib <= fits_mib)
monkeypatch.setattr(d, "plan_diffusion_memory", plan)
monkeypatch.setattr(d, "plan_keeps_transformer_resident", lambda p: p.resident)
monkeypatch.setattr(d, "comfy_fp8_storage", lambda target, lora = False: True)
monkeypatch.setattr(dm, "_available_system_memory_mib", lambda: 10**7)
monkeypatch.setattr(
d.DiffusionBackend, "_companion_cache_bytes", staticmethod(lambda *a, **k: 0)
)
monkeypatch.setattr(
d.DiffusionBackend, "_text_encoder_cache_bytes", staticmethod(lambda *a, **k: 0)
)
monkeypatch.setattr(d.DiffusionBackend, "_supplied_component_mib", lambda self, *a, **k: None)
if comfy_mib is not None:
monkeypatch.setattr(
d.DiffusionBackend, "_comfy_single_file_resident_mib", staticmethod(comfy_mib)
)
else:
monkeypatch.setattr(
d.DiffusionBackend,
"_comfy_single_file_resident_mib",
staticmethod(lambda *a, **k: None),
)
monkeypatch.setattr(d, "file_size_mib", lambda p: 1000)
monkeypatch.setattr(
d,
"estimate_safetensors_dense_mib",
lambda size, fp8_upcast = False: size * (2 if fp8_upcast else 1),
)
from core.inference.diffusion_device import resolve_diffusion_device_target
backend = d.DiffusionBackend()
final = backend._plan_memory(
resolve_diffusion_device_target(),
str(resident_at or path),
"base/repo",
fam,
None,
False,
kind = "single_file",
)
return backend, final, seen, str(resident_at or path)
def test_offloaded_plan_prices_fp8_with_the_offloaded_backend_and_picks_storage(
monkeypatch, tmp_path
):
"""Offload loses torchao fp8: the plan re-prices offloaded and picks storage."""
calls = []
def comfy_mib(
path,
fam,
target,
base,
*,
lora = False,
fp8_storage = False,
offload = False,
):
calls.append((fp8_storage, offload))
if not offload:
return 1000 # torchao keeps the codes either way
return 1000 if fp8_storage else 2000
fam = types.SimpleNamespace(name = "flux.1", single_file_is_pipeline = False)
# fits_mib below 1000: the plan offloads at every size
backend, final, seen, path = _plan_harness(
monkeypatch, tmp_path, fam = fam, resident_at = None, fits_mib = 500, comfy_mib = comfy_mib
)
assert (False, True) in calls and (True, True) in calls
assert backend._fp8_storage_picks[path] is True
assert seen[-1] == 1000 and 2000 in seen
assert final.resident is False
def test_offloaded_plan_without_storage_is_priced_widened(monkeypatch, tmp_path):
"""No storage possible (LoRA / kill switch): the offloaded plan carries the widened size the loader builds."""
from core.inference import diffusion as d
def comfy_mib(
path,
fam,
target,
base,
*,
lora = False,
fp8_storage = False,
offload = False,
):
return 2000 if offload and not fp8_storage else 1000
real_harness_storage = []
monkeypatch.setattr(
d.DiffusionBackend,
"_fp8_storage_off",
True,
raising = False,
)
fam = types.SimpleNamespace(name = "flux.1", single_file_is_pipeline = False)
orig_setattr = monkeypatch.setattr
def setattr_no_storage(target, name, value, *a, **k):
if target is d and name == "comfy_fp8_storage":
value = lambda target, lora = False: False # noqa: E731
real_harness_storage.append(name)
return orig_setattr(target, name, value, *a, **k)
monkeypatch.setattr = setattr_no_storage
try:
backend, final, seen, path = _plan_harness(
monkeypatch, tmp_path, fam = fam, resident_at = None, fits_mib = 500, comfy_mib = comfy_mib
)
finally:
monkeypatch.setattr = orig_setattr
assert seen[-1] == 2000
assert backend._fp8_storage_picks[path] is False
def test_a_resident_plan_keeps_the_resident_backend_price(monkeypatch, tmp_path):
"""Resident torchao fp8: no offload re-price, no storage."""
calls = []
def comfy_mib(
path,
fam,
target,
base,
*,
lora = False,
fp8_storage = False,
offload = False,
):
calls.append(offload)
return 1000 if not offload else 2000
fam = types.SimpleNamespace(name = "flux.1", single_file_is_pipeline = False)
backend, final, seen, path = _plan_harness(
monkeypatch, tmp_path, fam = fam, resident_at = None, fits_mib = 5000, comfy_mib = comfy_mib
)
assert True not in calls and seen == [1000]
assert backend._fp8_storage_picks[path] is False and final.resident is True
def test_fnuz_krea2_file_is_not_planned_at_the_stored_size(monkeypatch, tmp_path):
"""A Krea 2 FNUZ file widens under storage too, so the planner records no storage pick for it."""
from safetensors.torch import save_file
from core.inference.diffusion_krea2 import KREA2_FAMILY_NAME
fam = types.SimpleNamespace(name = KREA2_FAMILY_NAME, single_file_is_pipeline = False)
for dtype, picked in ((torch.float8_e4m3fnuz, False), (torch.float8_e4m3fn, True)):
path = tmp_path / f"krea2_{str(dtype).split('_')[-1]}.safetensors"
save_file({"blocks.0.weight": torch.zeros(4, 4, dtype = dtype)}, str(path))
backend, final, seen, p = _plan_harness(
monkeypatch, tmp_path, fam = fam, resident_at = path, fits_mib = 500
)
assert backend._fp8_storage_picks[p] is picked, dtype
assert backend._fp8_widen_picks[p] is False, dtype
assert seen[-1] == (1000 if picked else 2000)
def test_a_resident_widen_with_host_room_is_recorded_as_the_fp8_rules_choice(monkeypatch, tmp_path):
"""Fits resident and the host has room: the rule widens, and the load refuses against the rule's reserve; a LoRA
at load time drops that choice (it widens because it must)."""
from safetensors.torch import save_file
from core.inference import diffusion as d
from core.inference.diffusion_krea2 import KREA2_FAMILY_NAME
monkeypatch.delenv(cq.FP8_STORAGE_ENV, raising = False)
fam = types.SimpleNamespace(name = KREA2_FAMILY_NAME, single_file_is_pipeline = False)
path = tmp_path / "krea2_fn.safetensors"
save_file({"blocks.0.weight": torch.zeros(4, 4, dtype = torch.float8_e4m3fn)}, str(path))
backend, final, seen, p = _plan_harness(
monkeypatch, tmp_path, fam = fam, resident_at = path, fits_mib = 5000
)
assert backend._fp8_storage_picks[p] is False and final.resident is True
assert backend._fp8_widen_picks[p] is True
monkeypatch.setattr(d, "comfy_fp8_storage", lambda target, lora = False: not lora)
assert backend._fp8_widen_for(p, None, None) is True
monkeypatch.setattr(d, "_has_active_lora", lambda loras: True)
assert backend._fp8_widen_for(p, None, None) is False
def test_a_plan_priced_with_the_offload_only_backend_keeps_it_when_resident(monkeypatch, tmp_path):
"""Int8 priced via the offload twin: the re-plan fits resident but the loader must still probe offloaded."""
def comfy_mib(
path,
fam,
target,
base,
*,
lora = False,
fp8_storage = False,
offload = False,
):
return 1000 if offload else 2000
fam = types.SimpleNamespace(name = "flux.1", single_file_is_pipeline = False)
backend, final, seen, path = _plan_harness(
monkeypatch, tmp_path, fam = fam, resident_at = None, fits_mib = 1500, comfy_mib = comfy_mib
)
assert seen == [2000, 1000] and final.resident is True
assert backend._comfy_offload_for(final, path) is True
# Not priced offloaded: a resident plan probes resident.
backend._comfy_priced_offload[path] = False
assert backend._comfy_offload_for(final, path) is False
@pytest.mark.parametrize("budget, store", [(2000, True), (4000, False)])
def test_mps_stores_fp8_only_past_the_resident_margin(monkeypatch, tmp_path, budget, store):
"""MPS never offloads; fp8 codes are kept only if the widened DiT overruns 0.85 of budget."""
def comfy_mib(
path,
fam,
target,
base,
*,
lora = False,
fp8_storage = False,
offload = False,
):
return 1000 if fp8_storage else 2000
fam = types.SimpleNamespace(name = "flux.1", single_file_is_pipeline = False)
backend, _final, seen, path = _plan_harness(
monkeypatch,
tmp_path,
fam = fam,
resident_at = None,
fits_mib = 10**6,
comfy_mib = comfy_mib,
mps_budget_mib = budget,
)
assert backend._fp8_storage_picks[path] is store
assert seen[-1] == (1000 if store else 2000)
def test_a_plan_repriced_larger_under_offload_does_not_force_the_offload_probe(
monkeypatch, tmp_path
):
def comfy_mib(
path,
fam,
target,
base,
*,
lora = False,
fp8_storage = False,
offload = False,
):
return 2000 if offload else 1000
fam = types.SimpleNamespace(name = "flux.1", single_file_is_pipeline = False)
backend, final, seen, path = _plan_harness(
monkeypatch, tmp_path, fam = fam, resident_at = None, fits_mib = 500, comfy_mib = comfy_mib
)
assert backend._comfy_priced_offload[path] is False
assert backend._comfy_offload_for(types.SimpleNamespace(resident = True), path) is False
def test_fp8_stored_pricing_widens_keep_fp32_layers_as_the_loader_does(tmp_path):
"""``_keep_in_fp32_modules`` linears are never priced at one byte, under either name."""
n = 1024
names = ("blocks.0.ffn", "condition_embedder.time_embedder.linear_1", "time_embedding.0")
tensors, layers = {}, {}
for name in names:
tensors[f"{name}.weight"] = torch.zeros(n, n, dtype = torch.float8_e4m3fn)
tensors[f"{name}.weight_scale"] = torch.tensor(1.0)
layers[name] = {"format": "float8_e4m3fn"}
path = _save(
tmp_path / "wan.safetensors",
tensors,
{cq.QUANT_METADATA_KEY: json.dumps({"layers": layers})},
)
scan = cq.scan_comfy_quant(path)
kw = dict(keep_int8 = False, keep_fp8 = False, fp8_stored = True)
one = n * n / 2**20 # MiB per layer per byte
plain = cq.comfy_resident_mib(path, scan, **kw)
assert plain == pytest.approx(3 * one, abs = 1)
keep = ("time_embedder", "scale_shift_table")
fp16 = cq.comfy_resident_mib(path, scan, **kw, keep_fp32 = keep, keep_fp32_bytes = 4)
assert fp16 == pytest.approx(one + 2 * 4 * one, abs = 1)
bf16 = cq.comfy_resident_mib(path, scan, **kw, keep_fp32 = keep)
assert bf16 == pytest.approx(one + 2 * 2 * one, abs = 1)
for name in names[1:]:
assert not cq.fp8_stored_by_header("float8_e4m3fn", (n, n), name, keep)
assert cq.fp8_stored_by_header("float8_e4m3fn", (n, n), names[0], keep)
def _fake_diffusers(monkeypatch):
# CPU CI may run without diffusers (or with a stub): pin the two classes these tests read.
class WanTransformer3DModel:
_keep_in_fp32_modules = ["time_embedder", "scale_shift_table", "norm1", "norm2", "norm3"]
class FluxTransformer2DModel:
pass
fake = types.SimpleNamespace(
WanTransformer3DModel = WanTransformer3DModel, FluxTransformer2DModel = FluxTransformer2DModel
)
monkeypatch.setitem(sys.modules, "diffusers", fake)
def test_real_wan_transformer_keeps_its_time_embedder_in_fp32():
try:
import diffusers
cls = getattr(diffusers, "WanTransformer3DModel", None)
except Exception: # noqa: BLE001 -- no usable diffusers here
cls = None
if cls is None:
pytest.skip("this diffusers has no WanTransformer3DModel")
wan = types.SimpleNamespace(transformer_class = "WanTransformer3DModel")
assert "time_embedder" in cq.keep_fp32_modules_for(wan)
def test_keep_fp32_modules_come_from_the_family_transformer(monkeypatch):
_fake_diffusers(monkeypatch)
wan = types.SimpleNamespace(transformer_class = "WanTransformer3DModel")
assert "time_embedder" in cq.keep_fp32_modules_for(wan)
assert cq.keep_fp32_modules_for(types.SimpleNamespace(transformer_class = "NoSuchModel")) == ()
assert cq.keep_fp32_modules_for(types.SimpleNamespace()) == ()
def test_planner_prices_keep_fp32_layers_at_fp32_only_on_fp16(monkeypatch):
from core.inference import diffusion as d
_fake_diffusers(monkeypatch)
wan = types.SimpleNamespace(transformer_class = "WanTransformer3DModel")
fp16 = d._comfy_keep_fp32_pricing(wan, types.SimpleNamespace(dtype = torch.float16))
bf16 = d._comfy_keep_fp32_pricing(wan, types.SimpleNamespace(dtype = torch.bfloat16))
assert "time_embedder" in fp16["keep_fp32"] and fp16["keep_fp32_bytes"] == 4
assert bf16["keep_fp32_bytes"] is None
flux = types.SimpleNamespace(transformer_class = "FluxTransformer2DModel")
assert d._comfy_keep_fp32_pricing(flux, types.SimpleNamespace(dtype = torch.float16)) == {}
# ------------------------------------------------------------------------------------- int8 storage
import core.inference.diffusion_transformer_quant as tq # noqa: E402
_STORAGE_ENV = "UNSLOTH_DIFFUSION_INT8_STORAGE"
def _target(device, dtype):
return types.SimpleNamespace(device = device, dtype = dtype, backend = device)
def _nvidia(monkeypatch):
monkeypatch.setattr(tq, "torch_is_rocm", lambda: False)
monkeypatch.setattr(tq, "is_stubbed", lambda pkg: False)
# no torchao smoke probe here
monkeypatch.setattr(tq, "select_transformer_quant_scheme", lambda *a, **k: None)
@pytest.mark.parametrize(
"device, dtype, offload, env, want",
[
# pre-Ampere NVIDIA (fp16 compute) and XPU: stored under offload, or everywhere when forced
("cuda", torch.float16, True, None, "native"),
("cuda", torch.float16, False, None, None),
("cuda", torch.float16, False, "1", "native"),
("cuda", torch.float16, True, "0", None),
("xpu", torch.bfloat16, True, None, "native"),
("xpu", torch.float16, True, None, "native"),
("xpu", torch.bfloat16, False, None, None),
("xpu", torch.bfloat16, False, "1", "native"),
("xpu", torch.bfloat16, True, "0", None),
# untouched: Ampere+ keeps its W8A8 offload route, an fp32 XPU and MPS outside bf16 expand
("cuda", torch.bfloat16, True, None, "native"),
("cuda", torch.bfloat16, True, "0", "native"),
("cuda", torch.bfloat16, False, "1", None),
("xpu", torch.float32, True, None, None),
("mps", torch.float16, False, None, None),
("mps", torch.float32, False, "1", None),
# CPU (never offloaded): stored by default, expanded only under the kill switch
("cpu", torch.float32, False, None, "native"),
("cpu", torch.float32, False, "1", "native"),
("cpu", torch.float32, False, "0", None),
("cpu", torch.bfloat16, False, None, None),
],
)
def test_int8_storage_backend_choice(monkeypatch, device, dtype, offload, env, want):
_nvidia(monkeypatch)
if env is None:
monkeypatch.delenv(_STORAGE_ENV, raising = False)
else:
monkeypatch.setenv(_STORAGE_ENV, env)
assert cq.comfy_int8_backend(_target(device, dtype), "z-image", offload = offload) == want
def test_int8_storage_never_on_rocm_or_the_stub(monkeypatch):
monkeypatch.setenv(_STORAGE_ENV, "1")
monkeypatch.setattr(tq, "torch_is_rocm", lambda: True)
assert not tq.int8_storage_host(_target("cuda", torch.float16))
monkeypatch.setattr(tq, "torch_is_rocm", lambda: False)
monkeypatch.setattr(tq, "is_stubbed", lambda pkg: True)
assert not tq.int8_storage_host(_target("cuda", torch.float16))
def test_int8_storage_respects_the_family_deny(monkeypatch):
_nvidia(monkeypatch)
monkeypatch.setattr(tq, "_family_denied", lambda family, scheme, *a: scheme == "int8")
assert cq.comfy_int8_backend(_target("cuda", torch.float16), "z-image", offload = True) is None
def test_int8_storage_hosts_never_take_the_int8_gemm(monkeypatch):
_nvidia(monkeypatch)
monkeypatch.delenv(_STORAGE_ENV, raising = False)
assert tq.native_int8_act(_target("cuda", torch.float16)) is False
assert tq.native_int8_act(_target("xpu", torch.bfloat16)) is False
assert tq.native_int8_act(_target("cuda", torch.bfloat16)) is (not tq.int8_act_disabled())
assert tq.native_int8_act(_target("cpu", torch.float32)) is False
monkeypatch.setenv(_STORAGE_ENV, "0")
assert tq.native_int8_act(_target("cuda", torch.float16)) is (not tq.int8_act_disabled())
@pytest.fixture
def plain_int8_file(tmp_path, monkeypatch):
"""A ComfyUI int8_tensorwise file (no ConvRot) of ``_Tiny``."""
monkeypatch.setattr(cq, "_mapping", lambda cls: (_convert, _sfm()))
torch.manual_seed(1)
dense = _Tiny()
tensors = {}
for b, block in enumerate(dense.blocks):
qkv = torch.cat([block.to_q.weight, block.to_k.weight, block.to_v.weight]).detach()
bias = torch.cat([block.to_q.bias, block.to_k.bias, block.to_v.bias])
for name, (weight, b_) in {
f"blocks.{b}.qkv": (qkv, bias),
f"blocks.{b}.out": (block.to_out.weight.detach(), block.to_out.bias),
}.items():
q, s = _int8(weight)
tensors[f"{name}.weight"] = q
tensors[f"{name}.weight_scale"] = s
tensors[f"{name}.bias"] = b_.detach().clone()
tensors[f"{name}.comfy_quant"] = _conf(format = "int8_tensorwise")
tensors[f"blocks.{b}.adaLN_modulation.0.weight"] = (
block.adaLN_modulation[0].weight.detach().clone()
)
tensors[f"blocks.{b}.adaLN_modulation.0.bias"] = (
block.adaLN_modulation[0].bias.detach().clone()
)
tensors["norm.weight"] = dense.norm.weight.detach().clone()
tensors["norm.bias"] = dense.norm.bias.detach().clone()
return _save(tmp_path / "tiny_int8.safetensors", tensors)
def _load_as(path, target, backend):
return cq.load_comfy_quant_transformer(
_Tiny,
path,
cq.refuse_comfy_quant(path),
{"torch_dtype": target.dtype, "config": "base/repo", "subfolder": "transformer"},
int8_backend = backend,
family = "z-image",
target = target,
)
def _run(model, x):
outs = []
for block in model.blocks:
for name in ("to_q", "to_k", "to_v", "to_out"):
outs.append(getattr(block, name)(x))
return outs
def _held(model) -> int:
return sum(t.nbytes for t in (*model.parameters(), *model.buffers()))
@pytest.mark.parametrize("device, dtype", [("cuda", torch.float16), ("xpu", torch.bfloat16)])
def test_int8_storage_keeps_codes_and_matches_the_expanded_dit_bitwise(
monkeypatch, plain_int8_file, device, dtype
):
from core.inference.diffusion_native_quant import is_native_linear
_nvidia(monkeypatch)
monkeypatch.delenv(_STORAGE_ENV, raising = False)
target = _target(device, dtype)
backend = cq.comfy_int8_backend(target, "z-image", offload = True)
assert backend == "native"
expanded = _load_as(plain_int8_file, target, None)
stored = _load_as(plain_int8_file, target, backend)
assert (
stored._unsloth_comfy_quant["int8"] == 8
and stored._unsloth_comfy_quant["backend"] == "native"
)
for block in stored.blocks:
for name in ("to_q", "to_k", "to_v", "to_out"):
layer = getattr(block, name)
assert is_native_linear(layer) and not layer.act_int8 and not layer.rot_group
assert layer.weight_q.dtype == torch.int8
# 1 byte + a 4-byte row scale against itemsize per weight
assert _held(expanded) - _held(stored) == 8 * (DIM * DIM * (dtype.itemsize - 1) - 4 * DIM)
x = torch.randn(3, DIM).to(dtype)
for got, want in zip(_run(stored, x), _run(expanded, x)):
assert torch.equal(got, want)
def test_cpu_keeps_int8_codes_resident_and_matches_the_expanded_dit_bitwise(
monkeypatch, plain_int8_file
):
from core.inference.diffusion_native_quant import is_native_linear
_nvidia(monkeypatch)
monkeypatch.delenv(_STORAGE_ENV, raising = False)
dtype = torch.float32
target = _target("cpu", dtype)
backend = cq.comfy_int8_backend(target, "z-image")
assert backend == "native"
stored = _load_as(plain_int8_file, target, backend)
monkeypatch.setenv(_STORAGE_ENV, "0")
assert cq.comfy_int8_backend(target, "z-image") is None
expanded = _load_as(plain_int8_file, target, None)
assert stored._unsloth_comfy_quant["int8"] == 8
assert expanded._unsloth_comfy_quant["int8"] == 0
for block in stored.blocks:
for name in ("to_q", "to_k", "to_v", "to_out"):
layer = getattr(block, name)
assert is_native_linear(layer) and not layer.act_int8 and not layer.rot_group
assert layer.weight_q.dtype == torch.int8
for block in expanded.blocks:
assert not any(is_native_linear(m) for m in block.modules())
assert _held(expanded) - _held(stored) == 8 * (DIM * DIM * (dtype.itemsize - 1) - 4 * DIM)
for rows in (1, 3, 17):
x = torch.randn(rows, DIM).to(dtype)
for got, want in zip(_run(stored, x), _run(expanded, x)):
assert torch.equal(got, want)
def _cpu_native(
out_f,
in_f,
seed,
bias = True,
):
from core.inference.diffusion_native_quant import native_linear_class
torch.manual_seed(seed)
linear = torch.nn.Linear(in_f, out_f, bias = bias)
codes, scale = _int8(linear.weight.detach())
return native_linear_class()(linear, "int8", codes = codes, scale = scale)
def test_cpu_forward_widens_into_a_reused_scratch_bitwise():
import core.inference.diffusion_native_quant as nq
nq._CPU_WIDEN.buf = None
small, big = _cpu_native(48, 64, 0), _cpu_native(96, 128, 1, bias = False)
x_small, x_big = torch.randn(5, 64), torch.randn(7, 128)
with torch.no_grad():
want_small = torch.nn.functional.linear(
x_small, small._stored_weight(torch.float32), small.bias
)
want_big = torch.nn.functional.linear(x_big, big._stored_weight(torch.float32), None)
got_small = small(x_small)
buf = nq._CPU_WIDEN.buf
got_big = big(x_big)
grown = nq._CPU_WIDEN.buf
again = small(x_small)
assert torch.equal(got_small, want_small) and torch.equal(got_big, want_big)
assert torch.equal(again, want_small)
assert buf.numel() == 48 * 64 and grown.numel() == 96 * 128
assert nq._CPU_WIDEN.buf is grown
assert small.weight.data_ptr() != grown.data_ptr()
def test_cpu_scratch_made_under_inference_mode_still_serves_no_grad():
import core.inference.diffusion_native_quant as nq
nq._CPU_WIDEN.buf = None
layer = _cpu_native(40, 64, 5)
x = torch.randn(3, 64)
with torch.inference_mode():
first = layer(x)
with torch.no_grad():
second = layer(x)
second_grad_on = layer(x)
assert not nq._CPU_WIDEN.buf.is_inference()
assert torch.equal(first, second) and torch.equal(second, second_grad_on)
def test_cpu_forward_under_autograd_keeps_its_own_weight():
layer = _cpu_native(32, 64, 2)
other = _cpu_native(32, 64, 3)
x = torch.randn(4, 64, requires_grad = True)
out = layer(x)
other(torch.randn(4, 64)) # would overwrite a shared scratch before backward
out.sum().backward()
ref = x.detach().clone().requires_grad_()
torch.nn.functional.linear(
ref, layer._stored_weight(torch.float32), layer.bias
).sum().backward()
assert torch.equal(x.grad, ref.grad)
def test_cpu_native_linear_compiles_without_a_graph_break():
torch._dynamo.reset()
layer = _cpu_native(64, 64, 4)
x = torch.randn(9, 64)
with torch.no_grad():
want = layer(x)
got = torch.compile(layer, backend = "eager", fullgraph = True, dynamic = True)(x)
assert torch.equal(got, want)
def test_int8_storage_convrot_rotates_activations_like_the_native_route(monkeypatch, comfy_file):
_nvidia(monkeypatch)
monkeypatch.delenv(_STORAGE_ENV, raising = False)
path, dense, _ = comfy_file
target = _target("cuda", torch.float16)
stored = _load_as(path, target, cq.comfy_int8_backend(target, "z-image", offload = True))
expanded = _load_as(path, target, None)
assert stored._unsloth_comfy_quant["convrot"] == 8
layer = stored.blocks[0].to_k
assert layer.rot_group == GROUP and not layer.act_int8
x = torch.randn(5, DIM).to(torch.float16)
for got, want in zip(_run(stored, x), _run(expanded, x)):
assert _cos(got, want) > 0.9999
def test_int8_storage_stays_off_under_a_lora(monkeypatch):
_nvidia(monkeypatch)
monkeypatch.delenv(_STORAGE_ENV, raising = False)
t4 = _target("cuda", torch.float16)
assert cq.comfy_int8_backend(t4, "z-image", offload = True) == "native"
assert cq.comfy_int8_backend(t4, "z-image", offload = True, lora = True) is None
assert (
cq.comfy_int8_backend(_target("cuda", torch.bfloat16), "z-image", offload = True, lora = True)
== "native"
)