Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: load ComfyUI fp8mixed text encoders (nvfp4 / mxfp8 layers) (#14321)

Daniel Hanb6effa301bc2
diff --git a/studio/backend/core/inference/diffusion_comfy_components.py b/studio/backend/core/inference/diffusion_comfy_components.py
index 42950587b33..14c24d88646 100644
--- a/studio/backend/core/inference/diffusion_comfy_components.py
+++ b/studio/backend/core/inference/diffusion_comfy_components.py
@@ -5,7 +5,7 @@
Each file is classified from its header, assigned to the base pipeline slot whose class it fits, renamed under a
rule that loads strictly, and built from the base repo's config (which still supplies configs and tokenizers).
-fp8 dequantizes, int8 ConvRot stays int8, every other ComfyUI quant format is refused by name.
+fp8, nvfp4 and mxfp8 dequantize, int8 ConvRot stays int8, every other ComfyUI quant format is refused by name.
"""
from __future__ import annotations
@@ -424,8 +424,8 @@ def quant_layers(path: str) -> dict[str, Any]:
more = f" (and {len(scan.problems) - 3} more)" if len(scan.problems) > 3 else ""
raise ComponentFileError(
f"'{os.path.basename(path)}' uses a ComfyUI quantization Studio cannot load as a text encoder "
- f"or VAE: {shown}{more}. Supported: unquantized, scaled fp8 (float8_e4m3fn / float8_e5m2) and "
- "int8_tensorwise (with or without ConvRot); use the bf16 or fp8 file instead."
+ f"or VAE: {shown}{more}. Supported: unquantized, scaled fp8 (float8_e4m3fn / float8_e5m2), nvfp4, "
+ "mxfp8 and int8_tensorwise (with or without ConvRot); use the bf16 or fp8 file instead."
)
return dict(scan.layers)
@@ -463,7 +463,7 @@ def resident_bytes(
kind: str = "text_encoder",
) -> int:
"""What the loaded component holds: int8 ConvRot Linears at stored size (+ scales), every other weight
- (dequantized fp8 included) at the compute dtype. Header-only."""
+ (dequantized fp8 / nvfp4 / mxfp8 included) at the compute dtype, at its logical shape. Header-only."""
header, _ = read_safetensors_header(path)
layers = quant_layers(path) if kind != COMPONENT_VAE else {}
total = 0
@@ -474,6 +474,8 @@ def resident_bytes(
layer = layers.get(key[: -len(".weight")]) if key.endswith(".weight") else None
if layer is not None and layer.convrot and layer.format == "int8_tensorwise":
total += _numel(shape) + int(shape[0]) * 4
+ elif layer is not None:
+ total += _numel(_logical_shape(layer, shape)) * dtype_itemsize
elif dtype in ("F32", "F16", "BF16", "F8_E4M3", "F8_E5M2", "I8", "U8") or layer is not None:
total += _numel(shape) * dtype_itemsize
else:
@@ -502,8 +504,25 @@ def _expected_shapes(module: Any) -> dict[str, tuple]:
return {k: tuple(v.shape) for k, v in module.state_dict().items()}
-def _file_logical_shapes(header: dict) -> dict[str, tuple]:
- return {k: tuple(int(d) for d in (header[k].get("shape") or ())) for k in weight_keys(header)}
+def _logical_shape(layer: Any, shape: Sequence[int]) -> tuple:
+ """The dense shape of a quantized layer's stored weight: nvfp4 packs two columns per byte."""
+ from .diffusion_comfy_block import NVFP4
+
+ dims = tuple(int(d) for d in shape)
+ if layer is not None and layer.format == NVFP4 and len(dims) == 2:
+ return (dims[0], dims[1] * 2)
+ return dims
+
+
+def _file_logical_shapes(header: dict, layers: Optional[dict] = None) -> dict[str, tuple]:
+ layers = layers or {}
+ return {
+ k: _logical_shape(
+ layers.get(k[: -len(".weight")]) if k.endswith(".weight") else None,
+ header[k].get("shape") or (),
+ )
+ for k in weight_keys(header)
+ }
def _strip_meta(module: Any) -> list[str]:
@@ -525,11 +544,12 @@ def _fit_text_encoder(
if trim_lm_head:
trim_text_encoder(encoder)
expected = _expected_shapes(encoder)
+ layers = quant_layers(path)
tied = ()
if config_ties_lm_head(config) and "lm_head.weight" in expected:
tied = ("lm_head.weight",)
try:
- mapping = match_keys(_file_logical_shapes(header), expected, tied_missing = tied)
+ mapping = match_keys(_file_logical_shapes(header, layers), expected, tied_missing = tied)
except ComponentFileError as exc:
raise ComponentFileError(
f"'{os.path.basename(path)}' does not fit this pipeline's {encoder_cls.__name__}: {exc}"
@@ -558,6 +578,7 @@ def build_text_encoder(
layers = quant_layers(path)
encoder, mapping, tied = _fit_text_encoder(path, header, encoder_cls, config, trim_lm_head)
+ from .diffusion_comfy_block import BLOCK_SIZES, decode_layer, dequant_block
from .diffusion_comfy_quant import _dequant
convrot_cls = None
@@ -609,6 +630,33 @@ def build_text_encoder(
kept_int8 += 1
else:
state[dst] = _dequant(codes, scale, int(layer.group or 0), dtype)
+ elif layer.format in BLOCK_SIZES:
+ # nvfp4 / mxfp8: the DiT loader's decoder (nibble order, cuBLAS-tiled block scales), then dense.
+ if scale is None:
+ raise ComponentFileError(
+ f"'{name}': {stem} is {layer.format} with no weight_scale"
+ )
+ tensor_scale = (
+ handle.get_tensor(stem + ".weight_scale_2")
+ if stem + ".weight_scale_2" in header
+ else None
+ )
+ pre_quant_scale = (
+ handle.get_tensor(stem + ".pre_quant_scale")
+ if stem + ".pre_quant_scale" in header
+ else None
+ )
+ plain_codes, plain_scale = decode_layer(layer.format, codes, scale)
+ state[dst] = dequant_block(
+ layer.format,
+ plain_codes,
+ plain_scale,
+ tensor_scale = None
+ if tensor_scale is None
+ else float(tensor_scale.float().reshape(-1)[0]),
+ pre_quant_scale = pre_quant_scale,
+ dtype = dtype,
+ )
else:
state[dst] = _dequant_fp8(codes, scale, layer.format, dtype)
if convrot_cls is not None:
diff --git a/studio/backend/tests/test_diffusion_comfy_components.py b/studio/backend/tests/test_diffusion_comfy_components.py
index 7d5cba8930d..f9eec7baece 100644
--- a/studio/backend/tests/test_diffusion_comfy_components.py
+++ b/studio/backend/tests/test_diffusion_comfy_components.py
@@ -438,6 +438,7 @@ def test_qwen3_bf16_strip_model_prefix_and_dead_lm_head(tmp_path):
"conf",
[
{"format": "asym_w4a8_int8", "group_size": 16, "convrot": True, "convrot_groupsize": 256},
+ # Block formats load (below), but a layer not stored as one is refused naming the format.
{"format": "nvfp4"},
{"format": "mxfp8"},
],
@@ -456,6 +457,82 @@ def test_unsupported_comfy_quant_formats_refused_by_name(tmp_path, conf):
C.validate_component_specs([path], None, model_path = None)
+def _comfy_fp8mixed_qwen3(ref: "torch.nn.Module") -> tuple[dict, dict]:
+ """``ref`` in ComfyUI's fp8mixed text-encoder layout (qwen_3_8b_fp8mixed: float8_e4m3fn and nvfp4 Linears,
+ here with an mxfp8 one too), written by the independent encoder of test_diffusion_comfy_block. Returns the
+ file tensors and the dense weight each quantized key must load as."""
+ import importlib.util
+
+ spec = importlib.util.spec_from_file_location(
+ "_comfy_block_encoders", Path(__file__).with_name("test_diffusion_comfy_block.py")
+ )
+ module = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(module)
+ ENCODERS = module.ENCODERS
+
+ out, want = {}, {}
+ for key, value in ref.state_dict().items():
+ comfy_key = "model." + key
+ stem = comfy_key[: -len(".weight")]
+ fmt = None
+ if key.endswith(".weight") and value.dim() == 2 and "embed" not in key:
+ fmt = (
+ "nvfp4"
+ if ".mlp.gate_proj" in key or ".mlp.up_proj" in key or ".self_attn.k_proj" in key
+ else "mxfp8"
+ if ".mlp.down_proj" in key
+ else "float8_e4m3fn"
+ )
+ if fmt is None:
+ out[comfy_key] = value.to(torch.bfloat16)
+ elif fmt == "float8_e4m3fn":
+ scale = value.abs().amax().float() / 448.0
+ out[comfy_key] = (value.float() / scale).to(torch.float8_e4m3fn)
+ out[stem + ".weight_scale"] = scale.reshape(())
+ want[key] = out[comfy_key].float() * scale
+ else:
+ encode, reference = ENCODERS[fmt]
+ enc = encode(value)
+ for suffix, tensor in enc.items():
+ out[comfy_key if suffix == "weight" else f"{stem}.{suffix}"] = tensor
+ want[key] = reference(enc)
+ if fmt is not None:
+ out[stem + ".comfy_quant"] = _quant_blob({"format": fmt})
+ return out, want
+
+
+def test_qwen3_comfy_fp8mixed_with_nvfp4_and_mxfp8_dequantizes(tmp_path):
+ """ComfyUI's qwen_3_8b_fp8mixed (FLUX.2 klein 9B) mixes fp8 with nvfp4 Linears, whose packed weight is half
+ the logical width. It used to fail as "85 shape mismatches"; every layer now loads dense, bit for bit."""
+ cfg, ref = _tiny_qwen3()
+ out, want = _comfy_fp8mixed_qwen3(ref)
+ path = _save(tmp_path / "qwen_3_8b_fp8mixed.safetensors", out)
+ formats = {layer.format for layer in C.quant_layers(path).values()}
+ assert formats == {"nvfp4", "mxfp8", "float8_e4m3fn"}
+ assert C.validate_component_specs([path], None, model_path = None)
+ enc = C.build_text_encoder(
+ path, encoder_cls = transformers.Qwen3Model, config = cfg, dtype = torch.float32
+ )
+ state = enc.state_dict()
+ for key, dense in want.items():
+ assert state[key].shape == dense.shape, key
+ assert torch.equal(state[key], dense.to(torch.float32)), key
+ # The same model with the reference weights gives the same hidden states.
+ expected = transformers.Qwen3Model(cfg).eval()
+ # Unquantized tensors are stored bf16.
+ expected.load_state_dict(
+ {**{k: v.to(torch.bfloat16).float() for k, v in ref.state_dict().items()}, **want}
+ )
+ ids = torch.tensor([[1, 2, 3, 4, 5]])
+ with torch.no_grad():
+ assert torch.equal(
+ enc(input_ids = ids).last_hidden_state, expected(input_ids = ids).last_hidden_state
+ )
+ # Priced at the logical (dense) size, not the packed one.
+ dense_bytes = sum(v.numel() for k, v in ref.state_dict().items()) * 2
+ assert C.resident_bytes(path, dtype_itemsize = 2) == dense_bytes
+
+
def test_wrong_class_refused_with_diagnostics(tmp_path):
cfg, ref = _tiny_clip()
path = _save(tmp_path / "clip.safetensors", dict(ref.state_dict()))