Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: load ComfyUI fp8mixed text encoders (nvfp4 / mxfp8 layers) (#14321)
Daniel Hanb6effa301bc2
diff --git a/studio/backend/core/inference/diffusion_comfy_components.py b/studio/backend/core/inference/diffusion_comfy_components.pyindex 42950587b33..14c24d88646 100644--- a/studio/backend/core/inference/diffusion_comfy_components.py+++ b/studio/backend/core/inference/diffusion_comfy_components.py@@ -5,7 +5,7 @@Each file is classified from its header, assigned to the base pipeline slot whose class it fits, renamed under arule that loads strictly, and built from the base repo's config (which still supplies configs and tokenizers).-fp8 dequantizes, int8 ConvRot stays int8, every other ComfyUI quant format is refused by name.+fp8, nvfp4 and mxfp8 dequantize, int8 ConvRot stays int8, every other ComfyUI quant format is refused by name."""from __future__ import annotations@@ -424,8 +424,8 @@ def quant_layers(path: str) -> dict[str, Any]:more = f" (and {len(scan.problems) - 3} more)" if len(scan.problems) > 3 else ""raise ComponentFileError(f"'{os.path.basename(path)}' uses a ComfyUI quantization Studio cannot load as a text encoder "- f"or VAE: {shown}{more}. Supported: unquantized, scaled fp8 (float8_e4m3fn / float8_e5m2) and "- "int8_tensorwise (with or without ConvRot); use the bf16 or fp8 file instead."+ f"or VAE: {shown}{more}. Supported: unquantized, scaled fp8 (float8_e4m3fn / float8_e5m2), nvfp4, "+ "mxfp8 and int8_tensorwise (with or without ConvRot); use the bf16 or fp8 file instead.")return dict(scan.layers)@@ -463,7 +463,7 @@ def resident_bytes(kind: str = "text_encoder",) -> int:"""What the loaded component holds: int8 ConvRot Linears at stored size (+ scales), every other weight- (dequantized fp8 included) at the compute dtype. Header-only."""+ (dequantized fp8 / nvfp4 / mxfp8 included) at the compute dtype, at its logical shape. Header-only."""header, _ = read_safetensors_header(path)layers = quant_layers(path) if kind != COMPONENT_VAE else {}total = 0@@ -474,6 +474,8 @@ def resident_bytes(layer = layers.get(key[: -len(".weight")]) if key.endswith(".weight") else Noneif layer is not None and layer.convrot and layer.format == "int8_tensorwise":total += _numel(shape) + int(shape[0]) * 4+ elif layer is not None:+ total += _numel(_logical_shape(layer, shape)) * dtype_itemsizeelif dtype in ("F32", "F16", "BF16", "F8_E4M3", "F8_E5M2", "I8", "U8") or layer is not None:total += _numel(shape) * dtype_itemsizeelse:@@ -502,8 +504,25 @@ def _expected_shapes(module: Any) -> dict[str, tuple]:return {k: tuple(v.shape) for k, v in module.state_dict().items()}-def _file_logical_shapes(header: dict) -> dict[str, tuple]:- return {k: tuple(int(d) for d in (header[k].get("shape") or ())) for k in weight_keys(header)}+def _logical_shape(layer: Any, shape: Sequence[int]) -> tuple:+ """The dense shape of a quantized layer's stored weight: nvfp4 packs two columns per byte."""+ from .diffusion_comfy_block import NVFP4++ dims = tuple(int(d) for d in shape)+ if layer is not None and layer.format == NVFP4 and len(dims) == 2:+ return (dims[0], dims[1] * 2)+ return dims+++def _file_logical_shapes(header: dict, layers: Optional[dict] = None) -> dict[str, tuple]:+ layers = layers or {}+ return {+ k: _logical_shape(+ layers.get(k[: -len(".weight")]) if k.endswith(".weight") else None,+ header[k].get("shape") or (),+ )+ for k in weight_keys(header)+ }def _strip_meta(module: Any) -> list[str]:@@ -525,11 +544,12 @@ def _fit_text_encoder(if trim_lm_head:trim_text_encoder(encoder)expected = _expected_shapes(encoder)+ layers = quant_layers(path)tied = ()if config_ties_lm_head(config) and "lm_head.weight" in expected:tied = ("lm_head.weight",)try:- mapping = match_keys(_file_logical_shapes(header), expected, tied_missing = tied)+ mapping = match_keys(_file_logical_shapes(header, layers), expected, tied_missing = tied)except ComponentFileError as exc:raise ComponentFileError(f"'{os.path.basename(path)}' does not fit this pipeline's {encoder_cls.__name__}: {exc}"@@ -558,6 +578,7 @@ def build_text_encoder(layers = quant_layers(path)encoder, mapping, tied = _fit_text_encoder(path, header, encoder_cls, config, trim_lm_head)+ from .diffusion_comfy_block import BLOCK_SIZES, decode_layer, dequant_blockfrom .diffusion_comfy_quant import _dequantconvrot_cls = None@@ -609,6 +630,33 @@ def build_text_encoder(kept_int8 += 1else:state[dst] = _dequant(codes, scale, int(layer.group or 0), dtype)+ elif layer.format in BLOCK_SIZES:+ # nvfp4 / mxfp8: the DiT loader's decoder (nibble order, cuBLAS-tiled block scales), then dense.+ if scale is None:+ raise ComponentFileError(+ f"'{name}': {stem} is {layer.format} with no weight_scale"+ )+ tensor_scale = (+ handle.get_tensor(stem + ".weight_scale_2")+ if stem + ".weight_scale_2" in header+ else None+ )+ pre_quant_scale = (+ handle.get_tensor(stem + ".pre_quant_scale")+ if stem + ".pre_quant_scale" in header+ else None+ )+ plain_codes, plain_scale = decode_layer(layer.format, codes, scale)+ state[dst] = dequant_block(+ layer.format,+ plain_codes,+ plain_scale,+ tensor_scale = None+ if tensor_scale is None+ else float(tensor_scale.float().reshape(-1)[0]),+ pre_quant_scale = pre_quant_scale,+ dtype = dtype,+ )else:state[dst] = _dequant_fp8(codes, scale, layer.format, dtype)if convrot_cls is not None:diff --git a/studio/backend/tests/test_diffusion_comfy_components.py b/studio/backend/tests/test_diffusion_comfy_components.pyindex 7d5cba8930d..f9eec7baece 100644--- a/studio/backend/tests/test_diffusion_comfy_components.py+++ b/studio/backend/tests/test_diffusion_comfy_components.py@@ -438,6 +438,7 @@ def test_qwen3_bf16_strip_model_prefix_and_dead_lm_head(tmp_path):"conf",[{"format": "asym_w4a8_int8", "group_size": 16, "convrot": True, "convrot_groupsize": 256},+ # Block formats load (below), but a layer not stored as one is refused naming the format.{"format": "nvfp4"},{"format": "mxfp8"},],@@ -456,6 +457,82 @@ def test_unsupported_comfy_quant_formats_refused_by_name(tmp_path, conf):C.validate_component_specs([path], None, model_path = None)+def _comfy_fp8mixed_qwen3(ref: "torch.nn.Module") -> tuple[dict, dict]:+ """``ref`` in ComfyUI's fp8mixed text-encoder layout (qwen_3_8b_fp8mixed: float8_e4m3fn and nvfp4 Linears,+ here with an mxfp8 one too), written by the independent encoder of test_diffusion_comfy_block. Returns the+ file tensors and the dense weight each quantized key must load as."""+ import importlib.util++ spec = importlib.util.spec_from_file_location(+ "_comfy_block_encoders", Path(__file__).with_name("test_diffusion_comfy_block.py")+ )+ module = importlib.util.module_from_spec(spec)+ spec.loader.exec_module(module)+ ENCODERS = module.ENCODERS++ out, want = {}, {}+ for key, value in ref.state_dict().items():+ comfy_key = "model." + key+ stem = comfy_key[: -len(".weight")]+ fmt = None+ if key.endswith(".weight") and value.dim() == 2 and "embed" not in key:+ fmt = (+ "nvfp4"+ if ".mlp.gate_proj" in key or ".mlp.up_proj" in key or ".self_attn.k_proj" in key+ else "mxfp8"+ if ".mlp.down_proj" in key+ else "float8_e4m3fn"+ )+ if fmt is None:+ out[comfy_key] = value.to(torch.bfloat16)+ elif fmt == "float8_e4m3fn":+ scale = value.abs().amax().float() / 448.0+ out[comfy_key] = (value.float() / scale).to(torch.float8_e4m3fn)+ out[stem + ".weight_scale"] = scale.reshape(())+ want[key] = out[comfy_key].float() * scale+ else:+ encode, reference = ENCODERS[fmt]+ enc = encode(value)+ for suffix, tensor in enc.items():+ out[comfy_key if suffix == "weight" else f"{stem}.{suffix}"] = tensor+ want[key] = reference(enc)+ if fmt is not None:+ out[stem + ".comfy_quant"] = _quant_blob({"format": fmt})+ return out, want+++def test_qwen3_comfy_fp8mixed_with_nvfp4_and_mxfp8_dequantizes(tmp_path):+ """ComfyUI's qwen_3_8b_fp8mixed (FLUX.2 klein 9B) mixes fp8 with nvfp4 Linears, whose packed weight is half+ the logical width. It used to fail as "85 shape mismatches"; every layer now loads dense, bit for bit."""+ cfg, ref = _tiny_qwen3()+ out, want = _comfy_fp8mixed_qwen3(ref)+ path = _save(tmp_path / "qwen_3_8b_fp8mixed.safetensors", out)+ formats = {layer.format for layer in C.quant_layers(path).values()}+ assert formats == {"nvfp4", "mxfp8", "float8_e4m3fn"}+ assert C.validate_component_specs([path], None, model_path = None)+ enc = C.build_text_encoder(+ path, encoder_cls = transformers.Qwen3Model, config = cfg, dtype = torch.float32+ )+ state = enc.state_dict()+ for key, dense in want.items():+ assert state[key].shape == dense.shape, key+ assert torch.equal(state[key], dense.to(torch.float32)), key+ # The same model with the reference weights gives the same hidden states.+ expected = transformers.Qwen3Model(cfg).eval()+ # Unquantized tensors are stored bf16.+ expected.load_state_dict(+ {**{k: v.to(torch.bfloat16).float() for k, v in ref.state_dict().items()}, **want}+ )+ ids = torch.tensor([[1, 2, 3, 4, 5]])+ with torch.no_grad():+ assert torch.equal(+ enc(input_ids = ids).last_hidden_state, expected(input_ids = ids).last_hidden_state+ )+ # Priced at the logical (dense) size, not the packed one.+ dense_bytes = sum(v.numel() for k, v in ref.state_dict().items()) * 2+ assert C.resident_bytes(path, dtype_itemsize = 2) == dense_bytes++def test_wrong_class_refused_with_diagnostics(tmp_path):cfg, ref = _tiny_clip()path = _save(tmp_path / "clip.safetensors", dict(ref.state_dict()))