Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: keep nvfp4 / mxfp8 layers packed when no native runtime runs them (#17067)

Daniel Han285946a2295c
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.py
index d762f3cba5a..1bfe3ced321 100644
--- a/studio/backend/core/inference/diffusion.py
+++ b/studio/backend/core/inference/diffusion.py
@@ -11588,6 +11588,7 @@ class DiffusionBackend:
keep_int8 = comfy_int8_backend(target, name, base, offload = offload) is not None,
keep_fp8 = comfy_fp8_backend(target, name, base, offload = offload) is not None,
fp8_stored = fp8_storage and comfy_fp8_storage(target, lora = lora),
+ target = target,
**_comfy_keep_fp32_pricing(fam, target),
# the loader's runtime filter: layers it skips are priced dequantized
min_features = DEFAULT_MIN_LINEAR_FEATURES,
@@ -15925,6 +15926,14 @@ def _lora_load_block_reason(pipe: Any) -> Optional[str]:
"UNSLOTH_DIFFUSION_COMFY_MXFP8=0, or name the LoRA in the API load request (Unsloth then dequantizes "
"those layers)."
)
+ from .diffusion_packed_block import PACKED_BLOCK_ENV, packed_block_layers
+
+ if packed_block_layers(getattr(pipe, "transformer", None)):
+ return (
+ "LoRA is not available on this load: the file's nvfp4 / mxfp8 layers are kept packed and "
+ f"widened per layer (no native runtime on this GPU), which cannot carry adapters. Set {PACKED_BLOCK_ENV}=0, "
+ "or name the LoRA in the API load request (Unsloth then dequantizes those layers at load)."
+ )
if fp8_stored_layers(getattr(pipe, "transformer", None)):
return (
"LoRA is not available on this load: the file's fp8 layers are stored as fp8 and widened per layer "
diff --git a/studio/backend/core/inference/diffusion_comfy_block.py b/studio/backend/core/inference/diffusion_comfy_block.py
index 976fca07a06..5c0b2093a21 100644
--- a/studio/backend/core/inference/diffusion_comfy_block.py
+++ b/studio/backend/core/inference/diffusion_comfy_block.py
@@ -126,14 +126,22 @@ def _dequant_temp_bytes_per_row(fmt: str, cols: int, pre_quant: bool) -> int:
return peak + (4 * cols if pre_quant else 0)
-def _dequant_block_rows(fmt, codes, scale, tensor_scale, pre_quant_scale):
+def _dequant_block_rows(
+ fmt,
+ codes,
+ scale,
+ tensor_scale,
+ pre_quant_scale,
+ lut = None,
+):
import torch
rows = int(codes.shape[0])
if fmt == NVFP4:
- lut = torch.tensor(
- _E2M1 + tuple(-v for v in _E2M1), dtype = torch.float32, device = codes.device
- )
+ if lut is None:
+ lut = torch.tensor(
+ _E2M1 + tuple(-v for v in _E2M1), dtype = torch.float32, device = codes.device
+ )
c = codes.to(torch.int32)
values = torch.stack((lut[c & 0x0F], lut[c >> 4]), dim = -1).reshape(rows, -1, 16)
del c
@@ -161,12 +169,15 @@ def dequant_block(
tensor_scale: Any = None,
pre_quant_scale: Any = None,
dtype: Any = None,
+ chunk_bytes: Optional[int] = None,
+ lut: Any = None,
) -> Any:
"""Dense ``[N, K]`` weight; ``pre_quant_scale`` folds into the columns: ``(x * s) @ W.T == x @ (W * s).T``.
Dequantized in row chunks straight into the output, so the float32 intermediates stay bounded by
- ``_DEQUANT_CHUNK_BYTES`` instead of several times the layer; every op is per row, so the result is
- bit-identical to one pass."""
+ ``chunk_bytes`` (default ``_DEQUANT_CHUNK_BYTES``) instead of several times the layer; every op is per row, so
+ the result is bit-identical to one pass. ``tensor_scale``: one value or one per row. ``lut``: the e2m1 table
+ already on the device (a forward under CUDA graph capture must not upload one)."""
import torch
from .diffusion_single_file_converters import charge_host_ram
@@ -177,7 +188,7 @@ def dequant_block(
rows = int(codes.shape[0])
cols = logical_cols(fmt, codes)
per_row = _dequant_temp_bytes_per_row(fmt, cols, pre_quant_scale is not None)
- chunk = max(1, min(rows, _DEQUANT_CHUNK_BYTES // max(per_row, 1)))
+ chunk = max(1, min(rows, int(chunk_bytes or _DEQUANT_CHUNK_BYTES) // max(per_row, 1)))
charge_host_ram(rows * cols * torch.empty((), dtype = dtype).element_size() + chunk * per_row)
ts = None
if tensor_scale is not None:
@@ -190,7 +201,7 @@ def dequant_block(
stop = min(rows, start + chunk)
part_ts = ts if ts is None or ts.shape[0] == 1 else ts[start:stop]
out[start:stop] = _dequant_block_rows(
- fmt, codes[start:stop], scale[start:stop], part_ts, pqs
+ fmt, codes[start:stop], scale[start:stop], part_ts, pqs, lut
).to(dtype)
return out
@@ -549,8 +560,8 @@ def comfy_block_backends(
)
else:
logger.info(
- "diffusion.comfy_quant: %d %s layer(s) are dequantized to the compute dtype on load: correct, but no memory "
- "saving over the bf16 checkpoint (%s)",
+ "diffusion.comfy_quant: %d %s layer(s) have no native runtime here (%s): widened to the compute "
+ "dtype on load, or kept packed and widened per layer when the load stores its weights compactly",
counts[fmt],
fmt,
reason,
diff --git a/studio/backend/core/inference/diffusion_comfy_quant.py b/studio/backend/core/inference/diffusion_comfy_quant.py
index 36a220298f6..9f557c2293d 100644
--- a/studio/backend/core/inference/diffusion_comfy_quant.py
+++ b/studio/backend/core/inference/diffusion_comfy_quant.py
@@ -375,6 +375,28 @@ def fp8_stored_by_header(
return fmt in FP8_FORMATS and len(shape or ()) == 2 and not keeps_fp32(name, keep_fp32)
+def _packed_runs(fmt: str, target: Any) -> bool:
+ from .diffusion_packed_block import packed_block_runs_on
+ return packed_block_runs_on(fmt, target)
+
+
+def packed_block_by_header(
+ fmt: str,
+ shape: Any,
+ name: str = "",
+ keep_fp32: Any = (),
+) -> bool:
+ """Header rule for nvfp4 / mxfp8 layers that fp8 storage keeps packed when no native runtime runs them
+ (``diffusion_packed_block``); a key-mapped load and a split with mixed input smoothing still widen."""
+ from .diffusion_packed_block import packed_block_enabled
+ return (
+ fmt in BLOCK_SIZES
+ and len(shape or ()) == 2
+ and not keeps_fp32(name, keep_fp32)
+ and packed_block_enabled()
+ )
+
+
def comfy_resident_mib(
path: Optional[str],
scan: Optional[ComfyQuantScan] = None,
@@ -393,6 +415,7 @@ def comfy_resident_mib(
key_map: Any = None,
keep_fp32: Any = (),
keep_fp32_bytes: Optional[int] = None,
+ target: Any = None,
) -> Optional[int]:
"""What the loader leaves resident for a ComfyUI-quantized file, priced from its header: a quantized
weight a runtime keeps costs its stored bytes, one that is dequantized costs ``numel * compute_bytes``
@@ -403,8 +426,9 @@ def comfy_resident_mib(
leaves it, and so is a layer the runtime filter skips (in / out features under ``min_features``, or fp8 features
not multiples of ``fp8_divisible``). ``key_map`` names layers as the loader's filter sees them (an original-layout
file). ``keep_fp32``: the transformer's ``_keep_in_fp32_modules``; those layers are never fp8-stored and, widened,
- cost ``keep_fp32_bytes`` per element (4 on an fp16 load, default ``compute_bytes``). None when the header cannot
- be read. Torch-free."""
+ cost ``keep_fp32_bytes`` per element (4 on an fp16 load, default ``compute_bytes``). Under ``fp8_stored`` an
+ nvfp4 / mxfp8 layer no native runtime keeps is priced packed where it can widen on ``target``. None when the
+ header cannot be read. Torch-free."""
fp32_bytes = compute_bytes if keep_fp32_bytes is None else int(keep_fp32_bytes)
try:
scan = scan if scan is not None else scan_comfy_quant(path)
@@ -462,6 +486,12 @@ def comfy_resident_mib(
for n in _names(name + ".weight", shape)
)
)
+ or (
+ fp8_stored
+ and key_map is None
+ and packed_block_by_header(layer.format, shape, name + ".weight", keep_fp32)
+ and _packed_runs(layer.format, target)
+ )
or (
{NVFP4: keep_nvfp4, MXFP8: keep_mxfp8}.get(layer.format, False)
# input smoothing has no runtime Linear: the loader dequantizes such a layer
@@ -1011,8 +1041,12 @@ def fp8_stored_layers(model: Any) -> int:
def pipe_fp8_stored_layers(pipe: Any) -> int:
+ """fp8-stored plus packed nvfp4 / mxfp8 Linears of the denoisers: buffers widened per forward."""
+ from .diffusion_packed_block import packed_block_layers
return sum(
- fp8_stored_layers(getattr(pipe, attr, None)) for attr in ("transformer", "transformer_2")
+ fp8_stored_layers(getattr(pipe, attr, None))
+ + packed_block_layers(getattr(pipe, attr, None))
+ for attr in ("transformer", "transformer_2")
)
@@ -1259,6 +1293,16 @@ def load_comfy_quant_transformer(
mxfp8_backend = None
if fp8_storage is None:
fp8_storage = fp8_storage_forced() and comfy_fp8_storage(target)
+ from .diffusion_packed_block import (
+ PACKED_BLOCK_ATTR,
+ packed_block_args,
+ packed_block_enabled,
+ packed_block_linear_class,
+ packed_block_runs_on,
+ )
+
+ # fp8 storage also keeps nvfp4 / mxfp8 layers without a native runtime packed (widened per forward).
+ packed = bool(fp8_storage) and key_map is None and packed_block_enabled()
if keep_key is None:
state = load_file(str(path))
else:
@@ -1326,10 +1370,20 @@ def load_comfy_quant_transformer(
and fp8_stored_by_header(layer.format, shape, layer.name, keep_fp32)
)
+ def _packed(layer: Any) -> bool:
+ shape = header_shapes.get(layer.name + ".weight")
+ return (
+ packed
+ and shape is not None
+ and not backends.get(layer.format)
+ and packed_block_runs_on(layer.format, target)
+ and packed_block_by_header(layer.format, shape, layer.name, keep_fp32)
+ )
+
_widened = {
layer.name + ".weight": (
(2 if layer.format == NVFP4 or layer.packed_int4 else 1)
- if not (backends.get(layer.format) or _stored(layer))
+ if not (backends.get(layer.format) or _stored(layer) or _packed(layer))
or (layer.format in BLOCK_SIZES and key_map is not None)
else 0
)
@@ -1338,7 +1392,7 @@ def load_comfy_quant_transformer(
_stored_scale_bytes = sum(
4 * int(header_shapes[layer.name + ".weight"][0])
for layer in scan.layers.values()
- if _stored(layer) and not backends.get(layer.format)
+ if (_stored(layer) or _packed(layer)) and not backends.get(layer.format)
)
# The header estimate refuses early; the budget it activates is charged by every allocating helper
# below, so a layer the estimate cannot see (runtime filters, nibble swaps) still cannot OOM the host.
@@ -1391,7 +1445,10 @@ def load_comfy_quant_transformer(
codes = codes.view(getattr(torch, layer.format))
# A key-mapped (original-layout) family moves rows whole, which the tiled block scales do not survive.
if (
- backends.get(layer.format) or fp16_keep or (fp8_storage and layer.format in FP8_FORMATS)
+ backends.get(layer.format)
+ or fp16_keep
+ or (fp8_storage and layer.format in FP8_FORMATS)
+ or (packed and extra is not None and packed_block_runs_on(layer.format, target))
) and not (extra is not None and key_map is not None):
sources.append((layer, codes, scale, extra))
state[layer.name + ".weight"] = None # placeholder, tagged below
@@ -1537,6 +1594,7 @@ def load_comfy_quant_transformer(
native: dict = {}
stored: dict = {} # fqn -> (codes, scale)
block_native: dict = {} # fqn -> (fmt, codes, scale, tensor_scale, input_scale, bias)
+ block_packed: dict = {} # fqn -> (fmt, codes, scale, row_scale, pre_quant, bias)
built = {TQ_INT8: 0, TQ_FP8: 0, TQ_NVFP4: 0, TQ_MXFP8: 0}
for name, value in list(converted.items()):
if isinstance(value, _BlockWeight):
@@ -1560,6 +1618,23 @@ def load_comfy_quant_transformer(
del converted[name]
built[schemes[fmt]] += 1
else:
+ args = (
+ packed_block_args(value.parts)
+ if packed
+ and packed_block_runs_on(fmt, target)
+ and isinstance(module, torch.nn.Linear)
+ and name.endswith(".weight")
+ and _dtype_for(name) == dtype
+ and not any(m in fqn.split(".") for m in keep_fp32)
+ else None
+ )
+ if args is not None:
+ bias = converted.get(fqn + ".bias")
+ if torch.is_tensor(bias):
+ bias = bias.to(dtype)
+ block_packed[fqn] = (fmt, *args, bias)
+ del converted[name]
+ continue
converted[name] = _dequant_parts(fmt, value.parts, _dtype_for(name))
dequantized += 1
continue
@@ -1660,6 +1735,26 @@ def load_comfy_quant_transformer(
setattr(parent, leaf, layer)
for buffer_name, buffer in layer.named_buffers(recurse = False):
converted[f"{fqn}.{buffer_name}"] = buffer
+ if block_packed:
+ packed_cls = packed_block_linear_class()
+ for fqn, (fmt, codes, scale, row_scale, pre_quant, bias) in block_packed.items():
+ parent_name, _, leaf = fqn.rpartition(".")
+ parent = model.get_submodule(parent_name) if parent_name else model
+ linear = getattr(parent, leaf)
+ layer = packed_cls(
+ fmt,
+ linear.in_features,
+ linear.out_features,
+ codes = codes,
+ scale = scale,
+ row_scale = row_scale,
+ pre_quant = pre_quant,
+ bias = bias,
+ compute_dtype = dtype,
+ )
+ setattr(parent, leaf, layer)
+ for buffer_name, buffer in layer.named_buffers(recurse = False):
+ converted[f"{fqn}.{buffer_name}"] = buffer
_install_native(model)
missing, unexpected = model.load_state_dict(converted, strict = False, assign = True)
@@ -1685,7 +1780,7 @@ def load_comfy_quant_transformer(
del converted
# __init__ buffers (Wan's float64 rope) follow the compute dtype; checkpoint buffers keep theirs (H3 curve table)
for module_name, module in model.named_modules():
- if module_name in block_native:
+ if module_name in block_native or module_name in block_packed:
continue # fp32 scales / alpha the block kernels need; codes are not floating
for buffer_name, buffer in list(module._buffers.items()):
if (f"{module_name}.{buffer_name}" if module_name else buffer_name) in loaded:
@@ -1745,7 +1840,7 @@ def load_comfy_quant_transformer(
"diffusion.comfy_quant: %s loaded from a ComfyUI checkpoint (%s): %d int8 layers kept as "
"int8 (%s, %d ConvRot), %d fp8 layers kept as fp8 (%s), %d nvfp4 layers kept as nvfp4 (%s), "
"%d mxfp8 layers kept as mxfp8 (%s), %d fp8 layers stored as fp8 and widened per layer to %s, "
- "%d layers dequantized to %s",
+ "%d nvfp4 / mxfp8 layers kept packed and widened per layer, %d layers dequantized to %s",
transformer_cls.__name__,
", ".join(f"{k} x{v}" for k, v in sorted(scan.counts().items())),
built[TQ_INT8],
@@ -1759,6 +1854,7 @@ def load_comfy_quant_transformer(
f"{mxfp8_backend} runtime" if built[TQ_MXFP8] else "no mxfp8 runtime",
len(stored),
str(dtype).replace("torch.", ""),
+ len(block_packed),
dequantized,
str(dtype).replace("torch.", ""),
)
@@ -1774,6 +1870,12 @@ def load_comfy_quant_transformer(
if stored:
info["fp8_stored"] = len(stored)
setattr(model, FP8_STORED_ATTR, len(stored))
+ if block_packed:
+ for fmt in (NVFP4, MXFP8):
+ count = sum(1 for v in block_packed.values() if v[0] == fmt)
+ if count:
+ info[f"{fmt}_packed"] = count
+ setattr(model, PACKED_BLOCK_ATTR, len(block_packed))
for fmt, backend in ((NVFP4, nvfp4_backend), (MXFP8, mxfp8_backend)):
if scan.counts().get(fmt):
info[f"{fmt}_backend"] = backend if built[schemes[fmt]] else None
diff --git a/studio/backend/core/inference/diffusion_memory.py b/studio/backend/core/inference/diffusion_memory.py
index 43f19af897a..a1348c25fc5 100644
--- a/studio/backend/core/inference/diffusion_memory.py
+++ b/studio/backend/core/inference/diffusion_memory.py
@@ -2458,12 +2458,14 @@ def _pipe_denoisers_hold_torchao(pipe: Any) -> bool:
def _holds_fp8_stored(module: Any) -> bool:
- """Whether ``module`` holds fp8-STORED Linears; codes and scales are buffers, so a parameter scan reads it dense."""
+ """Whether ``module`` holds fp8-STORED or packed nvfp4 / mxfp8 Linears; codes and scales are buffers, so a
+ parameter scan reads it dense."""
if module is None:
return False
try:
from .diffusion_comfy_quant import fp8_stored_layers
- return fp8_stored_layers(module) > 0
+ from .diffusion_packed_block import packed_block_layers
+ return fp8_stored_layers(module) > 0 or packed_block_layers(module) > 0
except Exception: # noqa: BLE001 - unreadable: today's parameter verdict stands
return False
diff --git a/studio/backend/core/inference/diffusion_packed_block.py b/studio/backend/core/inference/diffusion_packed_block.py
new file mode 100644
index 00000000000..2ed25387a4b
--- /dev/null
+++ b/studio/backend/core/inference/diffusion_packed_block.py
@@ -0,0 +1,326 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""nvfp4 / mxfp8 layers with no native runtime, kept packed and widened per forward, bit-identical to
+``dequant_block`` at load. All state is integer buffers so a module-wide ``.to(dtype)`` never casts it."""
+
+from __future__ import annotations
+
+import contextlib
+import os
+import threading
+from typing import Any, Optional
+
+from .diffusion_comfy_block import BLOCK_SIZES, MXFP8, NVFP4
+
+PACKED_BLOCK_ENV = "UNSLOTH_DIFFUSION_PACKED_BLOCK_FALLBACK"
+PACKED_BLOCK_ATTR = "_unsloth_packed_block"
+_OFF = ("0", "off", "false", "no")
+
+# Torch widen intermediate cap per row chunk (host / accelerator).
+_HOST_CHUNK_BYTES = 16 << 20
+_DEVICE_CHUNK_BYTES = 256 << 20
+
+_TRITON_LOCK = threading.Lock()
+_TRITON: dict = {}
+_OP_LOCK = threading.Lock()
+_OP: dict = {}
+
+
+def packed_block_enabled() -> bool:
+ return (os.environ.get(PACKED_BLOCK_ENV) or "").strip().lower() not in _OFF
+
+
+def packed_block_layers(model: Any) -> int:
+ """How many nvfp4 / mxfp8 Linears of ``model`` are kept packed (they cannot carry LoRA adapters)."""
+ try:
+ return int(getattr(model, PACKED_BLOCK_ATTR, 0) or 0)
+ except (TypeError, ValueError):
+ return 0
+
+
+def fp8_casts(device: Any) -> bool:
+ """Whether float8 tensors cast on ``device`` (CPU / CUDA / ROCm); elsewhere nvfp4 scales go by table and mxfp8
+ layers are not kept packed."""
+ return getattr(device, "type", str(device).split(":")[0]) in ("cpu", "cuda")
+
+
+def packed_block_runs_on(fmt: str, target: Any) -> bool:
+ """Whether a ``fmt`` layer kept packed can widen on ``target``'s device (unknown: the host's rule)."""
+ device = getattr(target, "torch_device", None) or getattr(target, "device", None) or "cpu"
+ return fmt == NVFP4 or fp8_casts(device)
+
+
+def _torch_widen(
+ fmt: str, codes: Any, scale: Any, row_scale: Any, pre_quant: Any, dtype: Any
+) -> Any:
+ from .diffusion_comfy_block import dequant_block
+
+ import torch
+
+ lut = None
+ if fmt == MXFP8:
+ codes = codes.view(torch.float8_e4m3fn)
+ else:
+ from .diffusion_nvfp4_ops import e2m1_lut
+ if fp8_casts(codes.device):
+ scale = scale.view(torch.float8_e4m3fn)
+ else:
+ # no fp8 casts here (MPS before torch 2.15): the exact float32 of every e4m3 byte, by table
+ from .diffusion_native_quant import fp8_lookup_table
+ table = fp8_lookup_table("float8_e4m3fn", str(codes.device))
+ scale = table.index_select(0, scale.reshape(-1).int()).view(scale.shape)
+ lut = e2m1_lut(codes.device)
+ budget = _HOST_CHUNK_BYTES if codes.device.type == "cpu" else _DEVICE_CHUNK_BYTES
+ return dequant_block(
+ fmt,
+ codes,
+ scale,
+ tensor_scale = None if row_scale is None else row_scale.view(torch.float32),
+ pre_quant_scale = None if pre_quant is None else pre_quant.view(torch.float32),
+ dtype = dtype,
+ chunk_bytes = budget,
+ lut = lut,
+ )
+
+
+def _triton_kernel() -> Any:
+ with _TRITON_LOCK:
+ if "kernel" in _TRITON:
+ return _TRITON["kernel"]
+ try:
+ import triton
+
+ from .diffusion_packed_block_triton import nvfp4_widen_kernel
+ _TRITON["kernel"] = (triton, nvfp4_widen_kernel)
+ except Exception: # noqa: BLE001 -- no Triton: the torch widen is exact too
+ _TRITON["kernel"] = None
+ return _TRITON["kernel"]
+
+
+def _triton_failed(exc: BaseException) -> None:
+ with _TRITON_LOCK:
+ _TRITON["kernel"] = None
+ _TRITON["error"] = f"{type(exc).__name__}: {str(exc)[:200]}"
+
+
+def _triton_widen(
+ codes: Any, scale: Any, row_scale: Any, pre_quant: Any, dtype: Any
+) -> Optional[Any]:
+ import torch
+
+ built = _triton_kernel()
+ if built is None:
+ return None
+ triton, kernel = built
+ rows, half_k = int(codes.shape[0]), int(codes.shape[1])
+ out = torch.empty((rows, 2 * half_k), dtype = dtype, device = codes.device)
+ if rows == 0 or half_k == 0:
+ return out
+ block_r, block_c = 32, 64
+ grid = (triton.cdiv(rows, block_r), triton.cdiv(half_k, block_c))
+ rs = row_scale.view(torch.float32) if row_scale is not None else out
+ pq = pre_quant.view(torch.float32) if pre_quant is not None else out
+ guard = torch.cuda.device(codes.device) if codes.is_cuda else contextlib.nullcontext()
+ try:
+ with guard:
+ kernel[grid](
+ codes,
+ scale,
+ rs,
+ pq,
+ out,
+ rows,
+ half_k,
+ int(scale.shape[1]),
+ HAS_RS = row_scale is not None,
+ HAS_PQ = pre_quant is not None,
+ BLOCK_R = block_r,
+ BLOCK_C = block_c,
+ )
+ except Exception as exc: # noqa: BLE001 -- a kernel that cannot build here: the torch path is exact
+ _triton_failed(exc)
+ return None
+ return out
+
+
+def widen_packed(
+ fmt: str, codes: Any, scale: Any, row_scale: Any, pre_quant: Any, dtype: Any
+) -> Any:
+ """The dense ``[N, K]`` weight of packed ``codes`` / ``scale`` (uint8 buffers), bit-identical to the load-time
+ widen; Triton on CUDA / ROCm for nvfp4, row-chunked torch otherwise."""
+ if fmt not in BLOCK_SIZES:
+ raise ValueError(f"not a block-scaled format: {fmt!r}")
+ if (
+ fmt == NVFP4
+ and codes.is_cuda
+ and codes.is_contiguous()
+ and scale.is_contiguous()
+ and (row_scale is None or row_scale.is_contiguous())
+ and (pre_quant is None or pre_quant.is_contiguous())
+ ):
+ out = _triton_widen(codes, scale, row_scale, pre_quant, dtype)
+ if out is not None:
+ return out
+ return _torch_widen(fmt, codes, scale, row_scale, pre_quant, dtype)
+
+
+def _eager_widen(
+ codes: Any, scale: Any, row_scale: Any, pre_quant: Any, fmt: str, dtype: Any
+) -> Any:
+ return widen_packed(fmt, codes, scale, row_scale, pre_quant, dtype)
+
+
+def _widen_op() -> Any:
+ """``torch.ops.unsloth_packed_block.widen``: opaque to the compiler (shape-only fake), one per process."""
+ with _OP_LOCK:
+ if "op" in _OP:
+ return _OP["op"]
+ import torch
+
+ # explicit schema: this module's annotations are strings (no module-level torch to resolve them)
+ @torch.library.custom_op(
+ "unsloth_packed_block::widen",
+ mutates_args = (),
+ schema = "(Tensor codes, Tensor scale, Tensor? row_scale, Tensor? pre_quant, str fmt, "
+ "ScalarType dtype) -> Tensor",
+ )
+ def widen(codes, scale, row_scale, pre_quant, fmt, dtype):
+ return widen_packed(fmt, codes, scale, row_scale, pre_quant, dtype)
+
+ @widen.register_fake
+ def _(codes, scale, row_scale, pre_quant, fmt, dtype):
+ cols = int(codes.shape[1]) * (2 if fmt == NVFP4 else 1)
+ return codes.new_empty((codes.shape[0], cols), dtype = dtype)
+
+ _OP["op"] = widen
+ return widen
+
+
+def packed_block_linear_class():
+ import torch
+ from torch import nn
+ from torch.nn import functional as F
+
+ global _CLASS
+ if _CLASS is not None:
+ return _CLASS
+ _widen_op()
+
+ class PackedBlockLinear(nn.Module):
+ """A Linear whose nvfp4 / mxfp8 weight stays packed; each forward widens it to the compute dtype."""
+
+ def __init__(
+ self,
+ fmt: str,
+ in_features: int,
+ out_features: int,
+ *,
+ codes,
+ scale,
+ row_scale = None,
+ pre_quant = None,
+ bias = None,
+ compute_dtype = None,
+ ):
+ super().__init__()
+ if fmt not in BLOCK_SIZES:
+ raise ValueError(f"not a block-scaled format: {fmt!r}")
+ self.fmt = fmt
+ self.in_features = int(in_features)
+ self.out_features = int(out_features)
+ self.compute_dtype = compute_dtype or torch.bfloat16
+ self.register_buffer("weight_q", codes.contiguous().view(torch.uint8))
+ self.register_buffer("weight_sf", scale.contiguous().view(torch.uint8))
+ # float32 factors as int32 views: a module-wide .to(dtype) would round them
+ self.register_buffer(
+ "row_scale",
+ None
+ if row_scale is None
+ else row_scale.to(torch.float32).reshape(-1).contiguous().view(torch.int32),
+ )
+ self.register_buffer(
+ "pre_quant",
+ None
+ if pre_quant is None
+ else pre_quant.to(torch.float32).reshape(-1).contiguous().view(torch.int32),
+ )
+ self.register_buffer("bias", bias)
+
+ def dequantized_weight(self, dtype: Any = None) -> Any:
+ # eager calls the widen directly; a compiled graph sees the op (no graph break, no recompile)
+ widen = (
+ torch.ops.unsloth_packed_block.widen
+ if torch.compiler.is_compiling()
+ else _eager_widen
+ )
+ w = widen(
+ self.weight_q,
+ self.weight_sf,
+ self.row_scale,
+ self.pre_quant,
+ self.fmt,
+ self.compute_dtype,
+ )
+ return w if dtype is None or dtype == w.dtype else w.to(dtype)
+
+ @property
+ def weight(self) -> Any:
+ return self.dequantized_weight()
+
+ def forward(self, x: Any) -> Any:
+ return F.linear(x, self.dequantized_weight(), self.bias)
+
+ def extra_repr(self) -> str: # pragma: no cover - debug aid
+ return (
+ f"in_features={self.in_features}, out_features={self.out_features}, "
+ f"{self.fmt}=packed, bias={self.bias is not None}"
+ )
+
+ _CLASS = PackedBlockLinear
+ return _CLASS
+
+
+_CLASS = None
+
+
+def is_packed_block_linear(module: Any) -> bool:
+ return type(module).__name__ == "PackedBlockLinear" and hasattr(module, "weight_sf")
+
+
+def packed_block_args(parts: list) -> Optional[tuple]:
+ """``(codes, scale, row_scale, pre_quant)`` holding the row segments ``parts`` exactly, else None (segments with
+ different input smoothing need one weight each). ``row_scale`` / ``pre_quant`` are None when absent everywhere;
+ a segment without a tensor scale gets 1.0, which leaves its product bit-identical."""
+ import torch
+
+ from .diffusion_single_file_converters import charge_host_ram
+
+ smoothing = [extra.get("pre_quant_scale") for _c, _s, extra in parts]
+ first = smoothing[0]
+ for other in smoothing[1:]:
+ if (first is None) != (other is None):
+ return None
+ if first is not None and not torch.equal(first.to(torch.float32), other.to(torch.float32)):
+ return None
+ scales = [extra.get("tensor_scale") for _c, _s, extra in parts]
+ row_scale = None
+ if any(s is not None for s in scales):
+ row_scale = torch.cat(
+ [
+ torch.full((int(c.shape[0]),), 1.0 if s is None else float(s), dtype = torch.float32)
+ for (c, _s, _e), s in zip(parts, scales)
+ ]
+ )
+ if len(parts) > 1:
+ charge_host_ram(
+ sum(
+ int(c.numel()) * int(c.element_size()) + int(s.numel()) * int(s.element_size())
+ for c, s, _e in parts
+ )
+ )
+ codes = torch.cat([c for c, _s, _e in parts])
+ scale = torch.cat([s for _c, s, _e in parts])
+ else:
+ codes, scale = parts[0][0], parts[0][1]
+ return codes, scale, row_scale, None if first is None else first.to(torch.float32).reshape(-1)
diff --git a/studio/backend/core/inference/diffusion_packed_block_triton.py b/studio/backend/core/inference/diffusion_packed_block_triton.py
new file mode 100644
index 00000000000..1e6034f22c5
--- /dev/null
+++ b/studio/backend/core/inference/diffusion_packed_block_triton.py
@@ -0,0 +1,105 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Triton widen for packed nvfp4; helpers are module level because Triton resolves them via globals."""
+
+import triton
+import triton.language as tl
+
+
+def _e2m1(c):
+ # c: int32 nibble -> +-0, 0.5, 1, 1.5, 2, 3, 4, 6, built bit by bit (a float negation would lose -0.0)
+ k = c & 7
+ e = k >> 1
+ m = k & 1
+ bits = tl.where(e == 0, m * 0x3F000000, ((e + 126) << 23) | (m << 22))
+ return (bits | ((c & 8) << 28)).to(tl.float32, bitcast = True)
+
+
+def _e4m3(s):
+ # s: int32 byte of a float8_e4m3fn -> its exact float32 (NaN for 0x7f / 0xff), sign as a bit
+ e = (s >> 3) & 15
+ m = s & 7
+ normal = ((e + 120) << 23) | (m << 20)
+ sub = (m.to(tl.float32) * 0.001953125).to(tl.int32, bitcast = True)
+ bits = tl.where(e == 0, sub, normal)
+ bits = tl.where((e == 15) & (m == 7), 0x7FF00000, bits)
+ return (bits | ((s & 128) << 24)).to(tl.float32, bitcast = True)
+
+
+def _fp16_bits(value):
+ """float32 -> float16 bits, round to nearest even by integer ops (overflow to inf, subnormals kept), as torch
+ casts on every device; ROCm's own conversion differs."""
+ u = value.to(tl.uint32, bitcast = True)
+ sign = (u >> 16) & 0x8000
+ a = u & 0x7FFFFFFF
+ r = a - (112 << 23)
+ normal = (r + 0x0FFF + ((r >> 13) & 1)) >> 13
+ # below 2**-14: the subnormal count, |x| * 2**24 rounded to the nearest even integer (exact scaling, one add)
+ scaled = (a.to(tl.float32, bitcast = True) * 16777216.0 + 8388608.0) - 8388608.0
+ sub = scaled.to(tl.uint32)
+ bits = tl.where(a >= 0x38800000, normal, sub)
+ bits = tl.where(a >= 0x477FF000, 0x7C00, bits)
+ # NaN: quieted, top payload bits kept, as the hardware conversion torch uses
+ bits = tl.where(a > 0x7F800000, 0x7E00 | ((a >> 13) & 0x3FF), bits)
+ return (bits | sign).to(tl.uint16)
+
+
+def _store(dst, value, mask):
+ """``value`` in the output dtype; bf16 rounds to nearest even by integer ops, as torch does on every device
+ (the interpreter, pre-sm_80 and ROCm conversions are not guaranteed to); fp16 the same way."""
+ ty = dst.dtype.element_ty
+ if ty == tl.bfloat16:
+ u = value.to(tl.uint32, bitcast = True)
+ rounded = (u + 0x7FFF + ((u >> 16) & 1)) >> 16
+ rounded = tl.where(value != value, 0x7FC0, rounded)
+ tl.store(dst, rounded.to(tl.uint16).to(tl.bfloat16, bitcast = True), mask = mask)
+ elif ty == tl.float16:
+ tl.store(dst, _fp16_bits(value).to(tl.float16, bitcast = True), mask = mask)
+ else:
+ tl.store(dst, value.to(ty), mask = mask)
+
+
+def nvfp4_widen_kernel(
+ q_ptr,
+ sf_ptr,
+ rs_ptr,
+ pq_ptr,
+ out_ptr,
+ rows,
+ half_k,
+ blocks,
+ HAS_RS: tl.constexpr,
+ HAS_PQ: tl.constexpr,
+ BLOCK_R: tl.constexpr,
+ BLOCK_C: tl.constexpr,
+):
+ r = tl.program_id(0) * BLOCK_R + tl.arange(0, BLOCK_R)
+ j = tl.program_id(1) * BLOCK_C + tl.arange(0, BLOCK_C)
+ rmask = r < rows
+ jmask = j < half_k
+ mask = rmask[:, None] & jmask[None, :]
+ r64 = r.to(tl.int64)
+ j64 = j.to(tl.int64)
+ b = tl.load(q_ptr + r64[:, None] * half_k + j64[None, :], mask = mask, other = 0)
+ b = b.to(tl.int32)
+ s = tl.load(sf_ptr + r64[:, None] * blocks + (j64 // 8)[None, :], mask = mask, other = 0)
+ # same multiply order as dequant_block (bit-identical)
+ step = _e4m3(s.to(tl.int32))
+ if HAS_RS:
+ step = step * tl.load(rs_ptr + r64, mask = rmask, other = 1.0)[:, None]
+ lo = _e2m1(b & 15) * step
+ hi = _e2m1(b >> 4) * step
+ if HAS_PQ:
+ lo = lo * tl.load(pq_ptr + 2 * j64, mask = jmask, other = 1.0)[None, :]
+ hi = hi * tl.load(pq_ptr + 2 * j64 + 1, mask = jmask, other = 1.0)[None, :]
+ # even column = low nibble
+ w = tl.reshape(tl.join(lo, hi), (BLOCK_R, 2 * BLOCK_C))
+ c = tl.program_id(1).to(tl.int64) * (2 * BLOCK_C) + tl.arange(0, 2 * BLOCK_C)
+ dst = out_ptr + r64[:, None] * (2 * half_k) + c[None, :]
+ _store(dst, w, rmask[:, None] & (c < 2 * half_k)[None, :])
diff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.py
index b0405f93acd..5acf9bfdeb4 100644
--- a/studio/backend/core/inference/video.py
+++ b/studio/backend/core/inference/video.py
@@ -532,11 +532,14 @@ def _video_comfy_backends(
resident only; else fp8 storage under offload or a tight MPS plan). ``priced_offload``: the plan was sized with the
offloaded backends (``_video_offload_reprice``), so a load that now fits resident keeps them. ``widened_mib``: the
planned DiT price; a resident load stores fp8 when widening it would leave under the host reserve, as an image does.
- ``keep=False`` (the user asked for the bf16 DiT) dequantizes everything."""
+ ``keep=False`` (the user asked for the bf16 DiT) dequantizes everything. nvfp4 / mxfp8 layers take the image
+ path's native runtimes where their flag, device and family allow; without one, fp8 storage keeps them packed."""
if not keep:
return {
"int8_backend": None,
"fp8_backend": None,
+ "nvfp4_backend": None,
+ "mxfp8_backend": None,
"fp8_storage": False,
"fp8_widen_chosen": False,
}
@@ -547,12 +550,24 @@ def _video_comfy_backends(
return {
"int8_backend": comfy_int8_backend(target, name, base, offload = offload),
"fp8_backend": comfy_fp8_backend(target, name, base, offload = offload),
+ **_video_block_backends(target, name),
"fp8_storage": storage,
"fp8_widen_chosen": not storage
and _video_fp8_storage(fam, target, store_pays, widened_mib = widened_mib, widen_chosen = True),
}
+def _video_block_backends(target: Any, family: Optional[str]) -> dict:
+ """``comfy_block_backend`` per block format (None = no native runtime here)."""
+ from .diffusion_comfy_block import MXFP8, NVFP4, comfy_block_backend
+
+ dtype = getattr(target, "dtype", None)
+ return {
+ f"{fmt}_backend": comfy_block_backend(fmt, target, family, dtype = dtype)[0]
+ for fmt in (NVFP4, MXFP8)
+ }
+
+
def _video_comfy_reprice(plan: Any, bf16_plan: Any = None) -> bool:
"""Whether a ComfyUI DiT is re-priced with the offloaded backends: an offloaded plan (fp8 layers dequantize on
NVIDIA), or an MPS plan past its resident margin (MPS never offloads; its offloaded backends are its resident ones
@@ -614,10 +629,13 @@ def _video_comfy_resident_mib(
keep_int8 = keep_fp8 = fp8_stored = False
exclude: tuple = ()
filters: dict = {}
+ keep_block: dict = {}
if keep:
try:
from .diffusion_transformer_quant import (
DEFAULT_MIN_LINEAR_FEATURES,
+ TQ_MXFP8,
+ TQ_NVFP4,
divisible_for_scheme,
exclude_tokens_for_scheme,
)
@@ -625,6 +643,10 @@ def _video_comfy_resident_mib(
filters = {
"min_features": DEFAULT_MIN_LINEAR_FEATURES,
"fp8_divisible": divisible_for_scheme(TQ_FP8),
+ "block_divisible": {
+ "nvfp4": divisible_for_scheme(TQ_NVFP4),
+ "mxfp8": divisible_for_scheme(TQ_MXFP8),
+ },
}
except Exception: # noqa: BLE001 -- the estimate then keeps every int8 layer
exclude = ()
@@ -632,12 +654,19 @@ def _video_comfy_resident_mib(
keep_int8 = bool(comfy_int8_backend(target, name, base, offload = offload))
keep_fp8 = comfy_fp8_backend(target, name, base, offload = offload) is not None
fp8_stored = _video_fp8_storage(fam, target, offload)
+ blocks = _video_block_backends(target, name)
+ keep_block = {
+ "keep_nvfp4": blocks["nvfp4_backend"] is not None,
+ "keep_mxfp8": blocks["mxfp8_backend"] is not None,
+ }
return comfy_resident_mib(
str(path),
scan,
keep_int8 = keep_int8,
keep_fp8 = keep_fp8,
fp8_stored = fp8_stored,
+ **keep_block,
+ target = target,
keep_key = keep_key,
exclude_tokens = exclude,
key_map = _video_comfy_key_map(fam),
diff --git a/studio/backend/core/inference/video_ltx2.py b/studio/backend/core/inference/video_ltx2.py
index e2711f53117..89a5ee5ec7e 100644
--- a/studio/backend/core/inference/video_ltx2.py
+++ b/studio/backend/core/inference/video_ltx2.py
@@ -1259,6 +1259,8 @@ def load_ltx23_comfy_transformer(
local_files_only: bool = False,
int8_backend: Optional[str] = None,
fp8_backend: Optional[str] = None,
+ nvfp4_backend: Optional[str] = None,
+ mxfp8_backend: Optional[str] = None,
fp8_storage: Optional[bool] = None,
fp8_widen_chosen: bool = False,
family: Optional[str] = None,
@@ -1285,6 +1287,8 @@ def load_ltx23_comfy_transformer(
},
int8_backend = int8_backend,
fp8_backend = fp8_backend,
+ nvfp4_backend = nvfp4_backend,
+ mxfp8_backend = mxfp8_backend,
fp8_storage = fp8_storage,
fp8_widen_chosen = fp8_widen_chosen,
family = family,
diff --git a/studio/backend/core/inference/video_ltx25.py b/studio/backend/core/inference/video_ltx25.py
index 08a9aa1feae..beee958ccc9 100644
--- a/studio/backend/core/inference/video_ltx25.py
+++ b/studio/backend/core/inference/video_ltx25.py
@@ -399,6 +399,8 @@ def load_ltx25_comfy_transformer(
local_files_only: bool = False,
int8_backend: Optional[str] = None,
fp8_backend: Optional[str] = None,
+ nvfp4_backend: Optional[str] = None,
+ mxfp8_backend: Optional[str] = None,
fp8_storage: Optional[bool] = None,
fp8_widen_chosen: bool = False,
family: Optional[str] = None,
@@ -426,6 +428,8 @@ def load_ltx25_comfy_transformer(
},
int8_backend = int8_backend,
fp8_backend = fp8_backend,
+ nvfp4_backend = nvfp4_backend,
+ mxfp8_backend = mxfp8_backend,
fp8_storage = fp8_storage,
fp8_widen_chosen = fp8_widen_chosen,
family = family,
diff --git a/studio/backend/tests/test_diffusion_comfy_block.py b/studio/backend/tests/test_diffusion_comfy_block.py
index 49c08f2ab71..fd464d4fb51 100644
--- a/studio/backend/tests/test_diffusion_comfy_block.py
+++ b/studio/backend/tests/test_diffusion_comfy_block.py
@@ -582,7 +582,7 @@ def test_backends_are_logged_per_format_present(caplog):
with caplog.at_level(logging.INFO, logger = "comfy_block_test"):
out = cb.comfy_block_backends(scan, _target("cpu"), "z-image", logger = logger)
assert out == {"nvfp4_backend": None, "mxfp8_backend": None}
- assert "3 nvfp4 layer(s) are dequantized" in caplog.text and "mxfp8" not in caplog.text
+ assert "3 nvfp4 layer(s) have no native runtime" in caplog.text and "mxfp8" not in caplog.text
def test_a_failed_mxfp8_probe_is_not_cached(monkeypatch):
@@ -1069,3 +1069,30 @@ def test_untiling_padded_blocks_checks_both_live_copies(monkeypatch):
monkeypatch.setattr(conv, "charge_host_ram", lambda n: seen.append(int(n)))
assert torch.equal(cb.untile_scales(tiled, rows, blocks), plain)
assert seen == [_ceil(rows, 128) * _ceil(blocks, 4) + rows * blocks]
+
+
[email protected]("fmt", ["nvfp4", "mxfp8"])
+def test_a_compact_storage_load_holds_block_layers_at_their_file_size(block_file, fmt):
+ """The fp8-storage pick (widening would offload) keeps nvfp4 / mxfp8 layers no runtime takes near their file
+ size, not as a bf16 copy, and their weights stay the dense load's."""
+ path, _tensors, _expected = block_file(fmt)
+ dense = _load(path, dtype = torch.bfloat16)
+ compact = _load(path, dtype = torch.bfloat16, fp8_storage = True)
+ names = [f"blocks.{b}.{n}" for b in range(2) for n in ("to_q", "to_k", "to_v", "to_out")]
+
+ def held(model, name):
+ mod = model.get_submodule(name)
+ tensors = [
+ t for n, t in list(mod.named_parameters()) + list(mod.named_buffers()) if n != "bias"
+ ]
+ return sum(t.numel() * t.element_size() for t in tensors)
+
+ bf16 = sum(held(dense, n) for n in names)
+ kept = sum(held(compact, n) for n in names)
+ assert bf16 == 8 * DIM * DIM * 2
+ assert kept < (0.3 if fmt == "nvfp4" else 0.6) * bf16, (kept, bf16)
+ for name in names:
+ assert torch.equal(
+ compact.get_submodule(name).weight.view(torch.int16),
+ dense.get_submodule(name).weight.view(torch.int16),
+ ), name
diff --git a/studio/backend/tests/test_diffusion_packed_block.py b/studio/backend/tests/test_diffusion_packed_block.py
new file mode 100644
index 00000000000..3abb3fea75b
--- /dev/null
+++ b/studio/backend/tests/test_diffusion_packed_block.py
@@ -0,0 +1,311 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""nvfp4 / mxfp8 layers kept packed when no native runtime takes them: same weights as the load-time widen."""
+
+from __future__ import annotations
+
+import os
+import subprocess
+import sys
+import textwrap
+
+import pytest
+
+torch = pytest.importorskip("torch")
+
+import core.inference.diffusion_comfy_block as cb # noqa: E402
+import core.inference.diffusion_comfy_quant as cq # noqa: E402
+import core.inference.diffusion_packed_block as pb # noqa: E402
+from .test_diffusion_comfy_block import DIM, _Tiny, _load, block_file # noqa: E402,F401
+
+LINEARS = [f"blocks.{b}.{n}" for b in range(2) for n in ("to_q", "to_k", "to_v", "to_out")]
+
+
+_HELPERS = textwrap.dedent(
+ """
+ import torch
+
+
+ def _bits(t):
+ return t.view({2: torch.int16, 4: torch.int32}[t.element_size()])
+
+
+ def _exhaustive_nvfp4():
+ # every code byte against every e4m3 scale byte (NaN 0x7f / 0xff included), 16 blocks per row
+ rows = 256
+ codes = (torch.arange(128, dtype = torch.int64).view(1, -1) * 2 + torch.arange(rows).view(-1, 1)) % 256
+ scale = (torch.arange(rows).view(-1, 1) * 16 + torch.arange(16)) % 256
+ return codes.to(torch.uint8).contiguous(), scale.to(torch.uint8).contiguous()
+ """
+)
+exec(_HELPERS)
+
+
+def _same(a: torch.Tensor, b: torch.Tensor) -> bool:
+ """Bitwise, NaN positions included (payloads compared too)."""
+ return a.shape == b.shape and a.dtype == b.dtype and torch.equal(_bits(a), _bits(b))
+
+
[email protected]("fmt", ["nvfp4", "mxfp8"])
[email protected]("dtype", [torch.bfloat16, torch.float32])
+def test_a_packed_load_widens_to_the_dense_loads_exact_weights(block_file, fmt, dtype):
+ path, _tensors, _expected = block_file(fmt)
+ dense = _load(path, dtype = dtype)
+ packed = _load(path, dtype = dtype, fp8_storage = True)
+ info = packed._unsloth_comfy_quant
+ assert info[f"{fmt}_packed"] == 8 and info["dequantized"] == 0
+ assert pb.packed_block_layers(packed) == 8
+ state = dense.state_dict()
+ x = torch.randn(3, 5, DIM, generator = torch.Generator().manual_seed(1)).to(dtype)
+ for name in LINEARS:
+ layer = packed.get_submodule(name)
+ assert pb.is_packed_block_linear(layer), name
+ assert not any(b.is_floating_point() for n, b in layer.named_buffers() if n != "bias")
+ assert _same(layer.weight, state[name + ".weight"]), name
+ ref = dense.get_submodule(name)
+ assert _same(layer(x), ref(x)), name
+ # packed bytes: nvfp4 codes are half a byte per weight, mxfp8 one
+ q = packed.blocks[0].to_q
+ assert q.weight_q.numel() == DIM * DIM // (2 if fmt == "nvfp4" else 1)
+
+
[email protected]("fmt", ["nvfp4", "mxfp8"])
+def test_the_kill_switch_and_a_widening_pick_keep_todays_dense_load(block_file, fmt, monkeypatch):
+ path, _tensors, expected = block_file(fmt)
+ for kwargs, env in (({}, None), ({"fp8_storage": True}, "0")):
+ if env is None:
+ monkeypatch.delenv(pb.PACKED_BLOCK_ENV, raising = False)
+ else:
+ monkeypatch.setenv(pb.PACKED_BLOCK_ENV, env)
+ model = _load(path, **kwargs)
+ info = model._unsloth_comfy_quant
+ assert info["dequantized"] == 4 and f"{fmt}_packed" not in info
+ assert pb.packed_block_layers(model) == 0
+ state = model.state_dict()
+ for key, want in expected.items():
+ assert torch.equal(state[key], want), key
+
+
+def test_module_casts_and_moves_never_touch_the_packed_buffers(block_file):
+ path, _tensors, _expected = block_file("nvfp4")
+ model = _load(path, dtype = torch.bfloat16, fp8_storage = True)
+ layer = model.blocks[1].to_out
+ before = {n: b.clone() for n, b in layer.named_buffers() if n != "bias"}
+ weight = layer.weight.clone()
+ model.to(torch.float16)
+ for name, buf in layer.named_buffers():
+ if name != "bias":
+ assert buf.dtype == before[name].dtype and torch.equal(buf, before[name]), name
+ assert _same(layer.weight, weight) # the compute dtype is the load's, not the cast's
+
+
+def _parts(
+ fmt,
+ rows,
+ seed,
+ tensor_scale = 0.5,
+ smoothing = None,
+):
+ gen = torch.Generator().manual_seed(seed)
+ cols = 64
+ block = cb.BLOCK_SIZES[fmt]
+ if fmt == "nvfp4":
+ codes = torch.randint(0, 256, (rows, cols // 2), dtype = torch.uint8, generator = gen)
+ scale = torch.randint(0, 0x7F, (rows, cols // block), dtype = torch.uint8, generator = gen)
+ scale = scale.view(torch.float8_e4m3fn)
+ else:
+ codes = torch.randint(0, 0x7F, (rows, cols), dtype = torch.uint8, generator = gen)
+ codes = codes | (torch.randint(0, 2, codes.shape, dtype = torch.uint8, generator = gen) << 7)
+ codes = codes.view(torch.float8_e4m3fn)
+ scale = torch.randint(100, 150, (rows, cols // block), dtype = torch.uint8, generator = gen)
+ extra = {"tensor_scale": tensor_scale, "pre_quant_scale": smoothing}
+ return codes, scale, extra
+
+
[email protected]("fmt", ["nvfp4", "mxfp8"])
[email protected]("rows", [1, 17, 1023, 1024, 1025])
+def test_row_chunks_splits_and_smoothing_widen_bit_identically(fmt, rows, monkeypatch):
+ smoothing = torch.rand(64, generator = torch.Generator().manual_seed(7)) + 0.5
+ parts = [
+ _parts(fmt, rows, 1, tensor_scale = 0.25 if fmt == "nvfp4" else None, smoothing = smoothing),
+ _parts(fmt, 3, 2, tensor_scale = None, smoothing = smoothing),
+ _parts(fmt, 5, 3, tensor_scale = 3.0 if fmt == "nvfp4" else None, smoothing = smoothing),
+ ]
+ want = cq._dequant_parts(fmt, parts, torch.bfloat16)
+ codes, scale, row_scale, pre_quant = pb.packed_block_args(parts)
+ layer = pb.packed_block_linear_class()(
+ fmt,
+ 64,
+ int(codes.shape[0]),
+ codes = codes,
+ scale = scale,
+ row_scale = row_scale,
+ pre_quant = pre_quant,
+ compute_dtype = torch.bfloat16,
+ )
+ monkeypatch.setattr(pb, "_HOST_CHUNK_BYTES", 4096)
+ assert _same(layer.weight, want)
+ monkeypatch.setattr(pb, "_HOST_CHUNK_BYTES", 1 << 30)
+ assert _same(layer.weight, want)
+
+
+def test_segments_with_different_smoothing_stay_dense():
+ a = _parts("nvfp4", 4, 1, smoothing = torch.ones(64))
+ b = _parts("nvfp4", 4, 2, smoothing = torch.full((64,), 2.0))
+ c = _parts("nvfp4", 4, 3, smoothing = None)
+ assert pb.packed_block_args([a, b]) is None
+ assert pb.packed_block_args([a, c]) is None
+ assert pb.packed_block_args([a, a]) is not None
+
+
[email protected]("dtype", [torch.bfloat16, torch.float16, torch.float32])
+def test_exhaustive_codes_and_scales_through_the_torch_widen(dtype):
+ codes, scale = _exhaustive_nvfp4()
+ row_scale = torch.full((codes.shape[0],), 2.0**-9, dtype = torch.float32)
+ pre_quant = torch.linspace(0.25, 4.0, 2 * codes.shape[1], dtype = torch.float32)
+ want = cb.dequant_block(
+ "nvfp4",
+ codes,
+ scale.view(torch.float8_e4m3fn),
+ tensor_scale = row_scale,
+ pre_quant_scale = pre_quant,
+ dtype = dtype,
+ )
+ got = pb.widen_packed(
+ "nvfp4", codes, scale, row_scale.view(torch.int32), pre_quant.view(torch.int32), dtype
+ )
+ assert _same(got, want)
+
+
+_INTERPRETED = textwrap.dedent(
+ """
+ import sys, torch
+ sys.path.insert(0, {root!r})
+ import core.inference.diffusion_comfy_block as cb
+ import core.inference.diffusion_packed_block as pb
+ exec({helpers!r})
+ codes, scale = _exhaustive_nvfp4()
+ rows = codes.shape[0]
+ bad = []
+ for dtype in (torch.bfloat16, torch.float16, torch.float32):
+ for rs, pq in ((None, None), (torch.full((rows,), 0.75), None),
+ (torch.full((rows,), 2.0 ** -9), torch.linspace(0.25, 4.0, 2 * codes.shape[1]))):
+ want = cb.dequant_block("nvfp4", codes, scale.view(torch.float8_e4m3fn),
+ tensor_scale = rs, pre_quant_scale = pq, dtype = dtype)
+ got = pb._triton_widen(codes, scale, None if rs is None else rs.view(torch.int32),
+ None if pq is None else pq.view(torch.int32), dtype)
+ assert got is not None, pb._TRITON.get("error")
+ nan = torch.isnan(want.float())
+ if not torch.equal(nan, torch.isnan(got.float())):
+ bad.append((str(dtype), "nan positions"))
+ elif not torch.equal(_bits(got)[~nan], _bits(want)[~nan]):
+ bad.append((str(dtype), "values"))
+ print("BAD", bad)
+ """
+)
+
+
+def test_the_triton_widen_is_bit_identical_under_the_interpreter():
+ pytest.importorskip("triton")
+ root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+ env = dict(os.environ, TRITON_INTERPRET = "1", CUDA_VISIBLE_DEVICES = "")
+ proc = subprocess.run(
+ [sys.executable, "-c", _INTERPRETED.format(root = root, helpers = _HELPERS)],
+ cwd = root,
+ env = env,
+ capture_output = True,
+ text = True,
+ timeout = 600,
+ )
+ assert "BAD []" in proc.stdout, proc.stdout + proc.stderr[-2000:]
+
+
+def test_a_compiled_forward_has_one_graph_and_the_same_output(block_file):
+ path, _tensors, _expected = block_file("nvfp4")
+ model = _load(path, dtype = torch.float32, fp8_storage = True)
+ layer = model.blocks[0].to_q
+ x = torch.randn(4, DIM, generator = torch.Generator().manual_seed(3))
+ want = layer(x)
+ torch._dynamo.reset()
+ from torch._dynamo.testing import CompileCounterWithBackend
+
+ counter = CompileCounterWithBackend("aot_eager")
+ compiled = torch.compile(layer, backend = counter, fullgraph = True)
+ assert _same(compiled(x), want)
+ y = torch.randn(4, DIM, generator = torch.Generator().manual_seed(4))
+ assert _same(compiled(y), layer(y))
+ assert counter.frame_count == 1
+
+
+def test_fp8_storage_prices_block_layers_packed(block_file, monkeypatch):
+ path, _tensors, _expected = block_file("nvfp4")
+ scan = cq.scan_comfy_quant(path)
+ kw = dict(keep_int8 = False, keep_fp8 = False)
+ dense = cq.comfy_resident_mib(path, scan, **kw)
+ packed = cq.comfy_resident_mib(path, scan, fp8_stored = True, **kw)
+ kept = cq.comfy_resident_mib(path, scan, keep_nvfp4 = True, **kw)
+ assert packed == kept < dense
+ monkeypatch.setenv(pb.PACKED_BLOCK_ENV, "0")
+ assert cq.comfy_resident_mib(path, scan, fp8_stored = True, **kw) == dense
+
+
+def test_packed_layers_render_under_no_grad_and_refuse_a_late_lora(block_file):
+ import types
+
+ from core.inference.diffusion import _lora_load_block_reason
+
+ path, _tensors, _expected = block_file("nvfp4")
+ model = _load(path, fp8_storage = True)
+ pipe = types.SimpleNamespace(transformer = model)
+ assert cq.pipe_fp8_stored_layers(pipe) == 8
+ assert pb.PACKED_BLOCK_ENV in (_lora_load_block_reason(pipe) or "")
+
+
+def test_devices_without_fp8_casts_widen_nvfp4_by_table_and_keep_mxfp8_dense(monkeypatch):
+ import types
+
+ mps = types.SimpleNamespace(device = "mps", torch_device = "mps")
+ assert pb.packed_block_runs_on("nvfp4", mps) and not pb.packed_block_runs_on("mxfp8", mps)
+ assert pb.packed_block_runs_on(
+ "mxfp8", types.SimpleNamespace(device = "cuda", torch_device = "cuda:1")
+ )
+ # the table decode of the e4m3 scales gives the same weight as the cast
+ codes, scale = _exhaustive_nvfp4()
+ want = pb.widen_packed("nvfp4", codes, scale, None, None, torch.bfloat16)
+ monkeypatch.setattr(pb, "fp8_casts", lambda device: False)
+ assert _same(pb.widen_packed("nvfp4", codes, scale, None, None, torch.bfloat16), want)
+
+
+ not torch.cuda.is_available(), reason = "needs a CUDA / ROCm GPU to run the Triton widen"
+)
[email protected]("dtype", [torch.bfloat16, torch.float16, torch.float32])
+def test_the_triton_widen_on_the_gpu_matches_the_load_time_widen(dtype):
+ codes, scale = _exhaustive_nvfp4()
+ rows = codes.shape[0]
+ for rs, pq in (
+ (None, None),
+ (torch.full((rows,), 0.75), None),
+ (torch.full((rows,), 2.0**-9), torch.linspace(0.25, 4.0, 2 * codes.shape[1])),
+ ):
+ want = cb.dequant_block(
+ "nvfp4",
+ codes,
+ scale.view(torch.float8_e4m3fn),
+ tensor_scale = rs,
+ pre_quant_scale = pq,
+ dtype = dtype,
+ )
+ got = pb._triton_widen(
+ codes.cuda(),
+ scale.cuda(),
+ None if rs is None else rs.cuda().view(torch.int32),
+ None if pq is None else pq.cuda().view(torch.int32),
+ dtype,
+ )
+ assert got is not None, pb._TRITON.get("error")
+ nan = torch.isnan(want.float())
+ got = got.cpu()
+ assert torch.equal(torch.isnan(got.float()), nan)
+ assert torch.equal(_bits(got)[~nan], _bits(want)[~nan])
diff --git a/studio/backend/tests/test_video_comfy_quant.py b/studio/backend/tests/test_video_comfy_quant.py
index fde66dd9c3e..35c9b7a56a2 100644
--- a/studio/backend/tests/test_video_comfy_quant.py
+++ b/studio/backend/tests/test_video_comfy_quant.py
@@ -413,6 +413,8 @@ def test_video_backends_follow_studio_rules(monkeypatch):
assert vid._video_comfy_backends(fam, "b", None, _plan(True), _plan(True)) == {
"int8_backend": "torchao",
"fp8_backend": "torchao",
+ "nvfp4_backend": None,
+ "mxfp8_backend": None,
"fp8_storage": False,
"fp8_widen_chosen": False,
}
@@ -421,12 +423,16 @@ def test_video_backends_follow_studio_rules(monkeypatch):
assert vid._video_comfy_backends(fam, "b", None, _plan(True), _plan(False)) == {
"int8_backend": "native",
"fp8_backend": None,
+ "nvfp4_backend": None,
+ "mxfp8_backend": None,
"fp8_storage": True,
"fp8_widen_chosen": False,
}
assert vid._video_comfy_backends(fam, "b", None, _plan(True), keep = False) == {
"int8_backend": None,
"fp8_backend": None,
+ "nvfp4_backend": None,
+ "mxfp8_backend": None,
"fp8_storage": False,
"fp8_widen_chosen": False,
}
@@ -480,6 +486,8 @@ def test_a_smaller_offloaded_price_replans_and_keeps_the_offloaded_backends(monk
assert kept == {
"int8_backend": "native",
"fp8_backend": None,
+ "nvfp4_backend": None,
+ "mxfp8_backend": None,
"fp8_storage": True,
"fp8_widen_chosen": False,
}
@@ -1122,6 +1130,8 @@ def test_a_resident_video_dit_that_fits_only_stored_gets_fp8_storage(tmp_path, m
assert picked == {
"int8_backend": None,
"fp8_backend": None,
+ "nvfp4_backend": None,
+ "mxfp8_backend": None,
"fp8_storage": True,
"fp8_widen_chosen": False,
}