Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: keep nvfp4 / mxfp8 layers packed when no native runtime runs them (#17067)
Daniel Han285946a2295c
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.pyindex d762f3cba5a..1bfe3ced321 100644--- a/studio/backend/core/inference/diffusion.py+++ b/studio/backend/core/inference/diffusion.py@@ -11588,6 +11588,7 @@ class DiffusionBackend:keep_int8 = comfy_int8_backend(target, name, base, offload = offload) is not None,keep_fp8 = comfy_fp8_backend(target, name, base, offload = offload) is not None,fp8_stored = fp8_storage and comfy_fp8_storage(target, lora = lora),+ target = target,**_comfy_keep_fp32_pricing(fam, target),# the loader's runtime filter: layers it skips are priced dequantizedmin_features = DEFAULT_MIN_LINEAR_FEATURES,@@ -15925,6 +15926,14 @@ def _lora_load_block_reason(pipe: Any) -> Optional[str]:"UNSLOTH_DIFFUSION_COMFY_MXFP8=0, or name the LoRA in the API load request (Unsloth then dequantizes ""those layers).")+ from .diffusion_packed_block import PACKED_BLOCK_ENV, packed_block_layers++ if packed_block_layers(getattr(pipe, "transformer", None)):+ return (+ "LoRA is not available on this load: the file's nvfp4 / mxfp8 layers are kept packed and "+ f"widened per layer (no native runtime on this GPU), which cannot carry adapters. Set {PACKED_BLOCK_ENV}=0, "+ "or name the LoRA in the API load request (Unsloth then dequantizes those layers at load)."+ )if fp8_stored_layers(getattr(pipe, "transformer", None)):return ("LoRA is not available on this load: the file's fp8 layers are stored as fp8 and widened per layer "diff --git a/studio/backend/core/inference/diffusion_comfy_block.py b/studio/backend/core/inference/diffusion_comfy_block.pyindex 976fca07a06..5c0b2093a21 100644--- a/studio/backend/core/inference/diffusion_comfy_block.py+++ b/studio/backend/core/inference/diffusion_comfy_block.py@@ -126,14 +126,22 @@ def _dequant_temp_bytes_per_row(fmt: str, cols: int, pre_quant: bool) -> int:return peak + (4 * cols if pre_quant else 0)-def _dequant_block_rows(fmt, codes, scale, tensor_scale, pre_quant_scale):+def _dequant_block_rows(+ fmt,+ codes,+ scale,+ tensor_scale,+ pre_quant_scale,+ lut = None,+):import torchrows = int(codes.shape[0])if fmt == NVFP4:- lut = torch.tensor(- _E2M1 + tuple(-v for v in _E2M1), dtype = torch.float32, device = codes.device- )+ if lut is None:+ lut = torch.tensor(+ _E2M1 + tuple(-v for v in _E2M1), dtype = torch.float32, device = codes.device+ )c = codes.to(torch.int32)values = torch.stack((lut[c & 0x0F], lut[c >> 4]), dim = -1).reshape(rows, -1, 16)del c@@ -161,12 +169,15 @@ def dequant_block(tensor_scale: Any = None,pre_quant_scale: Any = None,dtype: Any = None,+ chunk_bytes: Optional[int] = None,+ lut: Any = None,) -> Any:"""Dense ``[N, K]`` weight; ``pre_quant_scale`` folds into the columns: ``(x * s) @ W.T == x @ (W * s).T``.Dequantized in row chunks straight into the output, so the float32 intermediates stay bounded by- ``_DEQUANT_CHUNK_BYTES`` instead of several times the layer; every op is per row, so the result is- bit-identical to one pass."""+ ``chunk_bytes`` (default ``_DEQUANT_CHUNK_BYTES``) instead of several times the layer; every op is per row, so+ the result is bit-identical to one pass. ``tensor_scale``: one value or one per row. ``lut``: the e2m1 table+ already on the device (a forward under CUDA graph capture must not upload one)."""import torchfrom .diffusion_single_file_converters import charge_host_ram@@ -177,7 +188,7 @@ def dequant_block(rows = int(codes.shape[0])cols = logical_cols(fmt, codes)per_row = _dequant_temp_bytes_per_row(fmt, cols, pre_quant_scale is not None)- chunk = max(1, min(rows, _DEQUANT_CHUNK_BYTES // max(per_row, 1)))+ chunk = max(1, min(rows, int(chunk_bytes or _DEQUANT_CHUNK_BYTES) // max(per_row, 1)))charge_host_ram(rows * cols * torch.empty((), dtype = dtype).element_size() + chunk * per_row)ts = Noneif tensor_scale is not None:@@ -190,7 +201,7 @@ def dequant_block(stop = min(rows, start + chunk)part_ts = ts if ts is None or ts.shape[0] == 1 else ts[start:stop]out[start:stop] = _dequant_block_rows(- fmt, codes[start:stop], scale[start:stop], part_ts, pqs+ fmt, codes[start:stop], scale[start:stop], part_ts, pqs, lut).to(dtype)return out@@ -549,8 +560,8 @@ def comfy_block_backends()else:logger.info(- "diffusion.comfy_quant: %d %s layer(s) are dequantized to the compute dtype on load: correct, but no memory "- "saving over the bf16 checkpoint (%s)",+ "diffusion.comfy_quant: %d %s layer(s) have no native runtime here (%s): widened to the compute "+ "dtype on load, or kept packed and widened per layer when the load stores its weights compactly",counts[fmt],fmt,reason,diff --git a/studio/backend/core/inference/diffusion_comfy_quant.py b/studio/backend/core/inference/diffusion_comfy_quant.pyindex 36a220298f6..9f557c2293d 100644--- a/studio/backend/core/inference/diffusion_comfy_quant.py+++ b/studio/backend/core/inference/diffusion_comfy_quant.py@@ -375,6 +375,28 @@ def fp8_stored_by_header(return fmt in FP8_FORMATS and len(shape or ()) == 2 and not keeps_fp32(name, keep_fp32)+def _packed_runs(fmt: str, target: Any) -> bool:+ from .diffusion_packed_block import packed_block_runs_on+ return packed_block_runs_on(fmt, target)+++def packed_block_by_header(+ fmt: str,+ shape: Any,+ name: str = "",+ keep_fp32: Any = (),+) -> bool:+ """Header rule for nvfp4 / mxfp8 layers that fp8 storage keeps packed when no native runtime runs them+ (``diffusion_packed_block``); a key-mapped load and a split with mixed input smoothing still widen."""+ from .diffusion_packed_block import packed_block_enabled+ return (+ fmt in BLOCK_SIZES+ and len(shape or ()) == 2+ and not keeps_fp32(name, keep_fp32)+ and packed_block_enabled()+ )++def comfy_resident_mib(path: Optional[str],scan: Optional[ComfyQuantScan] = None,@@ -393,6 +415,7 @@ def comfy_resident_mib(key_map: Any = None,keep_fp32: Any = (),keep_fp32_bytes: Optional[int] = None,+ target: Any = None,) -> Optional[int]:"""What the loader leaves resident for a ComfyUI-quantized file, priced from its header: a quantizedweight a runtime keeps costs its stored bytes, one that is dequantized costs ``numel * compute_bytes``@@ -403,8 +426,9 @@ def comfy_resident_mib(leaves it, and so is a layer the runtime filter skips (in / out features under ``min_features``, or fp8 featuresnot multiples of ``fp8_divisible``). ``key_map`` names layers as the loader's filter sees them (an original-layoutfile). ``keep_fp32``: the transformer's ``_keep_in_fp32_modules``; those layers are never fp8-stored and, widened,- cost ``keep_fp32_bytes`` per element (4 on an fp16 load, default ``compute_bytes``). None when the header cannot- be read. Torch-free."""+ cost ``keep_fp32_bytes`` per element (4 on an fp16 load, default ``compute_bytes``). Under ``fp8_stored`` an+ nvfp4 / mxfp8 layer no native runtime keeps is priced packed where it can widen on ``target``. None when the+ header cannot be read. Torch-free."""fp32_bytes = compute_bytes if keep_fp32_bytes is None else int(keep_fp32_bytes)try:scan = scan if scan is not None else scan_comfy_quant(path)@@ -462,6 +486,12 @@ def comfy_resident_mib(for n in _names(name + ".weight", shape)))+ or (+ fp8_stored+ and key_map is None+ and packed_block_by_header(layer.format, shape, name + ".weight", keep_fp32)+ and _packed_runs(layer.format, target)+ )or ({NVFP4: keep_nvfp4, MXFP8: keep_mxfp8}.get(layer.format, False)# input smoothing has no runtime Linear: the loader dequantizes such a layer@@ -1011,8 +1041,12 @@ def fp8_stored_layers(model: Any) -> int:def pipe_fp8_stored_layers(pipe: Any) -> int:+ """fp8-stored plus packed nvfp4 / mxfp8 Linears of the denoisers: buffers widened per forward."""+ from .diffusion_packed_block import packed_block_layersreturn sum(- fp8_stored_layers(getattr(pipe, attr, None)) for attr in ("transformer", "transformer_2")+ fp8_stored_layers(getattr(pipe, attr, None))+ + packed_block_layers(getattr(pipe, attr, None))+ for attr in ("transformer", "transformer_2"))@@ -1259,6 +1293,16 @@ def load_comfy_quant_transformer(mxfp8_backend = Noneif fp8_storage is None:fp8_storage = fp8_storage_forced() and comfy_fp8_storage(target)+ from .diffusion_packed_block import (+ PACKED_BLOCK_ATTR,+ packed_block_args,+ packed_block_enabled,+ packed_block_linear_class,+ packed_block_runs_on,+ )++ # fp8 storage also keeps nvfp4 / mxfp8 layers without a native runtime packed (widened per forward).+ packed = bool(fp8_storage) and key_map is None and packed_block_enabled()if keep_key is None:state = load_file(str(path))else:@@ -1326,10 +1370,20 @@ def load_comfy_quant_transformer(and fp8_stored_by_header(layer.format, shape, layer.name, keep_fp32))+ def _packed(layer: Any) -> bool:+ shape = header_shapes.get(layer.name + ".weight")+ return (+ packed+ and shape is not None+ and not backends.get(layer.format)+ and packed_block_runs_on(layer.format, target)+ and packed_block_by_header(layer.format, shape, layer.name, keep_fp32)+ )+_widened = {layer.name + ".weight": ((2 if layer.format == NVFP4 or layer.packed_int4 else 1)- if not (backends.get(layer.format) or _stored(layer))+ if not (backends.get(layer.format) or _stored(layer) or _packed(layer))or (layer.format in BLOCK_SIZES and key_map is not None)else 0)@@ -1338,7 +1392,7 @@ def load_comfy_quant_transformer(_stored_scale_bytes = sum(4 * int(header_shapes[layer.name + ".weight"][0])for layer in scan.layers.values()- if _stored(layer) and not backends.get(layer.format)+ if (_stored(layer) or _packed(layer)) and not backends.get(layer.format))# The header estimate refuses early; the budget it activates is charged by every allocating helper# below, so a layer the estimate cannot see (runtime filters, nibble swaps) still cannot OOM the host.@@ -1391,7 +1445,10 @@ def load_comfy_quant_transformer(codes = codes.view(getattr(torch, layer.format))# A key-mapped (original-layout) family moves rows whole, which the tiled block scales do not survive.if (- backends.get(layer.format) or fp16_keep or (fp8_storage and layer.format in FP8_FORMATS)+ backends.get(layer.format)+ or fp16_keep+ or (fp8_storage and layer.format in FP8_FORMATS)+ or (packed and extra is not None and packed_block_runs_on(layer.format, target))) and not (extra is not None and key_map is not None):sources.append((layer, codes, scale, extra))state[layer.name + ".weight"] = None # placeholder, tagged below@@ -1537,6 +1594,7 @@ def load_comfy_quant_transformer(native: dict = {}stored: dict = {} # fqn -> (codes, scale)block_native: dict = {} # fqn -> (fmt, codes, scale, tensor_scale, input_scale, bias)+ block_packed: dict = {} # fqn -> (fmt, codes, scale, row_scale, pre_quant, bias)built = {TQ_INT8: 0, TQ_FP8: 0, TQ_NVFP4: 0, TQ_MXFP8: 0}for name, value in list(converted.items()):if isinstance(value, _BlockWeight):@@ -1560,6 +1618,23 @@ def load_comfy_quant_transformer(del converted[name]built[schemes[fmt]] += 1else:+ args = (+ packed_block_args(value.parts)+ if packed+ and packed_block_runs_on(fmt, target)+ and isinstance(module, torch.nn.Linear)+ and name.endswith(".weight")+ and _dtype_for(name) == dtype+ and not any(m in fqn.split(".") for m in keep_fp32)+ else None+ )+ if args is not None:+ bias = converted.get(fqn + ".bias")+ if torch.is_tensor(bias):+ bias = bias.to(dtype)+ block_packed[fqn] = (fmt, *args, bias)+ del converted[name]+ continueconverted[name] = _dequant_parts(fmt, value.parts, _dtype_for(name))dequantized += 1continue@@ -1660,6 +1735,26 @@ def load_comfy_quant_transformer(setattr(parent, leaf, layer)for buffer_name, buffer in layer.named_buffers(recurse = False):converted[f"{fqn}.{buffer_name}"] = buffer+ if block_packed:+ packed_cls = packed_block_linear_class()+ for fqn, (fmt, codes, scale, row_scale, pre_quant, bias) in block_packed.items():+ parent_name, _, leaf = fqn.rpartition(".")+ parent = model.get_submodule(parent_name) if parent_name else model+ linear = getattr(parent, leaf)+ layer = packed_cls(+ fmt,+ linear.in_features,+ linear.out_features,+ codes = codes,+ scale = scale,+ row_scale = row_scale,+ pre_quant = pre_quant,+ bias = bias,+ compute_dtype = dtype,+ )+ setattr(parent, leaf, layer)+ for buffer_name, buffer in layer.named_buffers(recurse = False):+ converted[f"{fqn}.{buffer_name}"] = buffer_install_native(model)missing, unexpected = model.load_state_dict(converted, strict = False, assign = True)@@ -1685,7 +1780,7 @@ def load_comfy_quant_transformer(del converted# __init__ buffers (Wan's float64 rope) follow the compute dtype; checkpoint buffers keep theirs (H3 curve table)for module_name, module in model.named_modules():- if module_name in block_native:+ if module_name in block_native or module_name in block_packed:continue # fp32 scales / alpha the block kernels need; codes are not floatingfor buffer_name, buffer in list(module._buffers.items()):if (f"{module_name}.{buffer_name}" if module_name else buffer_name) in loaded:@@ -1745,7 +1840,7 @@ def load_comfy_quant_transformer("diffusion.comfy_quant: %s loaded from a ComfyUI checkpoint (%s): %d int8 layers kept as ""int8 (%s, %d ConvRot), %d fp8 layers kept as fp8 (%s), %d nvfp4 layers kept as nvfp4 (%s), ""%d mxfp8 layers kept as mxfp8 (%s), %d fp8 layers stored as fp8 and widened per layer to %s, "- "%d layers dequantized to %s",+ "%d nvfp4 / mxfp8 layers kept packed and widened per layer, %d layers dequantized to %s",transformer_cls.__name__,", ".join(f"{k} x{v}" for k, v in sorted(scan.counts().items())),built[TQ_INT8],@@ -1759,6 +1854,7 @@ def load_comfy_quant_transformer(f"{mxfp8_backend} runtime" if built[TQ_MXFP8] else "no mxfp8 runtime",len(stored),str(dtype).replace("torch.", ""),+ len(block_packed),dequantized,str(dtype).replace("torch.", ""),)@@ -1774,6 +1870,12 @@ def load_comfy_quant_transformer(if stored:info["fp8_stored"] = len(stored)setattr(model, FP8_STORED_ATTR, len(stored))+ if block_packed:+ for fmt in (NVFP4, MXFP8):+ count = sum(1 for v in block_packed.values() if v[0] == fmt)+ if count:+ info[f"{fmt}_packed"] = count+ setattr(model, PACKED_BLOCK_ATTR, len(block_packed))for fmt, backend in ((NVFP4, nvfp4_backend), (MXFP8, mxfp8_backend)):if scan.counts().get(fmt):info[f"{fmt}_backend"] = backend if built[schemes[fmt]] else Nonediff --git a/studio/backend/core/inference/diffusion_memory.py b/studio/backend/core/inference/diffusion_memory.pyindex 43f19af897a..a1348c25fc5 100644--- a/studio/backend/core/inference/diffusion_memory.py+++ b/studio/backend/core/inference/diffusion_memory.py@@ -2458,12 +2458,14 @@ def _pipe_denoisers_hold_torchao(pipe: Any) -> bool:def _holds_fp8_stored(module: Any) -> bool:- """Whether ``module`` holds fp8-STORED Linears; codes and scales are buffers, so a parameter scan reads it dense."""+ """Whether ``module`` holds fp8-STORED or packed nvfp4 / mxfp8 Linears; codes and scales are buffers, so a+ parameter scan reads it dense."""if module is None:return Falsetry:from .diffusion_comfy_quant import fp8_stored_layers- return fp8_stored_layers(module) > 0+ from .diffusion_packed_block import packed_block_layers+ return fp8_stored_layers(module) > 0 or packed_block_layers(module) > 0except Exception: # noqa: BLE001 - unreadable: today's parameter verdict standsreturn Falsediff --git a/studio/backend/core/inference/diffusion_packed_block.py b/studio/backend/core/inference/diffusion_packed_block.pynew file mode 100644index 00000000000..2ed25387a4b--- /dev/null+++ b/studio/backend/core/inference/diffusion_packed_block.py@@ -0,0 +1,326 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""nvfp4 / mxfp8 layers with no native runtime, kept packed and widened per forward, bit-identical to+``dequant_block`` at load. All state is integer buffers so a module-wide ``.to(dtype)`` never casts it."""++from __future__ import annotations++import contextlib+import os+import threading+from typing import Any, Optional++from .diffusion_comfy_block import BLOCK_SIZES, MXFP8, NVFP4++PACKED_BLOCK_ENV = "UNSLOTH_DIFFUSION_PACKED_BLOCK_FALLBACK"+PACKED_BLOCK_ATTR = "_unsloth_packed_block"+_OFF = ("0", "off", "false", "no")++# Torch widen intermediate cap per row chunk (host / accelerator).+_HOST_CHUNK_BYTES = 16 << 20+_DEVICE_CHUNK_BYTES = 256 << 20++_TRITON_LOCK = threading.Lock()+_TRITON: dict = {}+_OP_LOCK = threading.Lock()+_OP: dict = {}+++def packed_block_enabled() -> bool:+ return (os.environ.get(PACKED_BLOCK_ENV) or "").strip().lower() not in _OFF+++def packed_block_layers(model: Any) -> int:+ """How many nvfp4 / mxfp8 Linears of ``model`` are kept packed (they cannot carry LoRA adapters)."""+ try:+ return int(getattr(model, PACKED_BLOCK_ATTR, 0) or 0)+ except (TypeError, ValueError):+ return 0+++def fp8_casts(device: Any) -> bool:+ """Whether float8 tensors cast on ``device`` (CPU / CUDA / ROCm); elsewhere nvfp4 scales go by table and mxfp8+ layers are not kept packed."""+ return getattr(device, "type", str(device).split(":")[0]) in ("cpu", "cuda")+++def packed_block_runs_on(fmt: str, target: Any) -> bool:+ """Whether a ``fmt`` layer kept packed can widen on ``target``'s device (unknown: the host's rule)."""+ device = getattr(target, "torch_device", None) or getattr(target, "device", None) or "cpu"+ return fmt == NVFP4 or fp8_casts(device)+++def _torch_widen(+ fmt: str, codes: Any, scale: Any, row_scale: Any, pre_quant: Any, dtype: Any+) -> Any:+ from .diffusion_comfy_block import dequant_block++ import torch++ lut = None+ if fmt == MXFP8:+ codes = codes.view(torch.float8_e4m3fn)+ else:+ from .diffusion_nvfp4_ops import e2m1_lut+ if fp8_casts(codes.device):+ scale = scale.view(torch.float8_e4m3fn)+ else:+ # no fp8 casts here (MPS before torch 2.15): the exact float32 of every e4m3 byte, by table+ from .diffusion_native_quant import fp8_lookup_table+ table = fp8_lookup_table("float8_e4m3fn", str(codes.device))+ scale = table.index_select(0, scale.reshape(-1).int()).view(scale.shape)+ lut = e2m1_lut(codes.device)+ budget = _HOST_CHUNK_BYTES if codes.device.type == "cpu" else _DEVICE_CHUNK_BYTES+ return dequant_block(+ fmt,+ codes,+ scale,+ tensor_scale = None if row_scale is None else row_scale.view(torch.float32),+ pre_quant_scale = None if pre_quant is None else pre_quant.view(torch.float32),+ dtype = dtype,+ chunk_bytes = budget,+ lut = lut,+ )+++def _triton_kernel() -> Any:+ with _TRITON_LOCK:+ if "kernel" in _TRITON:+ return _TRITON["kernel"]+ try:+ import triton++ from .diffusion_packed_block_triton import nvfp4_widen_kernel+ _TRITON["kernel"] = (triton, nvfp4_widen_kernel)+ except Exception: # noqa: BLE001 -- no Triton: the torch widen is exact too+ _TRITON["kernel"] = None+ return _TRITON["kernel"]+++def _triton_failed(exc: BaseException) -> None:+ with _TRITON_LOCK:+ _TRITON["kernel"] = None+ _TRITON["error"] = f"{type(exc).__name__}: {str(exc)[:200]}"+++def _triton_widen(+ codes: Any, scale: Any, row_scale: Any, pre_quant: Any, dtype: Any+) -> Optional[Any]:+ import torch++ built = _triton_kernel()+ if built is None:+ return None+ triton, kernel = built+ rows, half_k = int(codes.shape[0]), int(codes.shape[1])+ out = torch.empty((rows, 2 * half_k), dtype = dtype, device = codes.device)+ if rows == 0 or half_k == 0:+ return out+ block_r, block_c = 32, 64+ grid = (triton.cdiv(rows, block_r), triton.cdiv(half_k, block_c))+ rs = row_scale.view(torch.float32) if row_scale is not None else out+ pq = pre_quant.view(torch.float32) if pre_quant is not None else out+ guard = torch.cuda.device(codes.device) if codes.is_cuda else contextlib.nullcontext()+ try:+ with guard:+ kernel[grid](+ codes,+ scale,+ rs,+ pq,+ out,+ rows,+ half_k,+ int(scale.shape[1]),+ HAS_RS = row_scale is not None,+ HAS_PQ = pre_quant is not None,+ BLOCK_R = block_r,+ BLOCK_C = block_c,+ )+ except Exception as exc: # noqa: BLE001 -- a kernel that cannot build here: the torch path is exact+ _triton_failed(exc)+ return None+ return out+++def widen_packed(+ fmt: str, codes: Any, scale: Any, row_scale: Any, pre_quant: Any, dtype: Any+) -> Any:+ """The dense ``[N, K]`` weight of packed ``codes`` / ``scale`` (uint8 buffers), bit-identical to the load-time+ widen; Triton on CUDA / ROCm for nvfp4, row-chunked torch otherwise."""+ if fmt not in BLOCK_SIZES:+ raise ValueError(f"not a block-scaled format: {fmt!r}")+ if (+ fmt == NVFP4+ and codes.is_cuda+ and codes.is_contiguous()+ and scale.is_contiguous()+ and (row_scale is None or row_scale.is_contiguous())+ and (pre_quant is None or pre_quant.is_contiguous())+ ):+ out = _triton_widen(codes, scale, row_scale, pre_quant, dtype)+ if out is not None:+ return out+ return _torch_widen(fmt, codes, scale, row_scale, pre_quant, dtype)+++def _eager_widen(+ codes: Any, scale: Any, row_scale: Any, pre_quant: Any, fmt: str, dtype: Any+) -> Any:+ return widen_packed(fmt, codes, scale, row_scale, pre_quant, dtype)+++def _widen_op() -> Any:+ """``torch.ops.unsloth_packed_block.widen``: opaque to the compiler (shape-only fake), one per process."""+ with _OP_LOCK:+ if "op" in _OP:+ return _OP["op"]+ import torch++ # explicit schema: this module's annotations are strings (no module-level torch to resolve them)+ @torch.library.custom_op(+ "unsloth_packed_block::widen",+ mutates_args = (),+ schema = "(Tensor codes, Tensor scale, Tensor? row_scale, Tensor? pre_quant, str fmt, "+ "ScalarType dtype) -> Tensor",+ )+ def widen(codes, scale, row_scale, pre_quant, fmt, dtype):+ return widen_packed(fmt, codes, scale, row_scale, pre_quant, dtype)++ @widen.register_fake+ def _(codes, scale, row_scale, pre_quant, fmt, dtype):+ cols = int(codes.shape[1]) * (2 if fmt == NVFP4 else 1)+ return codes.new_empty((codes.shape[0], cols), dtype = dtype)++ _OP["op"] = widen+ return widen+++def packed_block_linear_class():+ import torch+ from torch import nn+ from torch.nn import functional as F++ global _CLASS+ if _CLASS is not None:+ return _CLASS+ _widen_op()++ class PackedBlockLinear(nn.Module):+ """A Linear whose nvfp4 / mxfp8 weight stays packed; each forward widens it to the compute dtype."""++ def __init__(+ self,+ fmt: str,+ in_features: int,+ out_features: int,+ *,+ codes,+ scale,+ row_scale = None,+ pre_quant = None,+ bias = None,+ compute_dtype = None,+ ):+ super().__init__()+ if fmt not in BLOCK_SIZES:+ raise ValueError(f"not a block-scaled format: {fmt!r}")+ self.fmt = fmt+ self.in_features = int(in_features)+ self.out_features = int(out_features)+ self.compute_dtype = compute_dtype or torch.bfloat16+ self.register_buffer("weight_q", codes.contiguous().view(torch.uint8))+ self.register_buffer("weight_sf", scale.contiguous().view(torch.uint8))+ # float32 factors as int32 views: a module-wide .to(dtype) would round them+ self.register_buffer(+ "row_scale",+ None+ if row_scale is None+ else row_scale.to(torch.float32).reshape(-1).contiguous().view(torch.int32),+ )+ self.register_buffer(+ "pre_quant",+ None+ if pre_quant is None+ else pre_quant.to(torch.float32).reshape(-1).contiguous().view(torch.int32),+ )+ self.register_buffer("bias", bias)++ def dequantized_weight(self, dtype: Any = None) -> Any:+ # eager calls the widen directly; a compiled graph sees the op (no graph break, no recompile)+ widen = (+ torch.ops.unsloth_packed_block.widen+ if torch.compiler.is_compiling()+ else _eager_widen+ )+ w = widen(+ self.weight_q,+ self.weight_sf,+ self.row_scale,+ self.pre_quant,+ self.fmt,+ self.compute_dtype,+ )+ return w if dtype is None or dtype == w.dtype else w.to(dtype)++ @property+ def weight(self) -> Any:+ return self.dequantized_weight()++ def forward(self, x: Any) -> Any:+ return F.linear(x, self.dequantized_weight(), self.bias)++ def extra_repr(self) -> str: # pragma: no cover - debug aid+ return (+ f"in_features={self.in_features}, out_features={self.out_features}, "+ f"{self.fmt}=packed, bias={self.bias is not None}"+ )++ _CLASS = PackedBlockLinear+ return _CLASS+++_CLASS = None+++def is_packed_block_linear(module: Any) -> bool:+ return type(module).__name__ == "PackedBlockLinear" and hasattr(module, "weight_sf")+++def packed_block_args(parts: list) -> Optional[tuple]:+ """``(codes, scale, row_scale, pre_quant)`` holding the row segments ``parts`` exactly, else None (segments with+ different input smoothing need one weight each). ``row_scale`` / ``pre_quant`` are None when absent everywhere;+ a segment without a tensor scale gets 1.0, which leaves its product bit-identical."""+ import torch++ from .diffusion_single_file_converters import charge_host_ram++ smoothing = [extra.get("pre_quant_scale") for _c, _s, extra in parts]+ first = smoothing[0]+ for other in smoothing[1:]:+ if (first is None) != (other is None):+ return None+ if first is not None and not torch.equal(first.to(torch.float32), other.to(torch.float32)):+ return None+ scales = [extra.get("tensor_scale") for _c, _s, extra in parts]+ row_scale = None+ if any(s is not None for s in scales):+ row_scale = torch.cat(+ [+ torch.full((int(c.shape[0]),), 1.0 if s is None else float(s), dtype = torch.float32)+ for (c, _s, _e), s in zip(parts, scales)+ ]+ )+ if len(parts) > 1:+ charge_host_ram(+ sum(+ int(c.numel()) * int(c.element_size()) + int(s.numel()) * int(s.element_size())+ for c, s, _e in parts+ )+ )+ codes = torch.cat([c for c, _s, _e in parts])+ scale = torch.cat([s for _c, s, _e in parts])+ else:+ codes, scale = parts[0][0], parts[0][1]+ return codes, scale, row_scale, None if first is None else first.to(torch.float32).reshape(-1)diff --git a/studio/backend/core/inference/diffusion_packed_block_triton.py b/studio/backend/core/inference/diffusion_packed_block_triton.pynew file mode 100644index 00000000000..1e6034f22c5--- /dev/null+++ b/studio/backend/core/inference/diffusion_packed_block_triton.py@@ -0,0 +1,105 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Triton widen for packed nvfp4; helpers are module level because Triton resolves them via globals."""++import triton+import triton.language as tl+++def _e2m1(c):+ # c: int32 nibble -> +-0, 0.5, 1, 1.5, 2, 3, 4, 6, built bit by bit (a float negation would lose -0.0)+ k = c & 7+ e = k >> 1+ m = k & 1+ bits = tl.where(e == 0, m * 0x3F000000, ((e + 126) << 23) | (m << 22))+ return (bits | ((c & 8) << 28)).to(tl.float32, bitcast = True)+++def _e4m3(s):+ # s: int32 byte of a float8_e4m3fn -> its exact float32 (NaN for 0x7f / 0xff), sign as a bit+ e = (s >> 3) & 15+ m = s & 7+ normal = ((e + 120) << 23) | (m << 20)+ sub = (m.to(tl.float32) * 0.001953125).to(tl.int32, bitcast = True)+ bits = tl.where(e == 0, sub, normal)+ bits = tl.where((e == 15) & (m == 7), 0x7FF00000, bits)+ return (bits | ((s & 128) << 24)).to(tl.float32, bitcast = True)+++def _fp16_bits(value):+ """float32 -> float16 bits, round to nearest even by integer ops (overflow to inf, subnormals kept), as torch+ casts on every device; ROCm's own conversion differs."""+ u = value.to(tl.uint32, bitcast = True)+ sign = (u >> 16) & 0x8000+ a = u & 0x7FFFFFFF+ r = a - (112 << 23)+ normal = (r + 0x0FFF + ((r >> 13) & 1)) >> 13+ # below 2**-14: the subnormal count, |x| * 2**24 rounded to the nearest even integer (exact scaling, one add)+ scaled = (a.to(tl.float32, bitcast = True) * 16777216.0 + 8388608.0) - 8388608.0+ sub = scaled.to(tl.uint32)+ bits = tl.where(a >= 0x38800000, normal, sub)+ bits = tl.where(a >= 0x477FF000, 0x7C00, bits)+ # NaN: quieted, top payload bits kept, as the hardware conversion torch uses+ bits = tl.where(a > 0x7F800000, 0x7E00 | ((a >> 13) & 0x3FF), bits)+ return (bits | sign).to(tl.uint16)+++def _store(dst, value, mask):+ """``value`` in the output dtype; bf16 rounds to nearest even by integer ops, as torch does on every device+ (the interpreter, pre-sm_80 and ROCm conversions are not guaranteed to); fp16 the same way."""+ ty = dst.dtype.element_ty+ if ty == tl.bfloat16:+ u = value.to(tl.uint32, bitcast = True)+ rounded = (u + 0x7FFF + ((u >> 16) & 1)) >> 16+ rounded = tl.where(value != value, 0x7FC0, rounded)+ tl.store(dst, rounded.to(tl.uint16).to(tl.bfloat16, bitcast = True), mask = mask)+ elif ty == tl.float16:+ tl.store(dst, _fp16_bits(value).to(tl.float16, bitcast = True), mask = mask)+ else:+ tl.store(dst, value.to(ty), mask = mask)+++def nvfp4_widen_kernel(+ q_ptr,+ sf_ptr,+ rs_ptr,+ pq_ptr,+ out_ptr,+ rows,+ half_k,+ blocks,+ HAS_RS: tl.constexpr,+ HAS_PQ: tl.constexpr,+ BLOCK_R: tl.constexpr,+ BLOCK_C: tl.constexpr,+):+ r = tl.program_id(0) * BLOCK_R + tl.arange(0, BLOCK_R)+ j = tl.program_id(1) * BLOCK_C + tl.arange(0, BLOCK_C)+ rmask = r < rows+ jmask = j < half_k+ mask = rmask[:, None] & jmask[None, :]+ r64 = r.to(tl.int64)+ j64 = j.to(tl.int64)+ b = tl.load(q_ptr + r64[:, None] * half_k + j64[None, :], mask = mask, other = 0)+ b = b.to(tl.int32)+ s = tl.load(sf_ptr + r64[:, None] * blocks + (j64 // 8)[None, :], mask = mask, other = 0)+ # same multiply order as dequant_block (bit-identical)+ step = _e4m3(s.to(tl.int32))+ if HAS_RS:+ step = step * tl.load(rs_ptr + r64, mask = rmask, other = 1.0)[:, None]+ lo = _e2m1(b & 15) * step+ hi = _e2m1(b >> 4) * step+ if HAS_PQ:+ lo = lo * tl.load(pq_ptr + 2 * j64, mask = jmask, other = 1.0)[None, :]+ hi = hi * tl.load(pq_ptr + 2 * j64 + 1, mask = jmask, other = 1.0)[None, :]+ # even column = low nibble+ w = tl.reshape(tl.join(lo, hi), (BLOCK_R, 2 * BLOCK_C))+ c = tl.program_id(1).to(tl.int64) * (2 * BLOCK_C) + tl.arange(0, 2 * BLOCK_C)+ dst = out_ptr + r64[:, None] * (2 * half_k) + c[None, :]+ _store(dst, w, rmask[:, None] & (c < 2 * half_k)[None, :])diff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.pyindex b0405f93acd..5acf9bfdeb4 100644--- a/studio/backend/core/inference/video.py+++ b/studio/backend/core/inference/video.py@@ -532,11 +532,14 @@ def _video_comfy_backends(resident only; else fp8 storage under offload or a tight MPS plan). ``priced_offload``: the plan was sized with theoffloaded backends (``_video_offload_reprice``), so a load that now fits resident keeps them. ``widened_mib``: theplanned DiT price; a resident load stores fp8 when widening it would leave under the host reserve, as an image does.- ``keep=False`` (the user asked for the bf16 DiT) dequantizes everything."""+ ``keep=False`` (the user asked for the bf16 DiT) dequantizes everything. nvfp4 / mxfp8 layers take the image+ path's native runtimes where their flag, device and family allow; without one, fp8 storage keeps them packed."""if not keep:return {"int8_backend": None,"fp8_backend": None,+ "nvfp4_backend": None,+ "mxfp8_backend": None,"fp8_storage": False,"fp8_widen_chosen": False,}@@ -547,12 +550,24 @@ def _video_comfy_backends(return {"int8_backend": comfy_int8_backend(target, name, base, offload = offload),"fp8_backend": comfy_fp8_backend(target, name, base, offload = offload),+ **_video_block_backends(target, name),"fp8_storage": storage,"fp8_widen_chosen": not storageand _video_fp8_storage(fam, target, store_pays, widened_mib = widened_mib, widen_chosen = True),}+def _video_block_backends(target: Any, family: Optional[str]) -> dict:+ """``comfy_block_backend`` per block format (None = no native runtime here)."""+ from .diffusion_comfy_block import MXFP8, NVFP4, comfy_block_backend++ dtype = getattr(target, "dtype", None)+ return {+ f"{fmt}_backend": comfy_block_backend(fmt, target, family, dtype = dtype)[0]+ for fmt in (NVFP4, MXFP8)+ }++def _video_comfy_reprice(plan: Any, bf16_plan: Any = None) -> bool:"""Whether a ComfyUI DiT is re-priced with the offloaded backends: an offloaded plan (fp8 layers dequantize onNVIDIA), or an MPS plan past its resident margin (MPS never offloads; its offloaded backends are its resident ones@@ -614,10 +629,13 @@ def _video_comfy_resident_mib(keep_int8 = keep_fp8 = fp8_stored = Falseexclude: tuple = ()filters: dict = {}+ keep_block: dict = {}if keep:try:from .diffusion_transformer_quant import (DEFAULT_MIN_LINEAR_FEATURES,+ TQ_MXFP8,+ TQ_NVFP4,divisible_for_scheme,exclude_tokens_for_scheme,)@@ -625,6 +643,10 @@ def _video_comfy_resident_mib(filters = {"min_features": DEFAULT_MIN_LINEAR_FEATURES,"fp8_divisible": divisible_for_scheme(TQ_FP8),+ "block_divisible": {+ "nvfp4": divisible_for_scheme(TQ_NVFP4),+ "mxfp8": divisible_for_scheme(TQ_MXFP8),+ },}except Exception: # noqa: BLE001 -- the estimate then keeps every int8 layerexclude = ()@@ -632,12 +654,19 @@ def _video_comfy_resident_mib(keep_int8 = bool(comfy_int8_backend(target, name, base, offload = offload))keep_fp8 = comfy_fp8_backend(target, name, base, offload = offload) is not Nonefp8_stored = _video_fp8_storage(fam, target, offload)+ blocks = _video_block_backends(target, name)+ keep_block = {+ "keep_nvfp4": blocks["nvfp4_backend"] is not None,+ "keep_mxfp8": blocks["mxfp8_backend"] is not None,+ }return comfy_resident_mib(str(path),scan,keep_int8 = keep_int8,keep_fp8 = keep_fp8,fp8_stored = fp8_stored,+ **keep_block,+ target = target,keep_key = keep_key,exclude_tokens = exclude,key_map = _video_comfy_key_map(fam),diff --git a/studio/backend/core/inference/video_ltx2.py b/studio/backend/core/inference/video_ltx2.pyindex e2711f53117..89a5ee5ec7e 100644--- a/studio/backend/core/inference/video_ltx2.py+++ b/studio/backend/core/inference/video_ltx2.py@@ -1259,6 +1259,8 @@ def load_ltx23_comfy_transformer(local_files_only: bool = False,int8_backend: Optional[str] = None,fp8_backend: Optional[str] = None,+ nvfp4_backend: Optional[str] = None,+ mxfp8_backend: Optional[str] = None,fp8_storage: Optional[bool] = None,fp8_widen_chosen: bool = False,family: Optional[str] = None,@@ -1285,6 +1287,8 @@ def load_ltx23_comfy_transformer(},int8_backend = int8_backend,fp8_backend = fp8_backend,+ nvfp4_backend = nvfp4_backend,+ mxfp8_backend = mxfp8_backend,fp8_storage = fp8_storage,fp8_widen_chosen = fp8_widen_chosen,family = family,diff --git a/studio/backend/core/inference/video_ltx25.py b/studio/backend/core/inference/video_ltx25.pyindex 08a9aa1feae..beee958ccc9 100644--- a/studio/backend/core/inference/video_ltx25.py+++ b/studio/backend/core/inference/video_ltx25.py@@ -399,6 +399,8 @@ def load_ltx25_comfy_transformer(local_files_only: bool = False,int8_backend: Optional[str] = None,fp8_backend: Optional[str] = None,+ nvfp4_backend: Optional[str] = None,+ mxfp8_backend: Optional[str] = None,fp8_storage: Optional[bool] = None,fp8_widen_chosen: bool = False,family: Optional[str] = None,@@ -426,6 +428,8 @@ def load_ltx25_comfy_transformer(},int8_backend = int8_backend,fp8_backend = fp8_backend,+ nvfp4_backend = nvfp4_backend,+ mxfp8_backend = mxfp8_backend,fp8_storage = fp8_storage,fp8_widen_chosen = fp8_widen_chosen,family = family,diff --git a/studio/backend/tests/test_diffusion_comfy_block.py b/studio/backend/tests/test_diffusion_comfy_block.pyindex 49c08f2ab71..fd464d4fb51 100644--- a/studio/backend/tests/test_diffusion_comfy_block.py+++ b/studio/backend/tests/test_diffusion_comfy_block.py@@ -582,7 +582,7 @@ def test_backends_are_logged_per_format_present(caplog):with caplog.at_level(logging.INFO, logger = "comfy_block_test"):out = cb.comfy_block_backends(scan, _target("cpu"), "z-image", logger = logger)assert out == {"nvfp4_backend": None, "mxfp8_backend": None}- assert "3 nvfp4 layer(s) are dequantized" in caplog.text and "mxfp8" not in caplog.text+ assert "3 nvfp4 layer(s) have no native runtime" in caplog.text and "mxfp8" not in caplog.textdef test_a_failed_mxfp8_probe_is_not_cached(monkeypatch):@@ -1069,3 +1069,30 @@ def test_untiling_padded_blocks_checks_both_live_copies(monkeypatch):monkeypatch.setattr(conv, "charge_host_ram", lambda n: seen.append(int(n)))assert torch.equal(cb.untile_scales(tiled, rows, blocks), plain)assert seen == [_ceil(rows, 128) * _ceil(blocks, 4) + rows * blocks]++[email protected]("fmt", ["nvfp4", "mxfp8"])+def test_a_compact_storage_load_holds_block_layers_at_their_file_size(block_file, fmt):+ """The fp8-storage pick (widening would offload) keeps nvfp4 / mxfp8 layers no runtime takes near their file+ size, not as a bf16 copy, and their weights stay the dense load's."""+ path, _tensors, _expected = block_file(fmt)+ dense = _load(path, dtype = torch.bfloat16)+ compact = _load(path, dtype = torch.bfloat16, fp8_storage = True)+ names = [f"blocks.{b}.{n}" for b in range(2) for n in ("to_q", "to_k", "to_v", "to_out")]++ def held(model, name):+ mod = model.get_submodule(name)+ tensors = [+ t for n, t in list(mod.named_parameters()) + list(mod.named_buffers()) if n != "bias"+ ]+ return sum(t.numel() * t.element_size() for t in tensors)++ bf16 = sum(held(dense, n) for n in names)+ kept = sum(held(compact, n) for n in names)+ assert bf16 == 8 * DIM * DIM * 2+ assert kept < (0.3 if fmt == "nvfp4" else 0.6) * bf16, (kept, bf16)+ for name in names:+ assert torch.equal(+ compact.get_submodule(name).weight.view(torch.int16),+ dense.get_submodule(name).weight.view(torch.int16),+ ), namediff --git a/studio/backend/tests/test_diffusion_packed_block.py b/studio/backend/tests/test_diffusion_packed_block.pynew file mode 100644index 00000000000..3abb3fea75b--- /dev/null+++ b/studio/backend/tests/test_diffusion_packed_block.py@@ -0,0 +1,311 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""nvfp4 / mxfp8 layers kept packed when no native runtime takes them: same weights as the load-time widen."""++from __future__ import annotations++import os+import subprocess+import sys+import textwrap++import pytest++torch = pytest.importorskip("torch")++import core.inference.diffusion_comfy_block as cb # noqa: E402+import core.inference.diffusion_comfy_quant as cq # noqa: E402+import core.inference.diffusion_packed_block as pb # noqa: E402+from .test_diffusion_comfy_block import DIM, _Tiny, _load, block_file # noqa: E402,F401++LINEARS = [f"blocks.{b}.{n}" for b in range(2) for n in ("to_q", "to_k", "to_v", "to_out")]+++_HELPERS = textwrap.dedent(+ """+ import torch+++ def _bits(t):+ return t.view({2: torch.int16, 4: torch.int32}[t.element_size()])+++ def _exhaustive_nvfp4():+ # every code byte against every e4m3 scale byte (NaN 0x7f / 0xff included), 16 blocks per row+ rows = 256+ codes = (torch.arange(128, dtype = torch.int64).view(1, -1) * 2 + torch.arange(rows).view(-1, 1)) % 256+ scale = (torch.arange(rows).view(-1, 1) * 16 + torch.arange(16)) % 256+ return codes.to(torch.uint8).contiguous(), scale.to(torch.uint8).contiguous()+ """+)+exec(_HELPERS)+++def _same(a: torch.Tensor, b: torch.Tensor) -> bool:+ """Bitwise, NaN positions included (payloads compared too)."""+ return a.shape == b.shape and a.dtype == b.dtype and torch.equal(_bits(a), _bits(b))++[email protected]("fmt", ["nvfp4", "mxfp8"])[email protected]("dtype", [torch.bfloat16, torch.float32])+def test_a_packed_load_widens_to_the_dense_loads_exact_weights(block_file, fmt, dtype):+ path, _tensors, _expected = block_file(fmt)+ dense = _load(path, dtype = dtype)+ packed = _load(path, dtype = dtype, fp8_storage = True)+ info = packed._unsloth_comfy_quant+ assert info[f"{fmt}_packed"] == 8 and info["dequantized"] == 0+ assert pb.packed_block_layers(packed) == 8+ state = dense.state_dict()+ x = torch.randn(3, 5, DIM, generator = torch.Generator().manual_seed(1)).to(dtype)+ for name in LINEARS:+ layer = packed.get_submodule(name)+ assert pb.is_packed_block_linear(layer), name+ assert not any(b.is_floating_point() for n, b in layer.named_buffers() if n != "bias")+ assert _same(layer.weight, state[name + ".weight"]), name+ ref = dense.get_submodule(name)+ assert _same(layer(x), ref(x)), name+ # packed bytes: nvfp4 codes are half a byte per weight, mxfp8 one+ q = packed.blocks[0].to_q+ assert q.weight_q.numel() == DIM * DIM // (2 if fmt == "nvfp4" else 1)++[email protected]("fmt", ["nvfp4", "mxfp8"])+def test_the_kill_switch_and_a_widening_pick_keep_todays_dense_load(block_file, fmt, monkeypatch):+ path, _tensors, expected = block_file(fmt)+ for kwargs, env in (({}, None), ({"fp8_storage": True}, "0")):+ if env is None:+ monkeypatch.delenv(pb.PACKED_BLOCK_ENV, raising = False)+ else:+ monkeypatch.setenv(pb.PACKED_BLOCK_ENV, env)+ model = _load(path, **kwargs)+ info = model._unsloth_comfy_quant+ assert info["dequantized"] == 4 and f"{fmt}_packed" not in info+ assert pb.packed_block_layers(model) == 0+ state = model.state_dict()+ for key, want in expected.items():+ assert torch.equal(state[key], want), key+++def test_module_casts_and_moves_never_touch_the_packed_buffers(block_file):+ path, _tensors, _expected = block_file("nvfp4")+ model = _load(path, dtype = torch.bfloat16, fp8_storage = True)+ layer = model.blocks[1].to_out+ before = {n: b.clone() for n, b in layer.named_buffers() if n != "bias"}+ weight = layer.weight.clone()+ model.to(torch.float16)+ for name, buf in layer.named_buffers():+ if name != "bias":+ assert buf.dtype == before[name].dtype and torch.equal(buf, before[name]), name+ assert _same(layer.weight, weight) # the compute dtype is the load's, not the cast's+++def _parts(+ fmt,+ rows,+ seed,+ tensor_scale = 0.5,+ smoothing = None,+):+ gen = torch.Generator().manual_seed(seed)+ cols = 64+ block = cb.BLOCK_SIZES[fmt]+ if fmt == "nvfp4":+ codes = torch.randint(0, 256, (rows, cols // 2), dtype = torch.uint8, generator = gen)+ scale = torch.randint(0, 0x7F, (rows, cols // block), dtype = torch.uint8, generator = gen)+ scale = scale.view(torch.float8_e4m3fn)+ else:+ codes = torch.randint(0, 0x7F, (rows, cols), dtype = torch.uint8, generator = gen)+ codes = codes | (torch.randint(0, 2, codes.shape, dtype = torch.uint8, generator = gen) << 7)+ codes = codes.view(torch.float8_e4m3fn)+ scale = torch.randint(100, 150, (rows, cols // block), dtype = torch.uint8, generator = gen)+ extra = {"tensor_scale": tensor_scale, "pre_quant_scale": smoothing}+ return codes, scale, extra++[email protected]("fmt", ["nvfp4", "mxfp8"])[email protected]("rows", [1, 17, 1023, 1024, 1025])+def test_row_chunks_splits_and_smoothing_widen_bit_identically(fmt, rows, monkeypatch):+ smoothing = torch.rand(64, generator = torch.Generator().manual_seed(7)) + 0.5+ parts = [+ _parts(fmt, rows, 1, tensor_scale = 0.25 if fmt == "nvfp4" else None, smoothing = smoothing),+ _parts(fmt, 3, 2, tensor_scale = None, smoothing = smoothing),+ _parts(fmt, 5, 3, tensor_scale = 3.0 if fmt == "nvfp4" else None, smoothing = smoothing),+ ]+ want = cq._dequant_parts(fmt, parts, torch.bfloat16)+ codes, scale, row_scale, pre_quant = pb.packed_block_args(parts)+ layer = pb.packed_block_linear_class()(+ fmt,+ 64,+ int(codes.shape[0]),+ codes = codes,+ scale = scale,+ row_scale = row_scale,+ pre_quant = pre_quant,+ compute_dtype = torch.bfloat16,+ )+ monkeypatch.setattr(pb, "_HOST_CHUNK_BYTES", 4096)+ assert _same(layer.weight, want)+ monkeypatch.setattr(pb, "_HOST_CHUNK_BYTES", 1 << 30)+ assert _same(layer.weight, want)+++def test_segments_with_different_smoothing_stay_dense():+ a = _parts("nvfp4", 4, 1, smoothing = torch.ones(64))+ b = _parts("nvfp4", 4, 2, smoothing = torch.full((64,), 2.0))+ c = _parts("nvfp4", 4, 3, smoothing = None)+ assert pb.packed_block_args([a, b]) is None+ assert pb.packed_block_args([a, c]) is None+ assert pb.packed_block_args([a, a]) is not None++[email protected]("dtype", [torch.bfloat16, torch.float16, torch.float32])+def test_exhaustive_codes_and_scales_through_the_torch_widen(dtype):+ codes, scale = _exhaustive_nvfp4()+ row_scale = torch.full((codes.shape[0],), 2.0**-9, dtype = torch.float32)+ pre_quant = torch.linspace(0.25, 4.0, 2 * codes.shape[1], dtype = torch.float32)+ want = cb.dequant_block(+ "nvfp4",+ codes,+ scale.view(torch.float8_e4m3fn),+ tensor_scale = row_scale,+ pre_quant_scale = pre_quant,+ dtype = dtype,+ )+ got = pb.widen_packed(+ "nvfp4", codes, scale, row_scale.view(torch.int32), pre_quant.view(torch.int32), dtype+ )+ assert _same(got, want)+++_INTERPRETED = textwrap.dedent(+ """+ import sys, torch+ sys.path.insert(0, {root!r})+ import core.inference.diffusion_comfy_block as cb+ import core.inference.diffusion_packed_block as pb+ exec({helpers!r})+ codes, scale = _exhaustive_nvfp4()+ rows = codes.shape[0]+ bad = []+ for dtype in (torch.bfloat16, torch.float16, torch.float32):+ for rs, pq in ((None, None), (torch.full((rows,), 0.75), None),+ (torch.full((rows,), 2.0 ** -9), torch.linspace(0.25, 4.0, 2 * codes.shape[1]))):+ want = cb.dequant_block("nvfp4", codes, scale.view(torch.float8_e4m3fn),+ tensor_scale = rs, pre_quant_scale = pq, dtype = dtype)+ got = pb._triton_widen(codes, scale, None if rs is None else rs.view(torch.int32),+ None if pq is None else pq.view(torch.int32), dtype)+ assert got is not None, pb._TRITON.get("error")+ nan = torch.isnan(want.float())+ if not torch.equal(nan, torch.isnan(got.float())):+ bad.append((str(dtype), "nan positions"))+ elif not torch.equal(_bits(got)[~nan], _bits(want)[~nan]):+ bad.append((str(dtype), "values"))+ print("BAD", bad)+ """+)+++def test_the_triton_widen_is_bit_identical_under_the_interpreter():+ pytest.importorskip("triton")+ root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))+ env = dict(os.environ, TRITON_INTERPRET = "1", CUDA_VISIBLE_DEVICES = "")+ proc = subprocess.run(+ [sys.executable, "-c", _INTERPRETED.format(root = root, helpers = _HELPERS)],+ cwd = root,+ env = env,+ capture_output = True,+ text = True,+ timeout = 600,+ )+ assert "BAD []" in proc.stdout, proc.stdout + proc.stderr[-2000:]+++def test_a_compiled_forward_has_one_graph_and_the_same_output(block_file):+ path, _tensors, _expected = block_file("nvfp4")+ model = _load(path, dtype = torch.float32, fp8_storage = True)+ layer = model.blocks[0].to_q+ x = torch.randn(4, DIM, generator = torch.Generator().manual_seed(3))+ want = layer(x)+ torch._dynamo.reset()+ from torch._dynamo.testing import CompileCounterWithBackend++ counter = CompileCounterWithBackend("aot_eager")+ compiled = torch.compile(layer, backend = counter, fullgraph = True)+ assert _same(compiled(x), want)+ y = torch.randn(4, DIM, generator = torch.Generator().manual_seed(4))+ assert _same(compiled(y), layer(y))+ assert counter.frame_count == 1+++def test_fp8_storage_prices_block_layers_packed(block_file, monkeypatch):+ path, _tensors, _expected = block_file("nvfp4")+ scan = cq.scan_comfy_quant(path)+ kw = dict(keep_int8 = False, keep_fp8 = False)+ dense = cq.comfy_resident_mib(path, scan, **kw)+ packed = cq.comfy_resident_mib(path, scan, fp8_stored = True, **kw)+ kept = cq.comfy_resident_mib(path, scan, keep_nvfp4 = True, **kw)+ assert packed == kept < dense+ monkeypatch.setenv(pb.PACKED_BLOCK_ENV, "0")+ assert cq.comfy_resident_mib(path, scan, fp8_stored = True, **kw) == dense+++def test_packed_layers_render_under_no_grad_and_refuse_a_late_lora(block_file):+ import types++ from core.inference.diffusion import _lora_load_block_reason++ path, _tensors, _expected = block_file("nvfp4")+ model = _load(path, fp8_storage = True)+ pipe = types.SimpleNamespace(transformer = model)+ assert cq.pipe_fp8_stored_layers(pipe) == 8+ assert pb.PACKED_BLOCK_ENV in (_lora_load_block_reason(pipe) or "")+++def test_devices_without_fp8_casts_widen_nvfp4_by_table_and_keep_mxfp8_dense(monkeypatch):+ import types++ mps = types.SimpleNamespace(device = "mps", torch_device = "mps")+ assert pb.packed_block_runs_on("nvfp4", mps) and not pb.packed_block_runs_on("mxfp8", mps)+ assert pb.packed_block_runs_on(+ "mxfp8", types.SimpleNamespace(device = "cuda", torch_device = "cuda:1")+ )+ # the table decode of the e4m3 scales gives the same weight as the cast+ codes, scale = _exhaustive_nvfp4()+ want = pb.widen_packed("nvfp4", codes, scale, None, None, torch.bfloat16)+ monkeypatch.setattr(pb, "fp8_casts", lambda device: False)+ assert _same(pb.widen_packed("nvfp4", codes, scale, None, None, torch.bfloat16), want)+++ not torch.cuda.is_available(), reason = "needs a CUDA / ROCm GPU to run the Triton widen"+)[email protected]("dtype", [torch.bfloat16, torch.float16, torch.float32])+def test_the_triton_widen_on_the_gpu_matches_the_load_time_widen(dtype):+ codes, scale = _exhaustive_nvfp4()+ rows = codes.shape[0]+ for rs, pq in (+ (None, None),+ (torch.full((rows,), 0.75), None),+ (torch.full((rows,), 2.0**-9), torch.linspace(0.25, 4.0, 2 * codes.shape[1])),+ ):+ want = cb.dequant_block(+ "nvfp4",+ codes,+ scale.view(torch.float8_e4m3fn),+ tensor_scale = rs,+ pre_quant_scale = pq,+ dtype = dtype,+ )+ got = pb._triton_widen(+ codes.cuda(),+ scale.cuda(),+ None if rs is None else rs.cuda().view(torch.int32),+ None if pq is None else pq.cuda().view(torch.int32),+ dtype,+ )+ assert got is not None, pb._TRITON.get("error")+ nan = torch.isnan(want.float())+ got = got.cpu()+ assert torch.equal(torch.isnan(got.float()), nan)+ assert torch.equal(_bits(got)[~nan], _bits(want)[~nan])diff --git a/studio/backend/tests/test_video_comfy_quant.py b/studio/backend/tests/test_video_comfy_quant.pyindex fde66dd9c3e..35c9b7a56a2 100644--- a/studio/backend/tests/test_video_comfy_quant.py+++ b/studio/backend/tests/test_video_comfy_quant.py@@ -413,6 +413,8 @@ def test_video_backends_follow_studio_rules(monkeypatch):assert vid._video_comfy_backends(fam, "b", None, _plan(True), _plan(True)) == {"int8_backend": "torchao","fp8_backend": "torchao",+ "nvfp4_backend": None,+ "mxfp8_backend": None,"fp8_storage": False,"fp8_widen_chosen": False,}@@ -421,12 +423,16 @@ def test_video_backends_follow_studio_rules(monkeypatch):assert vid._video_comfy_backends(fam, "b", None, _plan(True), _plan(False)) == {"int8_backend": "native","fp8_backend": None,+ "nvfp4_backend": None,+ "mxfp8_backend": None,"fp8_storage": True,"fp8_widen_chosen": False,}assert vid._video_comfy_backends(fam, "b", None, _plan(True), keep = False) == {"int8_backend": None,"fp8_backend": None,+ "nvfp4_backend": None,+ "mxfp8_backend": None,"fp8_storage": False,"fp8_widen_chosen": False,}@@ -480,6 +486,8 @@ def test_a_smaller_offloaded_price_replans_and_keeps_the_offloaded_backends(monkassert kept == {"int8_backend": "native","fp8_backend": None,+ "nvfp4_backend": None,+ "mxfp8_backend": None,"fp8_storage": True,"fp8_widen_chosen": False,}@@ -1122,6 +1130,8 @@ def test_a_resident_video_dit_that_fits_only_stored_gets_fp8_storage(tmp_path, massert picked == {"int8_backend": None,"fp8_backend": None,+ "nvfp4_backend": None,+ "mxfp8_backend": None,"fp8_storage": True,"fp8_widen_chosen": False,}