Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: packed nvfp4 / mxfp8 layers follow a module dtype cast (#17131)

Daniel Han04cd919a5ed7
diff --git a/studio/backend/core/inference/diffusion_packed_block.py b/studio/backend/core/inference/diffusion_packed_block.py
index 2ed25387a4b..263cfce6dba 100644
--- a/studio/backend/core/inference/diffusion_packed_block.py
+++ b/studio/backend/core/inference/diffusion_packed_block.py
@@ -264,6 +264,19 @@ def packed_block_linear_class():
)
return w if dtype is None or dtype == w.dtype else w.to(dtype)
+ def _apply(self, fn, *args, **kwargs):
+ # the packed buffers are integer, so a module-wide .to(dtype) / .half() / .float() leaves them as they
+ # are: the widen target follows the cast the way a dense Linear's weight would
+ out = super()._apply(fn, *args, **kwargs)
+ probe = fn(torch.empty(0, dtype = self.compute_dtype, device = self.weight_q.device))
+ if isinstance(probe, torch.Tensor) and probe.dtype in (
+ torch.bfloat16,
+ torch.float16,
+ torch.float32,
+ ):
+ self.compute_dtype = probe.dtype
+ return out
+
@property
def weight(self) -> Any:
return self.dequantized_weight()
diff --git a/studio/backend/tests/test_diffusion_packed_block.py b/studio/backend/tests/test_diffusion_packed_block.py
index 6eb9426f233..94c498a7ff7 100644
--- a/studio/backend/tests/test_diffusion_packed_block.py
+++ b/studio/backend/tests/test_diffusion_packed_block.py
@@ -97,7 +97,34 @@ def test_module_casts_and_moves_never_touch_the_packed_buffers(block_file):
for name, buf in layer.named_buffers():
if name != "bias":
assert buf.dtype == before[name].dtype and torch.equal(buf, before[name]), name
- assert _same(layer.weight, weight) # the compute dtype is the load's, not the cast's
+ assert layer.compute_dtype == torch.float16 # the widen target follows the cast
+ model.to(torch.bfloat16)
+ assert _same(layer.weight, weight) # and back: the load's exact weight again
+
+
[email protected]("fmt", ["nvfp4", "mxfp8"])
[email protected]("cast", ["to", "method"])
[email protected]("dtype", [torch.float16, torch.float32])
+def test_a_cast_packed_model_forwards_like_a_dense_load_in_that_dtype(block_file, fmt, cast, dtype):
+ path, _tensors, _expected = block_file(fmt)
+ packed = _load(path, dtype = torch.bfloat16, fp8_storage = True)
+ if cast == "to":
+ packed.to(dtype)
+ else:
+ packed.half() if dtype == torch.float16 else packed.float()
+ dense = _load(path, dtype = dtype)
+ state = dense.state_dict()
+ x = torch.randn(3, 5, DIM, generator = torch.Generator().manual_seed(2)).to(dtype)
+ for name in LINEARS:
+ layer = packed.get_submodule(name)
+ assert pb.is_packed_block_linear(layer), name
+ assert layer.weight_q.dtype == torch.uint8 and layer.weight_sf.dtype == torch.uint8
+ assert _same(layer.weight, state[name + ".weight"]), name
+ out = layer(x)
+ assert out.dtype == dtype, name
+ # the bias went through the cast (bf16 then the new dtype), as a dense model's would
+ want = torch.nn.functional.linear(x, state[name + ".weight"], layer.bias)
+ assert _same(out, want), name
def _parts(