Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: packed nvfp4 / mxfp8 layers follow a module dtype cast (#17131)
Daniel Han04cd919a5ed7
diff --git a/studio/backend/core/inference/diffusion_packed_block.py b/studio/backend/core/inference/diffusion_packed_block.pyindex 2ed25387a4b..263cfce6dba 100644--- a/studio/backend/core/inference/diffusion_packed_block.py+++ b/studio/backend/core/inference/diffusion_packed_block.py@@ -264,6 +264,19 @@ def packed_block_linear_class():)return w if dtype is None or dtype == w.dtype else w.to(dtype)+ def _apply(self, fn, *args, **kwargs):+ # the packed buffers are integer, so a module-wide .to(dtype) / .half() / .float() leaves them as they+ # are: the widen target follows the cast the way a dense Linear's weight would+ out = super()._apply(fn, *args, **kwargs)+ probe = fn(torch.empty(0, dtype = self.compute_dtype, device = self.weight_q.device))+ if isinstance(probe, torch.Tensor) and probe.dtype in (+ torch.bfloat16,+ torch.float16,+ torch.float32,+ ):+ self.compute_dtype = probe.dtype+ return out+@propertydef weight(self) -> Any:return self.dequantized_weight()diff --git a/studio/backend/tests/test_diffusion_packed_block.py b/studio/backend/tests/test_diffusion_packed_block.pyindex 6eb9426f233..94c498a7ff7 100644--- a/studio/backend/tests/test_diffusion_packed_block.py+++ b/studio/backend/tests/test_diffusion_packed_block.py@@ -97,7 +97,34 @@ def test_module_casts_and_moves_never_touch_the_packed_buffers(block_file):for name, buf in layer.named_buffers():if name != "bias":assert buf.dtype == before[name].dtype and torch.equal(buf, before[name]), name- assert _same(layer.weight, weight) # the compute dtype is the load's, not the cast's+ assert layer.compute_dtype == torch.float16 # the widen target follows the cast+ model.to(torch.bfloat16)+ assert _same(layer.weight, weight) # and back: the load's exact weight again++[email protected]("fmt", ["nvfp4", "mxfp8"])[email protected]("cast", ["to", "method"])[email protected]("dtype", [torch.float16, torch.float32])+def test_a_cast_packed_model_forwards_like_a_dense_load_in_that_dtype(block_file, fmt, cast, dtype):+ path, _tensors, _expected = block_file(fmt)+ packed = _load(path, dtype = torch.bfloat16, fp8_storage = True)+ if cast == "to":+ packed.to(dtype)+ else:+ packed.half() if dtype == torch.float16 else packed.float()+ dense = _load(path, dtype = dtype)+ state = dense.state_dict()+ x = torch.randn(3, 5, DIM, generator = torch.Generator().manual_seed(2)).to(dtype)+ for name in LINEARS:+ layer = packed.get_submodule(name)+ assert pb.is_packed_block_linear(layer), name+ assert layer.weight_q.dtype == torch.uint8 and layer.weight_sf.dtype == torch.uint8+ assert _same(layer.weight, state[name + ".weight"]), name+ out = layer(x)+ assert out.dtype == dtype, name+ # the bias went through the cast (bf16 then the new dtype), as a dense model's would+ want = torch.nn.functional.linear(x, state[name + ".weight"], layer.bias)+ assert _same(out, want), namedef _parts(