Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio NVFP4: the protected branch compiles once from a cold decode table and accepts fp16 inputs (#17081)
Daniel Han461ba86945d5
diff --git a/studio/backend/core/inference/diffusion_comfy_block.py b/studio/backend/core/inference/diffusion_comfy_block.pyindex 5c0b2093a21..3c3e19a28a4 100644--- a/studio/backend/core/inference/diffusion_comfy_block.py+++ b/studio/backend/core/inference/diffusion_comfy_block.py@@ -313,7 +313,7 @@ def nvfp4_dynamic_linear_class():if self.protect.armed and self.protect.protected:out = F.linear(flat,- dequantize_nvfp4_weight(self.wq, self.w_sf, self.w_scale, dtype = torch.bfloat16),+ dequantize_nvfp4_weight(self.wq, self.w_sf, self.w_scale, dtype = flat.dtype),)else:with _device_guard(flat):diff --git a/studio/backend/core/inference/diffusion_nvfp4_linear.py b/studio/backend/core/inference/diffusion_nvfp4_linear.pyindex b1af17000b8..9de5f8b4b15 100644--- a/studio/backend/core/inference/diffusion_nvfp4_linear.py+++ b/studio/backend/core/inference/diffusion_nvfp4_linear.py@@ -97,9 +97,7 @@ def nvfp4_linear_class():*shape[:-1], self.out_features)if self.protect.armed and self.protect.protected:- weight = dequantize_nvfp4_weight(- self.wq, self.w_sf, self.w_scale, dtype = torch.bfloat16- )+ weight = dequantize_nvfp4_weight(self.wq, self.w_sf, self.w_scale, dtype = flat.dtype)out = F.linear(flat, weight)else:with _device_guard(flat):diff --git a/studio/backend/core/inference/diffusion_nvfp4_ops.py b/studio/backend/core/inference/diffusion_nvfp4_ops.pyindex 80997f99e83..70294984674 100644--- a/studio/backend/core/inference/diffusion_nvfp4_ops.py+++ b/studio/backend/core/inference/diffusion_nvfp4_ops.py@@ -269,6 +269,9 @@ def e2m1_lut(device: Any):"""The signed e2m1 decode table on ``device``, memoised so it is never first allocated during capture."""import torch+ # Traced: a graph constant. Reading the memo would guard on it, and its first insert would recompile the caller.+ if torch.compiler.is_compiling():+ return torch.tensor(_E2M1_LUT, device = device, dtype = torch.float32)key = (str(device.type), device.index)table = _LUT_CACHE.get(key)if table is None:diff --git a/studio/backend/tests/test_diffusion_nvfp4_protect.py b/studio/backend/tests/test_diffusion_nvfp4_protect.pyindex fa571363a40..3cae39e71ce 100644--- a/studio/backend/tests/test_diffusion_nvfp4_protect.py+++ b/studio/backend/tests/test_diffusion_nvfp4_protect.py@@ -629,3 +629,59 @@ def test_the_layer_registry_is_weak_so_an_unloaded_model_stops_counting():del layergc.collect()assert ctl.capable_layers() == 0+++def _cpu_protected_layer():+ torch = pytest.importorskip("torch")+ from core.inference.diffusion_nvfp4_protect import NVFP4StepController++ torch.manual_seed(7)+ q = torch.randint(0, 256, (32, 32), dtype = torch.uint8)+ scales = torch.full((32, 4), 0.5, dtype = torch.float8_e4m3fn)+ packed = ops.swizzle_sf(scales, 32, 64).reshape(ops.sf_matrix_shape(32, 4))+ layer = nl.nvfp4_linear_class()(+ 64,+ 32,+ wq = q,+ w_sf = packed,+ alpha = torch.ones(1),+ a_gsf = torch.ones(1),+ w_scale = torch.tensor([0.125]),+ )+ layer.protect = NVFP4StepController("auto")+ layer.protect.begin(9)+ assert layer.protect.armed and layer.protect.protected+ return torch, layer+++def test_a_cold_lut_compiles_the_protected_branch_once():+ torch, layer = _cpu_protected_layer()+ from torch._dynamo.testing import CompileCounter++ ops.reset_lut_cache()+ torch._dynamo.reset()+ counter = CompileCounter()+ compiled = torch.compile(layer, backend = counter, fullgraph = True, dynamic = True)+ x = torch.randn(4, 64, dtype = torch.bfloat16)+ with torch.inference_mode():+ ref = layer(x)+ ops.reset_lut_cache() # the eager reference memoised it; the compiled calls start cold+ outs = [compiled(x) for _ in range(3)]+ ops.e2m1_lut(x.device) # an eager dequantise between renders memoises the table+ outs.append(compiled(x))+ assert counter.frame_count == 1+ assert all(torch.equal(out, ref) for out in outs)+ torch._dynamo.reset()++[email protected]("dtype_name", ["float16", "bfloat16", "float32"])+def test_the_protected_branch_keeps_the_input_dtype(dtype_name):+ torch, layer = _cpu_protected_layer()+ dtype = getattr(torch, dtype_name)+ x = torch.randn(4, 64, dtype = dtype)+ compute = torch.bfloat16 if dtype is torch.float32 else dtype+ weight = ops.dequantize_nvfp4_weight(layer.wq, layer.w_sf, layer.w_scale, dtype = compute)+ with torch.inference_mode():+ out = layer(x)+ assert out.dtype == dtype+ assert torch.equal(out, torch.nn.functional.linear(x.to(compute), weight).to(dtype))