Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio NVFP4: tag the compile cache key with the one-pass activation quantiser so a warm cache cannot bypass its switch (#14853)
Daniel Han6fab68f54ac3
diff --git a/studio/backend/core/inference/diffusion_nvfp4_act_quant.py b/studio/backend/core/inference/diffusion_nvfp4_act_quant.pyindex a80e42bce1f..bba89c48d41 100644--- a/studio/backend/core/inference/diffusion_nvfp4_act_quant.py+++ b/studio/backend/core/inference/diffusion_nvfp4_act_quant.py@@ -120,6 +120,29 @@ def describe() -> dict:return dict(_STATE)+_CACHE_TAG_PREFIX = "unsloth_nvfp4_act_quant="+++def _set_compile_cache_tag(tag: str | None) -> None:+ """Key torch's FX/AOTAutograd caches on the quantiser: the Dynamo graph calls ``to_nvfp4`` by name, so untagged,+ a graph cached with the patch on is replayed with it off (the kill switch then still runs the kernel) and the+ reverse. ``None`` removes our tag; other tags (e.g. the fp8 dispatcher's) are kept."""+ try:+ import torch.compiler.config as _cc+ except Exception: # noqa: BLE001 - torch without compiler config: no persistent compile cache to split+ return+ current = getattr(_cc, "cache_key_tag", None)+ if not isinstance(current, str):+ return+ parts = [p for p in current.split(";") if p and not p.startswith(_CACHE_TAG_PREFIX)]+ if tag:+ parts.append(_CACHE_TAG_PREFIX + tag)+ try:+ _cc.cache_key_tag = ";".join(parts)+ except Exception: # noqa: BLE001+ pass++def _build_op():"""Register ``unsloth_nvfp4::act_quant(x, per_tensor_scale) -> (qdata u8 [M, K/2], scales u8 flat)``."""global _OP@@ -332,6 +355,7 @@ def install_nvfp4_act_quant_patch(logger: Any = None) -> bool:if ok:NVFP4Tensor.to_nvfp4 = staticmethod(_patched_to_nvfp4)_STATE["installed"] = True+ _set_compile_cache_tag("1")if logger is not None:try:logger.info(@@ -348,4 +372,5 @@ def uninstall_nvfp4_act_quant_patch() -> None:from torchao.prototype.mx_formats.nvfp4_tensor import NVFP4TensorNVFP4Tensor.to_nvfp4 = staticmethod(_ORIGINAL)_STATE["installed"] = False+ _set_compile_cache_tag(None)_STATE["verified"] = Nonediff --git a/studio/backend/tests/test_diffusion_nvfp4_act_quant.py b/studio/backend/tests/test_diffusion_nvfp4_act_quant.pyindex f850c97dbe6..77be37b647b 100644--- a/studio/backend/tests/test_diffusion_nvfp4_act_quant.py+++ b/studio/backend/tests/test_diffusion_nvfp4_act_quant.py@@ -89,3 +89,96 @@ def test_compiled_nvfp4_linear_launches_the_kernel():f(x)torch.cuda.synchronize()assert sum(1 for e in p.events() if "act_quant_kernel" in e.name) == 2+++def test_install_tags_the_compile_cache_and_uninstall_clears_it(monkeypatch):+ # The Dynamo graph is the same with or without the patch, so torch's persistent FX/AOT caches must be keyed on it.+ cc = pytest.importorskip("torch.compiler.config")+ if not isinstance(getattr(cc, "cache_key_tag", None), str):+ pytest.skip("torch without compiler cache_key_tag")+ nvfp4 = pytest.importorskip("torchao.prototype.mx_formats.nvfp4_tensor")+ monkeypatch.setattr(nvfp4.NVFP4Tensor, "to_nvfp4", nvfp4.NVFP4Tensor.__dict__["to_nvfp4"])+ monkeypatch.setattr(cc, "cache_key_tag", "other=1")+ monkeypatch.setattr(torch.cuda, "is_available", lambda: True)+ monkeypatch.setattr(aq, "verify", lambda: (True, "stub"))+ monkeypatch.setattr(+ aq, "_STATE", {"installed": False, "verified": None, "reason": "", "fallback": 0}+ )+ monkeypatch.setattr(aq, "_ORIGINAL", None)+ monkeypatch.delenv(aq.FAST_ACT_QUANT_ENV, raising = False)+ assert aq.install_nvfp4_act_quant_patch() is True+ assert cc.cache_key_tag == "other=1;unsloth_nvfp4_act_quant=1"+ assert aq.install_nvfp4_act_quant_patch() is True+ assert cc.cache_key_tag.count("unsloth_nvfp4_act_quant") == 1+ aq.uninstall_nvfp4_act_quant_patch()+ assert cc.cache_key_tag == "other=1"+++def test_failed_verification_leaves_the_cache_untagged(monkeypatch):+ cc = pytest.importorskip("torch.compiler.config")+ if not isinstance(getattr(cc, "cache_key_tag", None), str):+ pytest.skip("torch without compiler cache_key_tag")+ nvfp4 = pytest.importorskip("torchao.prototype.mx_formats.nvfp4_tensor")+ monkeypatch.setattr(nvfp4.NVFP4Tensor, "to_nvfp4", nvfp4.NVFP4Tensor.__dict__["to_nvfp4"])+ monkeypatch.setattr(cc, "cache_key_tag", "")+ monkeypatch.setattr(torch.cuda, "is_available", lambda: True)+ monkeypatch.setattr(aq, "verify", lambda: (False, "stub mismatch"))+ monkeypatch.setattr(+ aq, "_STATE", {"installed": False, "verified": None, "reason": "", "fallback": 0}+ )+ monkeypatch.setattr(aq, "_ORIGINAL", None)+ monkeypatch.delenv(aq.FAST_ACT_QUANT_ENV, raising = False)+ assert aq.install_nvfp4_act_quant_patch() is False+ assert cc.cache_key_tag == ""+++_CROSS_PROCESS_PROBE = """+import os, sys+import torch+from torch.profiler import ProfilerActivity, profile+from torchao.prototype.mx_formats import NVFP4DynamicActivationNVFP4WeightConfig+from torchao.quantization import quantize_+from core.inference import diffusion_nvfp4_act_quant as aq+aq.install_nvfp4_act_quant_patch()+torch.manual_seed(0)+m = torch.nn.Sequential(torch.nn.Linear(512, 512)).cuda().to(torch.bfloat16)+quantize_(m, NVFP4DynamicActivationNVFP4WeightConfig(use_triton_kernel = False))+f = torch.compile(m, fullgraph = True)+x = torch.randn(256, 512, device = "cuda", dtype = torch.bfloat16)+with torch.inference_mode():+ f(x)+ with profile(activities = [ProfilerActivity.CUDA]) as p:+ f(x)+ torch.cuda.synchronize()+print("LAUNCHES", sum(1 for e in p.events() if "act_quant_kernel" in e.name))+"""++[email protected](not _blackwell(), reason = "needs sm_100+ for torchao NVFP4")[email protected]("order", [("1", "0"), ("0", "1")])+def test_warm_compile_cache_follows_the_kill_switch(tmp_path, order):+ # A fresh process sharing a warm Inductor cache must run the quantiser its own switch selects.+ pytest.importorskip("triton")+ import os+ import subprocess+ import sys+ from pathlib import Path++ backend = str(Path(aq.__file__).resolve().parents[2])+ env = dict(os.environ)+ env["PYTHONPATH"] = backend + os.pathsep + env.get("PYTHONPATH", "")+ env["TORCHINDUCTOR_CACHE_DIR"] = str(tmp_path / "inductor")+ env["TRITON_CACHE_DIR"] = str(tmp_path / "triton")+ launches = {}+ for switch in order:+ env[aq.FAST_ACT_QUANT_ENV] = switch+ out = subprocess.run(+ [sys.executable, "-c", _CROSS_PROCESS_PROBE],+ env = env,+ capture_output = True,+ text = True,+ timeout = 600,+ )+ assert out.returncode == 0, out.stderr[-2000:]+ launches[switch] = int(out.stdout.split("LAUNCHES")[-1].split()[0])+ assert launches == {"1": 1, "0": 0}