Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio NVFP4: tag the compile cache key with the one-pass activation quantiser so a warm cache cannot bypass its switch (#14853)

Daniel Han6fab68f54ac3
diff --git a/studio/backend/core/inference/diffusion_nvfp4_act_quant.py b/studio/backend/core/inference/diffusion_nvfp4_act_quant.py
index a80e42bce1f..bba89c48d41 100644
--- a/studio/backend/core/inference/diffusion_nvfp4_act_quant.py
+++ b/studio/backend/core/inference/diffusion_nvfp4_act_quant.py
@@ -120,6 +120,29 @@ def describe() -> dict:
return dict(_STATE)
+_CACHE_TAG_PREFIX = "unsloth_nvfp4_act_quant="
+
+
+def _set_compile_cache_tag(tag: str | None) -> None:
+ """Key torch's FX/AOTAutograd caches on the quantiser: the Dynamo graph calls ``to_nvfp4`` by name, so untagged,
+ a graph cached with the patch on is replayed with it off (the kill switch then still runs the kernel) and the
+ reverse. ``None`` removes our tag; other tags (e.g. the fp8 dispatcher's) are kept."""
+ try:
+ import torch.compiler.config as _cc
+ except Exception: # noqa: BLE001 - torch without compiler config: no persistent compile cache to split
+ return
+ current = getattr(_cc, "cache_key_tag", None)
+ if not isinstance(current, str):
+ return
+ parts = [p for p in current.split(";") if p and not p.startswith(_CACHE_TAG_PREFIX)]
+ if tag:
+ parts.append(_CACHE_TAG_PREFIX + tag)
+ try:
+ _cc.cache_key_tag = ";".join(parts)
+ except Exception: # noqa: BLE001
+ pass
+
+
def _build_op():
"""Register ``unsloth_nvfp4::act_quant(x, per_tensor_scale) -> (qdata u8 [M, K/2], scales u8 flat)``."""
global _OP
@@ -332,6 +355,7 @@ def install_nvfp4_act_quant_patch(logger: Any = None) -> bool:
if ok:
NVFP4Tensor.to_nvfp4 = staticmethod(_patched_to_nvfp4)
_STATE["installed"] = True
+ _set_compile_cache_tag("1")
if logger is not None:
try:
logger.info(
@@ -348,4 +372,5 @@ def uninstall_nvfp4_act_quant_patch() -> None:
from torchao.prototype.mx_formats.nvfp4_tensor import NVFP4Tensor
NVFP4Tensor.to_nvfp4 = staticmethod(_ORIGINAL)
_STATE["installed"] = False
+ _set_compile_cache_tag(None)
_STATE["verified"] = None
diff --git a/studio/backend/tests/test_diffusion_nvfp4_act_quant.py b/studio/backend/tests/test_diffusion_nvfp4_act_quant.py
index f850c97dbe6..77be37b647b 100644
--- a/studio/backend/tests/test_diffusion_nvfp4_act_quant.py
+++ b/studio/backend/tests/test_diffusion_nvfp4_act_quant.py
@@ -89,3 +89,96 @@ def test_compiled_nvfp4_linear_launches_the_kernel():
f(x)
torch.cuda.synchronize()
assert sum(1 for e in p.events() if "act_quant_kernel" in e.name) == 2
+
+
+def test_install_tags_the_compile_cache_and_uninstall_clears_it(monkeypatch):
+ # The Dynamo graph is the same with or without the patch, so torch's persistent FX/AOT caches must be keyed on it.
+ cc = pytest.importorskip("torch.compiler.config")
+ if not isinstance(getattr(cc, "cache_key_tag", None), str):
+ pytest.skip("torch without compiler cache_key_tag")
+ nvfp4 = pytest.importorskip("torchao.prototype.mx_formats.nvfp4_tensor")
+ monkeypatch.setattr(nvfp4.NVFP4Tensor, "to_nvfp4", nvfp4.NVFP4Tensor.__dict__["to_nvfp4"])
+ monkeypatch.setattr(cc, "cache_key_tag", "other=1")
+ monkeypatch.setattr(torch.cuda, "is_available", lambda: True)
+ monkeypatch.setattr(aq, "verify", lambda: (True, "stub"))
+ monkeypatch.setattr(
+ aq, "_STATE", {"installed": False, "verified": None, "reason": "", "fallback": 0}
+ )
+ monkeypatch.setattr(aq, "_ORIGINAL", None)
+ monkeypatch.delenv(aq.FAST_ACT_QUANT_ENV, raising = False)
+ assert aq.install_nvfp4_act_quant_patch() is True
+ assert cc.cache_key_tag == "other=1;unsloth_nvfp4_act_quant=1"
+ assert aq.install_nvfp4_act_quant_patch() is True
+ assert cc.cache_key_tag.count("unsloth_nvfp4_act_quant") == 1
+ aq.uninstall_nvfp4_act_quant_patch()
+ assert cc.cache_key_tag == "other=1"
+
+
+def test_failed_verification_leaves_the_cache_untagged(monkeypatch):
+ cc = pytest.importorskip("torch.compiler.config")
+ if not isinstance(getattr(cc, "cache_key_tag", None), str):
+ pytest.skip("torch without compiler cache_key_tag")
+ nvfp4 = pytest.importorskip("torchao.prototype.mx_formats.nvfp4_tensor")
+ monkeypatch.setattr(nvfp4.NVFP4Tensor, "to_nvfp4", nvfp4.NVFP4Tensor.__dict__["to_nvfp4"])
+ monkeypatch.setattr(cc, "cache_key_tag", "")
+ monkeypatch.setattr(torch.cuda, "is_available", lambda: True)
+ monkeypatch.setattr(aq, "verify", lambda: (False, "stub mismatch"))
+ monkeypatch.setattr(
+ aq, "_STATE", {"installed": False, "verified": None, "reason": "", "fallback": 0}
+ )
+ monkeypatch.setattr(aq, "_ORIGINAL", None)
+ monkeypatch.delenv(aq.FAST_ACT_QUANT_ENV, raising = False)
+ assert aq.install_nvfp4_act_quant_patch() is False
+ assert cc.cache_key_tag == ""
+
+
+_CROSS_PROCESS_PROBE = """
+import os, sys
+import torch
+from torch.profiler import ProfilerActivity, profile
+from torchao.prototype.mx_formats import NVFP4DynamicActivationNVFP4WeightConfig
+from torchao.quantization import quantize_
+from core.inference import diffusion_nvfp4_act_quant as aq
+aq.install_nvfp4_act_quant_patch()
+torch.manual_seed(0)
+m = torch.nn.Sequential(torch.nn.Linear(512, 512)).cuda().to(torch.bfloat16)
+quantize_(m, NVFP4DynamicActivationNVFP4WeightConfig(use_triton_kernel = False))
+f = torch.compile(m, fullgraph = True)
+x = torch.randn(256, 512, device = "cuda", dtype = torch.bfloat16)
+with torch.inference_mode():
+ f(x)
+ with profile(activities = [ProfilerActivity.CUDA]) as p:
+ f(x)
+ torch.cuda.synchronize()
+print("LAUNCHES", sum(1 for e in p.events() if "act_quant_kernel" in e.name))
+"""
+
+
[email protected](not _blackwell(), reason = "needs sm_100+ for torchao NVFP4")
[email protected]("order", [("1", "0"), ("0", "1")])
+def test_warm_compile_cache_follows_the_kill_switch(tmp_path, order):
+ # A fresh process sharing a warm Inductor cache must run the quantiser its own switch selects.
+ pytest.importorskip("triton")
+ import os
+ import subprocess
+ import sys
+ from pathlib import Path
+
+ backend = str(Path(aq.__file__).resolve().parents[2])
+ env = dict(os.environ)
+ env["PYTHONPATH"] = backend + os.pathsep + env.get("PYTHONPATH", "")
+ env["TORCHINDUCTOR_CACHE_DIR"] = str(tmp_path / "inductor")
+ env["TRITON_CACHE_DIR"] = str(tmp_path / "triton")
+ launches = {}
+ for switch in order:
+ env[aq.FAST_ACT_QUANT_ENV] = switch
+ out = subprocess.run(
+ [sys.executable, "-c", _CROSS_PROCESS_PROBE],
+ env = env,
+ capture_output = True,
+ text = True,
+ timeout = 600,
+ )
+ assert out.returncode == 0, out.stderr[-2000:]
+ launches[switch] = int(out.stdout.split("LAUNCHES")[-1].split()[0])
+ assert launches == {"1": 1, "0": 0}