Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: carry FP8 / NVFP4 compile cache tags into render and compile threads (#16158)

Daniel Hand90ef5bf9c91
diff --git a/studio/backend/core/inference/diffusion_compile_cache.py b/studio/backend/core/inference/diffusion_compile_cache.py
index 4464eceef7f..49e888c6b51 100644
--- a/studio/backend/core/inference/diffusion_compile_cache.py
+++ b/studio/backend/core/inference/diffusion_compile_cache.py
@@ -204,8 +204,9 @@ def environment_fingerprint() -> dict[str, Any]:
fp["gpu_name"] = torch.cuda.get_device_name(index)
cap = torch.cuda.get_device_capability(index)
fp["gpu_capability"] = f"sm_{cap[0]}{cap[1]}"
- # torch keys artifacts on this tag (set by diffusion_fp8_unit_scale); a bundle from another tag would load,
- # miss, recompile and never be rewritten. Only a set tag joins the key, so untagged bundles stay valid.
+ from .diffusion_compile_config import apply
+
+ apply()
tag = getattr(getattr(torch, "compiler", None), "config", None)
tag = getattr(tag, "cache_key_tag", None)
if isinstance(tag, str) and tag:
diff --git a/studio/backend/core/inference/diffusion_compile_config.py b/studio/backend/core/inference/diffusion_compile_config.py
index 701ba603d6c..affd17c5979 100644
--- a/studio/backend/core/inference/diffusion_compile_config.py
+++ b/studio/backend/core/inference/diffusion_compile_config.py
@@ -119,6 +119,20 @@ def apply() -> None:
for (module_name, attr), value in knobs:
cfg = _module(module_name)
if cfg is not None:
+ if (module_name, attr) == ("torch.compiler.config", "cache_key_tag"):
+ current = getattr(cfg, attr, "")
+ value = ";".join(
+ [
+ p
+ for p in _cache_tag_parts(current)
+ if p and not p.startswith(_STUDIO_CACHE_TAG_PREFIXES)
+ ]
+ + [
+ p
+ for p in _cache_tag_parts(value)
+ if p.startswith(_STUDIO_CACHE_TAG_PREFIXES)
+ ]
+ )
_write(cfg, attr, value)
_applied.set(generation)
@@ -143,11 +157,35 @@ def _reset_for_tests() -> None:
_STUDIO_CACHE_TAG_PREFIXES = ("unsloth_fp8_unit_scale=", "unsloth_nvfp4_act_quant=")
+def _cache_tag_parts(tag: str) -> list[str]:
+ # Unsloth/Zoo append comma-delimited kernel and correctness tags.
+ return tag.replace(",", ";").split(";")
+
+
def canonical_compile_cache_tag(current: str, prefix: str, tag: str | None) -> str:
"""``prefix``'s token set to ``tag`` (None drops it); Studio's sorted so setter order never changes torch's cache key."""
- parts = [p for p in current.split(";") if p and not p.startswith(prefix)]
+ parts = [p for p in _cache_tag_parts(current) if p and not p.startswith(prefix)]
if tag:
parts.append(prefix + tag)
foreign = [p for p in parts if not p.startswith(_STUDIO_CACHE_TAG_PREFIXES)]
owned = sorted(p for p in parts if p.startswith(_STUDIO_CACHE_TAG_PREFIXES))
return ";".join(foreign + owned)
+
+
+def set_compile_cache_tag(prefix: str, tag: str | None) -> None:
+ cfg = _module("torch.compiler.config")
+ if cfg is None:
+ return
+ current = getattr(cfg, "cache_key_tag", None)
+ if not isinstance(current, str):
+ return
+ # Studio's patches outlive the load context. Keep their recorded tokens while
+ # preserving foreign tokens from the caller's context.
+ recorded_tag = get_knob("torch.compiler.config", "cache_key_tag", current)
+ current = ";".join(
+ [p for p in _cache_tag_parts(current) if not p.startswith(_STUDIO_CACHE_TAG_PREFIXES)]
+ + [p for p in _cache_tag_parts(recorded_tag) if p.startswith(_STUDIO_CACHE_TAG_PREFIXES)]
+ )
+ set_knob(
+ "torch.compiler.config", "cache_key_tag", canonical_compile_cache_tag(current, prefix, tag)
+ )
diff --git a/studio/backend/core/inference/diffusion_fp8_unit_scale.py b/studio/backend/core/inference/diffusion_fp8_unit_scale.py
index bda64dbdf78..d3dfc51ffbb 100644
--- a/studio/backend/core/inference/diffusion_fp8_unit_scale.py
+++ b/studio/backend/core/inference/diffusion_fp8_unit_scale.py
@@ -98,13 +98,12 @@ def _set_compile_cache_tag(tag: Optional[str]) -> None:
import torch.compiler.config as _cc
except Exception: # noqa: BLE001 - torch without compiler config: no persistent compile cache to split
return
- current = getattr(_cc, "cache_key_tag", None)
- if not isinstance(current, str):
+ if not isinstance(getattr(_cc, "cache_key_tag", None), str):
return
- from .diffusion_compile_config import canonical_compile_cache_tag
+ from .diffusion_compile_config import set_compile_cache_tag
try:
- _cc.cache_key_tag = canonical_compile_cache_tag(current, _CACHE_TAG_PREFIX, tag)
+ set_compile_cache_tag(_CACHE_TAG_PREFIX, tag)
except Exception: # noqa: BLE001
pass
@@ -126,6 +125,8 @@ def install(logger: Any = None, *, fast_accum: Optional[bool] = None) -> bool:
"""Patch torchao's fp8 addmm with the dispatcher; idempotent, False when off / no qualifying GPU / layout moved /
it cannot engage."""
if _STATE["installed"]:
+ from .diffusion_compile_config import apply
+ apply()
return True
mode = _mode()
if mode == "off" or not can_engage(fast_accum):
diff --git a/studio/backend/core/inference/diffusion_nvfp4_act_quant.py b/studio/backend/core/inference/diffusion_nvfp4_act_quant.py
index fe9e462ff67..cd229fe1ca3 100644
--- a/studio/backend/core/inference/diffusion_nvfp4_act_quant.py
+++ b/studio/backend/core/inference/diffusion_nvfp4_act_quant.py
@@ -125,13 +125,12 @@ def _set_compile_cache_tag(tag: str | None) -> None:
import torch.compiler.config as _cc
except Exception: # noqa: BLE001 - torch without compiler config: no persistent compile cache to split
return
- current = getattr(_cc, "cache_key_tag", None)
- if not isinstance(current, str):
+ if not isinstance(getattr(_cc, "cache_key_tag", None), str):
return
- from .diffusion_compile_config import canonical_compile_cache_tag
+ from .diffusion_compile_config import set_compile_cache_tag
try:
- _cc.cache_key_tag = canonical_compile_cache_tag(current, _CACHE_TAG_PREFIX, tag)
+ set_compile_cache_tag(_CACHE_TAG_PREFIX, tag)
except Exception: # noqa: BLE001
pass
diff --git a/studio/backend/tests/test_diffusion_cache_tag_contexts.py b/studio/backend/tests/test_diffusion_cache_tag_contexts.py
new file mode 100644
index 00000000000..e93c6b4c940
--- /dev/null
+++ b/studio/backend/tests/test_diffusion_cache_tag_contexts.py
@@ -0,0 +1,155 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+from __future__ import annotations
+
+import threading
+
+import pytest
+
+pytest.importorskip("torch")
+import torch.compiler.config as cc # noqa: E402
+
+from core.inference import diffusion_compile_config as compile_config # noqa: E402
+from core.inference import diffusion_fp8_unit_scale as fp8 # noqa: E402
+from core.inference import diffusion_nvfp4_act_quant as nvfp4 # noqa: E402
+
+
[email protected](autouse = True)
+def _clean():
+ saved = cc.cache_key_tag
+ fp8.uninstall()
+ compile_config._reset_for_tests()
+ yield
+ fp8.uninstall()
+ compile_config._reset_for_tests()
+ cc.cache_key_tag = saved
+
+
+def _in_thread(fn):
+ out = {}
+
+ def run():
+ try:
+ out["value"] = fn()
+ except BaseException as exc:
+ out["error"] = exc
+
+ t = threading.Thread(target = run)
+ t.start()
+ t.join()
+ if "error" in out:
+ raise out["error"]
+ return out["value"]
+
+
+def _render_context_tag():
+ def read():
+ compile_config.apply()
+ return cc.cache_key_tag
+
+ return _in_thread(read)
+
+
+def test_kill_switch_tag_set_on_the_load_thread_reaches_the_render_thread(monkeypatch):
+ monkeypatch.setenv(nvfp4.FAST_ACT_QUANT_ENV, "0")
+ assert _in_thread(lambda: (nvfp4.install_nvfp4_act_quant_patch(), cc.cache_key_tag)[1]) == (
+ "unsloth_nvfp4_act_quant=0"
+ )
+ assert _render_context_tag() == "unsloth_nvfp4_act_quant=0"
+
+
+def test_fp8_and_nvfp4_tags_set_on_the_load_thread_reach_the_render_thread():
+ _in_thread(
+ lambda: (fp8._set_compile_cache_tag("256"), nvfp4._set_compile_cache_tag(nvfp4._TAG_ON))
+ )
+ assert _render_context_tag() == "unsloth_fp8_unit_scale=256;unsloth_nvfp4_act_quant=1"
+
+
+def test_reset_on_a_later_load_reaches_the_render_thread():
+ _in_thread(lambda: nvfp4._set_compile_cache_tag(nvfp4._TAG_OFF))
+ assert _render_context_tag() == "unsloth_nvfp4_act_quant=0"
+ _in_thread(nvfp4.reset_compile_cache_tag)
+ assert _render_context_tag() == ""
+
+
+def test_render_context_keeps_its_foreign_cache_tokens():
+ def load():
+ cc.cache_key_tag = "loader-owned"
+ fp8._set_compile_cache_tag("256")
+
+ def render():
+ cc.cache_key_tag = "render-owned"
+ compile_config.apply()
+ return cc.cache_key_tag
+
+ _in_thread(load)
+ assert _in_thread(render) == "render-owned;unsloth_fp8_unit_scale=256"
+
+
+def test_setter_preserves_appended_correctness_tag():
+ nvfp4._set_compile_cache_tag("0")
+ cc.cache_key_tag += ",unsloth/inductor-rand4x-off"
+ fp8._set_compile_cache_tag("256")
+ assert "unsloth/inductor-rand4x-off" in cc.cache_key_tag
+ nvfp4.reset_compile_cache_tag()
+ assert cc.cache_key_tag == "unsloth/inductor-rand4x-off;unsloth_fp8_unit_scale=256"
+
+
+def test_bundle_fingerprint_uses_the_render_context_tag(monkeypatch):
+ from core.inference import diffusion_compile_cache as cache
+
+ monkeypatch.setattr(cache, "_triton_version", lambda: None)
+ monkeypatch.setattr(cache, "_diffusers_version", lambda: None)
+ _in_thread(lambda: fp8._set_compile_cache_tag("256"))
+
+ def request():
+ cc.cache_key_tag = "render-owned"
+ return cache.environment_fingerprint().get("torch_cache_key_tag")
+
+ assert _in_thread(request) == "render-owned;unsloth_fp8_unit_scale=256"
+
+
+def test_nvfp4_reset_in_a_fresh_context_keeps_the_recorded_fp8_tag():
+ _in_thread(lambda: (fp8._set_compile_cache_tag("256"), nvfp4._set_compile_cache_tag("1")))
+ _in_thread(nvfp4.reset_compile_cache_tag)
+ assert _render_context_tag() == "unsloth_fp8_unit_scale=256"
+
+
+def test_real_fp8_installer_keeps_the_tag_across_two_loads(monkeypatch):
+ f8t = pytest.importorskip("torchao.quantization.quantize_.workflows.float8.float8_tensor")
+ monkeypatch.delenv(fp8.UNIT_SCALE_ENV, raising = False)
+ monkeypatch.delenv(fp8.MIN_ROWS_ENV, raising = False)
+ monkeypatch.setattr(fp8, "_device_caps", lambda: {0: (8, 9)})
+
+ def load():
+ nvfp4.reset_compile_cache_tag()
+ return fp8.install()
+
+ assert _in_thread(load) is True
+ wrapped = f8t.addmm_float8_unwrapped_inference
+ assert _render_context_tag() == "unsloth_fp8_unit_scale=256"
+ assert _in_thread(load) is True
+ assert fp8.stats()["installed"] is True
+ assert f8t.addmm_float8_unwrapped_inference is wrapped
+ assert _render_context_tag() == "unsloth_fp8_unit_scale=256"
+
+
[email protected]("mode, rows, tag", [("auto", "256", "256"), ("force", "512", "512:force")])
+def test_repeated_install_restores_the_installed_tag_in_a_fresh_context(
+ monkeypatch, mode, rows, tag
+):
+ f8t = pytest.importorskip("torchao.quantization.quantize_.workflows.float8.float8_tensor")
+ monkeypatch.setenv(fp8.UNIT_SCALE_ENV, mode)
+ monkeypatch.setenv(fp8.MIN_ROWS_ENV, rows)
+ monkeypatch.setattr(fp8, "_device_caps", lambda: {0: (8, 9)})
+ assert _in_thread(fp8.install) is True
+ wrapped = f8t.addmm_float8_unwrapped_inference
+ # These settings are captured by the installed closure, so later env changes must not relabel it.
+ monkeypatch.setenv(fp8.UNIT_SCALE_ENV, "0")
+ monkeypatch.setenv(fp8.MIN_ROWS_ENV, "1024")
+ installed, current = _in_thread(lambda: (fp8.install(), cc.cache_key_tag))
+ assert installed is True
+ assert current == f"unsloth_fp8_unit_scale={tag}"
+ assert _render_context_tag() == current
+ assert f8t.addmm_float8_unwrapped_inference is wrapped
diff --git a/studio/backend/tests/test_diffusion_cache_tag_order.py b/studio/backend/tests/test_diffusion_cache_tag_order.py
index 55581db554d..ea974717bd4 100644
--- a/studio/backend/tests/test_diffusion_cache_tag_order.py
+++ b/studio/backend/tests/test_diffusion_cache_tag_order.py
@@ -10,6 +10,7 @@ import pytest
pytest.importorskip("torch")
import torch.compiler.config as cc # noqa: E402
+from core.inference import diffusion_compile_config as compile_config # noqa: E402
from core.inference import diffusion_fp8_unit_scale as fp8 # noqa: E402
from core.inference import diffusion_nvfp4_act_quant as nvfp4 # noqa: E402
@@ -17,7 +18,9 @@ from core.inference import diffusion_nvfp4_act_quant as nvfp4 # noqa: E402
@pytest.fixture(autouse = True)
def _restore_tag():
saved = cc.cache_key_tag
+ compile_config._reset_for_tests()
yield
+ compile_config._reset_for_tests()
cc.cache_key_tag = saved
diff --git a/studio/backend/tests/test_diffusion_fp8_unit_scale.py b/studio/backend/tests/test_diffusion_fp8_unit_scale.py
index a6c9b6b4dc4..13acdce01a1 100644
--- a/studio/backend/tests/test_diffusion_fp8_unit_scale.py
+++ b/studio/backend/tests/test_diffusion_fp8_unit_scale.py
@@ -11,12 +11,14 @@ import torch
pytest.importorskip("torchao")
from torchao.quantization.quantize_.workflows.float8 import float8_tensor as f8t # noqa: E402
+from core.inference import diffusion_compile_config as compile_config # noqa: E402
from core.inference import diffusion_fp8_unit_scale as us # noqa: E402
@pytest.fixture(autouse = True)
def _clean(monkeypatch):
us.uninstall()
+ compile_config._reset_for_tests()
monkeypatch.delenv(us.UNIT_SCALE_ENV, raising = False)
monkeypatch.delenv(us.MIN_ROWS_ENV, raising = False)
for k in us._COUNTS:
@@ -24,6 +26,7 @@ def _clean(monkeypatch):
original = f8t.addmm_float8_unwrapped_inference
yield
us.uninstall()
+ compile_config._reset_for_tests()
f8t.addmm_float8_unwrapped_inference = original