Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: carry FP8 / NVFP4 compile cache tags into render and compile threads (#16158)
Daniel Hand90ef5bf9c91
diff --git a/studio/backend/core/inference/diffusion_compile_cache.py b/studio/backend/core/inference/diffusion_compile_cache.pyindex 4464eceef7f..49e888c6b51 100644--- a/studio/backend/core/inference/diffusion_compile_cache.py+++ b/studio/backend/core/inference/diffusion_compile_cache.py@@ -204,8 +204,9 @@ def environment_fingerprint() -> dict[str, Any]:fp["gpu_name"] = torch.cuda.get_device_name(index)cap = torch.cuda.get_device_capability(index)fp["gpu_capability"] = f"sm_{cap[0]}{cap[1]}"- # torch keys artifacts on this tag (set by diffusion_fp8_unit_scale); a bundle from another tag would load,- # miss, recompile and never be rewritten. Only a set tag joins the key, so untagged bundles stay valid.+ from .diffusion_compile_config import apply++ apply()tag = getattr(getattr(torch, "compiler", None), "config", None)tag = getattr(tag, "cache_key_tag", None)if isinstance(tag, str) and tag:diff --git a/studio/backend/core/inference/diffusion_compile_config.py b/studio/backend/core/inference/diffusion_compile_config.pyindex 701ba603d6c..affd17c5979 100644--- a/studio/backend/core/inference/diffusion_compile_config.py+++ b/studio/backend/core/inference/diffusion_compile_config.py@@ -119,6 +119,20 @@ def apply() -> None:for (module_name, attr), value in knobs:cfg = _module(module_name)if cfg is not None:+ if (module_name, attr) == ("torch.compiler.config", "cache_key_tag"):+ current = getattr(cfg, attr, "")+ value = ";".join(+ [+ p+ for p in _cache_tag_parts(current)+ if p and not p.startswith(_STUDIO_CACHE_TAG_PREFIXES)+ ]+ + [+ p+ for p in _cache_tag_parts(value)+ if p.startswith(_STUDIO_CACHE_TAG_PREFIXES)+ ]+ )_write(cfg, attr, value)_applied.set(generation)@@ -143,11 +157,35 @@ def _reset_for_tests() -> None:_STUDIO_CACHE_TAG_PREFIXES = ("unsloth_fp8_unit_scale=", "unsloth_nvfp4_act_quant=")+def _cache_tag_parts(tag: str) -> list[str]:+ # Unsloth/Zoo append comma-delimited kernel and correctness tags.+ return tag.replace(",", ";").split(";")++def canonical_compile_cache_tag(current: str, prefix: str, tag: str | None) -> str:"""``prefix``'s token set to ``tag`` (None drops it); Studio's sorted so setter order never changes torch's cache key."""- parts = [p for p in current.split(";") if p and not p.startswith(prefix)]+ parts = [p for p in _cache_tag_parts(current) if p and not p.startswith(prefix)]if tag:parts.append(prefix + tag)foreign = [p for p in parts if not p.startswith(_STUDIO_CACHE_TAG_PREFIXES)]owned = sorted(p for p in parts if p.startswith(_STUDIO_CACHE_TAG_PREFIXES))return ";".join(foreign + owned)+++def set_compile_cache_tag(prefix: str, tag: str | None) -> None:+ cfg = _module("torch.compiler.config")+ if cfg is None:+ return+ current = getattr(cfg, "cache_key_tag", None)+ if not isinstance(current, str):+ return+ # Studio's patches outlive the load context. Keep their recorded tokens while+ # preserving foreign tokens from the caller's context.+ recorded_tag = get_knob("torch.compiler.config", "cache_key_tag", current)+ current = ";".join(+ [p for p in _cache_tag_parts(current) if not p.startswith(_STUDIO_CACHE_TAG_PREFIXES)]+ + [p for p in _cache_tag_parts(recorded_tag) if p.startswith(_STUDIO_CACHE_TAG_PREFIXES)]+ )+ set_knob(+ "torch.compiler.config", "cache_key_tag", canonical_compile_cache_tag(current, prefix, tag)+ )diff --git a/studio/backend/core/inference/diffusion_fp8_unit_scale.py b/studio/backend/core/inference/diffusion_fp8_unit_scale.pyindex bda64dbdf78..d3dfc51ffbb 100644--- a/studio/backend/core/inference/diffusion_fp8_unit_scale.py+++ b/studio/backend/core/inference/diffusion_fp8_unit_scale.py@@ -98,13 +98,12 @@ def _set_compile_cache_tag(tag: Optional[str]) -> None:import torch.compiler.config as _ccexcept Exception: # noqa: BLE001 - torch without compiler config: no persistent compile cache to splitreturn- current = getattr(_cc, "cache_key_tag", None)- if not isinstance(current, str):+ if not isinstance(getattr(_cc, "cache_key_tag", None), str):return- from .diffusion_compile_config import canonical_compile_cache_tag+ from .diffusion_compile_config import set_compile_cache_tagtry:- _cc.cache_key_tag = canonical_compile_cache_tag(current, _CACHE_TAG_PREFIX, tag)+ set_compile_cache_tag(_CACHE_TAG_PREFIX, tag)except Exception: # noqa: BLE001pass@@ -126,6 +125,8 @@ def install(logger: Any = None, *, fast_accum: Optional[bool] = None) -> bool:"""Patch torchao's fp8 addmm with the dispatcher; idempotent, False when off / no qualifying GPU / layout moved /it cannot engage."""if _STATE["installed"]:+ from .diffusion_compile_config import apply+ apply()return Truemode = _mode()if mode == "off" or not can_engage(fast_accum):diff --git a/studio/backend/core/inference/diffusion_nvfp4_act_quant.py b/studio/backend/core/inference/diffusion_nvfp4_act_quant.pyindex fe9e462ff67..cd229fe1ca3 100644--- a/studio/backend/core/inference/diffusion_nvfp4_act_quant.py+++ b/studio/backend/core/inference/diffusion_nvfp4_act_quant.py@@ -125,13 +125,12 @@ def _set_compile_cache_tag(tag: str | None) -> None:import torch.compiler.config as _ccexcept Exception: # noqa: BLE001 - torch without compiler config: no persistent compile cache to splitreturn- current = getattr(_cc, "cache_key_tag", None)- if not isinstance(current, str):+ if not isinstance(getattr(_cc, "cache_key_tag", None), str):return- from .diffusion_compile_config import canonical_compile_cache_tag+ from .diffusion_compile_config import set_compile_cache_tagtry:- _cc.cache_key_tag = canonical_compile_cache_tag(current, _CACHE_TAG_PREFIX, tag)+ set_compile_cache_tag(_CACHE_TAG_PREFIX, tag)except Exception: # noqa: BLE001passdiff --git a/studio/backend/tests/test_diffusion_cache_tag_contexts.py b/studio/backend/tests/test_diffusion_cache_tag_contexts.pynew file mode 100644index 00000000000..e93c6b4c940--- /dev/null+++ b/studio/backend/tests/test_diffusion_cache_tag_contexts.py@@ -0,0 +1,155 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++from __future__ import annotations++import threading++import pytest++pytest.importorskip("torch")+import torch.compiler.config as cc # noqa: E402++from core.inference import diffusion_compile_config as compile_config # noqa: E402+from core.inference import diffusion_fp8_unit_scale as fp8 # noqa: E402+from core.inference import diffusion_nvfp4_act_quant as nvfp4 # noqa: E402++[email protected](autouse = True)+def _clean():+ saved = cc.cache_key_tag+ fp8.uninstall()+ compile_config._reset_for_tests()+ yield+ fp8.uninstall()+ compile_config._reset_for_tests()+ cc.cache_key_tag = saved+++def _in_thread(fn):+ out = {}++ def run():+ try:+ out["value"] = fn()+ except BaseException as exc:+ out["error"] = exc++ t = threading.Thread(target = run)+ t.start()+ t.join()+ if "error" in out:+ raise out["error"]+ return out["value"]+++def _render_context_tag():+ def read():+ compile_config.apply()+ return cc.cache_key_tag++ return _in_thread(read)+++def test_kill_switch_tag_set_on_the_load_thread_reaches_the_render_thread(monkeypatch):+ monkeypatch.setenv(nvfp4.FAST_ACT_QUANT_ENV, "0")+ assert _in_thread(lambda: (nvfp4.install_nvfp4_act_quant_patch(), cc.cache_key_tag)[1]) == (+ "unsloth_nvfp4_act_quant=0"+ )+ assert _render_context_tag() == "unsloth_nvfp4_act_quant=0"+++def test_fp8_and_nvfp4_tags_set_on_the_load_thread_reach_the_render_thread():+ _in_thread(+ lambda: (fp8._set_compile_cache_tag("256"), nvfp4._set_compile_cache_tag(nvfp4._TAG_ON))+ )+ assert _render_context_tag() == "unsloth_fp8_unit_scale=256;unsloth_nvfp4_act_quant=1"+++def test_reset_on_a_later_load_reaches_the_render_thread():+ _in_thread(lambda: nvfp4._set_compile_cache_tag(nvfp4._TAG_OFF))+ assert _render_context_tag() == "unsloth_nvfp4_act_quant=0"+ _in_thread(nvfp4.reset_compile_cache_tag)+ assert _render_context_tag() == ""+++def test_render_context_keeps_its_foreign_cache_tokens():+ def load():+ cc.cache_key_tag = "loader-owned"+ fp8._set_compile_cache_tag("256")++ def render():+ cc.cache_key_tag = "render-owned"+ compile_config.apply()+ return cc.cache_key_tag++ _in_thread(load)+ assert _in_thread(render) == "render-owned;unsloth_fp8_unit_scale=256"+++def test_setter_preserves_appended_correctness_tag():+ nvfp4._set_compile_cache_tag("0")+ cc.cache_key_tag += ",unsloth/inductor-rand4x-off"+ fp8._set_compile_cache_tag("256")+ assert "unsloth/inductor-rand4x-off" in cc.cache_key_tag+ nvfp4.reset_compile_cache_tag()+ assert cc.cache_key_tag == "unsloth/inductor-rand4x-off;unsloth_fp8_unit_scale=256"+++def test_bundle_fingerprint_uses_the_render_context_tag(monkeypatch):+ from core.inference import diffusion_compile_cache as cache++ monkeypatch.setattr(cache, "_triton_version", lambda: None)+ monkeypatch.setattr(cache, "_diffusers_version", lambda: None)+ _in_thread(lambda: fp8._set_compile_cache_tag("256"))++ def request():+ cc.cache_key_tag = "render-owned"+ return cache.environment_fingerprint().get("torch_cache_key_tag")++ assert _in_thread(request) == "render-owned;unsloth_fp8_unit_scale=256"+++def test_nvfp4_reset_in_a_fresh_context_keeps_the_recorded_fp8_tag():+ _in_thread(lambda: (fp8._set_compile_cache_tag("256"), nvfp4._set_compile_cache_tag("1")))+ _in_thread(nvfp4.reset_compile_cache_tag)+ assert _render_context_tag() == "unsloth_fp8_unit_scale=256"+++def test_real_fp8_installer_keeps_the_tag_across_two_loads(monkeypatch):+ f8t = pytest.importorskip("torchao.quantization.quantize_.workflows.float8.float8_tensor")+ monkeypatch.delenv(fp8.UNIT_SCALE_ENV, raising = False)+ monkeypatch.delenv(fp8.MIN_ROWS_ENV, raising = False)+ monkeypatch.setattr(fp8, "_device_caps", lambda: {0: (8, 9)})++ def load():+ nvfp4.reset_compile_cache_tag()+ return fp8.install()++ assert _in_thread(load) is True+ wrapped = f8t.addmm_float8_unwrapped_inference+ assert _render_context_tag() == "unsloth_fp8_unit_scale=256"+ assert _in_thread(load) is True+ assert fp8.stats()["installed"] is True+ assert f8t.addmm_float8_unwrapped_inference is wrapped+ assert _render_context_tag() == "unsloth_fp8_unit_scale=256"++[email protected]("mode, rows, tag", [("auto", "256", "256"), ("force", "512", "512:force")])+def test_repeated_install_restores_the_installed_tag_in_a_fresh_context(+ monkeypatch, mode, rows, tag+):+ f8t = pytest.importorskip("torchao.quantization.quantize_.workflows.float8.float8_tensor")+ monkeypatch.setenv(fp8.UNIT_SCALE_ENV, mode)+ monkeypatch.setenv(fp8.MIN_ROWS_ENV, rows)+ monkeypatch.setattr(fp8, "_device_caps", lambda: {0: (8, 9)})+ assert _in_thread(fp8.install) is True+ wrapped = f8t.addmm_float8_unwrapped_inference+ # These settings are captured by the installed closure, so later env changes must not relabel it.+ monkeypatch.setenv(fp8.UNIT_SCALE_ENV, "0")+ monkeypatch.setenv(fp8.MIN_ROWS_ENV, "1024")+ installed, current = _in_thread(lambda: (fp8.install(), cc.cache_key_tag))+ assert installed is True+ assert current == f"unsloth_fp8_unit_scale={tag}"+ assert _render_context_tag() == current+ assert f8t.addmm_float8_unwrapped_inference is wrappeddiff --git a/studio/backend/tests/test_diffusion_cache_tag_order.py b/studio/backend/tests/test_diffusion_cache_tag_order.pyindex 55581db554d..ea974717bd4 100644--- a/studio/backend/tests/test_diffusion_cache_tag_order.py+++ b/studio/backend/tests/test_diffusion_cache_tag_order.py@@ -10,6 +10,7 @@ import pytestpytest.importorskip("torch")import torch.compiler.config as cc # noqa: E402+from core.inference import diffusion_compile_config as compile_config # noqa: E402from core.inference import diffusion_fp8_unit_scale as fp8 # noqa: E402from core.inference import diffusion_nvfp4_act_quant as nvfp4 # noqa: E402@@ -17,7 +18,9 @@ from core.inference import diffusion_nvfp4_act_quant as nvfp4 # noqa: E402@pytest.fixture(autouse = True)def _restore_tag():saved = cc.cache_key_tag+ compile_config._reset_for_tests()yield+ compile_config._reset_for_tests()cc.cache_key_tag = saveddiff --git a/studio/backend/tests/test_diffusion_fp8_unit_scale.py b/studio/backend/tests/test_diffusion_fp8_unit_scale.pyindex a6c9b6b4dc4..13acdce01a1 100644--- a/studio/backend/tests/test_diffusion_fp8_unit_scale.py+++ b/studio/backend/tests/test_diffusion_fp8_unit_scale.py@@ -11,12 +11,14 @@ import torchpytest.importorskip("torchao")from torchao.quantization.quantize_.workflows.float8 import float8_tensor as f8t # noqa: E402+from core.inference import diffusion_compile_config as compile_config # noqa: E402from core.inference import diffusion_fp8_unit_scale as us # noqa: E402@pytest.fixture(autouse = True)def _clean(monkeypatch):us.uninstall()+ compile_config._reset_for_tests()monkeypatch.delenv(us.UNIT_SCALE_ENV, raising = False)monkeypatch.delenv(us.MIN_ROWS_ENV, raising = False)for k in us._COUNTS:@@ -24,6 +26,7 @@ def _clean(monkeypatch):original = f8t.addmm_float8_unwrapped_inferenceyieldus.uninstall()+ compile_config._reset_for_tests()f8t.addmm_float8_unwrapped_inference = original