Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio diffusion: fuse the SwiGLU product into the down projection's act quant on untuned sm80+ archs (#16785)
Daniel Han268f414d6e68
diff --git a/studio/backend/core/inference/diffusion_convrot_quant.py b/studio/backend/core/inference/diffusion_convrot_quant.pyindex 8f59e662f29..d59b031272d 100644--- a/studio/backend/core/inference/diffusion_convrot_quant.py+++ b/studio/backend/core/inference/diffusion_convrot_quant.py@@ -1008,6 +1008,7 @@ INT8_ROTQUANT_UNLISTED_ENV = "UNSLOTH_DIFFUSION_INT8_ROTQUANT_UNLISTED"_UNLISTED_DEVICE: dict = {}_UNLISTED_TIMING: dict = {}_UNLISTED_PROBE: dict = {} # index -> _h16_probe result (mode independent)+_UNLISTED_SWIGLU: dict = {} # index -> _swiglu_probe result on an unlisted card_UNLISTED_LATENCY_SHAPES = ((2048, 4096), (1024, 12288))@@ -1090,7 +1091,7 @@ def _unlisted_latency(def unlisted_device_ok(index: int, stock_quant: Any = None) -> bool:"""Once per (device, mode), outside any trace / capture: arch gate, ``_h16_probe`` (once per device), latencycheck (auto only). Keyed by mode so switching ``..._UNLISTED`` between loads takes effect without a restart."""- global _ROTQ_HANDLE+ global _ROTQ_HANDLE, _SWIGLU_HANDLEmode = unlisted_mode()if mode == "off": # read per load: the switches act without a restartreturn False@@ -1124,6 +1125,19 @@ def unlisted_device_ok(index: int, stock_quant: Any = None) -> bool:if ok:_H16_DEVICE[index] = h16_ROTQ_HANDLE = _rotq_op()+ # SwiGLU prologue fused into rotq_h16; unlisted cards probe it once per device+ sw = False+ if rotquant_swiglu_enabled() and _swiglu_op() is not None:+ if index not in _UNLISTED_SWIGLU:+ try:+ _UNLISTED_SWIGLU[index] = bool(_swiglu_probe(index))+ except Exception: # noqa: BLE001 - no prologue+ _UNLISTED_SWIGLU[index] = False+ sw = _UNLISTED_SWIGLU[index]+ _SWIGLU_DEVICE[index] = sw+ if sw:+ _SWIGLU_HANDLE = _swiglu_op()elif h16: # probed unlisted card the latency check now rejects: drop a verdict a forced load left behind_H16_DEVICE.pop(index, None)+ _SWIGLU_DEVICE.pop(index, None)return okdiff --git a/studio/backend/tests/test_diffusion_convrot_unlisted.py b/studio/backend/tests/test_diffusion_convrot_unlisted.pyindex d31e5e4df4a..db9b7cbf171 100644--- a/studio/backend/tests/test_diffusion_convrot_unlisted.py+++ b/studio/backend/tests/test_diffusion_convrot_unlisted.py@@ -35,14 +35,20 @@ def _clean(monkeypatch):cq._UNLISTED_DEVICE.clear()cq._UNLISTED_TIMING.clear()cq._UNLISTED_PROBE.clear()+ cq._UNLISTED_SWIGLU.clear()saved = dict(cq._H16_DEVICE)+ saved_sw = dict(cq._SWIGLU_DEVICE)cq._H16_DEVICE.clear() # a real-card verdict from an earlier test file must not leak in+ cq._SWIGLU_DEVICE.clear()yieldcq._UNLISTED_DEVICE.clear()cq._UNLISTED_TIMING.clear()cq._UNLISTED_PROBE.clear()+ cq._UNLISTED_SWIGLU.clear()cq._H16_DEVICE.clear()cq._H16_DEVICE.update(saved)+ cq._SWIGLU_DEVICE.clear()+ cq._SWIGLU_DEVICE.update(saved_sw)@pytest.mark.parametrize(@@ -334,3 +340,52 @@ def test_compiled_no_graph_break_no_recompile_and_cuda_graph(armed):graph.replay()torch.cuda.synchronize()assert torch.equal(out, fn(static))+++def _swiglu_fakes(+ monkeypatch,+ timing,+ sw = True,+):+ _fake_device(monkeypatch, (8, 6))+ monkeypatch.setattr(cq, "_h16_probe", lambda index: "ieee")+ monkeypatch.setattr(cq, "_unlisted_latency", lambda index, tf32x3, stock = None: timing)+ monkeypatch.setattr(cq, "_swiglu_op", lambda: "swiglu")+ monkeypatch.setattr(cq, "_SWIGLU_HANDLE", None)+ probes = []+ monkeypatch.setattr(cq, "_swiglu_probe", lambda index: probes.append(index) or sw)+ return probes++[email protected]("sw", [True, False])+def test_unlisted_card_probes_the_swiglu_prologue_once(monkeypatch, sw):+ probes = _swiglu_fakes(monkeypatch, {"h16": 10.0, "stock": 20.0}, sw = sw)+ assert cq.unlisted_device_ok(0)+ assert cq.unlisted_device_ok(0)+ assert probes == [0] # once per device+ assert cq._SWIGLU_DEVICE[0] is sw+ assert (cq._SWIGLU_HANDLE == "swiglu") is sw+++def test_swiglu_prologue_needs_the_latency_check(monkeypatch):+ probes = _swiglu_fakes(monkeypatch, {"h16": 30.0, "stock": 20.0})+ assert not cq.unlisted_device_ok(0)+ assert probes == [] and not cq._SWIGLU_DEVICE.get(0)+++def test_swiglu_kill_switch_skips_the_prologue_probe(monkeypatch):+ monkeypatch.setenv(cq.INT8_ROTQUANT_SWIGLU_ENV, "0")+ probes = _swiglu_fakes(monkeypatch, {"h16": 10.0, "stock": 20.0})+ assert cq.unlisted_device_ok(0)+ assert probes == [] and cq._SWIGLU_DEVICE[0] is False+++def test_failed_swiglu_probe_keeps_rotq_h16(monkeypatch):+ _swiglu_fakes(monkeypatch, {"h16": 10.0, "stock": 20.0})++ def boom(index):+ raise RuntimeError("compile failed")++ monkeypatch.setattr(cq, "_swiglu_probe", boom)+ assert cq.unlisted_device_ok(0)+ assert cq._H16_DEVICE[0] == "ieee" and cq._SWIGLU_DEVICE[0] is False