Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio diffusion: fuse the SwiGLU product into the down projection's act quant on untuned sm80+ archs (#16785)

Daniel Han268f414d6e68
diff --git a/studio/backend/core/inference/diffusion_convrot_quant.py b/studio/backend/core/inference/diffusion_convrot_quant.py
index 8f59e662f29..d59b031272d 100644
--- a/studio/backend/core/inference/diffusion_convrot_quant.py
+++ b/studio/backend/core/inference/diffusion_convrot_quant.py
@@ -1008,6 +1008,7 @@ INT8_ROTQUANT_UNLISTED_ENV = "UNSLOTH_DIFFUSION_INT8_ROTQUANT_UNLISTED"
_UNLISTED_DEVICE: dict = {}
_UNLISTED_TIMING: dict = {}
_UNLISTED_PROBE: dict = {} # index -> _h16_probe result (mode independent)
+_UNLISTED_SWIGLU: dict = {} # index -> _swiglu_probe result on an unlisted card
_UNLISTED_LATENCY_SHAPES = ((2048, 4096), (1024, 12288))
@@ -1090,7 +1091,7 @@ def _unlisted_latency(
def unlisted_device_ok(index: int, stock_quant: Any = None) -> bool:
"""Once per (device, mode), outside any trace / capture: arch gate, ``_h16_probe`` (once per device), latency
check (auto only). Keyed by mode so switching ``..._UNLISTED`` between loads takes effect without a restart."""
- global _ROTQ_HANDLE
+ global _ROTQ_HANDLE, _SWIGLU_HANDLE
mode = unlisted_mode()
if mode == "off": # read per load: the switches act without a restart
return False
@@ -1124,6 +1125,19 @@ def unlisted_device_ok(index: int, stock_quant: Any = None) -> bool:
if ok:
_H16_DEVICE[index] = h16
_ROTQ_HANDLE = _rotq_op()
+ # SwiGLU prologue fused into rotq_h16; unlisted cards probe it once per device
+ sw = False
+ if rotquant_swiglu_enabled() and _swiglu_op() is not None:
+ if index not in _UNLISTED_SWIGLU:
+ try:
+ _UNLISTED_SWIGLU[index] = bool(_swiglu_probe(index))
+ except Exception: # noqa: BLE001 - no prologue
+ _UNLISTED_SWIGLU[index] = False
+ sw = _UNLISTED_SWIGLU[index]
+ _SWIGLU_DEVICE[index] = sw
+ if sw:
+ _SWIGLU_HANDLE = _swiglu_op()
elif h16: # probed unlisted card the latency check now rejects: drop a verdict a forced load left behind
_H16_DEVICE.pop(index, None)
+ _SWIGLU_DEVICE.pop(index, None)
return ok
diff --git a/studio/backend/tests/test_diffusion_convrot_unlisted.py b/studio/backend/tests/test_diffusion_convrot_unlisted.py
index d31e5e4df4a..db9b7cbf171 100644
--- a/studio/backend/tests/test_diffusion_convrot_unlisted.py
+++ b/studio/backend/tests/test_diffusion_convrot_unlisted.py
@@ -35,14 +35,20 @@ def _clean(monkeypatch):
cq._UNLISTED_DEVICE.clear()
cq._UNLISTED_TIMING.clear()
cq._UNLISTED_PROBE.clear()
+ cq._UNLISTED_SWIGLU.clear()
saved = dict(cq._H16_DEVICE)
+ saved_sw = dict(cq._SWIGLU_DEVICE)
cq._H16_DEVICE.clear() # a real-card verdict from an earlier test file must not leak in
+ cq._SWIGLU_DEVICE.clear()
yield
cq._UNLISTED_DEVICE.clear()
cq._UNLISTED_TIMING.clear()
cq._UNLISTED_PROBE.clear()
+ cq._UNLISTED_SWIGLU.clear()
cq._H16_DEVICE.clear()
cq._H16_DEVICE.update(saved)
+ cq._SWIGLU_DEVICE.clear()
+ cq._SWIGLU_DEVICE.update(saved_sw)
@pytest.mark.parametrize(
@@ -334,3 +340,52 @@ def test_compiled_no_graph_break_no_recompile_and_cuda_graph(armed):
graph.replay()
torch.cuda.synchronize()
assert torch.equal(out, fn(static))
+
+
+def _swiglu_fakes(
+ monkeypatch,
+ timing,
+ sw = True,
+):
+ _fake_device(monkeypatch, (8, 6))
+ monkeypatch.setattr(cq, "_h16_probe", lambda index: "ieee")
+ monkeypatch.setattr(cq, "_unlisted_latency", lambda index, tf32x3, stock = None: timing)
+ monkeypatch.setattr(cq, "_swiglu_op", lambda: "swiglu")
+ monkeypatch.setattr(cq, "_SWIGLU_HANDLE", None)
+ probes = []
+ monkeypatch.setattr(cq, "_swiglu_probe", lambda index: probes.append(index) or sw)
+ return probes
+
+
[email protected]("sw", [True, False])
+def test_unlisted_card_probes_the_swiglu_prologue_once(monkeypatch, sw):
+ probes = _swiglu_fakes(monkeypatch, {"h16": 10.0, "stock": 20.0}, sw = sw)
+ assert cq.unlisted_device_ok(0)
+ assert cq.unlisted_device_ok(0)
+ assert probes == [0] # once per device
+ assert cq._SWIGLU_DEVICE[0] is sw
+ assert (cq._SWIGLU_HANDLE == "swiglu") is sw
+
+
+def test_swiglu_prologue_needs_the_latency_check(monkeypatch):
+ probes = _swiglu_fakes(monkeypatch, {"h16": 30.0, "stock": 20.0})
+ assert not cq.unlisted_device_ok(0)
+ assert probes == [] and not cq._SWIGLU_DEVICE.get(0)
+
+
+def test_swiglu_kill_switch_skips_the_prologue_probe(monkeypatch):
+ monkeypatch.setenv(cq.INT8_ROTQUANT_SWIGLU_ENV, "0")
+ probes = _swiglu_fakes(monkeypatch, {"h16": 10.0, "stock": 20.0})
+ assert cq.unlisted_device_ok(0)
+ assert probes == [] and cq._SWIGLU_DEVICE[0] is False
+
+
+def test_failed_swiglu_probe_keeps_rotq_h16(monkeypatch):
+ _swiglu_fakes(monkeypatch, {"h16": 10.0, "stock": 20.0})
+
+ def boom(index):
+ raise RuntimeError("compile failed")
+
+ monkeypatch.setattr(cq, "_swiglu_probe", boom)
+ assert cq.unlisted_device_ok(0)
+ assert cq._H16_DEVICE[0] == "ieee" and cq._SWIGLU_DEVICE[0] is False