Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio diffusion: keep the compiled int8 W8A8 linear at or below the eager peak, honor force-eager (#15456)

Daniel Han7309599e1c66
diff --git a/studio/backend/core/inference/diffusion_native_quant.py b/studio/backend/core/inference/diffusion_native_quant.py
index b35b668d4f1..ced05c97b04 100644
--- a/studio/backend/core/inference/diffusion_native_quant.py
+++ b/studio/backend/core/inference/diffusion_native_quant.py
@@ -31,8 +31,8 @@ _INT8PACK_PROBED: dict = {}
# torch._int_mm needs M > 16 and K, N multiples of 8.
_INT_MM_MIN_ROWS = 17
_INT_MM_CHUNK_ELEMS = 1 << 25
-# Compiled linear holds only the int32 accumulator (no fp32 temporaries), so 4x the eager chunk.
-_INT_MM_FUSED_CHUNK_ELEMS = 1 << 27
+# Compiled linear holds int32 acc + output chunk (6 B/elem); eager's row budget keeps its peak <= eager's (#15183 used 4x).
+_INT_MM_FUSED_CHUNK_ELEMS = _INT_MM_CHUNK_ELEMS
# ``0`` keeps eager CUDA W8A8 kernels instead of one compiled function per linear.
NATIVE_INT8_FUSED_ENV = "UNSLOTH_NATIVE_INT8_FUSED"
# fp8 codes a native fp8 layer may hold; e5m2 only from a checkpoint (the quantise path writes e4m3fn).
@@ -505,6 +505,11 @@ def native_linear_class():
x.is_cuda and torch.cuda.is_current_stream_capturing()
):
return None
+ from .diffusion_bg_compile import eager_forced # noqa: PLC0415
+
+ # Detailer crops and in-flight background compiles must not compile on this thread (#15063).
+ if eager_forced():
+ return None
fn = _fused_w8a8()
if fn is None:
return None
@@ -517,12 +522,17 @@ def native_linear_class():
if rows <= chunk or chunk < 2 * _INT_MM_MIN_ROWS:
out = fn(x2, rot_h, self.weight_q, self.weight_scale, bias)
else:
- out = torch.cat(
- [
- fn(part.detach(), rot_h, self.weight_q, self.weight_scale, bias)
- for part in torch.tensor_split(x2, -(-rows // chunk))
- ]
- )
+ # Allocated from the first chunk to keep the compiled dtype (autocast can differ from x2's).
+ out = None
+ start = 0
+ for part in torch.tensor_split(x2, -(-rows // chunk)):
+ got = fn(part.detach(), rot_h, self.weight_q, self.weight_scale, bias)
+ if out is None:
+ out = got.new_empty(rows, self.out_features)
+ stop = start + part.shape[0]
+ out[start:stop].copy_(got)
+ del got
+ start = stop
except torch.cuda.OutOfMemoryError:
raise
except Exception as exc: # noqa: BLE001 - a compile failure falls back to the eager kernels
diff --git a/studio/backend/tests/test_diffusion_native_quant_fused.py b/studio/backend/tests/test_diffusion_native_quant_fused.py
index fa3f2a1eae2..84621b4447e 100644
--- a/studio/backend/tests/test_diffusion_native_quant_fused.py
+++ b/studio/backend/tests/test_diffusion_native_quant_fused.py
@@ -5,12 +5,19 @@
from __future__ import annotations
+import sys
+from pathlib import Path
+
import pytest
torch = pytest.importorskip("torch")
+import core.inference.diffusion_bg_compile as bg
import core.inference.diffusion_native_quant as nq
+sys.path.insert(0, str(Path(__file__).resolve().parents[3] / "tests" / "_shared"))
+from real_accelerator import has_real_cuda # noqa: E402
+
@pytest.fixture(autouse = True)
def fresh_state(monkeypatch):
@@ -137,6 +144,45 @@ def test_large_inputs_are_chunked_on_the_accumulator(monkeypatch):
assert sum(rows) == 100 and max(rows) <= 40
+def test_chunks_fill_one_output_in_row_order(monkeypatch):
+ layer = _layer()
+ monkeypatch.setattr(nq, "_INT_MM_FUSED_CHUNK_ELEMS", 512 * 40)
+ nq._FUSED_STATE["fn"] = lambda x2, *_a: x2[:, :128] * 2
+ x = torch.randn(100, 512, dtype = torch.bfloat16)
+ assert torch.equal(layer._forward_fused(x), x[:, :128] * 2)
+
+
[email protected]("in_f, out_f", [(3072, 14336), (14336, 3072), (512, 1536), (5120, 5120)])
+def test_fused_chunk_never_outgrows_the_eager_chunk(in_f, out_f):
+ # Same row budget as eager keeps the compiled peak <= eager's (#15183 used 4x and OOMed).
+ wide = max(in_f, out_f)
+ fused_rows = nq._INT_MM_FUSED_CHUNK_ELEMS // wide
+ eager_rows = nq._INT_MM_CHUNK_ELEMS // wide
+ assert fused_rows <= eager_rows
+ for rows in (eager_rows - 1, eager_rows + 1, 3 * eager_rows + 7, 9 * eager_rows):
+ if rows <= fused_rows:
+ fused, eager_peak = 6 * rows * out_f, 8 * rows * out_f
+ else:
+ n_f, n_e = -(-rows // fused_rows), -(-rows // eager_rows)
+ fused = 2 * rows * out_f + 6 * -(-rows // n_f) * out_f
+ last = rows - (n_e - 1) * (rows // n_e)
+ eager_peak = max(2 * (rows - last) * out_f + 8 * last * out_f, 4 * rows * out_f)
+ assert fused <= eager_peak, (rows, fused, eager_peak)
+
+
+def test_eager_forced_pass_never_enters_the_compiled_linear(monkeypatch):
+ layer = _layer()
+ monkeypatch.setattr(
+ nq, "_fused_w8a8", lambda: pytest.fail("compiled linear inside a force-eager pass")
+ )
+ x = torch.randn(64, 512, dtype = torch.bfloat16)
+ with bg.auxiliary_pass():
+ assert layer._forward_fused(x) is None
+ with bg.force_eager():
+ assert layer._forward_fused(x) is None
+ assert nq.fused_w8a8_state()["calls"] == 0
+
+
@pytest.mark.skipif(not _int_mm_on_cpu(), reason = "torch._int_mm has no CPU kernel in this build")
@pytest.mark.parametrize("rot", [256, 0])
@pytest.mark.parametrize("bias", [True, False])
@@ -153,7 +199,7 @@ def test_fused_maths_tracks_the_eager_kernels(rot, bias):
assert ((got.float() - ref.float()).norm() / ref.float().norm()).item() < 2e-2
[email protected](not torch.cuda.is_available(), reason = "needs CUDA torch._int_mm and inductor")
[email protected](not has_real_cuda(), reason = "needs CUDA torch._int_mm and inductor")
@pytest.mark.parametrize("rot", [256, 0])
def test_cuda_compiled_linear_is_bit_identical_to_eager(monkeypatch, rot):
torch.manual_seed(0)
@@ -180,3 +226,45 @@ def test_cuda_compiled_linear_is_bit_identical_to_eager(monkeypatch, rot):
assert nq.fused_w8a8_state()["calls"] == calls + 1
assert torch.equal(got, ref)
assert nq.fused_w8a8_state()["failed"] is None
+
+
[email protected](not has_real_cuda(), reason = "needs CUDA torch._int_mm and inductor")
[email protected]("rot", [256, 0])
+def test_cuda_compiled_linear_peak_stays_at_or_below_eager(monkeypatch, rot):
+ # 8192 rows x 16384 outputs: eager takes four chunks, the old 4x budget one call (768 vs 576 MiB).
+ torch.manual_seed(0)
+ layer = nq.native_linear_class()(
+ torch.nn.Linear(512, 16384).to("cuda", torch.bfloat16), "int8", act_int8 = True, rot_group = rot
+ )
+ x = torch.randn(8192, 512, device = "cuda", dtype = torch.bfloat16)
+ torch._dynamo.reset()
+
+ def peak():
+ layer(x)
+ torch.cuda.synchronize()
+ base = torch.cuda.memory_allocated()
+ torch.cuda.reset_peak_memory_stats()
+ out = layer(x)
+ torch.cuda.synchronize()
+ return out, torch.cuda.max_memory_allocated() - base
+
+ with torch.no_grad():
+ monkeypatch.setenv(nq.NATIVE_INT8_FUSED_ENV, "0")
+ ref, eager_peak = peak()
+ monkeypatch.delenv(nq.NATIVE_INT8_FUSED_ENV)
+ calls = nq.fused_w8a8_state()["calls"]
+ got, fused_peak = peak()
+ assert nq.fused_w8a8_state()["calls"] == calls + 2
+ assert torch.equal(got, ref)
+ assert fused_peak <= eager_peak, (fused_peak, eager_peak)
+
+
+def test_chunked_output_keeps_the_compiled_dtype(monkeypatch):
+ # Under autocast the compiled function can return another dtype than the input; chunking must not recast it.
+ layer = _layer()
+ monkeypatch.setattr(nq, "_INT_MM_FUSED_CHUNK_ELEMS", 512 * 40)
+ nq._FUSED_STATE["fn"] = lambda x2, *_a: (x2[:, :128] * 2).to(torch.float16)
+ x = torch.randn(100, 512, dtype = torch.float32)
+ got = layer._forward_fused(x)
+ assert got.dtype == torch.float16
+ assert torch.equal(got, (x[:, :128] * 2).to(torch.float16))