Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio diffusion: keep the compiled int8 W8A8 linear at or below the eager peak, honor force-eager (#15456)
Daniel Han7309599e1c66
diff --git a/studio/backend/core/inference/diffusion_native_quant.py b/studio/backend/core/inference/diffusion_native_quant.pyindex b35b668d4f1..ced05c97b04 100644--- a/studio/backend/core/inference/diffusion_native_quant.py+++ b/studio/backend/core/inference/diffusion_native_quant.py@@ -31,8 +31,8 @@ _INT8PACK_PROBED: dict = {}# torch._int_mm needs M > 16 and K, N multiples of 8._INT_MM_MIN_ROWS = 17_INT_MM_CHUNK_ELEMS = 1 << 25-# Compiled linear holds only the int32 accumulator (no fp32 temporaries), so 4x the eager chunk.-_INT_MM_FUSED_CHUNK_ELEMS = 1 << 27+# Compiled linear holds int32 acc + output chunk (6 B/elem); eager's row budget keeps its peak <= eager's (#15183 used 4x).+_INT_MM_FUSED_CHUNK_ELEMS = _INT_MM_CHUNK_ELEMS# ``0`` keeps eager CUDA W8A8 kernels instead of one compiled function per linear.NATIVE_INT8_FUSED_ENV = "UNSLOTH_NATIVE_INT8_FUSED"# fp8 codes a native fp8 layer may hold; e5m2 only from a checkpoint (the quantise path writes e4m3fn).@@ -505,6 +505,11 @@ def native_linear_class():x.is_cuda and torch.cuda.is_current_stream_capturing()):return None+ from .diffusion_bg_compile import eager_forced # noqa: PLC0415++ # Detailer crops and in-flight background compiles must not compile on this thread (#15063).+ if eager_forced():+ return Nonefn = _fused_w8a8()if fn is None:return None@@ -517,12 +522,17 @@ def native_linear_class():if rows <= chunk or chunk < 2 * _INT_MM_MIN_ROWS:out = fn(x2, rot_h, self.weight_q, self.weight_scale, bias)else:- out = torch.cat(- [- fn(part.detach(), rot_h, self.weight_q, self.weight_scale, bias)- for part in torch.tensor_split(x2, -(-rows // chunk))- ]- )+ # Allocated from the first chunk to keep the compiled dtype (autocast can differ from x2's).+ out = None+ start = 0+ for part in torch.tensor_split(x2, -(-rows // chunk)):+ got = fn(part.detach(), rot_h, self.weight_q, self.weight_scale, bias)+ if out is None:+ out = got.new_empty(rows, self.out_features)+ stop = start + part.shape[0]+ out[start:stop].copy_(got)+ del got+ start = stopexcept torch.cuda.OutOfMemoryError:raiseexcept Exception as exc: # noqa: BLE001 - a compile failure falls back to the eager kernelsdiff --git a/studio/backend/tests/test_diffusion_native_quant_fused.py b/studio/backend/tests/test_diffusion_native_quant_fused.pyindex fa3f2a1eae2..84621b4447e 100644--- a/studio/backend/tests/test_diffusion_native_quant_fused.py+++ b/studio/backend/tests/test_diffusion_native_quant_fused.py@@ -5,12 +5,19 @@from __future__ import annotations+import sys+from pathlib import Path+import pytesttorch = pytest.importorskip("torch")+import core.inference.diffusion_bg_compile as bgimport core.inference.diffusion_native_quant as nq+sys.path.insert(0, str(Path(__file__).resolve().parents[3] / "tests" / "_shared"))+from real_accelerator import has_real_cuda # noqa: E402+@pytest.fixture(autouse = True)def fresh_state(monkeypatch):@@ -137,6 +144,45 @@ def test_large_inputs_are_chunked_on_the_accumulator(monkeypatch):assert sum(rows) == 100 and max(rows) <= 40+def test_chunks_fill_one_output_in_row_order(monkeypatch):+ layer = _layer()+ monkeypatch.setattr(nq, "_INT_MM_FUSED_CHUNK_ELEMS", 512 * 40)+ nq._FUSED_STATE["fn"] = lambda x2, *_a: x2[:, :128] * 2+ x = torch.randn(100, 512, dtype = torch.bfloat16)+ assert torch.equal(layer._forward_fused(x), x[:, :128] * 2)++[email protected]("in_f, out_f", [(3072, 14336), (14336, 3072), (512, 1536), (5120, 5120)])+def test_fused_chunk_never_outgrows_the_eager_chunk(in_f, out_f):+ # Same row budget as eager keeps the compiled peak <= eager's (#15183 used 4x and OOMed).+ wide = max(in_f, out_f)+ fused_rows = nq._INT_MM_FUSED_CHUNK_ELEMS // wide+ eager_rows = nq._INT_MM_CHUNK_ELEMS // wide+ assert fused_rows <= eager_rows+ for rows in (eager_rows - 1, eager_rows + 1, 3 * eager_rows + 7, 9 * eager_rows):+ if rows <= fused_rows:+ fused, eager_peak = 6 * rows * out_f, 8 * rows * out_f+ else:+ n_f, n_e = -(-rows // fused_rows), -(-rows // eager_rows)+ fused = 2 * rows * out_f + 6 * -(-rows // n_f) * out_f+ last = rows - (n_e - 1) * (rows // n_e)+ eager_peak = max(2 * (rows - last) * out_f + 8 * last * out_f, 4 * rows * out_f)+ assert fused <= eager_peak, (rows, fused, eager_peak)+++def test_eager_forced_pass_never_enters_the_compiled_linear(monkeypatch):+ layer = _layer()+ monkeypatch.setattr(+ nq, "_fused_w8a8", lambda: pytest.fail("compiled linear inside a force-eager pass")+ )+ x = torch.randn(64, 512, dtype = torch.bfloat16)+ with bg.auxiliary_pass():+ assert layer._forward_fused(x) is None+ with bg.force_eager():+ assert layer._forward_fused(x) is None+ assert nq.fused_w8a8_state()["calls"] == 0++@pytest.mark.skipif(not _int_mm_on_cpu(), reason = "torch._int_mm has no CPU kernel in this build")@pytest.mark.parametrize("rot", [256, 0])@pytest.mark.parametrize("bias", [True, False])@@ -153,7 +199,7 @@ def test_fused_maths_tracks_the_eager_kernels(rot, bias):assert ((got.float() - ref.float()).norm() / ref.float().norm()).item() < 2e-2[email protected](not torch.cuda.is_available(), reason = "needs CUDA torch._int_mm and inductor")[email protected](not has_real_cuda(), reason = "needs CUDA torch._int_mm and inductor")@pytest.mark.parametrize("rot", [256, 0])def test_cuda_compiled_linear_is_bit_identical_to_eager(monkeypatch, rot):torch.manual_seed(0)@@ -180,3 +226,45 @@ def test_cuda_compiled_linear_is_bit_identical_to_eager(monkeypatch, rot):assert nq.fused_w8a8_state()["calls"] == calls + 1assert torch.equal(got, ref)assert nq.fused_w8a8_state()["failed"] is None++[email protected](not has_real_cuda(), reason = "needs CUDA torch._int_mm and inductor")[email protected]("rot", [256, 0])+def test_cuda_compiled_linear_peak_stays_at_or_below_eager(monkeypatch, rot):+ # 8192 rows x 16384 outputs: eager takes four chunks, the old 4x budget one call (768 vs 576 MiB).+ torch.manual_seed(0)+ layer = nq.native_linear_class()(+ torch.nn.Linear(512, 16384).to("cuda", torch.bfloat16), "int8", act_int8 = True, rot_group = rot+ )+ x = torch.randn(8192, 512, device = "cuda", dtype = torch.bfloat16)+ torch._dynamo.reset()++ def peak():+ layer(x)+ torch.cuda.synchronize()+ base = torch.cuda.memory_allocated()+ torch.cuda.reset_peak_memory_stats()+ out = layer(x)+ torch.cuda.synchronize()+ return out, torch.cuda.max_memory_allocated() - base++ with torch.no_grad():+ monkeypatch.setenv(nq.NATIVE_INT8_FUSED_ENV, "0")+ ref, eager_peak = peak()+ monkeypatch.delenv(nq.NATIVE_INT8_FUSED_ENV)+ calls = nq.fused_w8a8_state()["calls"]+ got, fused_peak = peak()+ assert nq.fused_w8a8_state()["calls"] == calls + 2+ assert torch.equal(got, ref)+ assert fused_peak <= eager_peak, (fused_peak, eager_peak)+++def test_chunked_output_keeps_the_compiled_dtype(monkeypatch):+ # Under autocast the compiled function can return another dtype than the input; chunking must not recast it.+ layer = _layer()+ monkeypatch.setattr(nq, "_INT_MM_FUSED_CHUNK_ELEMS", 512 * 40)+ nq._FUSED_STATE["fn"] = lambda x2, *_a: (x2[:, :128] * 2).to(torch.float16)+ x = torch.randn(100, 512, dtype = torch.float32)+ got = layer._forward_fused(x)+ assert got.dtype == torch.float16+ assert torch.equal(got, (x[:, :128] * 2).to(torch.float16))