Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: quantise an offloaded native int8 denoiser on the card, one linear at a time (#15936)
Daniel Hanfffb8f5eda88
diff --git a/studio/backend/core/inference/diffusion_native_quant.py b/studio/backend/core/inference/diffusion_native_quant.pyindex 05bc71ac261..c160968fdef 100644--- a/studio/backend/core/inference/diffusion_native_quant.py+++ b/studio/backend/core/inference/diffusion_native_quant.py@@ -6,6 +6,10 @@ hooks' Module.to(); torch imported lazily"""from __future__ import annotations+import contextlib+import contextvars+import os+import timefrom functools import lru_cachefrom typing import Any, Callable, Optional@@ -35,6 +39,11 @@ _INT_MM_FUSED_CHUNK_ELEMS = _INT_MM_CHUNK_ELEMSNATIVE_INT8_FUSED_ENV = "UNSLOTH_NATIVE_INT8_FUSED"# fp8 codes a native fp8 layer may hold; e5m2 only from a checkpoint (the quantise path writes e4m3fn).NATIVE_FP8_FORMATS = ("float8_e4m3fn", "float8_e5m2")+# ``0`` quantises host-resident linears on the host again instead of one at a time on the card.+NATIVE_QUANT_ON_DEVICE_ENV = "UNSLOTH_NATIVE_QUANT_ON_DEVICE"+_HOST_QUANT_DEVICE: contextvars.ContextVar = contextvars.ContextVar(+ "unsloth_native_host_quant_device", default = None+)def fp8_bits_to_float32(bits: Any, fmt: str = "float8_e4m3fn") -> Any:@@ -705,20 +714,25 @@ def apply_native_weight_quant(rot_group = int8_rotation_group() if act_int8 else 0# Names only: holding the old modules would keep every bf16 weight alive until the pass ends.names = [name for name, mod in transformer.named_modules() if name and filter_fn(mod, name)]++ def build(linear: Any) -> Any:+ return cls(+ linear,+ scheme,+ act_int8 = act_int8,+ rot_group = rot_group,+ scale_dtype = scale_dtype if scheme == NATIVE_INT8 and not act_int8 else None,+ )++ device = _HOST_QUANT_DEVICE.get()+ mover = _HostLinearMover(device) if device is not None else Nonefor name in names:parent_name, _, leaf = name.rpartition(".")parent = transformer.get_submodule(parent_name) if parent_name else transformer- setattr(- parent,- leaf,- cls(- getattr(parent, leaf),- scheme,- act_int8 = act_int8,- rot_group = rot_group,- scale_dtype = scale_dtype if scheme == NATIVE_INT8 and not act_int8 else None,- ),- )+ linear = getattr(parent, leaf)+ setattr(parent, leaf, mover.build(linear, build) if mover is not None else build(linear))+ if mover is not None:+ mover.close(logger)if logger is not None:logger.info("diffusion.transformer_quant: %s %s (torchao-free) on %d linears",@@ -731,6 +745,125 @@ def apply_native_weight_quant(return len(names)+def native_quant_on_device_enabled() -> bool:+ return os.environ.get(NATIVE_QUANT_ON_DEVICE_ENV, "").strip().lower() not in (+ "0",+ "false",+ "no",+ "off",+ )+++def quantise_host_linears_on(device: Any):+ """Inside, ``apply_native_weight_quant`` builds each host-resident Linear on ``device`` and returns it to the+ host. Same kernels as a layer staged on the card, so the codes match a fully staged build bit for bit."""+ token = _HOST_QUANT_DEVICE.set(device)+ try:+ yield+ finally:+ _HOST_QUANT_DEVICE.reset(token)+++class _HostLinearMover:+ """A card failure builds that layer and the rest on the host, as without this."""++ def __init__(self, device: Any) -> None:+ import torch++ self.device = torch.device(device)+ self._pinned: dict = {}+ self.layers = 0+ self.up_bytes = 0+ self.started = time.perf_counter()+ self.failed: Optional[str] = None++ def _staging(self, slot: str, nbytes: int) -> Any:+ import torch++ buf = self._pinned.get(slot)+ if buf is None or buf.numel() < nbytes:+ self._pinned[slot] = None+ try:+ buf = torch.empty(nbytes, dtype = torch.uint8, pin_memory = True)+ except Exception: # noqa: BLE001 -- no pinnable memory: pageable copies, same bytes+ buf = torch.empty(0, dtype = torch.uint8)+ self._pinned[slot] = buf+ return buf if buf.numel() >= nbytes else None++ def _to_device(self, tensor: Any) -> Any:+ import torch++ nbytes = tensor.numel() * tensor.element_size()+ buf = self._staging("up", nbytes) if tensor.is_contiguous() else None+ if buf is None:+ return tensor.to(self.device)+ host = buf[:nbytes].view(tensor.dtype).view(tensor.shape)+ host.copy_(tensor)+ out = torch.empty(tensor.shape, dtype = tensor.dtype, device = self.device)+ out.copy_(host, non_blocking = True)+ return out++ def _to_host(self, tensor: Any) -> Any:+ import torch++ nbytes = tensor.numel() * tensor.element_size()+ buf = self._staging("down", nbytes) if tensor.is_contiguous() else None+ if buf is None:+ return tensor.to("cpu")+ host = buf[:nbytes].view(tensor.dtype).view(tensor.shape)+ host.copy_(tensor)+ return torch.empty(tensor.shape, dtype = tensor.dtype).copy_(host)++ def build(self, linear: Any, make: Callable[[Any], Any]) -> Any:+ import torch++ weight = getattr(linear, "weight", None)+ if (+ self.failed is not None+ or not isinstance(weight, torch.nn.Parameter)+ or weight.device.type != "cpu"+ ):+ return make(linear)+ cuda = self.device.type == "cuda"+ try:+ linear.weight = torch.nn.Parameter(+ self._to_device(weight.detach()), requires_grad = False+ )+ built = make(linear)+ for key, buf in list(built._buffers.items()):+ if buf is not None and buf.device.type != "cpu":+ built._buffers[key] = self._to_host(buf)+ # The next upload reuses the pinned buffer.+ if cuda:+ torch.cuda.synchronize(self.device)+ except Exception as exc: # noqa: BLE001 -- out of memory or a device fault: the host build, as before+ self.failed = f"{type(exc).__name__}: {exc}".splitlines()[0][:200]+ linear.weight = weight+ if cuda:+ torch.cuda.empty_cache()+ return make(linear)+ finally:+ if linear.weight is not weight:+ linear.weight = weight+ self.layers += 1+ self.up_bytes += weight.numel() * weight.element_size()+ return built++ def close(self, logger: Any = None) -> None:+ self._pinned.clear()+ if logger is None or not (self.layers or self.failed):+ return+ logger.info(+ "diffusion.transformer_quant: built %d host linear(s) on %s one at a time (%.1f GB in %.1f s)%s",+ self.layers,+ self.device,+ self.up_bytes / 1e9,+ time.perf_counter() - self.started,+ f"; the rest on the host ({self.failed})" if self.failed else "",+ )++def _first_native_linear(module: Any) -> Any:try:return next((sub for sub in module.modules() if is_native_linear(sub)), None)diff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.pyindex 00e74810e4b..b5ef411cca6 100644--- a/studio/backend/core/inference/video.py+++ b/studio/backend/core/inference/video.py@@ -230,7 +230,11 @@ from .diffusion import (_restore_step_cache_after_layer_split,_step_cache_dropped_by_layer_split,)-from .diffusion_native_quant import native_quant_reason+from .diffusion_native_quant import (+ native_quant_on_device_enabled,+ native_quant_reason,+ quantise_host_linears_on,+)from .diffusion_batched import is_oom_errorfrom .diffusion_block_graph import compile_pipe_below_offload_hooksfrom .diffusion_offload_prefetch import (@@ -3773,6 +3777,20 @@ def _stage_denoiser_for_quant(return []+def _host_quant_device(target: Any) -> Any:+ try:+ import torch+ if (+ getattr(target, "device", None) != "cuda"+ or not torch.cuda.is_available()+ or not native_quant_on_device_enabled()+ ):+ return None+ return torch.device(getattr(target, "torch_device", None) or "cuda")+ except Exception: # noqa: BLE001 -- the host build is slower, never wrong+ return None++def _unstage_modules(modules: list) -> None:for module in modules:module.to("cpu")@@ -9457,21 +9475,30 @@ class VideoBackend:and (dense_transformer_supported(target) or native_scheme is not None)):engaged = []+ # An offloaded DiT goes back to the host anyway: build every linear on the card, one at a time.+ host_quant_device = (+ _host_quant_device(target) if video_offload and native_scheme else None+ )for view in views:# Quantise on the card: on the host int8 packing took 59 s (Wan2.2-5B) to 154 s (LTX-2) of the load.- staged = _stage_denoiser_for_quant(- getattr(view, "transformer", None), target, logger = logger+ staged = (+ []+ if host_quant_device is not None+ else _stage_denoiser_for_quant(+ getattr(view, "transformer", None), target, logger = logger+ ))scheme = Nonetry:- scheme = quantize_transformer(- view,- target,- mode = native_scheme or transformer_quant,- family = fam.name,- logger = logger,- **native_kwargs,- )+ with quantise_host_linears_on(host_quant_device):+ scheme = quantize_transformer(+ view,+ target,+ mode = native_scheme or transformer_quant,+ family = fam.name,+ logger = logger,+ **native_kwargs,+ )finally:# A declined quant can roll back to the bf16 offload plan, which must not start with dense blocks staged.if staged and (video_offload or scheme is None):diff --git a/studio/backend/tests/test_video_native_quant_on_device.py b/studio/backend/tests/test_video_native_quant_on_device.pynew file mode 100644index 00000000000..d7e51d567fe--- /dev/null+++ b/studio/backend/tests/test_video_native_quant_on_device.py@@ -0,0 +1,202 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Offloaded native int8 builds: host-resident linears are quantised on the card one at a time."""++from __future__ import annotations++import copy+import inspect+import logging+import types++import pytest++torch = pytest.importorskip("torch")++import core.inference.diffusion_native_quant as nq # noqa: E402++CUDA = torch.cuda.is_available()+++class _Block(torch.nn.Module):+ def __init__(self, width: int) -> None:+ super().__init__()+ self.norm = torch.nn.LayerNorm(width)+ self.q = torch.nn.Linear(width, width)+ self.ff = torch.nn.Linear(width, 2 * width, bias = False)+ self.out = torch.nn.Linear(2 * width, width)+++class _Dit(torch.nn.Module):+ def __init__(+ self,+ width: int = 512,+ depth: int = 3,+ ) -> None:+ super().__init__()+ self.blocks = torch.nn.ModuleList(_Block(width) for _ in range(depth))+ self.head = torch.nn.Linear(width, 64)+++def _dit(seed: int = 0) -> _Dit:+ torch.manual_seed(seed)+ return _Dit().to(torch.bfloat16)+++def _keep(mod, name) -> bool:+ return isinstance(mod, torch.nn.Linear) and mod.in_features >= 512+++def _quant(dit, logger = None) -> int:+ return nq.apply_native_weight_quant(+ dit, nq.NATIVE_INT8, filter_fn = _keep, act_int8 = True, logger = logger+ )+++def _tensors(module) -> dict:+ out = {}+ for name, sub in module.named_modules():+ for key, buf in list(sub._buffers.items()) + list(sub._parameters.items()):+ if buf is not None:+ out[f"{name}.{key}"] = buf.detach()+ return out+++def _assert_same_bytes(a: dict, b: dict) -> None:+ assert a.keys() == b.keys()+ for key in a:+ x, y = a[key], b[key]+ assert x.device.type == y.device.type == "cpu", key+ assert x.dtype == y.dtype and x.shape == y.shape, key+ assert torch.equal(+ x.contiguous().view(-1).view(torch.uint8), y.contiguous().view(-1).view(torch.uint8)+ ), key+++ "raw,on", [("", True), ("1", True), ("0", False), ("off", False), ("FALSE", False)]+)+def test_kill_switch(monkeypatch, raw, on):+ monkeypatch.setenv(nq.NATIVE_QUANT_ON_DEVICE_ENV, raw)+ assert nq.native_quant_on_device_enabled() is on+++def test_without_a_device_the_build_stays_on_the_host():+ a, b = _dit(), _dit()+ assert _quant(a) == 10+ with nq.quantise_host_linears_on(None):+ assert _quant(b) == 10+ _assert_same_bytes(_tensors(a), _tensors(b))+++def test_the_device_scope_ends_with_the_block():+ with nq.quantise_host_linears_on("cpu"):+ assert nq._HOST_QUANT_DEVICE.get() is not None+ assert nq._HOST_QUANT_DEVICE.get() is None+++def test_mover_restores_the_source_weight_and_keeps_the_bias_on_the_host(caplog):+ a, b = _dit(), _dit()+ _quant(a)+ log = logging.getLogger("nq-test")+ with caplog.at_level(logging.INFO, logger = "nq-test"), nq.quantise_host_linears_on("cpu"):+ _quant(b, logger = log)+ _assert_same_bytes(_tensors(a), _tensors(b))+ assert "built 10 host linear(s) on cpu one at a time" in caplog.text+ assert b.blocks[0].q.bias is not None and b.blocks[0].q.bias.device.type == "cpu"+++def test_a_device_failure_builds_that_layer_and_the_rest_on_the_host(monkeypatch, caplog):+ a, b = _dit(), _dit()+ _quant(a)+ calls = {"n": 0}+ real = nq._HostLinearMover._to_device++ def flaky(self, tensor):+ calls["n"] += 1+ if calls["n"] == 3:+ raise torch.OutOfMemoryError("CUDA out of memory (test)")+ return real(self, tensor)++ monkeypatch.setattr(nq._HostLinearMover, "_to_device", flaky)+ log = logging.getLogger("nq-test")+ with caplog.at_level(logging.INFO, logger = "nq-test"), nq.quantise_host_linears_on("cpu"):+ assert _quant(b, logger = log) == 10+ assert calls["n"] == 3+ _assert_same_bytes(_tensors(a), _tensors(b))+ assert (+ "built 2 host linear(s)" in caplog.text+ and "the rest on the host (OutOfMemoryError" in caplog.text+ )+++def test_video_load_routes_offloaded_native_builds_through_the_card():+ import core.inference.video as V++ src = inspect.getsource(V.VideoBackend)+ assert "_host_quant_device(target) if video_offload and native_scheme else None" in src+ assert "with quantise_host_linears_on(host_quant_device):" in src+++def test_host_quant_device_gates(monkeypatch):+ import core.inference.video as V++ monkeypatch.setattr(torch.cuda, "is_available", lambda: True)+ monkeypatch.delenv(nq.NATIVE_QUANT_ON_DEVICE_ENV, raising = False)+ assert V._host_quant_device(+ types.SimpleNamespace(device = "cuda", torch_device = "cuda:1")+ ) == torch.device("cuda:1")+ assert V._host_quant_device(types.SimpleNamespace(device = "mps")) is None+ assert V._host_quant_device(types.SimpleNamespace(device = "cpu")) is None+ monkeypatch.setenv(nq.NATIVE_QUANT_ON_DEVICE_ENV, "0")+ assert V._host_quant_device(types.SimpleNamespace(device = "cuda")) is None+ monkeypatch.delenv(nq.NATIVE_QUANT_ON_DEVICE_ENV)+ monkeypatch.setattr(torch.cuda, "is_available", lambda: False)+ assert V._host_quant_device(types.SimpleNamespace(device = "cuda")) is None++[email protected](not CUDA, reason = "needs a CUDA device")+def test_card_build_of_host_linears_matches_a_fully_staged_build_bit_for_bit():+ staged, host = _dit(), _dit()+ staged.to("cuda")+ _quant(staged)+ staged.to("cpu")+ with nq.quantise_host_linears_on(torch.device("cuda")):+ _quant(host)+ _assert_same_bytes(_tensors(staged), _tensors(host))+ assert all(t.device.type == "cpu" for t in _tensors(host).values())++[email protected](not CUDA, reason = "needs a CUDA device")+def test_card_build_runs_the_same_forward_as_a_staged_build():+ staged, host = _dit(), _dit()+ staged.to("cuda")+ _quant(staged)+ with nq.quantise_host_linears_on(torch.device("cuda")):+ _quant(host)+ host.to("cuda")+ x = torch.randn(64, 512, device = "cuda", dtype = torch.bfloat16)+ with torch.no_grad():+ assert torch.equal(+ staged.blocks[1].out(staged.blocks[1].ff(x)), host.blocks[1].out(host.blocks[1].ff(x))+ )++[email protected](not CUDA, reason = "needs a CUDA device")+def test_partially_staged_and_card_built_layers_agree():+ # Base staged what fit and quantised the rest on the host; the staged part must not change.+ base, head = _dit(), _dit()+ base.blocks[0].to("cuda")+ head.blocks[0].to("cuda")+ _quant(base)+ with nq.quantise_host_linears_on(torch.device("cuda")):+ _quant(head)+ base.to("cpu")+ head.to("cpu")+ a, b = _tensors(base.blocks[0]), _tensors(head.blocks[0])+ _assert_same_bytes(a, b)+ ref = copy.deepcopy(_dit()).to("cuda")+ _quant(ref)+ ref.to("cpu")+ _assert_same_bytes(_tensors(ref), _tensors(head))