Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: quantise an offloaded native int8 denoiser on the card, one linear at a time (#15936)

Daniel Hanfffb8f5eda88
diff --git a/studio/backend/core/inference/diffusion_native_quant.py b/studio/backend/core/inference/diffusion_native_quant.py
index 05bc71ac261..c160968fdef 100644
--- a/studio/backend/core/inference/diffusion_native_quant.py
+++ b/studio/backend/core/inference/diffusion_native_quant.py
@@ -6,6 +6,10 @@ hooks' Module.to(); torch imported lazily"""
from __future__ import annotations
+import contextlib
+import contextvars
+import os
+import time
from functools import lru_cache
from typing import Any, Callable, Optional
@@ -35,6 +39,11 @@ _INT_MM_FUSED_CHUNK_ELEMS = _INT_MM_CHUNK_ELEMS
NATIVE_INT8_FUSED_ENV = "UNSLOTH_NATIVE_INT8_FUSED"
# fp8 codes a native fp8 layer may hold; e5m2 only from a checkpoint (the quantise path writes e4m3fn).
NATIVE_FP8_FORMATS = ("float8_e4m3fn", "float8_e5m2")
+# ``0`` quantises host-resident linears on the host again instead of one at a time on the card.
+NATIVE_QUANT_ON_DEVICE_ENV = "UNSLOTH_NATIVE_QUANT_ON_DEVICE"
+_HOST_QUANT_DEVICE: contextvars.ContextVar = contextvars.ContextVar(
+ "unsloth_native_host_quant_device", default = None
+)
def fp8_bits_to_float32(bits: Any, fmt: str = "float8_e4m3fn") -> Any:
@@ -705,20 +714,25 @@ def apply_native_weight_quant(
rot_group = int8_rotation_group() if act_int8 else 0
# Names only: holding the old modules would keep every bf16 weight alive until the pass ends.
names = [name for name, mod in transformer.named_modules() if name and filter_fn(mod, name)]
+
+ def build(linear: Any) -> Any:
+ return cls(
+ linear,
+ scheme,
+ act_int8 = act_int8,
+ rot_group = rot_group,
+ scale_dtype = scale_dtype if scheme == NATIVE_INT8 and not act_int8 else None,
+ )
+
+ device = _HOST_QUANT_DEVICE.get()
+ mover = _HostLinearMover(device) if device is not None else None
for name in names:
parent_name, _, leaf = name.rpartition(".")
parent = transformer.get_submodule(parent_name) if parent_name else transformer
- setattr(
- parent,
- leaf,
- cls(
- getattr(parent, leaf),
- scheme,
- act_int8 = act_int8,
- rot_group = rot_group,
- scale_dtype = scale_dtype if scheme == NATIVE_INT8 and not act_int8 else None,
- ),
- )
+ linear = getattr(parent, leaf)
+ setattr(parent, leaf, mover.build(linear, build) if mover is not None else build(linear))
+ if mover is not None:
+ mover.close(logger)
if logger is not None:
logger.info(
"diffusion.transformer_quant: %s %s (torchao-free) on %d linears",
@@ -731,6 +745,125 @@ def apply_native_weight_quant(
return len(names)
+def native_quant_on_device_enabled() -> bool:
+ return os.environ.get(NATIVE_QUANT_ON_DEVICE_ENV, "").strip().lower() not in (
+ "0",
+ "false",
+ "no",
+ "off",
+ )
+
+
+def quantise_host_linears_on(device: Any):
+ """Inside, ``apply_native_weight_quant`` builds each host-resident Linear on ``device`` and returns it to the
+ host. Same kernels as a layer staged on the card, so the codes match a fully staged build bit for bit."""
+ token = _HOST_QUANT_DEVICE.set(device)
+ try:
+ yield
+ finally:
+ _HOST_QUANT_DEVICE.reset(token)
+
+
+class _HostLinearMover:
+ """A card failure builds that layer and the rest on the host, as without this."""
+
+ def __init__(self, device: Any) -> None:
+ import torch
+
+ self.device = torch.device(device)
+ self._pinned: dict = {}
+ self.layers = 0
+ self.up_bytes = 0
+ self.started = time.perf_counter()
+ self.failed: Optional[str] = None
+
+ def _staging(self, slot: str, nbytes: int) -> Any:
+ import torch
+
+ buf = self._pinned.get(slot)
+ if buf is None or buf.numel() < nbytes:
+ self._pinned[slot] = None
+ try:
+ buf = torch.empty(nbytes, dtype = torch.uint8, pin_memory = True)
+ except Exception: # noqa: BLE001 -- no pinnable memory: pageable copies, same bytes
+ buf = torch.empty(0, dtype = torch.uint8)
+ self._pinned[slot] = buf
+ return buf if buf.numel() >= nbytes else None
+
+ def _to_device(self, tensor: Any) -> Any:
+ import torch
+
+ nbytes = tensor.numel() * tensor.element_size()
+ buf = self._staging("up", nbytes) if tensor.is_contiguous() else None
+ if buf is None:
+ return tensor.to(self.device)
+ host = buf[:nbytes].view(tensor.dtype).view(tensor.shape)
+ host.copy_(tensor)
+ out = torch.empty(tensor.shape, dtype = tensor.dtype, device = self.device)
+ out.copy_(host, non_blocking = True)
+ return out
+
+ def _to_host(self, tensor: Any) -> Any:
+ import torch
+
+ nbytes = tensor.numel() * tensor.element_size()
+ buf = self._staging("down", nbytes) if tensor.is_contiguous() else None
+ if buf is None:
+ return tensor.to("cpu")
+ host = buf[:nbytes].view(tensor.dtype).view(tensor.shape)
+ host.copy_(tensor)
+ return torch.empty(tensor.shape, dtype = tensor.dtype).copy_(host)
+
+ def build(self, linear: Any, make: Callable[[Any], Any]) -> Any:
+ import torch
+
+ weight = getattr(linear, "weight", None)
+ if (
+ self.failed is not None
+ or not isinstance(weight, torch.nn.Parameter)
+ or weight.device.type != "cpu"
+ ):
+ return make(linear)
+ cuda = self.device.type == "cuda"
+ try:
+ linear.weight = torch.nn.Parameter(
+ self._to_device(weight.detach()), requires_grad = False
+ )
+ built = make(linear)
+ for key, buf in list(built._buffers.items()):
+ if buf is not None and buf.device.type != "cpu":
+ built._buffers[key] = self._to_host(buf)
+ # The next upload reuses the pinned buffer.
+ if cuda:
+ torch.cuda.synchronize(self.device)
+ except Exception as exc: # noqa: BLE001 -- out of memory or a device fault: the host build, as before
+ self.failed = f"{type(exc).__name__}: {exc}".splitlines()[0][:200]
+ linear.weight = weight
+ if cuda:
+ torch.cuda.empty_cache()
+ return make(linear)
+ finally:
+ if linear.weight is not weight:
+ linear.weight = weight
+ self.layers += 1
+ self.up_bytes += weight.numel() * weight.element_size()
+ return built
+
+ def close(self, logger: Any = None) -> None:
+ self._pinned.clear()
+ if logger is None or not (self.layers or self.failed):
+ return
+ logger.info(
+ "diffusion.transformer_quant: built %d host linear(s) on %s one at a time (%.1f GB in %.1f s)%s",
+ self.layers,
+ self.device,
+ self.up_bytes / 1e9,
+ time.perf_counter() - self.started,
+ f"; the rest on the host ({self.failed})" if self.failed else "",
+ )
+
+
def _first_native_linear(module: Any) -> Any:
try:
return next((sub for sub in module.modules() if is_native_linear(sub)), None)
diff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.py
index 00e74810e4b..b5ef411cca6 100644
--- a/studio/backend/core/inference/video.py
+++ b/studio/backend/core/inference/video.py
@@ -230,7 +230,11 @@ from .diffusion import (
_restore_step_cache_after_layer_split,
_step_cache_dropped_by_layer_split,
)
-from .diffusion_native_quant import native_quant_reason
+from .diffusion_native_quant import (
+ native_quant_on_device_enabled,
+ native_quant_reason,
+ quantise_host_linears_on,
+)
from .diffusion_batched import is_oom_error
from .diffusion_block_graph import compile_pipe_below_offload_hooks
from .diffusion_offload_prefetch import (
@@ -3773,6 +3777,20 @@ def _stage_denoiser_for_quant(
return []
+def _host_quant_device(target: Any) -> Any:
+ try:
+ import torch
+ if (
+ getattr(target, "device", None) != "cuda"
+ or not torch.cuda.is_available()
+ or not native_quant_on_device_enabled()
+ ):
+ return None
+ return torch.device(getattr(target, "torch_device", None) or "cuda")
+ except Exception: # noqa: BLE001 -- the host build is slower, never wrong
+ return None
+
+
def _unstage_modules(modules: list) -> None:
for module in modules:
module.to("cpu")
@@ -9457,21 +9475,30 @@ class VideoBackend:
and (dense_transformer_supported(target) or native_scheme is not None)
):
engaged = []
+ # An offloaded DiT goes back to the host anyway: build every linear on the card, one at a time.
+ host_quant_device = (
+ _host_quant_device(target) if video_offload and native_scheme else None
+ )
for view in views:
# Quantise on the card: on the host int8 packing took 59 s (Wan2.2-5B) to 154 s (LTX-2) of the load.
- staged = _stage_denoiser_for_quant(
- getattr(view, "transformer", None), target, logger = logger
+ staged = (
+ []
+ if host_quant_device is not None
+ else _stage_denoiser_for_quant(
+ getattr(view, "transformer", None), target, logger = logger
+ )
)
scheme = None
try:
- scheme = quantize_transformer(
- view,
- target,
- mode = native_scheme or transformer_quant,
- family = fam.name,
- logger = logger,
- **native_kwargs,
- )
+ with quantise_host_linears_on(host_quant_device):
+ scheme = quantize_transformer(
+ view,
+ target,
+ mode = native_scheme or transformer_quant,
+ family = fam.name,
+ logger = logger,
+ **native_kwargs,
+ )
finally:
# A declined quant can roll back to the bf16 offload plan, which must not start with dense blocks staged.
if staged and (video_offload or scheme is None):
diff --git a/studio/backend/tests/test_video_native_quant_on_device.py b/studio/backend/tests/test_video_native_quant_on_device.py
new file mode 100644
index 00000000000..d7e51d567fe
--- /dev/null
+++ b/studio/backend/tests/test_video_native_quant_on_device.py
@@ -0,0 +1,202 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Offloaded native int8 builds: host-resident linears are quantised on the card one at a time."""
+
+from __future__ import annotations
+
+import copy
+import inspect
+import logging
+import types
+
+import pytest
+
+torch = pytest.importorskip("torch")
+
+import core.inference.diffusion_native_quant as nq # noqa: E402
+
+CUDA = torch.cuda.is_available()
+
+
+class _Block(torch.nn.Module):
+ def __init__(self, width: int) -> None:
+ super().__init__()
+ self.norm = torch.nn.LayerNorm(width)
+ self.q = torch.nn.Linear(width, width)
+ self.ff = torch.nn.Linear(width, 2 * width, bias = False)
+ self.out = torch.nn.Linear(2 * width, width)
+
+
+class _Dit(torch.nn.Module):
+ def __init__(
+ self,
+ width: int = 512,
+ depth: int = 3,
+ ) -> None:
+ super().__init__()
+ self.blocks = torch.nn.ModuleList(_Block(width) for _ in range(depth))
+ self.head = torch.nn.Linear(width, 64)
+
+
+def _dit(seed: int = 0) -> _Dit:
+ torch.manual_seed(seed)
+ return _Dit().to(torch.bfloat16)
+
+
+def _keep(mod, name) -> bool:
+ return isinstance(mod, torch.nn.Linear) and mod.in_features >= 512
+
+
+def _quant(dit, logger = None) -> int:
+ return nq.apply_native_weight_quant(
+ dit, nq.NATIVE_INT8, filter_fn = _keep, act_int8 = True, logger = logger
+ )
+
+
+def _tensors(module) -> dict:
+ out = {}
+ for name, sub in module.named_modules():
+ for key, buf in list(sub._buffers.items()) + list(sub._parameters.items()):
+ if buf is not None:
+ out[f"{name}.{key}"] = buf.detach()
+ return out
+
+
+def _assert_same_bytes(a: dict, b: dict) -> None:
+ assert a.keys() == b.keys()
+ for key in a:
+ x, y = a[key], b[key]
+ assert x.device.type == y.device.type == "cpu", key
+ assert x.dtype == y.dtype and x.shape == y.shape, key
+ assert torch.equal(
+ x.contiguous().view(-1).view(torch.uint8), y.contiguous().view(-1).view(torch.uint8)
+ ), key
+
+
+ "raw,on", [("", True), ("1", True), ("0", False), ("off", False), ("FALSE", False)]
+)
+def test_kill_switch(monkeypatch, raw, on):
+ monkeypatch.setenv(nq.NATIVE_QUANT_ON_DEVICE_ENV, raw)
+ assert nq.native_quant_on_device_enabled() is on
+
+
+def test_without_a_device_the_build_stays_on_the_host():
+ a, b = _dit(), _dit()
+ assert _quant(a) == 10
+ with nq.quantise_host_linears_on(None):
+ assert _quant(b) == 10
+ _assert_same_bytes(_tensors(a), _tensors(b))
+
+
+def test_the_device_scope_ends_with_the_block():
+ with nq.quantise_host_linears_on("cpu"):
+ assert nq._HOST_QUANT_DEVICE.get() is not None
+ assert nq._HOST_QUANT_DEVICE.get() is None
+
+
+def test_mover_restores_the_source_weight_and_keeps_the_bias_on_the_host(caplog):
+ a, b = _dit(), _dit()
+ _quant(a)
+ log = logging.getLogger("nq-test")
+ with caplog.at_level(logging.INFO, logger = "nq-test"), nq.quantise_host_linears_on("cpu"):
+ _quant(b, logger = log)
+ _assert_same_bytes(_tensors(a), _tensors(b))
+ assert "built 10 host linear(s) on cpu one at a time" in caplog.text
+ assert b.blocks[0].q.bias is not None and b.blocks[0].q.bias.device.type == "cpu"
+
+
+def test_a_device_failure_builds_that_layer_and_the_rest_on_the_host(monkeypatch, caplog):
+ a, b = _dit(), _dit()
+ _quant(a)
+ calls = {"n": 0}
+ real = nq._HostLinearMover._to_device
+
+ def flaky(self, tensor):
+ calls["n"] += 1
+ if calls["n"] == 3:
+ raise torch.OutOfMemoryError("CUDA out of memory (test)")
+ return real(self, tensor)
+
+ monkeypatch.setattr(nq._HostLinearMover, "_to_device", flaky)
+ log = logging.getLogger("nq-test")
+ with caplog.at_level(logging.INFO, logger = "nq-test"), nq.quantise_host_linears_on("cpu"):
+ assert _quant(b, logger = log) == 10
+ assert calls["n"] == 3
+ _assert_same_bytes(_tensors(a), _tensors(b))
+ assert (
+ "built 2 host linear(s)" in caplog.text
+ and "the rest on the host (OutOfMemoryError" in caplog.text
+ )
+
+
+def test_video_load_routes_offloaded_native_builds_through_the_card():
+ import core.inference.video as V
+
+ src = inspect.getsource(V.VideoBackend)
+ assert "_host_quant_device(target) if video_offload and native_scheme else None" in src
+ assert "with quantise_host_linears_on(host_quant_device):" in src
+
+
+def test_host_quant_device_gates(monkeypatch):
+ import core.inference.video as V
+
+ monkeypatch.setattr(torch.cuda, "is_available", lambda: True)
+ monkeypatch.delenv(nq.NATIVE_QUANT_ON_DEVICE_ENV, raising = False)
+ assert V._host_quant_device(
+ types.SimpleNamespace(device = "cuda", torch_device = "cuda:1")
+ ) == torch.device("cuda:1")
+ assert V._host_quant_device(types.SimpleNamespace(device = "mps")) is None
+ assert V._host_quant_device(types.SimpleNamespace(device = "cpu")) is None
+ monkeypatch.setenv(nq.NATIVE_QUANT_ON_DEVICE_ENV, "0")
+ assert V._host_quant_device(types.SimpleNamespace(device = "cuda")) is None
+ monkeypatch.delenv(nq.NATIVE_QUANT_ON_DEVICE_ENV)
+ monkeypatch.setattr(torch.cuda, "is_available", lambda: False)
+ assert V._host_quant_device(types.SimpleNamespace(device = "cuda")) is None
+
+
[email protected](not CUDA, reason = "needs a CUDA device")
+def test_card_build_of_host_linears_matches_a_fully_staged_build_bit_for_bit():
+ staged, host = _dit(), _dit()
+ staged.to("cuda")
+ _quant(staged)
+ staged.to("cpu")
+ with nq.quantise_host_linears_on(torch.device("cuda")):
+ _quant(host)
+ _assert_same_bytes(_tensors(staged), _tensors(host))
+ assert all(t.device.type == "cpu" for t in _tensors(host).values())
+
+
[email protected](not CUDA, reason = "needs a CUDA device")
+def test_card_build_runs_the_same_forward_as_a_staged_build():
+ staged, host = _dit(), _dit()
+ staged.to("cuda")
+ _quant(staged)
+ with nq.quantise_host_linears_on(torch.device("cuda")):
+ _quant(host)
+ host.to("cuda")
+ x = torch.randn(64, 512, device = "cuda", dtype = torch.bfloat16)
+ with torch.no_grad():
+ assert torch.equal(
+ staged.blocks[1].out(staged.blocks[1].ff(x)), host.blocks[1].out(host.blocks[1].ff(x))
+ )
+
+
[email protected](not CUDA, reason = "needs a CUDA device")
+def test_partially_staged_and_card_built_layers_agree():
+ # Base staged what fit and quantised the rest on the host; the staged part must not change.
+ base, head = _dit(), _dit()
+ base.blocks[0].to("cuda")
+ head.blocks[0].to("cuda")
+ _quant(base)
+ with nq.quantise_host_linears_on(torch.device("cuda")):
+ _quant(head)
+ base.to("cpu")
+ head.to("cpu")
+ a, b = _tensors(base.blocks[0]), _tensors(head.blocks[0])
+ _assert_same_bytes(a, b)
+ ref = copy.deepcopy(_dit()).to("cuda")
+ _quant(ref)
+ ref.to("cpu")
+ _assert_same_bytes(_tensors(ref), _tensors(head))