Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"""A torchao int8 / fp8 denoiser stays quantised when the image plan offloads it."""
from __future__ import annotations
import types
from dataclasses import replace
import pytest
from core.inference import diffusion as dmod
from core.inference import diffusion_memory as mem
from core.inference.diffusion_denoiser_prequant import PIPELINE_SEED_DECLINED
from core.inference.diffusion_memory import (
OFFLOAD_GROUP,
OFFLOAD_MODEL,
OFFLOAD_NONE,
OFFLOAD_SEQUENTIAL,
OFFLOAD_STREAMING,
DeviceMemory,
plan_diffusion_memory,
torchao_offload_plan,
torchao_survives_plan,
)
T017, T018, T016 = (0, 17), (0, 18), (0, 16)
@pytest.fixture(autouse = True)
def _roomy_host(monkeypatch):
monkeypatch.setattr(mem, "_pin_budget_mib", lambda: 10**7)
monkeypatch.setattr(mem, "_pinned_memory_capped", lambda: False)
monkeypatch.delenv(mem.GROUP_OFFLOAD_PIN_ENV, raising = False)
# Studio's diffusers pin; without diffusers every streamed tier would be refused.
monkeypatch.setattr(mem, "_installed_diffusers_version", lambda: (0, 40))
def _plan(
policy,
*,
stream_transformer = True,
fits_model_offload = True,
):
device = DeviceMemory("cuda", "cuda:0", "discrete_vram", 15_000, 16_384)
plan = plan_diffusion_memory(
target = types.SimpleNamespace(supports_model_cpu_offload = True),
device_memory = device,
model_dense_mib = 12_000,
runtime_headroom_mib = 2_000,
companion_dense_mib = 4_000,
text_encoder_dense_mib = 3_800,
)
estimates = dict(plan.estimates)
if not fits_model_offload:
estimates["model_dense_mib"] = 40_000
return replace(
plan,
offload_policy = policy,
stream_transformer = stream_transformer,
estimates = estimates,
)
PLACEMENTS = {
"resident": _plan(OFFLOAD_NONE),
"group_dit_resident": _plan(OFFLOAD_GROUP, stream_transformer = False),
"group_dit_streamed": _plan(OFFLOAD_GROUP),
"model_fits": _plan(OFFLOAD_MODEL),
"model_too_big": _plan(OFFLOAD_MODEL, fits_model_offload = False),
"streaming": _plan(OFFLOAD_STREAMING),
"sequential": _plan(OFFLOAD_SEQUENTIAL),
}
# placement -> scheme -> torchao version -> (survives, policy it runs on)
SURVIVAL = {
"resident": {
s: {v: (True, OFFLOAD_NONE) for v in (T016, T017, T018)} for s in ("int8", "fp8", "nvfp4")
},
"group_dit_resident": {
s: {v: (True, OFFLOAD_GROUP) for v in (T016, T017, T018)} for s in ("int8", "fp8", "nvfp4")
},
"group_dit_streamed": {
# 0.17 streams int8 through its pinnable Int8Tensor (UNSLOTH_DIFFUSION_INT8_STREAM_TORCHAO17)
"int8": {T016: (False, None), T017: (True, OFFLOAD_GROUP), T018: (True, OFFLOAD_GROUP)},
"fp8": {T016: (False, None), T017: (True, OFFLOAD_GROUP), T018: (True, OFFLOAD_GROUP)},
"nvfp4": {T016: (False, None), T017: (False, None), T018: (False, None)},
},
"model_fits": {
s: {v: (True, OFFLOAD_MODEL) for v in (T016, T017, T018)} for s in ("int8", "fp8", "nvfp4")
},
"model_too_big": {
"int8": {
T016: (False, None),
T017: (True, OFFLOAD_STREAMING),
T018: (True, OFFLOAD_STREAMING),
},
"fp8": {
T016: (False, None),
T017: (True, OFFLOAD_STREAMING),
T018: (True, OFFLOAD_STREAMING),
},
"nvfp4": {T016: (False, None), T017: (False, None), T018: (False, None)},
},
"streaming": {
"int8": {
T016: (False, None),
T017: (True, OFFLOAD_STREAMING),
T018: (True, OFFLOAD_STREAMING),
},
"fp8": {
T016: (False, None),
T017: (True, OFFLOAD_STREAMING),
T018: (True, OFFLOAD_STREAMING),
},
"nvfp4": {T016: (False, None), T017: (False, None), T018: (False, None)},
},
"sequential": {
s: {v: (False, None) for v in (T016, T017, T018)} for s in ("int8", "fp8", "nvfp4")
},
}
CASES = [
(placement, scheme, version, *expected)
for placement, by_scheme in SURVIVAL.items()
for scheme, by_version in by_scheme.items()
for version, expected in by_version.items()
]
@pytest.mark.parametrize("placement, scheme, version, survives, policy", CASES)
def test_survival_table(monkeypatch, placement, scheme, version, survives, policy):
# Hermetic: the table is about versions, not this host's torchao.
monkeypatch.setattr(mem, "_int8_tensor_pinnable", lambda: True, raising = False)
monkeypatch.delenv("UNSLOTH_DIFFUSION_INT8_STREAM_TORCHAO17", raising = False)
plan = PLACEMENTS[placement]
assert torchao_survives_plan(plan, scheme, torchao_version = version) is survives
placed = torchao_offload_plan(plan, scheme, torchao_version = version)
assert (placed.offload_policy if placed is not None else None) == policy
@pytest.mark.parametrize(
"diffusers_version, streams",
[((0, 36), False), ((0, 37), False), ((0, 38), True), (None, False)],
)
def test_diffusers_before_the_torchao_swap_keeps_the_resident_rule(
monkeypatch, diffusers_version, streams
):
monkeypatch.setattr(mem, "_installed_diffusers_version", lambda: diffusers_version)
assert (
torchao_survives_plan(PLACEMENTS["group_dit_streamed"], "int8", torchao_version = T018)
is streams
)
assert torchao_survives_plan(PLACEMENTS["group_dit_resident"], "int8", torchao_version = T018)
def test_swap_retry_collects_once_then_gives_up(monkeypatch):
import sys
calls: list = []
failures = {"n": 1}
def _swap(param, source):
calls.append(param)
if failures["n"] > 0:
failures["n"] -= 1
raise RuntimeError("Cannot swap t1 because it has weakref associated with it")
go = types.SimpleNamespace(_swap_torchao_tensor = _swap)
hooks = types.ModuleType("diffusers.hooks")
hooks.group_offloading = go
monkeypatch.setitem(sys.modules, "diffusers.hooks", hooks)
monkeypatch.setitem(sys.modules, "diffusers.hooks.group_offloading", go)
assert mem.install_group_offload_torchao_swap_retry()
assert not mem.install_group_offload_torchao_swap_retry()
go._swap_torchao_tensor("p", "s")
assert calls == ["p", "p"]
failures["n"] = 2
with pytest.raises(RuntimeError, match = "weakref"):
go._swap_torchao_tensor("p", "s")
@pytest.mark.parametrize(
"capped, budget, env, streams",
[
(False, 10**7, None, True),
(True, 10**7, None, False),
(False, 7_999, None, False),
(False, None, None, False),
(False, 10**7, "0", False),
(True, 0, "1", True),
],
)
def test_a_streamed_torchao_denoiser_needs_its_pin(monkeypatch, capped, budget, env, streams):
monkeypatch.setattr(mem, "_pinned_memory_capped", lambda: capped)
monkeypatch.setattr(mem, "_pin_budget_mib", lambda: budget)
if env is not None:
monkeypatch.setenv(mem.GROUP_OFFLOAD_PIN_ENV, env)
for name in ("group_dit_streamed", "streaming", "model_too_big"):
assert (
torchao_survives_plan(PLACEMENTS[name], "int8", torchao_version = T018) is streams
), name
assert torchao_survives_plan(PLACEMENTS["group_dit_resident"], "int8", torchao_version = T018)
assert torchao_survives_plan(PLACEMENTS["model_fits"], "int8", torchao_version = T018)
def test_no_torchao_keeps_the_resident_rule():
assert not torchao_survives_plan(PLACEMENTS["group_dit_streamed"], "int8", torchao_version = None)
assert torchao_survives_plan(PLACEMENTS["group_dit_resident"], "int8", torchao_version = None)
def test_never_moves_still_means_resident_only():
keeps = mem.plan_keeps_transformer_resident
assert keeps(PLACEMENTS["resident"]) and keeps(PLACEMENTS["group_dit_resident"])
for name in ("group_dit_streamed", "model_fits", "streaming"):
assert not keeps(PLACEMENTS[name]), name
class _Weight:
def __init__(
self,
cls_name: str,
module: str = "torchao.quantization.fake",
):
self.data = None
self.__class__ = type(cls_name, (_Weight,), {"__module__": module})
class _Module:
def __init__(
self,
*class_names: str,
plain: bool = False,
):
self._params = [_Weight(name) for name in class_names]
if plain:
self._params.append(_Weight("Parameter", module = "torch.nn.parameter"))
def parameters(self):
return iter(self._params)
STREAM_KW = {
"onload_device": "cuda",
"use_stream": True,
"non_blocking": True,
"record_stream": True,
}
NO_STREAM = {"onload_device": "cuda", "use_stream": False}
@pytest.mark.parametrize(
"classes, low_cpu_mem_usage, pin_budget, expected",
[
(("Int8Tensor",), False, 10**7, "as is"),
(("Float8Tensor",), False, 10**7, "as is"),
(("Int8Tensor", "Float8Tensor"), False, 10**7, "as is"),
(("Int8Tensor",), True, 10**7, "pinned"),
(("Float8Tensor",), True, 10**7, "pinned"),
(("Int8Tensor",), True, None, "no stream"),
# torchao 0.17 default int8 has no aten.is_pinned; streams once Studio registers the pin ops.
(("LinearActivationQuantizedTensor",), False, 10**7, "as is"),
(("LinearActivationQuantizedTensor",), False, 10**7, "no stream, no pin ops"),
],
)
def test_group_offload_kwargs_per_weight_class(
monkeypatch, classes, low_cpu_mem_usage, pin_budget, expected
):
monkeypatch.setattr(mem, "_pin_budget_mib", lambda: pin_budget)
# Whether this host's torchao ships the v1 classes must not decide the table.
pin_ops = expected != "no stream, no pin ops"
monkeypatch.setattr(mem, "install_torchao_v1_int8_pin_ops", lambda: pin_ops, raising = False)
kwargs = {**STREAM_KW, "low_cpu_mem_usage": low_cpu_mem_usage}
out = mem._torchao_group_offload_kwargs(_Module(*classes, plain = True), kwargs)
if expected == "as is":
assert out is kwargs
elif expected == "pinned":
assert out == {**STREAM_KW, "low_cpu_mem_usage": False}
else:
assert out == NO_STREAM
def test_group_offload_kwargs_leave_dense_and_stream_free_modules_alone():
dense = {**STREAM_KW, "low_cpu_mem_usage": True}
assert mem._torchao_group_offload_kwargs(_Module(plain = True), dense) is dense
assert mem._torchao_group_offload_kwargs(_Module("Int8Tensor"), NO_STREAM) is NO_STREAM
@pytest.mark.parametrize(
"policy, transformer_quant, text_encoder_quant, no_grad",
[
(OFFLOAD_NONE, "int8", None, False),
(OFFLOAD_NONE, "int8", "fp8", False),
(OFFLOAD_MODEL, "int8", None, True),
(OFFLOAD_GROUP, "int8", None, True),
(OFFLOAD_GROUP, "fp8", None, True),
(OFFLOAD_STREAMING, "int8", None, True),
(OFFLOAD_GROUP, None, "fp8", True),
(OFFLOAD_GROUP, None, None, False),
(OFFLOAD_MODEL, None, None, False),
],
)
def test_render_grad_mode(policy, transformer_quant, text_encoder_quant, no_grad):
state = types.SimpleNamespace(
offload_policy = policy,
transformer_quant = transformer_quant,
text_encoder_quant = text_encoder_quant,
)
assert dmod._torchao_render_needs_no_grad(state) is no_grad
@pytest.mark.parametrize(
"policy, no_grad",
[(OFFLOAD_NONE, False), (OFFLOAD_MODEL, True), (OFFLOAD_GROUP, True)],
)
def test_offloaded_stored_fp8_renders_under_no_grad(policy, no_grad):
"""Stored fp8 codes are plain buffers: moved under inference_mode they are inference tensors the keep-host
offload cannot version, so each offload would copy 12 GB back to fresh host memory (Krea 2)."""
transformer = types.SimpleNamespace(_unsloth_fp8_stored = 256)
state = types.SimpleNamespace(
offload_policy = policy,
transformer_quant = None,
text_encoder_quant = None,
pipe = types.SimpleNamespace(transformer = transformer),
)
assert dmod._torchao_render_needs_no_grad(state) is no_grad
from core.inference import video as vid
assert vid._render_under_no_grad(state) is no_grad
@pytest.mark.parametrize(
"policy, info, no_grad",
[
(OFFLOAD_STREAMING, {"fp8_backend": "torchao", "fp8": 229}, True),
(OFFLOAD_GROUP, {"backend": "torchao", "int8": 8}, True),
(OFFLOAD_NONE, {"fp8_backend": "torchao", "fp8": 229}, False),
(OFFLOAD_STREAMING, {"fp8_backend": None, "fp8": 0}, False),
],
)
def test_streamed_single_file_torchao_renders_under_no_grad(policy, info, no_grad):
"""A single file's torchao layers carry no transformer_quant; a bf16 text encoder must not drop no_grad."""
transformer = types.SimpleNamespace(_unsloth_comfy_quant = info)
state = types.SimpleNamespace(
offload_policy = policy,
transformer_quant = None,
text_encoder_quant = None,
pipe = types.SimpleNamespace(transformer = transformer),
)
assert dmod._torchao_render_needs_no_grad(state) is no_grad
def test_render_uses_the_widened_switch():
import inspect
source = inspect.getsource(dmod.DiffusionBackend)
assert "if _torchao_render_needs_no_grad(state)" in source
assert "and state.offload_policy == OFFLOAD_MODEL" not in source
def _estimate(steady):
return types.SimpleNamespace(steady_transformer_mib = steady)
@pytest.mark.parametrize(
"placement, estimate_mib, largest_te_mib, version, policy, declined",
[
("resident", 6_000, 4_000, T018, OFFLOAD_NONE, False),
("group_dit_streamed", 6_000, 4_000, T018, OFFLOAD_GROUP, False),
("group_dit_streamed", 6_000, 4_000, T016, OFFLOAD_GROUP, True),
("model_fits", 6_000, 4_000, T018, OFFLOAD_MODEL, False),
("model_fits", 60_000, 4_000, T018, OFFLOAD_STREAMING, False),
("model_fits", 6_000, 60_000, T018, OFFLOAD_STREAMING, False),
("model_fits", 60_000, 4_000, T016, OFFLOAD_MODEL, True),
("sequential", 6_000, 4_000, T018, OFFLOAD_SEQUENTIAL, True),
],
)
def test_inplace_quant_placement(
monkeypatch, placement, estimate_mib, largest_te_mib, version, policy, declined
):
monkeypatch.setattr(mem, "_installed_torchao_version", lambda: version)
plan, why = dmod._inplace_torchao_placement(
PLACEMENTS[placement], "int8", _estimate(estimate_mib), lambda: largest_te_mib
)
assert plan.offload_policy == policy
assert (why is not None) is declined
@pytest.mark.parametrize("estimate_mib, largest_te_mib", [(60_000, 4_000), (6_000, 60_000)])
def test_inplace_quant_declines_streaming_it_cannot_pin(monkeypatch, estimate_mib, largest_te_mib):
"""Unpinnable (Windows, capped host RAM) streaming runs without a copy stream, ~35x slower: decline it."""
monkeypatch.setattr(mem, "_installed_torchao_version", lambda: T018)
monkeypatch.setattr(mem, "_pin_budget_mib", lambda: 0)
plan, why = dmod._inplace_torchao_placement(
PLACEMENTS["model_fits"], "int8", _estimate(estimate_mib), lambda: largest_te_mib
)
assert plan.offload_policy == OFFLOAD_MODEL
assert why is not None
def test_inplace_quant_no_longer_calls_group_offload_wrong():
import inspect
source = inspect.getsource(dmod)
assert "Group offload is WRONG for torchao" not in source
GIB = 1024
@pytest.fixture
def planner(monkeypatch):
import torch
from core.inference import diffusion_auto_policy as ap
from core.inference import diffusion_transformer_quant as tq
monkeypatch.setattr(mem, "_installed_torchao_version", lambda: T018)
monkeypatch.setattr(torch.cuda, "get_device_capability", lambda *_a, **_k: (12, 0))
monkeypatch.setattr(torch.cuda, "is_available", lambda: True)
def _supported(
scheme,
device = None,
unproven_ok = False,
):
return scheme in ("int8", "fp8")
monkeypatch.setattr(tq, "_scheme_supported", _supported)
monkeypatch.setattr(tq, "_is_consumer_gpu", lambda *_a, **_k: True)
monkeypatch.setattr(ap, "_hf_cache_free_mib", lambda: 10**7)
card: dict = {}
for module in (dmod, mem):
for fn in (
"snapshot_device_memory",
"settled_snapshot_device_memory",
"reclaimable_snapshot_device_memory",
):
if hasattr(module, fn):
monkeypatch.setattr(module, fn, lambda *_a, **_k: card["memory"])
target = types.SimpleNamespace(
device = "cuda",
dtype = torch.bfloat16,
backend = "cuda",
supports_model_cpu_offload = True,
supports_default_torch_compile = True,
ordinal = None,
capability = (12, 0),
)
backend = dmod.DiffusionBackend.__new__(dmod.DiffusionBackend)
backend._resolve_device_target = lambda *_a, **_k: target
backend._target_for_ordinal = lambda *_a, **_k: target
backend._cache_bytes = lambda *_a, **_k: 0
backend._released_transformer_cached = lambda *_a, **_k: False
def pick(family_name: str, base: str, gib: int):
from core.inference.diffusion_families import _FAMILIES
from core.inference.diffusion_transformer_quant import select_transformer_quant_scheme
fam = next(f for f in _FAMILIES if f.name == family_name)
card["memory"] = DeviceMemory("cuda", "cuda", "discrete_vram", gib * GIB - 600, gib * GIB)
seed = backend._pipeline_planned_denoiser_scheme(
fam,
base = base,
kind = "pipeline",
transformer_quant = None,
speed_mode = None,
repo_id = base,
fetch_base = base,
)
if seed not in (None, PIPELINE_SEED_DECLINED):
seeded = backend._seeded_pipeline_plan(
seed,
target,
base,
fam,
None,
False,
repo_id = base,
base_local_dir = None,
fetch_base = base,
)
placed = torchao_offload_plan(seeded, seed)
assert placed is not None
return f"hosted {seed}", placed.offload_policy
plan = backend._bf16_table_plan(
target, fam, base, None, False, kind = "pipeline", repo_id = base, fetch_base = base
)
if dmod._auto_quant_eager_reason(fam, plan, None, "pipeline") is not None:
return "bf16", plan.offload_policy
scheme = select_transformer_quant_scheme(target, "auto", family = fam.name, base_repo = base)
estimate = ap.estimate_dense_quant(fam, scheme, base_repo = base)
if plan.offload_policy != OFFLOAD_NONE:
replanned = backend._plan_memory(
target,
None,
base,
fam,
None,
False,
kind = "pipeline",
repo_id = base,
fetch_base = base,
transformer_resident_override_mib = estimate.steady_transformer_mib,
**backend._candidate_companion_overrides(estimate, fam, base, target, None),
)
placed = torchao_offload_plan(replanned, scheme)
if placed is not None:
plan = placed
largest_te = int(ap.family_bf16_components_gb(fam, base)[1] * 1000**3 / 1024**2)
plan, why = dmod._inplace_torchao_placement(plan, scheme, estimate, lambda: largest_te)
return ("bf16" if why else f"on-the-fly {scheme}"), plan.offload_policy
return pick
FAMILY_TABLE = {
("krea-2", "krea/Krea-2-Turbo"): {
8: ("hosted int8", OFFLOAD_STREAMING),
12: ("hosted int8", OFFLOAD_GROUP),
16: ("hosted int8", OFFLOAD_GROUP),
24: ("hosted int8", OFFLOAD_GROUP),
32: ("hosted int8", OFFLOAD_GROUP),
},
("lumina-2", "Alpha-VLLM/Lumina-Image-2.0"): {
8: ("hosted int8", OFFLOAD_STREAMING),
12: ("hosted int8", OFFLOAD_STREAMING),
16: ("hosted int8", OFFLOAD_GROUP),
24: ("hosted int8", OFFLOAD_NONE),
32: ("bf16", OFFLOAD_NONE),
},
("hunyuanimage-2.1", "hunyuanvideo-community/HunyuanImage-2.1-Diffusers"): {
8: ("hosted int8", OFFLOAD_STREAMING),
12: ("hosted int8", OFFLOAD_STREAMING),
16: ("hosted int8", OFFLOAD_GROUP),
24: ("hosted int8", OFFLOAD_GROUP),
32: ("hosted int8", OFFLOAD_GROUP),
},
("hidream-i1", "HiDream-ai/HiDream-I1-Full"): {
8: ("hosted int8", OFFLOAD_STREAMING),
12: ("hosted int8", OFFLOAD_STREAMING),
16: ("hosted int8", OFFLOAD_GROUP),
24: ("hosted int8", OFFLOAD_GROUP),
32: ("hosted int8", OFFLOAD_GROUP),
},
("ideogram-4", "ideogram-ai/ideogram-4-fp8"): {
8: ("on-the-fly int8", OFFLOAD_STREAMING),
12: ("on-the-fly int8", OFFLOAD_STREAMING),
16: ("on-the-fly int8", OFFLOAD_GROUP),
24: ("on-the-fly int8", OFFLOAD_GROUP),
32: ("on-the-fly int8", OFFLOAD_GROUP),
},
}
@pytest.mark.parametrize(
"family, base, gib, expected",
[
(family, base, gib, expected)
for (family, base), by_gib in FAMILY_TABLE.items()
for gib, expected in by_gib.items()
],
)
def test_auto_keeps_int8_on_the_offload_tier(planner, family, base, gib, expected):
assert planner(family, base, gib) == expected
@pytest.mark.parametrize("version, expected", [(T016, "bf16"), (T017, "hosted int8")])
def test_older_torchao_streamed_seeds(planner, monkeypatch, version, expected):
monkeypatch.setattr(mem, "_installed_torchao_version", lambda: version)
monkeypatch.setattr(mem, "_int8_tensor_pinnable", lambda: True, raising = False)
monkeypatch.delenv("UNSLOTH_DIFFUSION_INT8_STREAM_TORCHAO17", raising = False)
precision, _policy = planner(
"hunyuanimage-2.1", "hunyuanvideo-community/HunyuanImage-2.1-Diffusers", 16
)
assert precision == expected
def _gpu_stack():
torch = pytest.importorskip("torch")
if not torch.cuda.is_available():
pytest.skip("needs CUDA")
pytest.importorskip("torchao")
hooks = pytest.importorskip("diffusers.hooks")
return torch, hooks
@pytest.mark.parametrize("scheme", ["int8", "fp8"])
@pytest.mark.parametrize("low_cpu_mem_usage", [False, True])
@pytest.mark.parametrize("requires_grad", [False, True])
def test_quantised_blocks_survive_group_offload_under_no_grad(
scheme, low_cpu_mem_usage, requires_grad
):
import copy
import inspect
torch, hooks = _gpu_stack()
from torchao.quantization import (
Float8DynamicActivationFloat8WeightConfig,
Int8DynamicActivationInt8WeightConfig,
PerRow,
quantize_,
)
if scheme == "fp8" and torch.cuda.get_device_capability() < (8, 9):
pytest.skip("fp8 needs sm_89+")
torch.manual_seed(0)
blocks = torch.nn.Sequential(
*[
torch.nn.Sequential(
torch.nn.Linear(256, 512), torch.nn.GELU(), torch.nn.Linear(512, 256)
)
for _ in range(3)
]
).to(torch.bfloat16)
config = (
Int8DynamicActivationInt8WeightConfig
if scheme == "int8"
else lambda: Float8DynamicActivationFloat8WeightConfig(granularity = PerRow())
)
offloaded = copy.deepcopy(blocks)
quantize_(offloaded, config())
# Same CPU-quantised weights: fp8 quantised on CPU can be an ulp off CUDA (torch 2.12).
resident = copy.deepcopy(offloaded).cuda()
for param in offloaded.parameters():
param.requires_grad_(requires_grad)
params = inspect.signature(hooks.apply_group_offloading).parameters
kwargs = {
"onload_device": torch.device("cuda"),
"offload_device": torch.device("cpu"),
"offload_type": "block_level",
"num_blocks_per_group": 1,
"use_stream": True,
}
for name, value in (
("non_blocking", True),
("record_stream", True),
("low_cpu_mem_usage", low_cpu_mem_usage),
):
if name in params:
kwargs[name] = value
hooks.apply_group_offloading(offloaded, **mem._torchao_group_offload_kwargs(offloaded, kwargs))
x = torch.randn(2, 16, 256, dtype = torch.bfloat16, device = "cuda")
with torch.no_grad():
want = resident(x)
for _ in range(3):
assert torch.equal(offloaded(x), want)
if type(next(offloaded.parameters())).__name__ == "LinearActivationQuantizedTensor":
return # torchao <= 0.17 v1 int8: onload is a plain re-wrap, which inference_mode allows
with pytest.raises(Exception), torch.inference_mode():
offloaded(x)
@pytest.mark.parametrize("offload_graphs", [True, False])
@pytest.mark.parametrize(
"offload_active, denoiser_offloaded, expected",
[(False, None, False), (True, None, True), (True, False, False), (True, True, True)],
)
def test_graph_gate_follows_the_denoiser(
monkeypatch, offload_active, denoiser_offloaded, expected, offload_graphs
):
"""With offload graphs on, a moved denoiser is judged on everything else and armed after placement; with the
kill switch, a moved denoiser is refused as before."""
from core.inference import diffusion_cuda_graph as dcg
from core.inference import diffusion_speed
if offload_graphs:
monkeypatch.delenv(dcg.OFFLOAD_CUDA_GRAPH_ENV, raising = False)
expected = False
else:
monkeypatch.setenv(dcg.OFFLOAD_CUDA_GRAPH_ENV, "0")
seen: list = []
def _eligible(target, **kwargs):
seen.append(kwargs["offload_active"])
return False, "stub"
monkeypatch.setattr(dcg, "graph_eligible", _eligible)
pipe = types.SimpleNamespace(components = {})
target = types.SimpleNamespace(device = "cpu", backend = "cpu", dtype = None)
kwargs = {} if denoiser_offloaded is None else {"denoiser_offloaded": denoiser_offloaded}
diffusion_speed.apply_speed_optims(
pipe,
target,
is_gguf = False,
family = types.SimpleNamespace(name = "z-image"),
speed_mode = "default",
offload_active = offload_active,
**kwargs,
)
assert seen == [expected]
class _Hooked:
def __init__(
self,
keys = (),
hf_hook = None,
):
self._diffusers_hook = types.SimpleNamespace(hooks = {k: object() for k in keys})
self._hf_hook = hf_hook
@pytest.mark.parametrize(
"transformer, hooked",
[
(_Hooked(), False),
(_Hooked(("group_offloading", "lazy_prefetch_group_offloading")), True),
(_Hooked(hf_hook = object()), True),
(_Hooked(("first_block_cache",)), False),
],
)
def test_denoiser_hooked(transformer, hooked):
assert dmod._denoiser_hooked(types.SimpleNamespace(transformer = transformer)) is hooked
def _int8_linear(rows = 1024, cols = 1024):
torch = pytest.importorskip("torch")
quant = pytest.importorskip("torchao.quantization")
linear = torch.nn.Linear(cols, rows, bias = False).to(torch.bfloat16)
try:
quant.quantize_(linear, quant.Int8DynamicActivationInt8WeightConfig(version = 2))
except Exception as exc: # noqa: BLE001 - torchao without the v2 int8 tensor
pytest.skip(f"int8 v2 quantisation unavailable: {exc}")
if type(linear.weight).__name__ != "Int8Tensor":
pytest.skip("this torchao does not produce Int8Tensor")
return linear
def test_host_size_counts_packed_torchao_storage_not_the_logical_bf16_size():
linear = _int8_linear(2048, 2048)
# 2048 x 2048 int8 = 4 MiB qdata plus per-row scales; logical bf16 would be 8 MiB.
assert mem._module_host_mib(linear) < 8
assert mem._module_host_mib(linear) >= 4
def test_stream_kept_when_the_packed_weights_fit_the_pin_budget(monkeypatch):
linear = _int8_linear(2048, 2048)
monkeypatch.setattr(mem, "_pinned_memory_capped", lambda: False)
monkeypatch.setattr(mem, "install_group_offload_torchao_swap_retry", lambda: None)
# Between the packed (~4 MiB) and logical bf16 (8 MiB) sizes.
monkeypatch.setattr(mem, "_pin_budget_mib", lambda: 6)
kwargs = mem._torchao_group_offload_kwargs(
linear, {"use_stream": True, "low_cpu_mem_usage": True}
)
assert kwargs["use_stream"] is True
assert kwargs["low_cpu_mem_usage"] is False
def test_storage_size_recurses_through_nested_wrappers():
class Inner:
def __init__(self, n):
self._n = n
def numel(self):
return self._n
def element_size(self):
return 1
class Wrapper:
def __init__(self, *inner):
self._inner = inner
for i, t in enumerate(inner):
setattr(self, f"t{i}", t)
def __tensor_flatten__(self):
return [f"t{i}" for i in range(len(self._inner))], None
def numel(self):
return 10**9
def element_size(self):
return 2
nested = Wrapper(Wrapper(Inner(100), Inner(4)), Inner(7))
assert mem._storage_nbytes(nested) == [100, 4, 7]
@pytest.mark.parametrize(
"env, capped, budget, keeps_stream",
[
("1", True, None, True),
("1", False, 0, True),
("0", False, 10**7, False),
("", False, 10**7, True),
],
)
def test_runtime_pinning_honours_the_same_override_as_the_planner(
monkeypatch, env, capped, budget, keeps_stream
):
linear = _int8_linear(256, 256)
monkeypatch.setenv(mem.GROUP_OFFLOAD_PIN_ENV, env)
monkeypatch.setattr(mem, "_pinned_memory_capped", lambda: capped)
monkeypatch.setattr(mem, "_pin_budget_mib", lambda: budget)
monkeypatch.setattr(mem, "install_group_offload_torchao_swap_retry", lambda: None)
kwargs = mem._torchao_group_offload_kwargs(
linear, {"use_stream": True, "low_cpu_mem_usage": True}
)
assert kwargs["use_stream"] is keeps_stream
def test_several_torchao_denoisers_share_one_pin_budget(monkeypatch):
"""Ideogram 4 streams two torchao transformers: each fits the budget alone, together they do not."""
first, second = _int8_linear(2048, 2048), _int8_linear(2048, 2048)
monkeypatch.delenv(mem.GROUP_OFFLOAD_PIN_ENV, raising = False)
monkeypatch.setattr(mem, "_pinned_memory_capped", lambda: False)
monkeypatch.setattr(mem, "install_group_offload_torchao_swap_retry", lambda: None)
monkeypatch.setattr(mem, "_pin_budget_mib", lambda: 6)
base = {"use_stream": True, "low_cpu_mem_usage": True}
pinned = [0]
kept = [mem._torchao_group_offload_kwargs(m, dict(base), pinned) for m in (first, second)]
assert kept[0]["low_cpu_mem_usage"] is False
assert kept[1].get("low_cpu_mem_usage") is not False
assert pinned[0] <= 6
def test_every_group_offload_site_shares_the_pin_total():
import inspect
import re
source = inspect.getsource(mem)
calls = re.findall(r"(?<!def )_torchao_group_offload_kwargs\(([^)]*)\)", source)
assert calls and all("pinned_mib" in args for args in calls)
@pytest.mark.parametrize("fits_whole, raises", [(False, True), (True, False)])
def test_failed_group_setup_never_onloads_an_oversized_quantised_transformer(
monkeypatch, fits_whole, raises
):
"""Streaming was the only placement the quantised transformer fits; model offload would OOM on first onload."""
calls = []
pipe = types.SimpleNamespace(
enable_model_cpu_offload = lambda device = None: calls.append("model_offload")
)
monkeypatch.setattr(mem, "_apply_group_offload", lambda *a, **k: False)
monkeypatch.setattr(mem, "_pipe_denoisers_hold_torchao", lambda pipe: True)
monkeypatch.setattr(mem, "_model_offload_fits_quantised", lambda plan: fits_whole)
monkeypatch.setattr(mem, "keep_cpu_weights_on_offload", lambda *a, **k: 0)
monkeypatch.setattr(mem, "_enable_vae_saver", lambda *a, **k: False)
plan = _plan(OFFLOAD_GROUP)
if raises:
with pytest.raises(RuntimeError, match = "does not fit the GPU whole"):
mem.apply_memory_plan(pipe, plan, device = "cuda")
assert "model_offload" not in calls
else:
effective, _ = mem.apply_memory_plan(pipe, plan, device = "cuda")
assert effective == OFFLOAD_MODEL
def test_pipeline_host_mib_counts_only_host_resident_weights():
torch = pytest.importorskip("torch")
cpu = torch.nn.Linear(1024, 1024, bias = False)
pipe = types.SimpleNamespace(components = {"transformer": cpu, "scheduler": object()})
assert mem.pipeline_host_mib(pipe) == 4
assert mem.pipeline_host_mib(None) == 0
if torch.cuda.is_available():
pipe.components["transformer"] = cpu.to("cuda")
assert mem.pipeline_host_mib(pipe) == 0
def test_reclaimable_host_ram_widens_the_pin_budget(monkeypatch):
monkeypatch.setattr(mem, "_pin_budget_mib", lambda: 4_000)
plan = types.SimpleNamespace(
estimates = {"model_dense_mib": 20_000, "companion_dense_mib": 8_000}
)
assert not mem._torchao_stream_pinnable(plan)
assert mem._torchao_stream_pinnable(plan, 8_000)