Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

24.8 KB
Raw Blame
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0

"""CPU tests onload to ``meta``: a kept weight returns to its host storage, a stock copy raises (no data)."""

from __future__ import annotations

import types

import pytest

torch = pytest.importorskip("torch")
hooks = pytest.importorskip("accelerate.hooks")

from core.inference import diffusion_memory as dm  # noqa: E402


def _pipe(device, *names):
    """Diffusers-shaped pipe whose enable_model_cpu_offload rebuilds chained CpuOffload hooks each call."""
    torch.manual_seed(0)
    comps = {name: torch.nn.Linear(8, 8) for name in names}
    pipe = types.SimpleNamespace(components = comps, enabled = 0)

    def enable_model_cpu_offload(device = device):
        pipe.enabled += 1
        prev = None
        for module in comps.values():
            hooks.remove_hook_from_module(module)
            module.to("cpu")
            hook = hooks.CpuOffload(execution_device = device, prev_module_hook = prev)
            hooks.add_hook_to_module(module, hook)
            prev = hooks.UserCpuOffloadHook(module, hook)

    pipe.enable_model_cpu_offload = enable_model_cpu_offload
    for name, module in comps.items():
        setattr(pipe, name, module)
    return pipe


def _ptrs(module):
    return [p.data_ptr() for p in module.parameters()]


def _wrapped(module):
    return bool(getattr(module._hf_hook, dm._KEEP_ATTR, False))


def test_every_offload_hook_is_wrapped():
    pipe = _pipe("meta", "text_encoder", "transformer")
    pipe.enable_model_cpu_offload()
    assert dm.keep_cpu_weights_on_offload(pipe) == 2
    assert _wrapped(pipe.text_encoder) and _wrapped(pipe.transformer)
    assert dm.keep_cpu_weights_on_offload(pipe) == 0


def test_hooks_rebuilt_by_a_later_enable_are_wrapped_too():
    pipe = _pipe("meta", "transformer")
    pipe.enable_model_cpu_offload()
    dm.keep_cpu_weights_on_offload(pipe)
    for _ in range(3):
        pipe.enable_model_cpu_offload()
        assert _wrapped(pipe.transformer)


def test_a_device_that_hands_back_new_parameters_takes_the_stock_path():
    pipe = _pipe("meta", "transformer")
    pipe.enable_model_cpu_offload()
    dm.keep_cpu_weights_on_offload(pipe)
    pipe.transformer(torch.ones(1, 8))
    assert pipe.transformer.weight.device.type == "meta"
    with pytest.raises(NotImplementedError):
        pipe.transformer._hf_hook.init_hook(pipe.transformer)


def test_kill_switch_and_missing_hooks_leave_the_stock_path(monkeypatch):
    pipe = _pipe("meta", "transformer")
    pipe.enable_model_cpu_offload()
    monkeypatch.setenv(dm.OFFLOAD_KEEP_CPU_ENV, "0")
    assert dm.keep_cpu_weights_on_offload(pipe) == 0
    assert not _wrapped(pipe.transformer)
    monkeypatch.delenv(dm.OFFLOAD_KEEP_CPU_ENV)
    assert dm.keep_cpu_weights_on_offload(types.SimpleNamespace(components = {})) == 0


def test_subclass_weights_are_not_kept():
    class Sub(torch.Tensor):
        pass

    lin = torch.nn.Linear(2, 2)
    assert dm._keepable(lin.weight)
    lin.weight = torch.nn.Parameter(lin.weight.detach().as_subclass(Sub))
    assert not dm._keepable(lin.weight)


def _gguf_utils():
    pytest.importorskip("gguf")
    return pytest.importorskip("diffusers.quantizers.gguf.utils")


def _gguf_linear(
    utils,
    out_features = 8,
    in_features = 64,
):
    """A diffusers GGUFLinear holding a real Q8_0 weight: per 32-value block, an fp16 scale then 32 int8 codes."""
    import numpy as np

    rng = np.random.default_rng(0)
    blocks = out_features * in_features // 32
    scale = np.full((blocks, 1), 0.01, dtype = np.float16).view(np.uint8)
    codes = rng.integers(-127, 128, size = (blocks, 32), dtype = np.int8).view(np.uint8)
    raw = torch.from_numpy(np.concatenate([scale, codes], axis = 1).reshape(out_features, -1).copy())
    lin = utils.GGUFLinear(in_features, out_features, bias = True, compute_dtype = torch.float32)
    lin.weight = utils.GGUFParameter(raw, quant_type = utils.gguf.GGMLQuantizationType.Q8_0)
    return lin


def test_gguf_weights_are_kept_but_other_subclasses_are_not():
    utils = _gguf_utils()
    lin = _gguf_linear(utils)
    assert type(lin.weight) is utils.GGUFParameter
    assert dm._keepable(lin.weight) and dm._keepable(lin.bias)

    class Sub(torch.Tensor):
        pass

    other = torch.nn.Parameter(torch.zeros(2).as_subclass(Sub))
    assert not dm._keepable(other)


def test_an_all_subclass_module_is_not_rescanned_every_forward():
    class Sub(torch.Tensor):
        pass

    pipe = _pipe("meta", "transformer")
    lin = pipe.transformer
    lin.weight = torch.nn.Parameter(lin.weight.detach().as_subclass(Sub))
    lin.bias = torch.nn.Parameter(lin.bias.detach().as_subclass(Sub))
    pipe.enable_model_cpu_offload()
    dm.keep_cpu_weights_on_offload(pipe)
    scans = []
    stock = type(lin).named_parameters

    def named_parameters(self, *args, **kwargs):
        # accelerate's own device probe passes recurse=; the wrapper's scans pass nothing.
        if not args and not kwargs:
            scans.append(1)
        return stock(self, *args, **kwargs)

    lin.named_parameters = types.MethodType(named_parameters, lin)
    for _ in range(5):
        lin._hf_hook.pre_forward(lin, torch.zeros(1, 8))
    assert scans == []


def test_model_offload_plan_wraps_the_pipeline():
    pipe = _pipe("meta", "transformer")
    plan = types.SimpleNamespace(
        offload_policy = dm.OFFLOAD_MODEL, vae_tiling = False, vae_slicing = False
    )
    policy, _ = dm.apply_memory_plan(pipe, plan, device = "meta")
    assert policy == dm.OFFLOAD_MODEL
    assert _wrapped(pipe.transformer)


cuda = pytest.mark.skipif(not torch.cuda.is_available(), reason = "needs a CUDA device")


def _render(pipe, x):
    out = x
    for module in pipe.components.values():
        out = module(out)
    out = out.detach().cpu()
    pipe.transformer._hf_hook.init_hook(pipe.transformer)
    pipe.enable_model_cpu_offload()
    return out


@cuda
def test_real_offload_keeps_host_storage_and_is_bit_identical(monkeypatch):
    monkeypatch.setenv(dm.OFFLOAD_PIN_ENV, "0")
    x = torch.randn(4, 8)
    stock = _pipe("cuda", "text_encoder", "transformer")
    stock.enable_model_cpu_offload()
    ref = _render(stock, x)
    pipe = _pipe("cuda", "text_encoder", "transformer")
    pipe.enable_model_cpu_offload()
    dm.keep_cpu_weights_on_offload(pipe)
    host = {n: _ptrs(m) for n, m in pipe.components.items()}
    for _ in range(3):
        assert torch.equal(_render(pipe, x), ref)
        assert {n: _ptrs(m) for n, m in pipe.components.items()} == host


@cuda
def test_a_weight_written_on_the_device_is_copied_back():
    pipe = _pipe("cuda", "transformer")
    pipe.enable_model_cpu_offload()
    dm.keep_cpu_weights_on_offload(pipe)
    before = pipe.transformer.weight.detach().clone()
    pipe.transformer(torch.ones(1, 8))
    with torch.no_grad():
        pipe.transformer.weight.add_(1)
    pipe.transformer._hf_hook.init_hook(pipe.transformer)
    assert pipe.transformer.weight.device.type == "cpu"
    assert torch.equal(pipe.transformer.weight.detach(), before + 1)


@cuda
def test_weights_are_pinned_on_first_onload_into_shared_chunks(monkeypatch):
    monkeypatch.setenv(dm.OFFLOAD_PIN_ENV, "1")
    x = torch.randn(4, 8)
    stock = _pipe("cuda", "text_encoder", "transformer")
    stock.enable_model_cpu_offload()
    ref = _render(stock, x)
    pipe = _pipe("cuda", "text_encoder", "transformer")
    pipe.enable_model_cpu_offload()
    dm.keep_cpu_weights_on_offload(pipe)
    assert not pipe.transformer.weight.is_pinned()
    for _ in range(2):
        assert torch.equal(_render(pipe, x), ref)
    for module in pipe.components.values():
        assert all(p.is_pinned() for p in module.parameters())
        assert (
            module.weight.untyped_storage().data_ptr() == module.bias.untyped_storage().data_ptr()
        )


@cuda
def test_pinning_is_off_by_switch_and_when_ram_is_short(monkeypatch):
    monkeypatch.setenv(dm.OFFLOAD_PIN_ENV, "0")
    pipe = _pipe("cuda", "transformer")
    pipe.enable_model_cpu_offload()
    dm.keep_cpu_weights_on_offload(pipe)
    _render(pipe, torch.randn(4, 8))
    assert not pipe.transformer.weight.is_pinned()

    psutil = pytest.importorskip("psutil")
    monkeypatch.delenv(dm.OFFLOAD_PIN_ENV)
    short = types.SimpleNamespace(total = 16 << 30, available = 2 << 30)
    monkeypatch.setattr(psutil, "virtual_memory", lambda: short)
    pipe = _pipe("cuda", "transformer")
    pipe.enable_model_cpu_offload()
    dm.keep_cpu_weights_on_offload(pipe)
    _render(pipe, torch.randn(4, 8))
    assert not pipe.transformer.weight.is_pinned()


@cuda
def test_a_non_contiguous_weight_keeps_its_layout(monkeypatch):
    monkeypatch.setenv(dm.OFFLOAD_PIN_ENV, "1")
    pipe = _pipe("cuda", "transformer")
    lin = pipe.transformer
    lin.weight = torch.nn.Parameter(lin.weight.detach().t().contiguous().t())
    assert not lin.weight.is_contiguous()
    pipe.enable_model_cpu_offload()
    dm.keep_cpu_weights_on_offload(pipe)
    _render(pipe, torch.randn(4, 8))
    assert not lin.weight.is_pinned() and lin.bias.is_pinned()
    assert lin.weight.stride() == (1, 8)


@cuda
@pytest.mark.parametrize(
    "conv, fmt, shape",
    [
        (torch.nn.Conv2d, torch.channels_last, (1, 4, 6, 6)),
        (torch.nn.Conv3d, torch.channels_last_3d, (1, 4, 3, 6, 6)),
    ],
)
def test_a_channels_last_conv_weight_is_pinned_in_its_layout(monkeypatch, conv, fmt, shape):
    monkeypatch.setenv(dm.OFFLOAD_PIN_ENV, "1")
    x = torch.randn(*shape)
    seen = []

    def build():
        torch.manual_seed(0)
        module = conv(4, 4, 3, padding = 1).to(memory_format = fmt)
        module.register_forward_pre_hook(
            lambda mod, args: seen.append(mod.weight.is_contiguous(memory_format = fmt))
        )
        pipe = types.SimpleNamespace(components = {"vae": module}, vae = module)

        def enable_model_cpu_offload(device = "cuda"):
            hooks.remove_hook_from_module(module)
            module.to("cpu")
            hooks.add_hook_to_module(module, hooks.CpuOffload(execution_device = device))

        pipe.enable_model_cpu_offload = enable_model_cpu_offload
        pipe.enable_model_cpu_offload()
        return pipe

    def render(pipe):
        out = pipe.vae(x).detach().cpu()
        pipe.vae._hf_hook.init_hook(pipe.vae)
        pipe.enable_model_cpu_offload()
        return out

    ref = render(build())
    pipe = build()
    dm.keep_cpu_weights_on_offload(pipe)
    for _ in range(2):
        assert torch.equal(render(pipe), ref)
    weight = pipe.vae.weight
    assert weight.device.type == "cpu" and weight.is_pinned()
    assert weight.is_contiguous(memory_format = fmt)
    assert all(seen)


@cuda
def test_a_parameter_replaced_on_the_device_is_copied_back():
    pipe = _pipe("cuda", "transformer")
    pipe.enable_model_cpu_offload()
    dm.keep_cpu_weights_on_offload(pipe)
    pipe.transformer(torch.ones(1, 8))
    # Same name and version as the onloaded weight: only identity tells it apart.
    old = pipe.transformer.weight
    new = torch.nn.Parameter(torch.full_like(old, 3.0))
    with torch.no_grad():
        while new._version < old._version:
            new.mul_(1)
    assert new._version == old._version
    pipe.transformer.weight = new
    pipe.transformer._hf_hook.init_hook(pipe.transformer)
    assert pipe.transformer.weight.device.type == "cpu"
    assert torch.equal(pipe.transformer.weight.detach(), torch.full((8, 8), 3.0))


def _host_of(module):
    return {name: p.data for name, p in module.named_parameters()}


@cuda
def test_the_ram_gate_counts_the_chunks_really_allocated(monkeypatch):
    # Weight 256 B and bias 32 B (256 aligned) do not share a 384 B chunk: 384 + 256 B are allocated for 512.
    monkeypatch.setattr(dm, "_PIN_CHUNK_BYTES", 384)
    # The gate reads MiB (system available capped by the cgroup headroom), so pin the readings and move the
    # reserve by bytes instead: 600 B of room is short of the 640 B allocated, 640 B is enough.
    monkeypatch.setattr(dm, "_available_system_memory_mib", lambda: 5 << 10)
    monkeypatch.setattr(dm, "_host_ram_capacity_mib", lambda: 16 << 10)
    lin = torch.nn.Linear(8, 8)
    monkeypatch.setattr(dm, "_PIN_RESERVE_MIN_BYTES", (5 << 30) - 600)
    assert dm._pin_host_weights(lin, _host_of(lin)) == 0
    assert not lin.weight.is_pinned()
    monkeypatch.setattr(dm, "_PIN_RESERVE_MIN_BYTES", (5 << 30) - 640)
    assert dm._pin_host_weights(lin, _host_of(lin)) == 640
    assert lin.weight.is_pinned() and lin.bias.is_pinned()


@cuda
def test_a_failed_pin_hands_the_partial_chunks_back(monkeypatch):
    monkeypatch.setenv(dm.OFFLOAD_PIN_ENV, "1")
    monkeypatch.setattr(dm, "_PIN_CHUNK_BYTES", 384)
    real_empty, calls, emptied = torch.empty, [], []

    def empty(*args, **kwargs):
        calls.append(args)
        if len(calls) == 2:
            raise RuntimeError("cudaHostAlloc failed")
        return real_empty(*args, **kwargs)

    lin = torch.nn.Linear(8, 8)
    host = _host_of(lin)
    monkeypatch.setattr(torch, "empty", empty)
    monkeypatch.setattr(dm, "_host_empty_cache", lambda: emptied.append(True))
    assert dm._pin_host_weights(lin, host) == 0
    assert len(calls) == 2 and emptied
    assert not lin.weight.is_pinned() and not lin.bias.is_pinned()


@cuda
@pytest.mark.parametrize("how", ["parameter", "data"])
def test_a_parameter_replaced_while_offloaded_is_not_restored_to_the_old_weight(how):
    pipe = _pipe("cuda", "transformer")
    pipe.enable_model_cpu_offload()
    dm.keep_cpu_weights_on_offload(pipe)
    pipe.transformer(torch.ones(1, 8))
    pipe.transformer._hf_hook.init_hook(pipe.transformer)
    assert pipe.transformer.weight.device.type == "cpu"
    fresh = torch.full((8, 8), 5.0)
    if how == "parameter":
        pipe.transformer.weight = torch.nn.Parameter(fresh)
    else:
        pipe.transformer.weight.data = fresh
    out = pipe.transformer(torch.ones(1, 8))
    assert torch.allclose(
        out.cpu(), torch.full((1, 8), 40.0) + pipe.transformer.bias.detach().cpu()
    )
    pipe.transformer._hf_hook.init_hook(pipe.transformer)
    assert pipe.transformer.weight.device.type == "cpu"
    assert torch.equal(pipe.transformer.weight.detach(), torch.full((8, 8), 5.0))


@cuda
def test_a_weight_reassigned_through_data_on_the_device_is_copied_back():
    pipe = _pipe("cuda", "transformer")
    pipe.enable_model_cpu_offload()
    dm.keep_cpu_weights_on_offload(pipe)
    pipe.transformer(torch.ones(1, 8))
    version = pipe.transformer.weight._version
    pipe.transformer.weight.data = torch.full((8, 8), 7.0, device = "cuda")
    assert pipe.transformer.weight._version == version
    pipe.transformer._hf_hook.init_hook(pipe.transformer)
    assert pipe.transformer.weight.device.type == "cpu"
    assert torch.equal(pipe.transformer.weight.detach(), torch.full((8, 8), 7.0))


def _fake_cgroup(
    tmp_path,
    monkeypatch,
    limit,
    current,
    version = 2,
):
    import core.inference.llama_cpp as llama_cpp_module

    root = tmp_path / "cgroup"
    leaf = root / ("studio.slice" if version == 2 else "memory/studio.slice")
    leaf.mkdir(parents = True)
    names = (
        ("memory.max", "memory.current")
        if version == 2
        else ("memory.limit_in_bytes", "memory.usage_in_bytes")
    )
    if limit is not None:
        (leaf / names[0]).write_text(str(limit), encoding = "utf-8")
        (leaf / names[1]).write_text(str(current), encoding = "utf-8")
    proc = tmp_path / "self.cgroup"
    line = "0::/studio.slice" if version == 2 else "4:memory:/studio.slice"
    proc.write_text(line + "\n", encoding = "utf-8")
    monkeypatch.setattr(llama_cpp_module, "_CGROUP_ROOT", str(root))
    monkeypatch.setattr(llama_cpp_module, "_PROC_SELF_CGROUP", str(proc))


@pytest.mark.parametrize(
    "version, limit, current, host_available, pinned",
    [
        # 64 GiB container on a 512 GiB host, 10 GiB left: the host reading would pin 16 GiB.
        (2, 64 << 30, 54 << 30, 400 << 30, False),
        (1, 64 << 30, 54 << 30, 400 << 30, False),
        # 40 GiB left: fits the container's 15% reserve; 15% of the host would refuse it.
        (2, 64 << 30, 24 << 30, 400 << 30, True),
        (1, 64 << 30, 24 << 30, 400 << 30, True),
        # 24 GiB left: 8 GiB would remain, under the container's 9.6 GiB reserve.
        (2, 64 << 30, 40 << 30, 400 << 30, False),
        (2, 1 << 50, 0, 400 << 30, True),
        (2, 1 << 50, 0, 90 << 30, False),
        (2, None, 0, 400 << 30, True),
        (2, None, 0, 90 << 30, False),
    ],
)
def test_the_ram_gate_is_sized_from_the_container(
    tmp_path, monkeypatch, version, limit, current, host_available, pinned
):
    psutil = pytest.importorskip("psutil")
    _fake_cgroup(tmp_path, monkeypatch, limit, current, version)
    monkeypatch.setattr(
        psutil,
        "virtual_memory",
        lambda: types.SimpleNamespace(total = 512 << 30, available = host_available),
    )
    # 16 GiB (power of two, no rounding): the gate refuses before any allocation.
    data = types.SimpleNamespace(
        nbytes = 16 << 30, is_contiguous = lambda: True, is_pinned = lambda: False
    )
    weight = types.SimpleNamespace(device = torch.device("cpu"), data = data)
    module = types.SimpleNamespace(named_parameters = lambda: [("weight", weight)])
    allocated = []

    def empty(*args, **kwargs):
        allocated.append(args)
        raise RuntimeError("not allocating in a test")

    monkeypatch.setattr(torch.cuda, "is_available", lambda: True)
    monkeypatch.setattr(dm, "_keepable", lambda p: True)
    monkeypatch.setattr(dm, "_host_empty_cache", lambda: None)
    monkeypatch.setattr(torch, "empty", empty)
    assert dm._pin_host_weights(module, {"weight": data}) == 0
    assert bool(allocated) is pinned


def _gguf_pipe(device):
    utils = _gguf_utils()
    pipe = _pipe(device, "text_encoder", "transformer", "vae")
    comps = pipe.components
    comps["text_encoder"] = pipe.text_encoder = torch.nn.Linear(8, 64)
    comps["transformer"] = pipe.transformer = _gguf_linear(utils)
    return pipe, utils


def _run(pipe, x):
    """One call the way a diffusers pipeline makes it: the last module stays onloaded until the re-enable."""
    out = x
    for module in pipe.components.values():
        out = module(out)
    out = out.detach().cpu()
    pipe.enable_model_cpu_offload()
    return out


class _BufferLinear(torch.nn.Module):
    """A weight held as plain buffers, as the torchao-free int8 linear stores it."""

    def __init__(self):
        super().__init__()
        torch.manual_seed(0)
        self.register_buffer("weight_q", torch.randint(-127, 127, (8, 8), dtype = torch.int8))
        self.register_buffer("weight_scale", torch.rand(8))
        self.bias = torch.nn.Parameter(torch.zeros(8))

    def forward(self, x):
        return x @ (self.weight_q.float() * self.weight_scale[:, None]).t() + self.bias


def _buffer_pipe(device):
    pipe = _pipe(device, "transformer")
    pipe.components["transformer"] = pipe.transformer = _BufferLinear()
    return pipe


def _buffer_ptrs(module):
    return [b.data_ptr() for b in module.buffers()]


@cuda
@pytest.mark.parametrize("pin", ["0", "1"])
def test_a_gguf_transformer_keeps_its_host_bytes_and_quant_type(monkeypatch, pin):
    monkeypatch.setenv(dm.OFFLOAD_PIN_ENV, pin)
    x = torch.randn(4, 8)
    stock, _ = _gguf_pipe("cuda")
    stock.enable_model_cpu_offload()
    ref = _run(stock, x)
    pipe, utils = _gguf_pipe("cuda")
    raw = pipe.transformer.weight.detach().clone()
    pipe.enable_model_cpu_offload()
    dm.keep_cpu_weights_on_offload(pipe)
    _run(pipe, x)
    # Held so a stock copy can never land at a freed address and pass for the kept tensor.
    held = [p.data for p in pipe.transformer.parameters()]
    host = [t.data_ptr() for t in held]
    for _ in range(3):
        assert torch.equal(_run(pipe, x), ref)
        weight = pipe.transformer.weight
        assert weight.device.type == "cpu" and _ptrs(pipe.transformer) == host
        assert type(weight) is utils.GGUFParameter
        assert weight.quant_type == utils.gguf.GGMLQuantizationType.Q8_0
        assert torch.equal(weight.detach().view(torch.uint8), raw.view(torch.uint8))
        assert weight.is_pinned() is (pin == "1")


@cuda
def test_the_module_left_onloaded_by_a_call_is_offloaded_through_its_kept_hook(monkeypatch):
    monkeypatch.setenv(dm.OFFLOAD_PIN_ENV, "0")
    pipe = _pipe("cuda", "text_encoder", "transformer", "vae")
    pipe.enable_model_cpu_offload()
    dm.keep_cpu_weights_on_offload(pipe)
    held = {n: [p.data for p in m.parameters()] for n, m in pipe.components.items()}
    host = {n: [t.data_ptr() for t in ts] for n, ts in held.items()}
    for _ in range(3):
        _run(pipe, torch.randn(4, 8))
        assert pipe.vae.weight.device.type == "cpu"
        assert {n: _ptrs(m) for n, m in pipe.components.items()} == host


@cuda
@pytest.mark.parametrize("pin", ["0", "1"])
def test_buffer_weights_keep_their_host_storage_and_are_pinned(monkeypatch, pin):
    monkeypatch.setenv(dm.OFFLOAD_PIN_ENV, pin)
    x = torch.randn(4, 8)
    stock = _buffer_pipe("cuda")
    stock.enable_model_cpu_offload()
    ref = _render(stock, x)
    pipe = _buffer_pipe("cuda")
    pipe.enable_model_cpu_offload()
    dm.keep_cpu_weights_on_offload(pipe)
    assert torch.equal(_render(pipe, x), ref)
    host = _buffer_ptrs(pipe.transformer)
    for _ in range(3):
        assert torch.equal(_render(pipe, x), ref)
        assert _buffer_ptrs(pipe.transformer) == host
    assert all(b.is_pinned() == (pin == "1") for b in pipe.transformer.buffers())


@cuda
def test_a_buffer_written_on_the_device_is_copied_back():
    pipe = _buffer_pipe("cuda")
    pipe.enable_model_cpu_offload()
    dm.keep_cpu_weights_on_offload(pipe)
    before = pipe.transformer.weight_scale.clone()
    pipe.transformer(torch.ones(1, 8))
    pipe.transformer.weight_scale.add_(1)
    pipe.transformer._hf_hook.init_hook(pipe.transformer)
    assert pipe.transformer.weight_scale.device.type == "cpu"
    assert torch.equal(pipe.transformer.weight_scale, before + 1)


@cuda
def test_buffers_onloaded_under_inference_mode_do_not_break_the_hook():
    """A buffer moved under inference_mode is an inference tensor with no version counter; it takes the stock copy."""
    x = torch.randn(4, 8)
    stock = _buffer_pipe("cuda")
    stock.enable_model_cpu_offload()
    with torch.inference_mode():
        ref = _render(stock, x)
    pipe = _buffer_pipe("cuda")
    pipe.enable_model_cpu_offload()
    dm.keep_cpu_weights_on_offload(pipe)
    for _ in range(3):
        with torch.inference_mode():
            assert torch.equal(_render(pipe, x), ref)


def _int8_pipe(device):
    """The _pipe layout with torchao Int8Tensor weights (Studio's int8 denoisers under low_vram)."""
    quant = pytest.importorskip("torchao.quantization")
    pipe = _pipe(device, "text_encoder", "transformer")
    lin = torch.nn.Linear(64, 64, dtype = torch.bfloat16)
    torch.manual_seed(1)
    torch.nn.init.normal_(lin.weight)
    try:
        config = quant.Int8DynamicActivationInt8WeightConfig(version = 2)
    except TypeError:
        config = quant.Int8DynamicActivationInt8WeightConfig()
    quant.quantize_(torch.nn.Sequential(lin), config)
    if _inner_of(lin.weight.data) is None:
        pytest.skip("this torchao build stores int8 weights without a traceable wrapper")
    pipe.components["text_encoder"] = torch.nn.Linear(8, 64, dtype = torch.bfloat16)
    pipe.components["transformer"] = lin
    pipe.text_encoder, pipe.transformer = pipe.components["text_encoder"], lin
    return pipe


def _inner_ptrs(module):
    return [x.data_ptr() for _, x in _inner_of(module.weight.data)]


def _inner_of(t):
    """A torchao wrapper weight's inner tensors, read without the module under test."""
    from torch.utils._python_dispatch import is_traceable_wrapper_subclass

    if not is_traceable_wrapper_subclass(t):
        return None
    names, _ctx = t.__tensor_flatten__()
    inner = [(n, getattr(t, n)) for n in names]
    return inner if all(type(x) is torch.Tensor for _, x in inner) else None


@cuda
@pytest.mark.parametrize("pin", ["0", "1"])
def test_torchao_int8_weights_keep_their_host_storage_and_are_pinned(monkeypatch, pin):
    """Each offload re-points the int8 weight at its kept host tensors (no device-to-host copy, no new host
    allocation), the inner tensors are pinned once, and every render matches the stock offload bit for bit."""
    monkeypatch.setenv(dm.OFFLOAD_PIN_ENV, pin)
    x = torch.randn(4, 8, dtype = torch.bfloat16)
    stock = _int8_pipe("cuda")
    stock.enable_model_cpu_offload()
    ref = _render(stock, x)
    pipe = _int8_pipe("cuda")
    pipe.enable_model_cpu_offload()
    dm.keep_cpu_weights_on_offload(pipe)
    assert _wrapped(pipe.transformer)
    outs, host = [], None
    for _ in range(3):
        outs.append(_render(pipe, x))
        assert pipe.transformer.weight.device.type == "cpu"
        now = _inner_ptrs(pipe.transformer)
        assert host is None or now == host  # the same host tensors every render
        host = now
    for out in outs:
        assert torch.equal(out, ref)
    inner = _inner_of(pipe.transformer.weight.data)
    assert all(t.is_pinned() for _, t in inner) == (pin == "1")