Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

24.9 KB
Raw Blame
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0

"""Event-fenced prefetch: CPU checks order and window; CUDA checks bit identity, no host sync."""

from __future__ import annotations

import copy
import types
import warnings

import pytest
import torch

import core.inference.diffusion_memory as dm
import core.inference.diffusion_offload_prefetch as op


@pytest.fixture(autouse = True)
def _clean_env(monkeypatch):
    for name in (
        op.ASYNC_PREFETCH_ENV,
        op.PREFETCH_DEPTH_ENV,
        "UNSLOTH_DIFFUSION_PARTIAL_RESIDENT",
        "UNSLOTH_DIFFUSION_GROUP_OFFLOAD_PIN",
        "UNSLOTH_DIFFUSION_PIN_TOP_GROUP",
    ):
        monkeypatch.delenv(name, raising = False)


class _Group:
    def __init__(self, name: str):
        self.name = name


def _cpu_prefetcher(
    names,
    nbytes = 10,
    depth = 2,
    window = None,
):
    groups = [_Group(n) for n in names]
    pf = op.GroupPrefetcher.__new__(op.GroupPrefetcher)
    pf.module = None
    pf.device = torch.device("cpu")
    pf.depth = depth
    pf.groups = groups
    pf.by_id = {id(g): g for g in groups}
    pf.nbytes = {id(g): nbytes for g in groups}
    pf.window = window if window is not None else (depth + 1) * nbytes
    pf.ready, pf.inflight_bytes, pf.peak_inflight_bytes = {}, 0, 0
    pf.order, pf.seen, pf.pos, pf.active, pf.on_order, pf.pending = [], [], 0, False, False, False
    pf.stream, pf.fence_first = None, False
    pf.stats = {"forwards": 0, "copies": 0, "prefetched": 0, "missed": 0, "dropped": 0}
    log: list = []

    def issue(group):
        log.append(("copy", group.name))
        pf.ready[id(group)] = object()
        pf.inflight_bytes += pf.nbytes[id(group)]
        pf.peak_inflight_bytes = max(pf.peak_inflight_bytes, pf.inflight_bytes)

    def release(
        group,
        event,
        counted = True,
    ):
        log.append(("free", group.name))
        if counted:
            pf.inflight_bytes -= pf.nbytes[id(group)]

    class _Compute:
        def wait_event(self, event):
            log.append(("wait", None))

    pf._issue = issue
    pf._release = release
    pf._compute = lambda: _Compute()
    pf.owns = lambda g: not getattr(g, "resident", False)
    return pf, groups, log


def _forward(
    pf,
    groups,
    skip = (),
):
    pf.begin()
    for g in groups:
        if g.name in skip:
            continue
        pf.onload(g)
        pf.offload(g)
    pf.end()


def test_first_forward_records_the_order_and_copies_each_group_when_it_runs():
    pf, groups, log = _cpu_prefetcher("abcd")
    _forward(pf, groups)
    assert pf.order == [id(g) for g in groups]
    assert [e for e in log if e[0] == "copy"] == [("copy", n) for n in "abcd"]
    assert pf.stats["missed"] == 0 and pf.inflight_bytes == 0


def test_later_forwards_keep_depth_groups_in_flight_ahead():
    pf, groups, log = _cpu_prefetcher("abcdef", depth = 2)
    _forward(pf, groups)
    log.clear()
    pf.begin()
    # no copy at forward start: the top-level upload must reach the copy engine first
    assert not log
    pf.onload(groups[0])
    assert [n for k, n in log if k == "copy"] == ["a", "b", "c"]
    for g in groups:
        if g is not groups[0]:
            pf.onload(g)
        pf.offload(g)
    pf.end()
    assert pf.stats["missed"] == 0 and pf.stats["prefetched"] == 6
    assert pf.peak_inflight_bytes <= pf.window


def test_window_bounds_bytes_in_flight():
    pf, groups, log = _cpu_prefetcher("abcdef", nbytes = 10, depth = 4, window = 25)
    _forward(pf, groups)
    for _ in range(2):
        _forward(pf, groups)
        assert pf.peak_inflight_bytes <= 25


def test_one_group_larger_than_the_window_still_progresses():
    pf, groups, log = _cpu_prefetcher("abc", nbytes = 10, depth = 2, window = 5)
    _forward(pf, groups)
    _forward(pf, groups)
    assert pf.stats["forwards"] == 2 and pf.inflight_bytes == 0


class _FrozenGpu:
    """Card whose events complete only when the host waits; tracks pinned staging buffers whose copy has not run."""

    def __init__(self):
        self.outstanding = 0
        self.peak = 0
        self.unattributed = 0
        self.waits = 0
        gpu = self

        class Event:
            def __init__(self):
                self.staged = 0
                self.done = False

            def record(self, stream = None):
                self.staged, gpu.unattributed = gpu.unattributed, 0

            def query(self):
                return self.done

            def synchronize(self):
                gpu.waits += 1
                if not self.done:
                    self.done = True
                    gpu.outstanding -= self.staged

        class HostTensor:
            def __init__(
                self,
                nbytes: int,
                pinned: bool = False,
            ):
                self.nbytes = nbytes
                self.pinned = pinned

            def is_pinned(self):
                return self.pinned

            def numel(self):
                return self.nbytes

            def element_size(self):
                return 1

            def pin_memory(self):
                gpu.outstanding += self.nbytes
                gpu.unattributed += self.nbytes
                gpu.peak = max(gpu.peak, gpu.outstanding)
                return HostTensor(self.nbytes, pinned = True)

            def to(
                self,
                device,
                non_blocking = False,
            ):
                return self

        self.Event = Event
        self.HostTensor = HostTensor


class _Param:
    data = None


def _frozen_gpu_prefetcher(
    monkeypatch,
    pinned: bool,
    groups_n: int = 10,
    tensors: int = 3,
):
    import contextlib

    gpu = _FrozenGpu()
    monkeypatch.setattr(torch.cuda, "Event", gpu.Event)
    monkeypatch.setattr(torch.cuda, "stream", lambda s: contextlib.nullcontext())
    monkeypatch.setattr(op, "_go", lambda: types.SimpleNamespace())
    groups = []
    for _ in range(groups_n):
        params = [_Param() for _ in range(tensors)]
        groups.append(
            types.SimpleNamespace(
                modules = [],
                parameters = params,
                buffers = [],
                cpu_param_dict = {p: gpu.HostTensor(100, pinned) for p in params},
                stream = object(),
            )
        )
    pf = op.GroupPrefetcher(None, groups, "cpu", depth = 1)
    return pf, groups, gpu


def test_unpinned_staging_stays_in_the_window_on_a_slow_card(monkeypatch):
    """Host a whole forward ahead of the card: unbounded, staging held the whole denoiser in pinned RAM."""
    pf, groups, gpu = _frozen_gpu_prefetcher(monkeypatch, pinned = False)
    group_bytes = 300
    assert pf.window == 2 * group_bytes
    for g in groups:
        pf._issue(g)
    assert gpu.peak <= pf.window, (gpu.peak, pf.window)
    assert gpu.waits > 0
    assert pf.stats["copies"] == len(groups)
    assert all(p.data is not None for g in groups for p in g.parameters)


def test_landed_copies_free_staging_without_a_wait(monkeypatch):
    pf, groups, gpu = _frozen_gpu_prefetcher(monkeypatch, pinned = False)
    for g in groups:
        pf._issue(g)
        for event, _n in list(pf.staged):
            event.done = True
    assert gpu.waits == 0


def test_pinned_host_copies_never_wait(monkeypatch):
    pf, groups, gpu = _frozen_gpu_prefetcher(monkeypatch, pinned = True)
    for g in groups:
        pf._issue(g)
    assert gpu.waits == 0 and gpu.peak == 0


def test_resident_groups_are_skipped_and_not_counted():
    pf, groups, log = _cpu_prefetcher("abcdef", depth = 2)
    _forward(pf, groups)
    for g in groups[:3]:
        g.resident = True
    log.clear()
    pf.begin()
    assert not log
    pf.kick()
    assert [n for k, n in log if k == "copy"] == ["d", "e"]
    pf.kick()
    assert [n for k, n in log if k == "copy"] == ["d", "e"]
    pf.end()


def test_off_order_forward_stops_prefetch_and_rerecords():
    pf, groups, log = _cpu_prefetcher("abcdef", depth = 2)
    _forward(pf, groups)
    _forward(pf, groups, skip = "c")
    assert ("free", "c") in log
    assert pf.order == [id(g) for g in groups if g.name != "c"]
    assert pf.inflight_bytes == 0 and not pf.ready


def test_exception_mid_forward_releases_everything_on_the_device():
    pf, groups, log = _cpu_prefetcher("abcdef", depth = 2)
    _forward(pf, groups)
    pf.begin()
    pf.onload(groups[0])
    pf.end()
    assert not pf.ready and pf.inflight_bytes == 0


def test_kill_switch_and_cpu_leave_diffusers_alone(monkeypatch):
    m = torch.nn.Linear(4, 4)
    assert op.install_group_prefetch(m, "cpu") == 0
    monkeypatch.setenv(op.ASYNC_PREFETCH_ENV, "0")
    assert op.install_group_prefetch(m, "cuda") == 0


def test_depth_env(monkeypatch):
    assert op.prefetch_depth() == op.DEFAULT_PREFETCH_DEPTH
    monkeypatch.setenv(op.PREFETCH_DEPTH_ENV, "1")
    assert op.prefetch_depth() == 1
    monkeypatch.setenv(op.PREFETCH_DEPTH_ENV, "99")
    assert op.prefetch_depth() == op.MAX_PREFETCH_DEPTH
    monkeypatch.setenv(op.PREFETCH_DEPTH_ENV, "x")
    assert op.prefetch_depth() == op.DEFAULT_PREFETCH_DEPTH


def test_resident_onload_skips_the_copy_stream_wait_when_fenced(monkeypatch):
    pytest.importorskip("diffusers.hooks")  # _keep_groups_resident is a no-op without this import

    class Stream:
        waits = 0

        def synchronize(self):
            Stream.waits += 1

    module = types.SimpleNamespace(_unsloth_stream_state = {"streamed": 1, "fenced": True})
    stream = Stream()
    groups = [
        types.SimpleNamespace(
            modules = [],
            parameters = [],
            buffers = [],
            offload_leader = object(),
            stream = stream,
            cpu_param_dict = {},
            offload_to_disk_path = None,
        )
        for _ in range(2)
    ]
    monkeypatch.setattr(dm, "_offload_groups", lambda m: groups)
    dm._keep_groups_resident(module, 1, "cpu")
    module._unsloth_stream_state["streamed"] = 1
    for g in groups:
        g.onload_()
    assert Stream.waits == 0
    kicks = []
    module._unsloth_stream_state["kick"] = lambda: kicks.append(1)
    for g in groups:
        g.onload_()
    assert Stream.waits == 0 and len(kicks) == 2
    del module._unsloth_stream_state["kick"]
    module._unsloth_stream_state["fenced"] = False
    for g in groups:
        g.onload_()
    assert Stream.waits == 2


def _cuda():
    if not torch.cuda.is_available():
        pytest.skip("needs CUDA: diffusers stream group offload")
    pytest.importorskip("diffusers.hooks")


class _Net(torch.nn.Module):
    def __init__(
        self,
        width = 1024,
        blocks = 8,
        sleep_cycles = 0,
    ):
        super().__init__()
        self.proj_in = torch.nn.Linear(64, width)
        self.blocks = torch.nn.ModuleList(torch.nn.Linear(width, width) for _ in range(blocks))
        self.proj_out = torch.nn.Linear(width, 64)
        self.sleep_cycles = sleep_cycles

    def forward(
        self,
        x,
        upto = None,
    ):
        x = self.proj_in(x)
        for i, block in enumerate(self.blocks):
            if upto is not None and i >= upto:
                break
            if self.sleep_cycles:
                # simulate a slow GPU so the host queues later copies before this block reads
                torch.cuda._sleep(self.sleep_cycles)
            x = torch.nn.functional.gelu(block(x))
        return self.proj_out(x)


def _streamed(
    net,
    resident_mib = None,
    **kwargs,
):
    pipe = types.SimpleNamespace(transformer = net, components = {"transformer": net})
    if resident_mib:
        kwargs["resident_transformer_mib"] = resident_mib
    assert dm._apply_group_offload(pipe, "cuda", None, **kwargs)
    return pipe


def _count_syncs(fn):
    """Host synchronizations torch reports (stream / device synchronize, blocking copies) while ``fn`` runs."""
    n = 0
    prev = torch.cuda.get_sync_debug_mode()
    torch.cuda.set_sync_debug_mode("warn")
    try:
        with warnings.catch_warnings(record = True) as caught:
            warnings.simplefilter("always")
            out = fn()
        n = sum(1 for w in caught if "synchroniz" in str(w.message).lower())
    finally:
        torch.cuda.set_sync_debug_mode(prev)
    return out, n


@pytest.mark.parametrize("depth", ["1", "2", "3"])
@pytest.mark.parametrize("resident_mib", [None, 13])
def test_streamed_forward_is_bit_identical_with_no_host_sync(depth, resident_mib, monkeypatch):
    _cuda()
    monkeypatch.setenv(op.PREFETCH_DEPTH_ENV, depth)
    torch.manual_seed(0)
    net = _Net()
    ref = copy.deepcopy(net).cuda()
    _streamed(net, resident_mib)
    pf = op.module_prefetcher(net)
    assert pf is not None and pf.depth == int(depth)
    x = torch.randn(4, 64, device = "cuda")
    with torch.no_grad():
        want = ref(x)
        net(x)
        for _ in range(3):
            got, syncs = _count_syncs(lambda: net(x))
            assert syncs == 0
            torch.cuda.synchronize()
            assert torch.equal(got, want)
    assert pf.stats["prefetched"] > 0 and pf.stats["missed"] == 0
    assert pf.peak_inflight_bytes <= pf.window
    assert net.blocks[-1].weight.device.type == "cpu"


def test_dense_top_level_upload_never_waits_behind_a_block_copy(tmp_path):
    """The dense top-level group uploads on the compute stream at the forward start. Block copies sharing the copy
    engine with it would hold it back, and the previous forward's last offload already lets the copy stream run, so
    the first block copies of a forward are fenced behind it: no compute-stream upload overlaps a block copy."""
    _cuda()
    import json

    from torch.profiler import ProfilerActivity, profile

    torch.manual_seed(0)
    net = _Net(width = 4096, blocks = 6)
    _streamed(net)
    pf = op.module_prefetcher(net)
    top = net._diffusers_hook.get_hook("group_offloading").group
    assert top not in pf.groups and "onload_" in top.__dict__ and pf.fence_first
    x = torch.randn(4, 64, device = "cuda")
    with torch.no_grad():
        net(x)
        net(x)
        torch.cuda.synchronize()
        with profile(activities = [ProfilerActivity.CPU, ProfilerActivity.CUDA]) as prof:
            for _ in range(4):
                net(x)
            torch.cuda.synchronize()
    trace = tmp_path / "trace.json"
    prof.export_chrome_trace(str(trace))
    events = [e for e in json.loads(trace.read_text())["traceEvents"] if e.get("ph") == "X"]
    compute = {e["args"].get("stream") for e in events if e.get("cat") == "kernel"}
    h2d = [e for e in events if e.get("cat") == "gpu_memcpy" and "HtoD" in e["name"]]
    top_up = [(e["ts"], e["ts"] + e["dur"]) for e in h2d if e["args"].get("stream") in compute]
    blocks = [(e["ts"], e["ts"] + e["dur"]) for e in h2d if e["args"].get("stream") not in compute]
    assert len(top_up) == 4 * 4 and len(blocks) == 4 * 2 * len(net.blocks)
    assert not [(a, b) for a in top_up for b in blocks if a[0] < b[1] and b[0] < a[1]]
    assert pf.stats["missed"] == 0


def test_diffusers_stream_path_synchronizes_per_group(monkeypatch):
    """The base this replaces: the same streamed model through diffusers' own prefetch syncs the host per group."""
    _cuda()
    monkeypatch.setenv(op.ASYNC_PREFETCH_ENV, "0")
    torch.manual_seed(0)
    net = _Net()
    _streamed(net)
    assert op.module_prefetcher(net) is None
    x = torch.randn(4, 64, device = "cuda")
    with torch.no_grad():
        net(x)
        _out, syncs = _count_syncs(lambda: net(x))
    assert syncs >= len(net.blocks)


def test_slow_gpu_keeps_memory_in_the_window_and_reads_the_right_weights(monkeypatch):
    """The copy stream waits on the offload event, so queued copies never overwrite blocks still read."""
    _cuda()
    monkeypatch.setenv(op.PREFETCH_DEPTH_ENV, "1")
    torch.manual_seed(0)
    net = _Net(width = 2048, blocks = 12, sleep_cycles = 20_000_000)
    with torch.no_grad():
        for b in net.blocks:
            b.weight.mul_(
                1.0 + torch.rand(())
            )  # distinct blocks: a wrong-weight read changes the output
    ref = copy.deepcopy(net).cuda()
    _streamed(net)
    pf = op.module_prefetcher(net)
    x = torch.randn(4, 64, device = "cuda")
    with torch.no_grad():
        want = ref(x)
        net(x)
        torch.cuda.synchronize()
        torch.cuda.empty_cache()
        base = torch.cuda.memory_reserved()
        torch.cuda.reset_peak_memory_stats()
        got = net(x)
        host_ahead = not torch.cuda.current_stream().query()
        torch.cuda.synchronize()
        # reserved, not allocated: record_stream frees drop allocated bytes but stay reserved
        peak = torch.cuda.max_memory_reserved() - base
    assert torch.equal(got, want)
    assert host_ahead
    block_bytes = 2048 * 2048 * 4 + 2048 * 4
    # window (2 groups) + activations + top-level group
    assert peak <= pf.window + 2 * block_bytes, (peak, pf.window)


@pytest.mark.parametrize("drop", ["exception", "prefix"])
def test_dropped_prefetch_never_lands_in_a_reused_block(drop, monkeypatch):
    _cuda()
    monkeypatch.setenv(op.PREFETCH_DEPTH_ENV, "3")
    H = 2048

    class Net(torch.nn.Module):
        def __init__(self):
            super().__init__()
            self.blocks = torch.nn.ModuleList(torch.nn.Linear(H, H, bias = False) for _ in range(8))

        def forward(
            self,
            x,
            upto = 8,
            boom = False,
        ):
            for i, m in enumerate(self.blocks):
                if i >= upto:
                    break
                if boom and i == 2:
                    raise RuntimeError("boom")
                x = m(x)
            return x

    net = Net().to(torch.bfloat16).eval()
    with torch.no_grad():
        for p in net.parameters():
            p.fill_(1.0)
    _streamed(net)
    x = torch.zeros(1, H, dtype = torch.bfloat16, device = "cuda")
    with torch.no_grad():
        net(x)
        for _ in range(6):
            try:
                net(x, upto = 2, boom = drop == "exception")
            except RuntimeError:
                pass
            outs = [torch.empty(H, H, dtype = torch.bfloat16, device = "cuda") for _ in range(8)]
            for o in outs:
                o.fill_(-2.0)
            torch.cuda.synchronize()
            assert all(bool((o == -2.0).all()) for o in outs)
            del outs
            net(x)
            torch.cuda.synchronize()
    pf = op.module_prefetcher(net)
    assert not pf.ready and pf.inflight_bytes == 0


def test_partial_residency_release_and_restore_stay_bit_identical():
    _cuda()
    torch.manual_seed(0)
    net = _Net()
    ref = copy.deepcopy(net).cuda()
    pipe = _streamed(net, resident_mib = 17)
    pf = op.module_prefetcher(net)
    groups = dm._offload_groups(net)
    resident = [g for g in groups if getattr(g, "_unsloth_resident", False)]
    assert resident and len(resident) < len(groups)
    x = torch.randn(4, 64, device = "cuda")
    with torch.no_grad():
        want = ref(x)
        for _ in range(2):
            assert torch.equal(net(x), want)
        restore = dm.release_resident_groups(pipe, 100, None)
        assert restore is not None
        for _ in range(2):
            got, syncs = _count_syncs(lambda: net(x))
            assert torch.equal(got, want)
        assert syncs == 0
        restore()
        for _ in range(2):
            assert torch.equal(net(x), want)
    assert pf.stats["missed"] <= len(groups)


def test_unpinned_host_copies_stream_bit_identical(monkeypatch):
    """low_cpu_mem_usage (host RAM too small to pin): each onload pins through the host allocator."""
    _cuda()
    monkeypatch.setenv("UNSLOTH_DIFFUSION_GROUP_OFFLOAD_PIN", "0")
    torch.manual_seed(0)
    net = _Net()
    ref = copy.deepcopy(net).cuda()
    _streamed(net, stream_text_encoders = True)
    assert not any(
        t.is_pinned() for g in dm._offload_groups(net) for t in g.cpu_param_dict.values()
    )
    x = torch.randn(4, 64, device = "cuda")
    with torch.no_grad():
        want = ref(x)
        for _ in range(3):
            assert torch.equal(net(x), want)
    assert op.module_prefetcher(net).stats["prefetched"] > 0


def test_slow_gpu_unpinned_staging_stays_in_the_window(monkeypatch):
    """Unpinned host copies on a card slower than the host thread: pinned staging is capped, outputs bit-identical."""
    _cuda()
    monkeypatch.setenv("UNSLOTH_DIFFUSION_GROUP_OFFLOAD_PIN", "0")
    monkeypatch.setenv(op.PREFETCH_DEPTH_ENV, "1")
    torch.manual_seed(0)
    # ~0.1 s per block on any card: the host queues the whole forward before the first blocks have run
    net = _Net(width = 2048, blocks = 12, sleep_cycles = 200_000_000)
    ref = copy.deepcopy(net).cuda()
    _streamed(net, stream_text_encoders = True)
    pf = op.module_prefetcher(net)
    x = torch.randn(4, 64, device = "cuda")
    with torch.no_grad():
        want = ref(x)
        net(x)
        torch.cuda.synchronize()
        torch.cuda.reset_peak_host_memory_stats()
        before = torch.cuda.host_memory_stats().get("allocated_bytes.current", 0)
        got = net(x)
        host_ahead = not torch.cuda.current_stream().query()
        torch.cuda.synchronize()
    peak = torch.cuda.host_memory_stats().get("allocated_bytes.peak", 0) - before
    assert torch.equal(got, want)
    assert host_ahead
    block_bytes = 2048 * 2048 * 4 + 2048 * 4
    # window (2 groups) + the group being staged; unbounded it was every block of the forward
    assert peak <= pf.window + 2 * block_bytes, (peak, pf.window)


@pytest.mark.parametrize("version", [None, 2])
def test_torchao_int8_weights_stream_bit_identical(version):
    """torchao's default int8 weight (v1 on 0.17) and ``Int8Tensor`` (version 2), whose 0.17 ``to`` drops
    ``non_blocking`` in its common ``_to_copy``."""
    _cuda()
    torchao = pytest.importorskip("torchao")
    from torchao.quantization import quantize_

    try:
        from torchao.quantization import Int8WeightOnlyConfig as Cfg
    except ImportError:
        pytest.skip(f"torchao {torchao.__version__} has no Int8WeightOnlyConfig")
    torch.manual_seed(0)
    net = _Net().to(torch.bfloat16)
    try:
        cfg = Cfg() if version is None else Cfg(version = version)
    except TypeError:
        pytest.skip(f"torchao {torchao.__version__} has no versioned Int8WeightOnlyConfig")
    quantize_(net, cfg)
    ref = copy.deepcopy(net).cuda()
    _streamed(net)
    pf = op.module_prefetcher(net)
    assert pf is not None
    top = net._diffusers_hook.get_hook("group_offloading").group
    assert top.stream is not None and pf.owns(top)
    assert all(c.is_pinned() for c in top.cpu_param_dict.values())
    x = torch.randn(4, 64, device = "cuda", dtype = torch.bfloat16)
    with torch.no_grad():
        ref(x)
        want, ref_syncs = _count_syncs(
            lambda: ref(x)
        )  # syncs the resident int8 op does itself (torchao 0.17: 1)
        net(x)
        for _ in range(3):
            got, syncs = _count_syncs(lambda: net(x))
            torch.cuda.synchronize()
            assert torch.equal(got, want)
            # torchao 0.17 int8 `to` drops non_blocking; offload must add no syncs
            assert syncs == ref_syncs
    assert pf.stats["prefetched"] > 0
    go = pytest.importorskip("diffusers.hooks.group_offloading")
    # the torchao wrapper reports its onload device, so check the inner tensors
    assert dm._placed_on(net.proj_in.weight, "cpu", go._is_torchao_tensor)


def test_streamed_onload_of_a_compiled_wrapper_weight_does_not_swap(monkeypatch):
    """A compile keeps a weakref to every wrapper-subclass weight it traced (its fake-mode memo outlives it), and
    swap_tensors refuses such a tensor: FLUX.3 INT8 streamed at 6 GiB failed every render with 'Cannot swap t1 because
    it has weakref associated with it'. A weight whose wrapper already reports the device needs no swap."""
    import weakref

    two = pytest.importorskip("torch.testing._internal.two_tensor")
    pf, groups, gpu = _frozen_gpu_prefetcher(monkeypatch, pinned = True, groups_n = 1, tensors = 1)
    lin = torch.nn.Linear(8, 8)
    w = lin.weight.detach().clone()
    lin.weight = torch.nn.Parameter(two.TwoTensor(w, w.clone()), requires_grad = False)
    torch._dynamo.reset()
    compiled = torch.compile(lin, backend = "eager")
    x = torch.randn(2, 8)
    with torch.no_grad():
        compiled(x)
    # compile's own weakref is gone by torch 2.12; hold one so swap_tensors would refuse, as in a render.
    held = weakref.ref(lin.weight)
    assert weakref.getweakrefs(lin.weight)
    host = two.TwoTensor(w + 1, w + 2)
    monkeypatch.setattr(type(host), "is_pinned", lambda self: True, raising = False)
    group = groups[0]
    group.parameters = [lin.weight]
    group.cpu_param_dict = {lin.weight: host}
    monkeypatch.setattr(
        op,
        "_go",
        lambda: types.SimpleNamespace(
            _is_torchao_tensor = lambda t: isinstance(t, two.TwoTensor),
            _swap_torchao_tensor = torch.utils.swap_tensors,
        ),
    )
    param = lin.weight
    pf._issue(group)
    assert lin.weight is param
    assert torch.equal(param.a, w + 1) and torch.equal(param.b, w + 2)
    assert held() is param
    with torch.no_grad():
        got = compiled(x)
    assert torch.equal(got, torch.nn.functional.linear(x, two.TwoTensor(w + 1, w + 2), lin.bias))
    torch._dynamo.reset()