Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

11.4 KB
Raw Blame
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0

"""A cast-cache write that runs while the published pipeline renders reads block-streamed tensors from their offload
group's host copy, never from a device destination whose upload has not landed yet."""

from __future__ import annotations

import json
import sys
from pathlib import Path

import pytest

torch = pytest.importorskip("torch")
diffusers = pytest.importorskip("diffusers")
pytest.importorskip("transformers")
pytest.importorskip("safetensors")

BACKEND = Path(__file__).resolve().parents[1]
if str(BACKEND) not in sys.path:
    sys.path.insert(0, str(BACKEND))

from core.inference import diffusion_cast_cache as CC  # noqa: E402
from core.inference import diffusion_prequant_cache as C  # noqa: E402

FP16_DEFAULT = {"vae": torch.float32, "default": torch.float16}


@pytest.fixture(autouse = True)
def cache_root(tmp_path, monkeypatch):
    root = tmp_path / "cache"
    monkeypatch.setenv(C.ENV_DIR, str(root))
    monkeypatch.delenv(CC.CAST_CACHE_ENV, raising = False)
    monkeypatch.delenv(C.ENV_MODE, raising = False)
    monkeypatch.setattr(C, "_hub_source_stale", lambda shards, logger = None: False)
    with C._lock:
        C._REJECTED.clear()
    return root


@pytest.fixture
def pipe_dir(tmp_path):
    torch.manual_seed(0)
    model = diffusers.WanTransformer3DModel(
        patch_size = (1, 2, 2),
        num_attention_heads = 2,
        attention_head_dim = 16,
        in_channels = 4,
        out_channels = 4,
        text_dim = 32,
        freq_dim = 16,
        ffn_dim = 64,
        num_layers = 2,
        cross_attn_norm = True,
        qk_norm = "rms_norm_across_heads",
        eps = 1e-6,
        rope_max_seq_len = 32,
    ).to(torch.float32)
    root = tmp_path / "Wan2.1-T2V-tiny-Diffusers"
    model.save_pretrained(root / "transformer", max_shard_size = "60KB")
    (root / "model_index.json").write_text(
        json.dumps(
            {"_class_name": "WanPipeline", "transformer": ["diffusers", "WanTransformer3DModel"]}
        )
    )
    return root


def _cand(pipe_dir):
    return {c.component: c for c in CC.candidates(pipe_dir, torch_dtype = FP16_DEFAULT)}[
        "transformer"
    ]


def _dense(cand):
    kwargs = CC._load_kwargs(cand)
    kwargs.pop("output_loading_info")
    return cand.cls.from_pretrained(str(Path(cand.source) / cand.component), **kwargs)


def _bits(t):
    return t.reshape(-1).view(torch.uint8) if t.numel() else t


def _assert_hit_equals(cand, reference: dict):
    hit = CC.read(cand)
    assert hit is not None
    for k, v in hit.state_dict().items():
        assert v.dtype == reference[k].dtype and torch.equal(_bits(v), _bits(reference[k])), k


def _unsampled_cast_key(cand) -> str:
    """A float16 tensor the pre-write source comparison does not sample, so only the write itself can catch it."""
    names = {n: d for n, d, _s in cand.names}
    sampled = set()
    for shard in cand.shards:
        sampled.update(CC._sample_keys(CC._shard_header(shard) or {}, names))
    keys = sorted(
        n for n, d in names.items() if d == torch.float16 and n not in sampled and "weight" in n
    )
    assert keys
    return keys[0]


class _Group:
    def __init__(self, cpu_param_dict):
        self.cpu_param_dict = cpu_param_dict


class _Hook:
    def __init__(self, group):
        self.group = group


class _Registry:
    def __init__(self, hook):
        self.hooks = {"group_offloading": hook}


def test_streamed_tensor_is_written_from_its_host_copy(pipe_dir):
    cand = _cand(pipe_dir)
    dense = _dense(cand)
    reference = {k: v.clone() for k, v in dense.state_dict().items()}
    host = {t: t.data.clone() for t in [*dense.parameters(), *dense.buffers()]}
    dense._diffusers_hook = _Registry(_Hook(_Group(host)))
    CC.capture(type("P", (), {"transformer": dense})(), [cand])
    key = _unsampled_cast_key(cand)
    live = dense.get_parameter(key)
    live.data = torch.full_like(live.data, float("nan"))
    assert CC.save(cand, dense) is not None
    _assert_hit_equals(cand, reference)


def test_unstreamed_module_still_reads_its_live_tensors(pipe_dir):
    cand = _cand(pipe_dir)
    dense = _dense(cand)
    reference = {k: v.clone() for k, v in dense.state_dict().items()}
    CC.capture(type("P", (), {"transformer": dense})(), [cand])
    assert CC.save(cand, dense) is not None
    _assert_hit_equals(cand, reference)


def test_layout_version_retires_entries_written_by_the_racy_reader(pipe_dir):
    assert CC.CAST_LAYOUT_VERSION >= 2
    assert _cand(pipe_dir).parts["layout"] == CC.CAST_LAYOUT_VERSION


@pytest.mark.skipif(not torch.cuda.is_available(), reason = "needs a CUDA device")
def test_block_streamed_write_during_a_refill_is_bit_identical(pipe_dir, monkeypatch):
    """Before every read each group is re-pointed to fresh NaN device tensors whose upload has not landed."""
    from diffusers.hooks import apply_group_offloading

    monkeypatch.setattr(CC, "_ONLOAD_WAIT_SECONDS", 0.0)
    cand = _cand(pipe_dir)
    dense = _dense(cand)
    reference = {k: v.clone() for k, v in dense.state_dict().items()}
    apply_group_offloading(
        dense,
        onload_device = torch.device("cuda"),
        offload_type = "block_level",
        num_blocks_per_group = 1,
        use_stream = True,
    )
    CC.capture(type("P", (), {"transformer": dense})(), [cand])
    groups = []
    for m in dense.modules():
        for hook in getattr(getattr(m, "_diffusers_hook", None), "hooks", {}).values():
            g = getattr(hook, "group", None)
            if g is not None and all(g is not x for x in groups):
                groups.append(g)
    # a group holding no tensors (e.g. the unmatched-modules group) keeps an empty host copy
    groups = [g for g in groups if g.cpu_param_dict]
    assert groups

    def onload():
        for g in groups:
            with torch.cuda.stream(g.stream):
                for t, src in g.cpu_param_dict.items():
                    t.data = src.to("cuda", non_blocking = True)
        torch.cuda.synchronize()

    names = {n: d for n, d, _s in cand.names}
    sampled = sum(len(CC._sample_keys(CC._shard_header(sh) or {}, names)) for sh in cand.shards)
    state = {"calls": 0}

    def prefetch():
        # After the sampled reads (so the write itself is what goes wrong), as the prefetcher does: `.data`
        # re-pointed now, its upload queued behind work that has not finished.
        state["calls"] += 1
        if state["calls"] <= sampled:
            return
        moved = {t: torch.full_like(t.data, float("nan")) for g in groups for t in g.cpu_param_dict}
        torch.cuda.synchronize()
        for g in groups:
            for t in g.cpu_param_dict:
                t.data = moved[t]
            with torch.cuda.stream(g.stream):
                torch.cuda._sleep(50_000_000)
                for t, src in g.cpu_param_dict.items():
                    t.data.copy_(src, non_blocking = True)

    onload()
    assert CC.save(cand, dense, check = prefetch) is not None
    torch.cuda.synchronize()
    _assert_hit_equals(cand, reference)


needs_swap = pytest.mark.skipif(
    not hasattr(torch.utils, "swap_tensors"), reason = "needs torch.utils.swap_tensors (torch >= 2.3)"
)


class _TopGroup:
    """diffusers' unmatched top-level group: no stream, so its cpu_param_dict stays empty."""

    def __init__(self, module):
        self.cpu_param_dict = {}
        self.stream = None
        self.offload_to_disk_path = None
        self.onload_device = torch.device("cuda")
        self.modules = [module]
        self.parameters = []
        self.buffers = []


class _TopRegistry:
    def __init__(self, group):
        self.hooks = {"group_offloading": _Hook(group)}

    def get_hook(self, name):
        return self.hooks["group_offloading"]


def _onload(module):
    """Onload keeping each Parameter / buffer object; meta stands in for a device copy the writer must not read."""
    twins = []
    for t in [*module.parameters(), *module.buffers()]:
        twin = torch.empty_like(t, device = "meta")
        if isinstance(t, torch.nn.Parameter):
            twin = torch.nn.Parameter(twin, requires_grad = t.requires_grad)
        torch.utils.swap_tensors(t, twin)
        twins.append((t, twin))
    return twins


def _offload(twins):
    for t, twin in twins:
        torch.utils.swap_tensors(t, twin)


class _CpuOffloadHook:
    """accelerate's CpuOffload without a device: pre_forward onloads, init_hook offloads."""

    twins: list = []

    def init_hook(self, mod):
        _offload(self.twins)
        self.twins = []
        return mod

    def pre_forward(self, mod, *args, **kwargs):
        self.twins = _onload(mod)
        return args, kwargs


def _written(cand, module, path):
    import hashlib
    CC._write(str(path), cand, {"k": "v"}, hashlib.sha256(), CC._Reader(module, lambda: None))
    return path.read_bytes()


@pytest.fixture
def sleeps(monkeypatch):
    """Counted, not slept."""
    calls = []

    class _Time:
        def __getattr__(self, name):
            return getattr(__import__("time"), name)

        def sleep(self, s):
            calls.append(s)

    monkeypatch.setattr(CC, "time", _Time())
    return calls


@needs_swap
def test_onloaded_top_level_group_is_written_from_its_kept_host_copy(pipe_dir, tmp_path, sleeps):
    from core.inference import diffusion_memory as dm

    cand = _cand(pipe_dir)
    dense = _dense(cand)
    CC.capture(type("P", (), {"transformer": dense})(), [cand])
    want = _written(cand, dense, tmp_path / "plain.safetensors")
    group = _TopGroup(dense)
    dense._diffusers_hook = _TopRegistry(group)
    assert dm._skip_top_level_copy_back(dense)
    assert group.cpu_param_dict == {}
    twins = _onload(dense)
    assert _written(cand, dense, tmp_path / "onloaded.safetensors") == want
    assert sleeps == []
    # replaced on the host while offloaded: the live tensor is read
    _offload(twins)
    live = dense.get_parameter(_unsampled_cast_key(cand))
    live.data = live.data + 1
    assert _written(cand, dense, tmp_path / "replaced.safetensors") != want
    assert sleeps == []


@needs_swap
def test_onloaded_model_offload_is_written_from_its_kept_host_copy(pipe_dir, tmp_path, sleeps):
    from core.inference import diffusion_memory as dm

    cand = _cand(pipe_dir)
    dense = _dense(cand)
    CC.capture(type("P", (), {"transformer": dense})(), [cand])
    want = _written(cand, dense, tmp_path / "plain.safetensors")
    hook = _CpuOffloadHook()
    dense._hf_hook = hook
    dm._wrap_cpu_offload_hook(hook, dense)
    hook.pre_forward(dense)
    assert all(p.device.type == "meta" for p in dense.parameters())
    assert _written(cand, dense, tmp_path / "onloaded.safetensors") == want
    assert sleeps == []


@needs_swap
def test_a_weight_changed_on_the_device_is_never_read_from_the_host(pipe_dir, sleeps):
    from core.inference import diffusion_memory as dm

    cand = _cand(pipe_dir)
    dense = _dense(cand)
    CC.capture(type("P", (), {"transformer": dense})(), [cand])
    hook = _CpuOffloadHook()
    dense._hf_hook = hook
    dm._wrap_cpu_offload_hook(hook, dense)
    hook.pre_forward(dense)
    key = _unsampled_cast_key(cand)
    with torch.no_grad():
        dense.get_parameter(key).add_(1)  # the kept host copy is stale now
    reader = CC._Reader(dense, lambda: None)
    other = next(n for n, _d, _s in cand.names if n != key and "weight" in n)
    assert reader.tensor(other).device.type == "cpu" and sleeps == []
    with pytest.raises(NotImplementedError):
        reader.tensor(key)  # waits, then reads the device copy
    assert sleeps