Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

13.4 KB
Raw Blame
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0

"""Local float16 cast cache of fp32-shard components: candidates, bit-identical round trip, refusals, video wiring."""

from __future__ import annotations

import json
import sys
from pathlib import Path

import pytest

torch = pytest.importorskip("torch")
diffusers = pytest.importorskip("diffusers")
transformers = pytest.importorskip("transformers")
pytest.importorskip("safetensors")

BACKEND = Path(__file__).resolve().parents[1]
if str(BACKEND) not in sys.path:
    sys.path.insert(0, str(BACKEND))

from core.inference import diffusion_cast_cache as CC  # noqa: E402
from core.inference import diffusion_prequant_cache as C  # noqa: E402

FP16_DEFAULT = {"vae": torch.float32, "default": torch.float16}


def _tiny_wan():
    torch.manual_seed(0)
    return diffusers.WanTransformer3DModel(
        patch_size = (1, 2, 2),
        num_attention_heads = 2,
        attention_head_dim = 16,
        in_channels = 4,
        out_channels = 4,
        text_dim = 32,
        freq_dim = 16,
        ffn_dim = 64,
        num_layers = 2,
        cross_attn_norm = True,
        qk_norm = "rms_norm_across_heads",
        eps = 1e-6,
        rope_max_seq_len = 32,
    ).to(torch.float32)


def _tiny_umt5():
    torch.manual_seed(1)
    config = transformers.UMT5Config(
        vocab_size = 64,
        d_model = 32,
        d_kv = 8,
        d_ff = 64,
        num_layers = 2,
        num_heads = 4,
        relative_attention_num_buckets = 8,
    )
    return transformers.UMT5EncoderModel(config).to(torch.float32)


@pytest.fixture(autouse = True)
def cache_root(tmp_path, monkeypatch):
    root = tmp_path / "cache"
    monkeypatch.setenv(C.ENV_DIR, str(root))
    monkeypatch.delenv(CC.CAST_CACHE_ENV, raising = False)
    monkeypatch.delenv(C.ENV_MODE, raising = False)
    # No Hub probe from a local directory source.
    monkeypatch.setattr(C, "_hub_source_stale", lambda shards, logger = None: False)
    with C._lock:
        C._REJECTED.clear()
    return root


@pytest.fixture
def pipe_dir(tmp_path):
    root = tmp_path / "Wan2.1-T2V-tiny-Diffusers"
    _tiny_wan().save_pretrained(root / "transformer", max_shard_size = "60KB")
    _tiny_umt5().save_pretrained(root / "text_encoder")
    (root / "vae").mkdir()
    (root / "model_index.json").write_text(
        json.dumps(
            {
                "_class_name": "WanPipeline",
                "transformer": ["diffusers", "WanTransformer3DModel"],
                "text_encoder": ["transformers", "UMT5EncoderModel"],
                "vae": ["diffusers", "AutoencoderKLWan"],
                "tokenizer": ["transformers", "T5TokenizerFast"],
                "scheduler": ["diffusers", "UniPCMultistepScheduler"],
            }
        )
    )
    return root


def _dense(cand):
    kwargs = CC._load_kwargs(cand)
    kwargs.pop("output_loading_info")
    return cand.cls.from_pretrained(str(Path(cand.source) / cand.component), **kwargs)


class _Pipe:
    pass


def _save(cand, module):
    pipe = _Pipe()
    setattr(pipe, cand.component, module)
    writes = CC.defer_writes(pipe, [cand])
    assert writes is not None and writes.start()
    assert writes.wait(60)
    return writes.results.get(cand.component)


def _bits(t):
    return t.reshape(-1).view(torch.uint8) if t.numel() else t


def _identical(a, b):
    sa, sb = a.state_dict(), b.state_dict()
    assert sa.keys() == sb.keys()
    for k in sa:
        assert sa[k].dtype == sb[k].dtype, k
        assert torch.equal(_bits(sa[k]), _bits(sb[k])), k
    ba, bb = dict(a.named_buffers()), dict(b.named_buffers())
    assert ba.keys() == bb.keys()
    for k in ba:
        assert ba[k].dtype == bb[k].dtype and torch.equal(ba[k], bb[k]), k


def _by_name(cands):
    return {c.component: c for c in cands}


def test_candidates_are_fp32_shards_loaded_at_fp16(pipe_dir):
    cands = _by_name(CC.candidates(pipe_dir, torch_dtype = FP16_DEFAULT))
    assert set(cands) == {
        "transformer",
        "text_encoder",
    }  # the vae stays fp32; the rest hold no weights
    assert len(cands["transformer"].shards) > 1
    assert cands["text_encoder"].parts["keep_in_fp32"] == ["wo"]
    assert cands["transformer"].key != cands["text_encoder"].key


def test_no_candidates_at_float32(pipe_dir):
    assert CC.candidates(pipe_dir, torch_dtype = torch.float32) == []


def test_kill_switches(pipe_dir, monkeypatch):
    monkeypatch.setenv(CC.CAST_CACHE_ENV, "0")
    assert CC.candidates(pipe_dir, torch_dtype = FP16_DEFAULT) == []
    monkeypatch.delenv(CC.CAST_CACHE_ENV)
    monkeypatch.setenv(C.ENV_MODE, "0")
    assert CC.candidates(pipe_dir, torch_dtype = FP16_DEFAULT) == []


def test_skip_and_narrow_shards(pipe_dir):
    assert set(
        _by_name(CC.candidates(pipe_dir, torch_dtype = FP16_DEFAULT, skip = ("text_encoder",)))
    ) == {"transformer"}
    _tiny_umt5().to(torch.float16).save_pretrained(pipe_dir / "text_encoder")
    assert set(_by_name(CC.candidates(pipe_dir, torch_dtype = FP16_DEFAULT))) == {"transformer"}


@pytest.mark.parametrize("component", ["transformer", "text_encoder"])
def test_round_trip_is_bit_identical(pipe_dir, component):
    cand = _by_name(CC.candidates(pipe_dir, torch_dtype = FP16_DEFAULT))[component]
    assert CC.read(cand) is None  # cold: a miss
    dense = _dense(cand)
    path = _save(cand, dense)
    assert path is not None and C.path_in_cache(path)
    stored = CC._shard_header(path)
    assert {e["dtype"] for e in stored.values()} == {"F16", "F32"}  # fp32 keep modules stay fp32
    fresh = _by_name(CC.candidates(pipe_dir, torch_dtype = FP16_DEFAULT))[component]
    hit = CC.read(fresh)
    assert hit is not None
    _identical(dense, hit)
    _identical(_dense(fresh), hit)
    if component == "text_encoder":
        # the tied embedding is written once and re-tied by from_pretrained
        assert "encoder.embed_tokens.weight" not in stored
        assert hit.shared.weight.data_ptr() == hit.encoder.embed_tokens.weight.data_ptr()


def test_module_changed_in_place_is_not_cached(pipe_dir):
    cand = _by_name(CC.candidates(pipe_dir, torch_dtype = FP16_DEFAULT))["transformer"]
    dense = _dense(cand)
    pipe = _Pipe()
    pipe.transformer = dense
    writes = CC.defer_writes(pipe, [cand])
    with torch.no_grad():
        for p in dense.parameters():
            p.mul_(2)
    writes.start()
    writes.wait(60)
    assert writes.results["transformer"] is None
    assert C._entry(cand.key) is None


def test_module_restructured_after_load_is_not_cached(pipe_dir):
    cand = _by_name(CC.candidates(pipe_dir, torch_dtype = FP16_DEFAULT))["transformer"]
    dense = _dense(cand)
    pipe = _Pipe()
    pipe.transformer = dense
    writes = CC.defer_writes(pipe, [cand])
    dense.proj_out = torch.nn.Linear(4, 4)
    writes.start()
    writes.wait(60)
    assert writes.results["transformer"] is None


def test_new_source_bytes_change_the_key(pipe_dir):
    cand = _by_name(CC.candidates(pipe_dir, torch_dtype = FP16_DEFAULT))["text_encoder"]
    assert _save(cand, _dense(cand)) is not None
    model = _tiny_umt5()
    with torch.no_grad():
        model.shared.weight.add_(1)
    model.save_pretrained(pipe_dir / "text_encoder")
    fresh = _by_name(CC.candidates(pipe_dir, torch_dtype = FP16_DEFAULT))["text_encoder"]
    assert fresh.key != cand.key
    assert CC.read(fresh) is None


def test_corrupt_entry_is_a_miss(pipe_dir):
    cand = _by_name(CC.candidates(pipe_dir, torch_dtype = FP16_DEFAULT))["transformer"]
    path = Path(_save(cand, _dense(cand)))
    raw = bytearray(path.read_bytes())
    raw[-1] ^= 0xFF
    path.write_bytes(bytes(raw))
    (path.parent / C.VERIFIED_NAME).unlink()
    assert CC.read(cand) is None
    with C._lock:
        assert str(path.resolve()) in C._REJECTED


def test_video_reader_injects_hits_and_returns_misses(pipe_dir):
    from core.inference import video as V

    cand = _by_name(CC.candidates(pipe_dir, torch_dtype = FP16_DEFAULT))["text_encoder"]
    _save(cand, _dense(cand))
    pipe_kwargs = {"torch_dtype": dict(FP16_DEFAULT), "cache_dir": None, "local_files_only": True}
    missed = V._read_cast_cache(pipe_kwargs, source = str(pipe_dir), revision = None, base = None)
    assert [c.component for c in missed] == ["transformer"]
    assert isinstance(pipe_kwargs["text_encoder"], transformers.UMT5EncoderModel)
    assert "transformer" not in pipe_kwargs
    # an injected component is never a candidate
    again = V._read_cast_cache(dict(pipe_kwargs), source = str(pipe_dir), revision = None, base = None)
    assert [c.component for c in again] == ["transformer"]


def test_teardown_waits_then_clears(pipe_dir):
    from core.inference import video as V

    cand = _by_name(CC.candidates(pipe_dir, torch_dtype = FP16_DEFAULT))["transformer"]
    pipe = _Pipe()
    pipe.transformer = _dense(cand)
    writes = CC.defer_writes(pipe, [cand])
    pipe._unsloth_cast_writes = writes
    writes.start()
    V._finish_cast_writes(pipe)
    assert pipe._unsloth_cast_writes is None
    assert writes.done()
    assert C._entry(cand.key) is not None


def test_sample_keys_tolerate_one_kind_per_shard():
    names = {"a": torch.float16, "b": torch.float16, "w": torch.float32}
    entry = lambda n: {"dtype": "F32", "data_offsets": [0, n]}  # noqa: E731
    assert CC._sample_keys({"a": entry(8), "b": entry(16)}, names) == ["b"]
    assert CC._sample_keys({"w": entry(8)}, names) == ["w"]
    assert CC._sample_keys({"a": entry(CC._SAMPLE_MAX_BYTES + 1)}, names) == []


def test_save_when_kept_fp32_tensors_exceed_the_sample_size(pipe_dir, monkeypatch):
    cand = _by_name(CC.candidates(pipe_dir, torch_dtype = FP16_DEFAULT))["text_encoder"]
    dense = _dense(cand)
    wide = min(t.numel() * 4 for n, t in dense.state_dict().items() if t.dtype == torch.float32)
    monkeypatch.setattr(CC, "_SAMPLE_MAX_BYTES", wide - 1)
    assert _save(cand, dense) is not None
    _identical(dense, CC.read(cand))


@pytest.mark.skipif(not torch.cuda.is_available(), reason = "needs a CUDA device")
def test_resident_device_module_is_read_back_bit_identical(pipe_dir):
    cand = _by_name(CC.candidates(pipe_dir, torch_dtype = FP16_DEFAULT))["transformer"]
    dense = _dense(cand)
    reference = {k: v.clone() for k, v in dense.state_dict().items()}
    dense.to("cuda")
    assert _save(cand, dense) is not None
    hit = CC.read(cand)
    for k, v in hit.state_dict().items():
        assert v.dtype == reference[k].dtype and torch.equal(_bits(v), _bits(reference[k])), k


def test_source_cast_matches_the_live_module_cast_bit_for_bit(pipe_dir, tmp_path, monkeypatch):
    """A component too big for host RAM is cast from its source shards one tensor at a time: the cached file is the
    same file the live-module write produces (same key, same bytes), and reads back as the dense load."""
    cand = _by_name(CC.candidates(pipe_dir, torch_dtype = FP16_DEFAULT))["transformer"]
    path = CC.save_from_source(cand)
    assert path is not None and C.path_in_cache(path)
    from_source = Path(path).read_bytes()
    hit = CC.read(_by_name(CC.candidates(pipe_dir, torch_dtype = FP16_DEFAULT))["transformer"])
    assert hit is not None
    dense = _dense(cand)
    _identical(dense, hit)
    # the live-module write of the same component, into a second cache root: identical bytes (bar the timestamp-free
    # metadata, which is the same too)
    monkeypatch.setenv(C.ENV_DIR, str(tmp_path / "cache_live"))
    live = _by_name(CC.candidates(pipe_dir, torch_dtype = FP16_DEFAULT))["transformer"]
    live_path = _save(live, dense)
    assert live_path is not None
    assert Path(live_path).read_bytes() == from_source


def test_source_cast_refuses_a_class_its_shards_do_not_match(pipe_dir):
    cand = _by_name(CC.candidates(pipe_dir, torch_dtype = FP16_DEFAULT))["transformer"]
    cand.cls = (
        diffusers.UNet2DModel
    )  # a different architecture: names / shapes disagree with the shards
    cand.config_path = str(pipe_dir / "transformer" / "config.json")
    assert CC.save_from_source(cand) is None
    # transformers components are not cast from source (their dense load is the small one)
    te = _by_name(CC.candidates(pipe_dir, torch_dtype = FP16_DEFAULT))["text_encoder"]
    assert CC.save_from_source(te) is None


def test_source_cast_of_a_hunyuanvideo15_denoiser_reads_back_as_the_dense_load(tmp_path):
    """The HunyuanVideo-1.5 case: its fp32 DiT (31 GB) does not fit a 31 GB host beside the converted encoder."""
    torch.manual_seed(0)
    model = diffusers.HunyuanVideo15Transformer3DModel(
        in_channels = 9,
        out_channels = 4,
        num_attention_heads = 2,
        attention_head_dim = 32,
        num_layers = 1,
        num_refiner_layers = 1,
        mlp_ratio = 2.0,
        text_embed_dim = 64,
        text_embed_2_dim = 32,
        image_embed_dim = 32,
        rope_axes_dim = (8, 12, 12),
        target_size = 64,
    ).to(torch.float32)
    root = tmp_path / "HunyuanVideo-1.5-tiny"
    model.save_pretrained(root / "transformer", max_shard_size = "200KB")
    (root / "model_index.json").write_text(
        json.dumps(
            {
                "_class_name": "HunyuanVideo15Pipeline",
                "transformer": ["diffusers", "HunyuanVideo15Transformer3DModel"],
            }
        )
    )
    cand = _by_name(CC.candidates(root, torch_dtype = FP16_DEFAULT))["transformer"]
    assert len(cand.shards) > 1
    assert CC.save_from_source(cand) is not None
    hit = CC.read(_by_name(CC.candidates(root, torch_dtype = FP16_DEFAULT))["transformer"])
    assert hit is not None
    _identical(_dense(cand), hit)