Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

16.8 KB
Raw Blame
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0

"""A single-file DiT widened on load beyond free host RAM is refused before widening (it used to OOM
the app); files already at the compute dtype stay memory-mapped and are never refused."""

from __future__ import annotations

import re
import sys
from pathlib import Path

import pytest
import torch

from core.inference import diffusion_comfy_quant as cq
from core.inference import diffusion_single_file_converters as conv
from core.inference import diffusion_small_host as small_host

sys.path.insert(0, str(Path(__file__).resolve().parent))
from test_diffusion_single_file_krea2_hunyuanimage import (  # noqa: E402
    KREA2_CFG,
    _class,
    _tiny,
    krea2_original,
)

safetensors_torch = pytest.importorskip("safetensors.torch")

_LINEAR = re.compile(r"^blocks\.\d+\.(attn\.(wq|wk|wv|wo|gate)|mlp\.(gate|up|down))\.weight$")
_KWARGS = {"torch_dtype": torch.bfloat16, "config": "krea/Krea-2-Turbo", "subfolder": "transformer"}


def _write(tmp_path, kind):
    model = _tiny("Krea2Transformer2DModel", KREA2_CFG)
    tensors = {}
    for key, value in krea2_original(model.state_dict()).items():
        if kind != "bf16" and _LINEAR.match(key):
            if kind == "fp8_scaled":
                scale = value.abs().max().clamp_min(1e-8) / 448.0
                tensors[key] = (value / scale).to(torch.float8_e4m3fn)
                tensors[key[: -len(".weight")] + ".scale_weight"] = scale.reshape(1).float()
            else:
                tensors[key] = value.to(torch.float8_e4m3fn)
        else:
            tensors[key] = value.to(torch.bfloat16).contiguous()
    if kind == "fp8_scaled":
        tensors["scaled_fp8"] = torch.zeros(0, dtype = torch.float8_e4m3fn)
    path = tmp_path / f"krea2_{kind}.safetensors"
    safetensors_torch.save_file(tensors, str(path))
    widened = sum(
        v.numel() * 2 for k, v in tensors.items() if v.dtype == torch.float8_e4m3fn and v.numel()
    )
    return path, widened


@pytest.fixture
def krea2(monkeypatch):
    cls = _class("Krea2Transformer2DModel")
    monkeypatch.setattr(cls, "load_config", classmethod(lambda c, *a, **k: dict(KREA2_CFG)))
    # The tiny model widens a few KB; price every byte so the real check runs on it.
    monkeypatch.setattr(conv, "HOST_RAM_CHECK_MIN_MIB", 0, raising = False)
    monkeypatch.delenv(small_host.HOST_RAM_CHECK_ENV, raising = False)
    return cls


def _host(
    monkeypatch,
    available_mib,
    total_mib = 32768,
):
    monkeypatch.setattr(small_host, "host_ram_mib", lambda: (total_mib, available_mib))


def _no_read(monkeypatch):
    def boom(*a, **k):
        raise AssertionError("the weights were read before the host RAM check")

    monkeypatch.setattr(safetensors_torch, "load_file", boom)


def test_fp8_file_widened_past_free_host_ram_is_refused_before_reading(
    tmp_path, monkeypatch, krea2
):
    path, _ = _write(tmp_path, "fp8")
    _host(monkeypatch, available_mib = 2048)  # under the 3 GB reserve alone
    _no_read(monkeypatch)
    with pytest.raises(RuntimeError, match = r"Not enough system RAM to load krea2_fp8"):
        conv.load_original_layout_transformer(krea2, str(path), dict(_KWARGS))


def test_fp8_scaled_comfy_file_without_fp8_runtime_is_refused(tmp_path, monkeypatch, krea2):
    path, _ = _write(tmp_path, "fp8_scaled")
    scan = cq.refuse_comfy_quant(str(path))
    assert scan is not None and scan.legacy_scaled_fp8
    _host(monkeypatch, available_mib = 2048)
    with pytest.raises(RuntimeError, match = "expanding its quantized weights"):
        cq.load_comfy_quant_transformer(
            krea2, str(path), scan, dict(_KWARGS), int8_backend = None, fp8_backend = None
        )


def test_same_files_load_when_the_host_has_room(tmp_path, monkeypatch, krea2):
    _host(monkeypatch, available_mib = 64 * 1024)
    path, _ = _write(tmp_path, "fp8")
    model = conv.load_original_layout_transformer(krea2, str(path), dict(_KWARGS))
    assert model.transformer_blocks[0].attn.to_q.weight.dtype == torch.bfloat16
    path, _ = _write(tmp_path, "fp8_scaled")
    scan = cq.refuse_comfy_quant(str(path))
    model = cq.load_comfy_quant_transformer(
        krea2, str(path), scan, dict(_KWARGS), int8_backend = None, fp8_backend = None
    )
    assert model.transformer_blocks[0].attn.to_q.weight.dtype == torch.bfloat16


def test_bf16_file_stays_mapped_and_is_never_refused(tmp_path, monkeypatch, krea2):
    """A bf16 file at a bf16 compute dtype writes no new host memory (only the fp32-kept norms)."""
    path, _ = _write(tmp_path, "bf16")
    need = conv.converted_host_bytes(
        str(path), torch.bfloat16, keep_fp32 = krea2._keep_in_fp32_modules
    )
    sizes = path.stat().st_size
    assert need < sizes // 10
    monkeypatch.setattr(conv, "HOST_RAM_CHECK_MIN_MIB", 2)  # its few KB stay under any real floor
    _host(monkeypatch, available_mib = 0)
    conv.load_original_layout_transformer(krea2, str(path), dict(_KWARGS))


def test_widened_bytes_count_the_fp8_weights_at_bf16(tmp_path):
    path, widened = _write(tmp_path, "fp8")
    need = conv.converted_host_bytes(str(path), torch.bfloat16)
    assert need == widened
    # fp32 compute (CPU) widens the bf16 tensors too
    assert conv.converted_host_bytes(str(path), torch.float32) > 2 * widened


def test_kill_switch_and_unreadable_ram_never_refuse(tmp_path, monkeypatch, krea2):
    path, _ = _write(tmp_path, "fp8")
    _host(monkeypatch, available_mib = 1)
    monkeypatch.setenv(small_host.HOST_RAM_CHECK_ENV, "0")
    conv.load_original_layout_transformer(krea2, str(path), dict(_KWARGS))
    monkeypatch.delenv(small_host.HOST_RAM_CHECK_ENV)
    _host(monkeypatch, available_mib = None)
    conv.load_original_layout_transformer(krea2, str(path), dict(_KWARGS))


def _spy_need(monkeypatch):
    """The header estimate each load checks up front, recorded."""
    seen = []
    real = conv.HostRamBudget.check

    def spy(self, need_bytes):
        seen.append(need_bytes)
        return real(self, need_bytes)

    monkeypatch.setattr(conv.HostRamBudget, "check", spy)
    return seen


def test_fp16_pipeline_counts_the_deferred_dequantization(tmp_path, monkeypatch, krea2):
    """fp16 with ``_keep_in_fp32_modules`` defers the dtype choice until after key conversion, but
    with no fp8 runtime (always the case off bf16) every fp8 layer is still expanded to fp16."""
    path, widened = _write(tmp_path, "fp8_scaled")
    scan = cq.refuse_comfy_quant(str(path))
    seen = _spy_need(monkeypatch)
    _host(monkeypatch, available_mib = 64 * 1024)
    model = cq.load_comfy_quant_transformer(
        krea2,
        str(path),
        scan,
        {**_KWARGS, "torch_dtype": torch.float16},
        int8_backend = None,
        fp8_backend = None,
    )
    assert model.transformer_blocks[0].attn.to_q.weight.dtype == torch.float16
    # Everything else in the file is bf16, widened to fp16 (or fp32 for the kept norms) as well.
    keep = krea2._keep_in_fp32_modules
    rest = sum(
        v.numel() * (4 if any(m in k.split(".") for m in keep) else 2)
        for k, v in safetensors_torch.load_file(str(path)).items()
        if v.dtype == torch.bfloat16
    )
    assert seen and seen[0] >= widened + rest


def test_fnuz_float8_weights_are_priced_as_widened(tmp_path):
    """ROCm FNUZ float8 tensors are widened by ``value.to(want)`` like any other float8."""
    model = _tiny("Krea2Transformer2DModel", KREA2_CFG)
    tensors = {}
    for key, value in krea2_original(model.state_dict()).items():
        if _LINEAR.match(key):
            tensors[key] = value.to(torch.float8_e4m3fnuz)
        else:
            tensors[key] = value.to(torch.bfloat16).contiguous()
    path = tmp_path / "krea2_fnuz.safetensors"
    safetensors_torch.save_file(tensors, str(path))
    widened = sum(v.numel() * 2 for v in tensors.values() if v.dtype == torch.float8_e4m3fnuz)
    assert conv.converted_host_bytes(str(path), torch.bfloat16) == widened


@pytest.mark.parametrize("kind", ["fp8", "fp8_scaled"])
def test_kill_switch_charges_nothing(tmp_path, monkeypatch, krea2, kind):
    """``UNSLOTH_DIFFUSION_HOST_RAM_CHECK=0`` installs no budget, so no helper charges one."""
    path, _ = _write(tmp_path, kind)
    charged = []
    monkeypatch.setattr(conv.HostRamBudget, "charge", lambda self, n: charged.append(n))
    monkeypatch.setenv(small_host.HOST_RAM_CHECK_ENV, "0")
    _host(monkeypatch, available_mib = 1)
    if kind == "fp8":
        conv.load_original_layout_transformer(krea2, str(path), dict(_KWARGS))
    else:
        cq.load_comfy_quant_transformer(
            krea2,
            str(path),
            cq.refuse_comfy_quant(str(path)),
            dict(_KWARGS),
            int8_backend = None,
            fp8_backend = None,
        )
    assert charged == []


@pytest.mark.parametrize("kind", ["fp8", "fp8_scaled"])
def test_a_file_that_fits_only_stored_loads_with_fp8_storage_and_is_refused_widened(
    tmp_path, monkeypatch, krea2, kind
):
    """fp8 storage stays memory-mapped and is priced at stored bytes plus fp32 row scales; with
    UNSLOTH_DIFFUSION_FP8_STORAGE=0 the load widens and is refused."""
    path, _ = _write(tmp_path, kind)
    keep = krea2._keep_in_fp32_modules
    # Price in bytes, no reserve: the tiny model's needs are a few KB apart.
    monkeypatch.setattr(conv, "_MIB", 1)
    monkeypatch.setattr(small_host, "host_reserve_mib", lambda total: 0)
    if kind == "fp8":
        widened = conv.converted_host_bytes(str(path), torch.bfloat16, keep_fp32 = keep)
        stored = conv.converted_host_bytes(
            str(path), torch.bfloat16, keep_fp32 = keep, dequantized = conv.fp8_stored_estimate(keep)
        ) + conv._fp8_stored_scale_bytes(str(path), keep)
    else:
        scan = cq.refuse_comfy_quant(str(path))
        widened = conv.converted_host_bytes(
            str(path), torch.bfloat16, dequantized = {n + ".weight": 1 for n in scan.layers}.get
        )
        stored = conv.converted_host_bytes(
            str(path), torch.bfloat16, dequantized = {n + ".weight": 0 for n in scan.layers}.get
        )
    assert 0 < stored < widened
    _host(monkeypatch, available_mib = (stored + widened) // 2)

    def load(**kw):
        if kind == "fp8":
            return conv.load_original_layout_transformer(krea2, str(path), dict(_KWARGS), **kw)
        return cq.load_comfy_quant_transformer(
            krea2,
            str(path),
            cq.refuse_comfy_quant(str(path)),
            dict(_KWARGS),
            int8_backend = None,
            fp8_backend = None,
            **kw,
        )

    model = load(fp8_storage = True)
    assert model._unsloth_fp8_stored > 0
    assert model.transformer_blocks[0].attn.to_q.weight_q.dtype == torch.uint8
    monkeypatch.setenv(cq.FP8_STORAGE_ENV, "0")
    with pytest.raises(RuntimeError, match = "Not enough system RAM"):
        load(fp8_storage = cq.comfy_fp8_storage(None))


_GIB = 1024
_KREA2_WIDENED_MIB = 24 * _GIB  # Krea 2 fp8 widened to bf16


def _budget(
    tmp_path,
    monkeypatch,
    total_mib,
    available_mib,
    fp8_widen_chosen = False,
):
    monkeypatch.delenv(small_host.HOST_RAM_CHECK_ENV, raising = False)
    _host(monkeypatch, available_mib, total_mib)
    path = tmp_path / "krea2.safetensors"
    path.write_bytes(b"\0")
    return conv.HostRamBudget(
        str(path), "widening its weights to bfloat16", fp8_widen_chosen = fp8_widen_chosen
    )


@pytest.mark.parametrize("available_gib", [28.5, 29.0, 30.0, 30.3])
def test_big_host_loads_what_the_fp8_storage_rule_widens(tmp_path, monkeypatch, available_gib):
    """64 GB host, Krea 2 (~24 GB widened), 28-30.4 GB free: the fp8-storage rule widens (the widened load fits its
    reserve), so the single-file refusal must not reserve more and refuse the load the rule just chose."""
    monkeypatch.delenv(cq.FP8_STORAGE_ENV, raising = False)
    available = int(available_gib * _GIB)
    cuda = type("T", (), {"device": "cuda", "dtype": torch.bfloat16})()
    assert (
        cq.fp8_storage_wanted(cuda, widened_mib = _KREA2_WIDENED_MIB, host_available_mib = available)
        is False
    )
    chosen = cq.fp8_widen_chosen(cuda, widened_mib = _KREA2_WIDENED_MIB, host_available_mib = available)
    assert chosen is True
    budget = _budget(tmp_path, monkeypatch, 64 * _GIB, available, fp8_widen_chosen = chosen)
    budget.check(_KREA2_WIDENED_MIB * conv._MIB)  # no refusal


@pytest.mark.parametrize("available_gib", [28.5, 29.0, 30.0, 30.3])
def test_big_host_keeps_the_proportional_reserve_without_an_fp8_widen_choice(
    tmp_path, monkeypatch, available_gib
):
    """Without fp8 storage the load keeps the 10% reserve of issue 14510 and is refused."""
    budget = _budget(tmp_path, monkeypatch, 64 * _GIB, int(available_gib * _GIB))
    assert (
        budget.reserve_mib
        == small_host.host_reserve_mib(64 * _GIB)
        > cq.FP8_STORAGE_HOST_RESERVE_MIB
    )
    with pytest.raises(RuntimeError, match = "Not enough system RAM"):
        budget.check(_KREA2_WIDENED_MIB * conv._MIB)


def test_fp8_widen_chosen_only_when_the_rule_had_the_choice(monkeypatch):
    monkeypatch.delenv(cq.FP8_STORAGE_ENV, raising = False)
    cuda = type("T", (), {"device": "cuda", "dtype": torch.bfloat16})()
    mps = type("T", (), {"device": "mps", "dtype": torch.bfloat16})()
    room = dict(widened_mib = _KREA2_WIDENED_MIB, host_available_mib = 29 * _GIB)
    assert cq.fp8_widen_chosen(cuda, **room) is True
    # no fp8 storage option: LoRA, offload forces storage, unknown sizes, crowded host (stores instead)
    assert cq.fp8_widen_chosen(cuda, lora = True, **room) is False
    assert cq.fp8_widen_chosen(mps, **room) is True
    assert cq.fp8_widen_chosen(cuda, offload = True, **room) is False
    assert cq.fp8_widen_chosen(cuda, widened_mib = _KREA2_WIDENED_MIB) is False
    assert (
        cq.fp8_widen_chosen(cuda, widened_mib = _KREA2_WIDENED_MIB, host_available_mib = 26 * _GIB)
        is False
    )
    for env in ("0", "1"):
        monkeypatch.setenv(cq.FP8_STORAGE_ENV, env)
        assert cq.fp8_widen_chosen(cuda, **room) is False


def _reserve_seen(monkeypatch):
    seen = []
    real = conv.HostRamBudget.check

    def spy(self, need_bytes):
        seen.append(self.reserve_mib)
        return real(self, need_bytes)

    monkeypatch.setattr(conv.HostRamBudget, "check", spy)
    return seen


def test_loaders_cap_the_reserve_only_for_an_fp8_widen_choice(tmp_path, monkeypatch, krea2):
    """Both single-file loaders refuse against the capped reserve only when the planner chose to widen fp8."""
    total = 64 * _GIB
    proportional = small_host.host_reserve_mib(total)
    _host(monkeypatch, 60 * _GIB, total)
    seen = _reserve_seen(monkeypatch)
    path, _ = _write(tmp_path, "fp8")
    conv.load_original_layout_transformer(krea2, str(path), dict(_KWARGS))
    conv.load_original_layout_transformer(krea2, str(path), dict(_KWARGS), fp8_widen_chosen = True)
    path, _ = _write(tmp_path, "fp8_scaled")
    scan = cq.refuse_comfy_quant(str(path))
    for chosen in (False, True):
        cq.load_comfy_quant_transformer(
            krea2,
            str(path),
            scan,
            dict(_KWARGS),
            int8_backend = None,
            fp8_backend = None,
            fp8_widen_chosen = chosen,
        )
    cap = cq.FP8_STORAGE_HOST_RESERVE_MIB
    assert seen == [proportional, cap, proportional, cap]


def test_reserves_cannot_drift(monkeypatch):
    """The single-file refusal never reserves more than the fp8-storage rule keeps free when it widens."""
    for total in (8 * _GIB, 16 * _GIB, 31 * _GIB, 40 * _GIB, 64 * _GIB, 256 * _GIB, 2048 * _GIB):
        assert small_host.single_file_host_reserve_mib(total) <= cq.FP8_STORAGE_HOST_RESERVE_MIB
    assert cq.FP8_STORAGE_HOST_RESERVE_MIB == small_host.FP8_STORAGE_HOST_RESERVE_MIB


def test_big_host_still_refuses_a_load_that_cannot_fit(tmp_path, monkeypatch):
    """Too little free RAM for even the cheapest path is still refused before any read."""
    budget = _budget(tmp_path, monkeypatch, 64 * _GIB, 20 * _GIB)
    with pytest.raises(RuntimeError, match = "Not enough system RAM"):
        budget.check(_KREA2_WIDENED_MIB * conv._MIB)
    # fp8-stored (~12 GB) fits 20 GB free with the reserve; the same stored need is refused at 15 GB free
    budget.check(12 * _GIB * conv._MIB)
    budget = _budget(tmp_path, monkeypatch, 64 * _GIB, 15 * _GIB)
    with pytest.raises(RuntimeError, match = "Not enough system RAM"):
        budget.check(12 * _GIB * conv._MIB)


@pytest.mark.parametrize("total_gib", [8, 16, 24, 31])
def test_small_host_reserve_unchanged(tmp_path, monkeypatch, total_gib):
    """Hosts up to 40 GB keep the small-host reserve max(3 GB, 10%) on the single-file refusal."""
    total = total_gib * _GIB
    reserve = max(3072, int(total * 0.10))
    assert small_host.host_reserve_mib(total) == reserve
    budget = _budget(tmp_path, monkeypatch, total, 6 * _GIB + reserve)
    assert budget.reserve_mib == reserve
    budget.check(6 * _GIB * conv._MIB)
    with pytest.raises(RuntimeError, match = "Not enough system RAM"):
        budget.check((6 * _GIB + 1) * conv._MIB)