Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

18.9 KB
Raw Blame
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0

"""MiniMax-H3 keeps its quantised denoiser on a small card: the fit picks resident, pinned or streamed."""

from __future__ import annotations

import copy
import types

import pytest

torch = pytest.importorskip("torch")


def _h3_family():
    from core.inference.video_families import detect_video_family
    return detect_video_family("minimax-h3")


@pytest.fixture(autouse = True)
def _hosted_checkpoint_readable(monkeypatch):
    import core.inference.diffusion_prequant as pq
    monkeypatch.setattr(pq, "restricted_prequant_load_supported", lambda *a, **k: True)
    monkeypatch.setattr(pq, "torchao_group_offload_supported", lambda: True)


def _precision(vid, fam, tq, te_scheme, free_gb, monkeypatch):
    if tq in ("int8", "fp8"):
        return tq
    if tq == "none":
        return None
    monkeypatch.setattr(vid, "_h3_auto_precision_ok", lambda target = None: True, raising = False)
    monkeypatch.setattr(vid, "_h3_free_device_bytes", lambda device: int(free_gb * 1000**3))
    return vid._h3_auto_denoiser_scheme(
        fam,
        target = None,
        dtype = torch.bfloat16,
        device = "cuda",
        te_scheme = te_scheme,
        task = "fl2va",
        base_repo = fam.base_repo,
    )


# (free GB, conditioner, request) -> (engaged denoiser, tier). Denoiser 20.3 GB quantised / 66.3 GB,
# conditioner 27.2 GB int8 / 66.7 GB dense, VAEs 11.1 GB, plus activation headroom.
_LADDER = {
    (24, "int8", "auto"): ("int8", "stream"),
    (32, "int8", "auto"): ("int8", "stream"),
    (48, "int8", "auto"): ("int8", "stream"),
    (64, "int8", "auto"): ("int8", "pinned"),
    (80, "int8", "auto"): ("int8", "resident"),
    (141, "int8", "auto"): ("int8", "resident"),
    (24, None, "auto"): ("int8", "stream"),
    (32, None, "auto"): ("int8", "stream"),
    (48, None, "auto"): ("int8", "stream"),
    (64, None, "auto"): ("int8", "stream"),
    (80, None, "auto"): ("int8", "stream"),
    (141, None, "auto"): ("int8", "resident"),
    (24, "int8", "none"): (None, "rotation"),
    (32, "int8", "none"): (None, "rotation"),
    (48, "int8", "none"): (None, "rotation"),
    (64, "int8", "none"): (None, "rotation"),
    (80, "int8", "none"): (None, "rotation"),
    (141, "int8", "none"): (None, "resident"),
    (24, None, "none"): (None, "rotation"),
    (80, None, "none"): (None, "rotation"),
    (141, None, "none"): (None, "rotation"),
}


@pytest.mark.parametrize("free_gb", [24, 32, 48, 64, 80, 141])
@pytest.mark.parametrize("te_scheme", ["int8", None])
@pytest.mark.parametrize("tq", ["auto", "int8", "fp8", "none"])
def test_the_placement_ladder(monkeypatch, free_gb, te_scheme, tq):
    from core.inference import video as vid

    fam = _h3_family()
    engaged = _precision(vid, fam, tq, te_scheme, free_gb, monkeypatch)
    assert engaged == (None if tq == "none" else ("int8" if tq == "auto" else tq))

    from core.inference.video_minimax_h3 import h3_transformer_resident_gb

    denoiser = int(h3_transformer_resident_gb(engaged) * 1000**3)
    whole = vid._h3_planned_denoiser_bytes(fam, te_scheme = te_scheme, dtype = torch.bfloat16)
    pinned = vid._h3_planned_denoiser_bytes(
        fam, te_scheme = te_scheme, dtype = torch.bfloat16, rotating = True
    )
    tier = vid._h3_placement_tier(
        quantised = engaged is not None,
        whole_set_sizes = (denoiser, whole[1]),
        pinned_sizes = (denoiser, pinned[1]),
        free_bytes = int(free_gb * 1000**3),
        speed_off = False,
    )
    expected = _LADDER.get((free_gb, te_scheme, "none" if engaged is None else "auto"))
    if expected is not None:
        assert tier == expected[1]
    if engaged is not None:
        assert tier in ("resident", "pinned", "stream")


def test_speed_off_and_unreadable_cards_keep_their_placements():
    from core.inference import video as vid

    sizes = (20_300_000_000, 36_000_000_000)
    whole = (20_300_000_000, 47_000_000_000)
    assert (
        vid._h3_placement_tier(
            quantised = True,
            whole_set_sizes = whole,
            pinned_sizes = sizes,
            free_bytes = 500 * 1000**3,
            speed_off = True,
        )
        == "pinned"
    )
    assert (
        vid._h3_placement_tier(
            quantised = False,
            whole_set_sizes = whole,
            pinned_sizes = sizes,
            free_bytes = 500 * 1000**3,
            speed_off = True,
        )
        == "rotation"
    )
    for quantised, tier in ((True, "pinned"), (False, "rotation")):
        assert (
            vid._h3_placement_tier(
                quantised = quantised,
                whole_set_sizes = whole,
                pinned_sizes = sizes,
                free_bytes = None,
                speed_off = False,
            )
            == tier
        )


def test_the_quantised_denoiser_is_sized_on_its_payload_not_its_logical_shape():
    """A torchao weight reports the bf16 shape through numel / element_size, which priced the
    20.3 GB hosted denoiser at ~40 GB and pushed a card that could pin it onto the streamed tier."""
    from core.inference.diffusion_prequant import tensor_payload_bytes

    class _Inner:
        def __init__(self, n):
            self._n = n

        def numel(self):
            return self._n

        def element_size(self):
            return 1

    class QuantisedWeight:
        def __init__(self):
            self.qdata = _Inner(1000)
            self.scale = types.SimpleNamespace(numel = lambda: 10, element_size = lambda: 4)

        def __tensor_flatten__(self):
            return ["qdata", "scale"], None

        def numel(self):
            return 1000

        def element_size(self):
            return 2

    assert tensor_payload_bytes(QuantisedWeight()) == 1000 + 40
    assert tensor_payload_bytes(torch.empty(10, dtype = torch.bfloat16)) == 20


def test_a_32gb_card_is_admitted_with_the_streamed_quantised_denoiser():
    from core.inference.video_minimax_h3 import estimate_h3_diffusers_vram_gb

    free_5090_gb = 33.56
    streamed = estimate_h3_diffusers_vram_gb(
        960, 544, 124, text_encoder_gb = 27.2, transformer_gb = 20.3, transformer_streamed = True
    )
    assert streamed + 0 <= free_5090_gb + 0.25
    assert streamed == pytest.approx(29.0, abs = 0.01)
    pinned = estimate_h3_diffusers_vram_gb(
        960, 544, 124, text_encoder_gb = 27.2, transformer_gb = 20.3, transformer_pinned = True
    )
    rotating_bf16 = estimate_h3_diffusers_vram_gb(960, 544, 124, text_encoder_gb = 27.2)
    assert pinned > free_5090_gb + 0.25 and rotating_bf16 > free_5090_gb + 0.25
    assert estimate_h3_diffusers_vram_gb(960, 544, 124) == pytest.approx(73.68, abs = 0.02)
    long_clip = estimate_h3_diffusers_vram_gb(
        1344, 768, 345, text_encoder_gb = 27.2, transformer_gb = 20.3, transformer_streamed = True
    )
    assert long_clip > streamed


def test_the_host_floor_counts_the_pinned_staging_copy_of_a_streamed_denoiser():
    """Group offloading stages the streamed denoiser in pinned host memory beside the copy it was
    built as: measured 80.2 GB peak host RSS against the 64.5 GB a single count asks for."""
    from core.inference.video_minimax_h3 import estimate_h3_diffusers_host_ram_gb

    single = estimate_h3_diffusers_host_ram_gb(33.5, text_encoder_gb = 27.2, transformer_gb = 20.3)
    streamed = estimate_h3_diffusers_host_ram_gb(
        33.5, text_encoder_gb = 27.2, transformer_gb = 20.3, transformer_streamed = True
    )
    assert single == pytest.approx(64.5, abs = 0.01)
    assert streamed == pytest.approx(84.8, abs = 0.01) and streamed >= 80.2
    assert estimate_h3_diffusers_host_ram_gb(33.5) == pytest.approx(150.0, abs = 0.01)


def test_a_streamed_load_keeps_its_host_floor_when_free_vram_later_climbs():
    """Streamed while another tenant held the card; that tenant leaves before the render."""
    from core.inference.video_minimax_h3 import estimate_h3_diffusers_host_ram_gb

    streamed = estimate_h3_diffusers_host_ram_gb(
        140.0, text_encoder_gb = 66.7, transformer_gb = 20.3, transformer_streamed = True
    )
    assert streamed == pytest.approx(124.3, abs = 0.01)
    assert (
        estimate_h3_diffusers_host_ram_gb(140.0, text_encoder_gb = 66.7, transformer_gb = 20.3) == 85.0
    )


def test_the_generate_preflight_reads_the_streamed_fact_off_the_state():
    import inspect

    from core.inference import video as vid

    fields = set(vid._VideoLoadState.__dataclass_fields__)
    assert {"denoiser_streamed", "denoiser_host_copy"} <= fields
    source = inspect.getsource(vid.VideoBackend.generate)
    assert (
        source.count('transformer_streamed = bool(getattr(state, "denoiser_streamed", False))') == 1
    )
    assert (
        source.count('transformer_streamed = bool(getattr(state, "denoiser_host_copy", False))')
        == 1
    )
    load = inspect.getsource(vid)
    assert 'denoiser_host_copy = denoiser_streamed == "stream"' in load


class _Net(torch.nn.Module):
    def __init__(
        self,
        d = 256,
        n = 4,
    ):
        super().__init__()
        self.blocks = torch.nn.ModuleList(
            [
                torch.nn.Sequential(torch.nn.Linear(d, d), torch.nn.GELU(), torch.nn.Linear(d, d))
                for _ in range(n)
            ]
        )
        self.proj_out = torch.nn.Linear(d, d)

    def pad_small_m(self):
        from core.inference.diffusion_quant_pad import PadToMinM
        self.proj_out = PadToMinM(self.proj_out)
        return self

    def forward(self, x):
        for block in self.blocks:
            x = x + block(x)
        return self.proj_out(x)


class _UserHook:
    def __init__(self, model):
        self.model = model
        self.hook = types.SimpleNamespace(other_hooks = [])
        self.removed = False

    def remove(self):
        self.removed = True

    def offload(self):
        self.model.to("cpu")


def _torchao_configs():
    torchao = pytest.importorskip("torchao")
    from torchao.quantization import (
        Float8DynamicActivationFloat8WeightConfig,
        Int8DynamicActivationInt8WeightConfig,
        PerRow,
    )

    configs = {}
    import inspect

    int8_params = inspect.signature(Int8DynamicActivationInt8WeightConfig).parameters
    if "version" in int8_params:
        configs["int8_v2"] = lambda: Int8DynamicActivationInt8WeightConfig(version = 2)
        try:
            Int8DynamicActivationInt8WeightConfig(version = 1)
            from torchao.quantization import linear_activation_quantized_tensor  # noqa: F401
            configs["int8_v1"] = lambda: Int8DynamicActivationInt8WeightConfig(version = 1)
        except Exception:  # noqa: BLE001 -- torchao >= 0.18 removed the v1 classes
            pass
    else:
        configs["int8_v1"] = lambda: Int8DynamicActivationInt8WeightConfig()
    if torch.cuda.get_device_capability() >= (8, 9):
        configs["fp8_row"] = lambda: Float8DynamicActivationFloat8WeightConfig(granularity = PerRow())
    del torchao
    return configs


def _stream(module, stream_prequantized_module):
    rotating = torch.nn.Linear(8, 8).to("cuda")
    hooks = [_UserHook(module), _UserHook(rotating)]
    manager = types.SimpleNamespace(model_hooks = list(hooks))
    mode = stream_prequantized_module(manager, module, "cuda")
    assert hooks[0].removed and [h.model for h in manager.model_hooks] == [rotating]
    return mode, rotating


@pytest.mark.skipif(not torch.cuda.is_available(), reason = "needs CUDA")
@pytest.mark.parametrize("grad_mode", ["inference_mode", "no_grad"])
def test_a_streamed_torchao_denoiser_matches_the_resident_one_bit_for_bit(monkeypatch, grad_mode):
    """Studio renders under inference_mode, and under no_grad where a torchao denoiser streams."""
    pytest.importorskip("diffusers")
    pytest.importorskip("torchao")
    monkeypatch.undo()
    from torchao.quantization import quantize_

    import core.inference.prequant_legacy_int8 as legacy
    from core.inference.diffusion_prequant import (
        stream_prequantized_module,
        torchao_group_offload_supported,
    )

    if not torchao_group_offload_supported():
        pytest.skip("this diffusers cannot group-offload torchao weights")
    render = torch.inference_mode if grad_mode == "inference_mode" else torch.no_grad

    torch.manual_seed(0)
    base = _Net().to(torch.bfloat16)
    x = torch.randn(64, 256, dtype = torch.bfloat16, device = "cuda")
    ran = []
    for name, config in _torchao_configs().items():
        resident = copy.deepcopy(base)
        quantize_(resident, config())
        resident.pad_small_m()
        v1 = type(resident.blocks[0][0].weight).__name__ == "LinearActivationQuantizedTensor"
        if v1:
            assert legacy.convert_legacy_int8_weights(resident) > 0
        resident.to("cuda")
        with render():
            expected = resident(x)
        del resident

        streamed = copy.deepcopy(base)
        quantize_(streamed, config())
        streamed.pad_small_m()
        mode, rotating = _stream(streamed, stream_prequantized_module)
        assert mode == "stream", (name, mode)
        with render():
            outs = [streamed(x) for _ in range(3)]
        for out in outs:
            assert torch.equal(out, expected), name
        assert rotating.weight.device.type == "cpu"
        ran.append((name, type(streamed.blocks[0][0].weight).__name__, mode))
    print("RAN", ran)
    assert ran


@pytest.mark.skipif(not torch.cuda.is_available(), reason = "needs CUDA")
def test_a_hosted_denoiser_left_trainable_by_its_load_still_streams(monkeypatch):
    """load_state_dict(assign=True) leaves weights trainable; Int8Tensor lacks aten.view."""
    pytest.importorskip("diffusers")
    pytest.importorskip("torchao")
    monkeypatch.undo()
    from torchao.quantization import quantize_

    from core.inference.diffusion_prequant import (
        stream_prequantized_module,
        torchao_group_offload_supported,
    )

    if not torchao_group_offload_supported():
        pytest.skip("this diffusers cannot group-offload torchao weights")
    configs = {k: v for k, v in _torchao_configs().items() if k != "int8_v1"}
    torch.manual_seed(0)
    base = _Net().to(torch.bfloat16)
    x = torch.randn(64, 256, dtype = torch.bfloat16, device = "cuda")
    for name, config in configs.items():
        resident = copy.deepcopy(base)
        quantize_(resident, config())
        resident.to("cuda")
        with torch.no_grad():
            expected = resident(x)
        del resident

        streamed = copy.deepcopy(base)
        quantize_(streamed, config())
        streamed.requires_grad_(True)
        mode, _ = _stream(streamed, stream_prequantized_module)
        assert mode == "stream", (name, mode)
        with torch.no_grad():
            outs = [streamed(x) for _ in range(2)]
        for out in outs:
            assert torch.equal(out, expected), name


@pytest.mark.skipif(not torch.cuda.is_available(), reason = "needs CUDA")
def test_a_v1_int8_denoiser_that_cannot_be_rebuilt_moves_synchronously_and_exactly(monkeypatch):
    """The fallback when the Int8Tensor rebuild declines: synchronous copies, same values as the
    resident v1 module, including under inference_mode (the first thing it used to fail on)."""
    pytest.importorskip("diffusers")
    pytest.importorskip("torchao")
    monkeypatch.undo()
    from torchao.quantization import quantize_

    import core.inference.prequant_legacy_int8 as legacy
    from core.inference.diffusion_prequant import (
        stream_prequantized_module,
        torchao_group_offload_supported,
    )

    configs = _torchao_configs()
    if "int8_v1" not in configs or not torchao_group_offload_supported():
        pytest.skip("needs torchao <= 0.17 (v1 int8) and torchao-aware group offload")
    monkeypatch.setattr(legacy, "convert_legacy_int8_weights", lambda module: 0)
    torch.manual_seed(0)
    base = _Net().to(torch.bfloat16)
    x = torch.randn(64, 256, dtype = torch.bfloat16, device = "cuda")
    resident = copy.deepcopy(base)
    quantize_(resident, configs["int8_v1"]())
    resident.to("cuda")
    with torch.inference_mode():
        expected = resident(x)
    streamed = copy.deepcopy(base)
    quantize_(streamed, configs["int8_v1"]())
    mode, _ = _stream(streamed, stream_prequantized_module)
    assert mode == "sync"
    assert type(streamed.blocks[0][0].weight).__name__ == "LinearActivationQuantizedTensor"
    with torch.inference_mode():
        for _ in range(2):
            assert torch.equal(streamed(x), expected)
    assert streamed.blocks[-1][0].weight.device.type == "cpu"


def test_a_streamed_torchao_render_runs_under_no_grad_and_the_rest_keep_inference_mode():
    """torchao's aliasing check fails a weight move under inference_mode, and a streamed denoiser
    moves weights inside the render; the image loader switches for the same reason."""
    import inspect

    from core.inference import video as vid

    source = inspect.getsource(vid.VideoBackend.generate)
    assert (
        'if state.transformer_quant and getattr(state, "denoiser_streamed", False)\n'
        "                    else torch.inference_mode()"
    ) in source
    assert "with grad_ctx, protect_ctx, progress_ctx(), sigma_ctx:" in source


def test_a_failed_streaming_setup_raises_instead_of_pinning(monkeypatch):
    """Streaming is only chosen when pinning does not fit, so a failed setup must not fall back to pinning."""
    pytest.importorskip("diffusers")
    import diffusers.hooks
    import core.inference.diffusion_prequant as prequant

    def _boom(*args, **kwargs):
        raise ValueError("no group offload here")

    monkeypatch.setattr(prequant, "torchao_group_offload_supported", lambda: True)
    monkeypatch.setattr(diffusers.hooks, "apply_group_offloading", _boom)
    module = torch.nn.Linear(8, 8)
    hook = _UserHook(module)
    manager = types.SimpleNamespace(model_hooks = [hook])
    with pytest.raises(RuntimeError, match = "group offloading could not be set up"):
        prequant.stream_prequantized_module(manager, module, "cpu")
    assert hook.removed and module.weight.device.type == "cpu"


@pytest.mark.skipif(not torch.cuda.is_available(), reason = "needs CUDA")
@pytest.mark.parametrize("pin_all, expected", [(True, "stream"), (False, "stream_lazy")])
def test_the_stream_mode_says_whether_a_full_pinned_host_copy_exists(
    monkeypatch, pin_all, expected
):
    """The host floor doubles the denoiser only for a full up-front pin; lazy pinning holds one group at a time."""
    pytest.importorskip("diffusers")
    from torchao.quantization import quantize_

    import core.inference.diffusion_memory as mem
    from core.inference.diffusion_prequant import (
        stream_prequantized_module,
        torchao_group_offload_supported,
    )

    configs = _torchao_configs()
    if "int8_v2" not in configs or not torchao_group_offload_supported():
        pytest.skip("needs Int8Tensor and torchao-aware group offload")
    monkeypatch.setattr(mem, "_streamed_pin_plan", lambda *a, **k: (pin_all, pin_all))
    streamed = _Net().to(torch.bfloat16)
    quantize_(streamed, configs["int8_v2"]())
    mode, _ = _stream(streamed, stream_prequantized_module)
    assert mode == expected