Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

8.9 KB
Raw Blame
# SPDX-License-Identifier: AGPL-3.0-or-later
"""head_dim 256 routes to flex only until SDPA reaches cuDNN with a mask (torch 2.14, sm100).
Above 256 no cuDNN build helps, so it routes on every version."""

import inspect

import pytest

import unsloth  # noqa: F401  (must precede transformers)
import unsloth.models._utils as u


class _Cfg:
    def __init__(self, **kw):
        for k, v in kw.items():
            setattr(self, k, v)


def _cfg(head_dim):
    return _Cfg(model_type = "fake", head_dim = head_dim, num_attention_heads = 8)


@pytest.fixture(autouse = True)
def _no_env_override(monkeypatch):
    # The env var short-circuits before the gate, so it must be clear for these.
    monkeypatch.delenv(u._FLEX_LARGE_HEAD_DIM_ENV_VAR, raising = False)
    monkeypatch.setattr(u, "_flex_kernels_fit_large_head_dim", lambda: True)


@pytest.fixture
def cudnn_reaches_256(monkeypatch):
    def _set(value):
        monkeypatch.setattr(u, "_sdpa_reaches_cudnn_at_head_dim_256", lambda: value)

    return _set


def test_head_dim_256_takes_flex_while_sdpa_cannot_reach_cudnn(cudnn_reaches_256):
    cudnn_reaches_256(False)
    assert u._prefers_flex_for_head_dim(_cfg(256)) is True


def test_head_dim_256_stays_on_sdpa_once_cudnn_takes_the_mask(cudnn_reaches_256):
    cudnn_reaches_256(True)
    assert u._prefers_flex_for_head_dim(_cfg(256)) is False


@pytest.mark.parametrize("head_dim", [264, 512])
def test_above_256_takes_flex_on_every_version(head_dim, cudnn_reaches_256):
    for reaches in (False, True):
        cudnn_reaches_256(reaches)
        assert u._prefers_flex_for_head_dim(_cfg(head_dim)) is True


@pytest.mark.parametrize("head_dim", [64, 128])
def test_at_or_below_128_never_takes_flex(head_dim, cudnn_reaches_256):
    for reaches in (False, True):
        cudnn_reaches_256(reaches)
        assert u._prefers_flex_for_head_dim(_cfg(head_dim)) is False


def test_no_head_dim_means_no_routing(cudnn_reaches_256):
    cudnn_reaches_256(False)
    assert u._prefers_flex_for_head_dim(_Cfg(model_type = "fake")) is False


def test_env_var_forces_flex_even_when_cudnn_would_serve_it(monkeypatch, cudnn_reaches_256):
    cudnn_reaches_256(True)
    monkeypatch.setenv(u._FLEX_LARGE_HEAD_DIM_ENV_VAR, "1")
    assert u._prefers_flex_for_head_dim(_cfg(256)) is True


def test_env_var_keeps_sdpa_even_above_256(monkeypatch, cudnn_reaches_256):
    cudnn_reaches_256(False)
    monkeypatch.setenv(u._FLEX_LARGE_HEAD_DIM_ENV_VAR, "0")
    assert u._prefers_flex_for_head_dim(_cfg(512)) is False


def test_gate_is_off_below_torch_2_14(monkeypatch):
    monkeypatch.setattr(u.torch, "__version__", "2.13.0+cu130")
    assert u._sdpa_reaches_cudnn_at_head_dim_256() is False


def test_gate_is_off_on_rocm(monkeypatch):
    monkeypatch.setattr(u.torch, "__version__", "2.14.0+cu130")
    monkeypatch.setattr(u.torch.version, "hip", "6.2.0", raising = False)
    assert u._sdpa_reaches_cudnn_at_head_dim_256() is False


def test_gate_is_off_with_no_cuda(monkeypatch):
    monkeypatch.setattr(u.torch, "__version__", "2.14.0+cu130")
    monkeypatch.setattr(u.torch.version, "hip", None, raising = False)
    monkeypatch.setattr(u.torch.cuda, "is_available", lambda: False)
    assert u._sdpa_reaches_cudnn_at_head_dim_256() is False


def test_gate_is_off_below_blackwell(monkeypatch):
    monkeypatch.setattr(u.torch, "__version__", "2.14.0+cu130")
    monkeypatch.setattr(u.torch.version, "hip", None, raising = False)
    monkeypatch.setattr(u.torch.cuda, "is_available", lambda: True)
    monkeypatch.setattr(u.torch.cuda, "device_count", lambda: 1)
    monkeypatch.setattr(u.torch.cuda, "get_device_capability", lambda index: (9, 0))
    assert u._sdpa_reaches_cudnn_at_head_dim_256() is False


def _plain_sdpa(monkeypatch):
    # Import-time wrappers (the SM100 cuDNN head_dim 256 detour, the torch 2.6 - 2.13 cuDNN grad
    # layout fix on top of it) can stack on a B200; these test plain SDPA, so unwrap them all.
    sdpa = u.torch.nn.functional.scaled_dot_product_attention
    monkeypatch.setattr(u.torch.nn.functional, "scaled_dot_product_attention", inspect.unwrap(sdpa))


def _blackwell_torch_2_14(monkeypatch):
    monkeypatch.setattr(u.torch, "__version__", "2.14.0+cu130")
    monkeypatch.setattr(u.torch.version, "hip", None, raising = False)
    monkeypatch.setattr(u.torch.cuda, "is_available", lambda: True)
    monkeypatch.setattr(u.torch.cuda, "device_count", lambda: 1)
    monkeypatch.setattr(u.torch.cuda, "get_device_capability", lambda index: (10, 0))


def test_gate_is_on_for_blackwell_on_torch_2_14(monkeypatch):
    _plain_sdpa(monkeypatch)
    _blackwell_torch_2_14(monkeypatch)
    assert u._sdpa_reaches_cudnn_at_head_dim_256() is True


def test_plain_sdpa_strips_both_stacked_sdpa_wrappers(monkeypatch):
    # A B200 on torch 2.13 installs the d256 detour, then the grad-layout wrapper over it.
    from unsloth import import_fixes

    F = u.torch.nn.functional
    monkeypatch.setattr(
        F, "scaled_dot_product_attention", u.torch._C._nn.scaled_dot_product_attention
    )
    monkeypatch.delenv("UNSLOTH_ALLOW_CUDNN_SDPA_D256", raising = False)
    monkeypatch.delenv("UNSLOTH_SKIP_CUDNN_SDPA_GRAD_LAYOUT_FIX", raising = False)
    monkeypatch.setattr(import_fixes, "_sdpa_cudnn_d256_sm100_devices", lambda: {0})
    monkeypatch.setattr(u.torch.cuda, "is_available", lambda: True)
    monkeypatch.setattr(u.torch.version, "hip", None, raising = False)
    monkeypatch.setattr(u.torch, "__version__", "2.13.0+cu130")
    monkeypatch.setattr(u.torch.backends.cudnn, "version", lambda: 92000)
    import_fixes.fix_cudnn_sdpa_d256_masked_backward()
    import_fixes.fix_cudnn_sdpa_backward_grad_layout()
    sdpa = F.scaled_dot_product_attention
    assert getattr(sdpa, import_fixes._SDPA_GRAD_LAYOUT_FLAG, False)
    assert getattr(sdpa.__wrapped__, "_unsloth_avoids_cudnn_d256_masked_backward", False)
    test_gate_is_on_for_blackwell_on_torch_2_14(monkeypatch)


def test_gate_is_off_while_the_cudnn_d256_detour_is_installed(monkeypatch):
    # Masked head_dim 256 training then runs the efficient kernel, not cuDNN, so flex is faster.
    _blackwell_torch_2_14(monkeypatch)
    sdpa = u.torch.nn.functional.scaled_dot_product_attention
    original = getattr(sdpa, "__wrapped__", sdpa)

    def detour(*args, **kwargs):
        return original(*args, **kwargs)

    detour._unsloth_avoids_cudnn_d256_masked_backward = True
    monkeypatch.setattr(u.torch.nn.functional, "scaled_dot_product_attention", detour)
    assert u._sdpa_reaches_cudnn_at_head_dim_256() is False
    assert u._prefers_flex_for_head_dim(_cfg(256)) is True


def test_a_mixed_box_falls_back_to_the_weakest_card(monkeypatch):
    monkeypatch.setattr(u.torch, "__version__", "2.14.0+cu130")
    monkeypatch.setattr(u.torch.version, "hip", None, raising = False)
    monkeypatch.setattr(u.torch.cuda, "is_available", lambda: True)
    monkeypatch.setattr(u.torch.cuda, "device_count", lambda: 2)
    monkeypatch.setattr(
        u.torch.cuda, "get_device_capability", lambda index: (10, 0) if index == 0 else (9, 0)
    )
    assert u._sdpa_reaches_cudnn_at_head_dim_256() is False


def _cuda_box(monkeypatch, shared_memory):
    class _Props:
        def __init__(self, index):
            self.shared_memory_per_block_optin = shared_memory[index]

    monkeypatch.setattr(u.torch.version, "hip", None, raising = False)
    monkeypatch.setattr(u.torch.cuda, "is_available", lambda: True)
    monkeypatch.setattr(u.torch.cuda, "device_count", lambda: len(shared_memory))
    monkeypatch.setattr(u.torch.cuda, "get_device_properties", _Props)


@pytest.mark.parametrize("shared_memory", [101376, 166912])  # sm86/sm89/sm120, A100
def test_flex_kernels_do_not_fit_small_shared_memory(monkeypatch, shared_memory):
    monkeypatch.undo()
    _cuda_box(monkeypatch, [shared_memory])
    assert u._flex_kernels_fit_large_head_dim() is False


def test_flex_kernels_fit_the_measured_class(monkeypatch):
    monkeypatch.undo()
    _cuda_box(monkeypatch, [232448])
    assert u._flex_kernels_fit_large_head_dim() is True


def test_flex_kernels_fit_needs_every_card(monkeypatch):
    monkeypatch.undo()
    _cuda_box(monkeypatch, [232448, 101376])
    assert u._flex_kernels_fit_large_head_dim() is False


def test_flex_kernels_fit_is_off_on_rocm_and_without_cuda(monkeypatch):
    monkeypatch.undo()
    _cuda_box(monkeypatch, [232448])
    monkeypatch.setattr(u.torch.version, "hip", "6.2.0", raising = False)
    assert u._flex_kernels_fit_large_head_dim() is False
    monkeypatch.setattr(u.torch.version, "hip", None, raising = False)
    monkeypatch.setattr(u.torch.cuda, "is_available", lambda: False)
    assert u._flex_kernels_fit_large_head_dim() is False


@pytest.mark.parametrize("head_dim", [256, 512])
def test_small_shared_memory_keeps_sdpa(monkeypatch, cudnn_reaches_256, head_dim):
    cudnn_reaches_256(False)
    monkeypatch.setattr(u, "_flex_kernels_fit_large_head_dim", lambda: False)
    assert u._prefers_flex_for_head_dim(_cfg(head_dim)) is False
    monkeypatch.setenv(u._FLEX_LARGE_HEAD_DIM_ENV_VAR, "1")
    assert u._prefers_flex_for_head_dim(_cfg(head_dim)) is True