Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

382 KB
Raw Blame
"""AMD ROCm support tests across install pathways (all mocked, no AMD HW)."""

import contextlib
import importlib.util
import io
import json
import os
import re
import shutil
import subprocess
import sys
import tempfile
from pathlib import Path
from types import SimpleNamespace
from unittest.mock import MagicMock, mock_open, patch, PropertyMock

import pytest
from unsloth_pwsh_runner import run_pwsh


PACKAGE_ROOT = Path(__file__).resolve().parents[3]

_PREBUILT_PATH = PACKAGE_ROOT / "studio" / "install_llama_prebuilt.py"
_PREBUILT_SPEC = importlib.util.spec_from_file_location(
    "studio_install_llama_prebuilt", _PREBUILT_PATH
)
assert _PREBUILT_SPEC is not None and _PREBUILT_SPEC.loader is not None
prebuilt_mod = importlib.util.module_from_spec(_PREBUILT_SPEC)
sys.modules[_PREBUILT_SPEC.name] = prebuilt_mod
_PREBUILT_SPEC.loader.exec_module(prebuilt_mod)

HostInfo = prebuilt_mod.HostInfo
AssetChoice = prebuilt_mod.AssetChoice
PrebuiltFallback = prebuilt_mod.PrebuiltFallback
resolve_upstream_asset_choice = prebuilt_mod.resolve_upstream_asset_choice
runtime_patterns_for_choice = prebuilt_mod.runtime_patterns_for_choice
_apply_host_overrides = prebuilt_mod._apply_host_overrides
_normalize_forwarded_gfx = prebuilt_mod._normalize_forwarded_gfx

_STACK_PATH = PACKAGE_ROOT / "studio" / "install_python_stack.py"
_STACK_SPEC = importlib.util.spec_from_file_location("studio_install_python_stack", _STACK_PATH)
assert _STACK_SPEC is not None and _STACK_SPEC.loader is not None
stack_mod = importlib.util.module_from_spec(_STACK_SPEC)
sys.modules[_STACK_SPEC.name] = stack_mod
_STACK_SPEC.loader.exec_module(stack_mod)

# The probe prints its answer behind this marker; mocked stdout must carry it.
_MARK = stack_mod._TORCH_PROBE_MARKER

_detect_rocm_version = stack_mod._detect_rocm_version
_ensure_rocm_torch = stack_mod._ensure_rocm_torch
_has_rocm_gpu = stack_mod._has_rocm_gpu
_has_usable_nvidia_gpu = stack_mod._has_usable_nvidia_gpu
_ROCM_TORCH_INDEX = stack_mod._ROCM_TORCH_INDEX
_windows_rocm_index_url = stack_mod._windows_rocm_index_url
_detect_windows_gfx_arch = stack_mod._detect_windows_gfx_arch
_install_bnb_windows_rocm = stack_mod._install_bnb_windows_rocm


@pytest.fixture(autouse = True)
def _reset_torch_runtime_probe():
    """Both probes are memoized per install run, so they must not leak between tests."""
    stack_mod._invalidate_torch_runtime_probe()
    stack_mod._invalidate_rocm_version_probe()
    yield
    stack_mod._invalidate_torch_runtime_probe()
    stack_mod._invalidate_rocm_version_probe()


def _extract_sh_function_body(source: str, name: str) -> str:
    """Return a shell function body from `source` by brace matching."""
    needle = f"{name}() {{"
    start = source.find(needle)
    # Raise rather than return '': an empty body makes callers fail on the wrong subject.
    assert start >= 0, f"install.sh defines no {name}()"
    depth = 0
    i = start + len(needle) - 1  # land on the opening brace
    n = len(source)
    while i < n:
        ch = source[i]
        if ch == "{":
            depth += 1
        elif ch == "}":
            depth -= 1
            if depth == 0:
                return source[start : i + 1]
        i += 1
    return source[start:]


def _gpu_record_helpers(source: str) -> str:
    """A missing helper fails silently: || true swallows the error and the assertion sees no GPUs."""
    return "\n".join(
        _extract_sh_function_body(source, name)
        for name in (
            "_rocminfo_gpu_records",
            "_amd_smi_gpu_records",
            "_gfx_arch_slots",
            "_amd_smi_hip_order",
            "_amd_prefer_discrete_gfx",
            "_hsa_override_gfx_arch",
            "_amd_gfx_is_shadowing_integrated",
            "_amd_gfx_has_wheel_route",
            "_amd_arch_index_family_for_gfx",
            "_amd_generic_tag_carries_gfx",
        )
    )


def _visibility_mask_names(source: str) -> tuple[str, ...]:
    """Reads the mask list from install.sh so tests clear every mask production honours, not the host's."""
    match = re.search(r'^\s*_vis_masks="([^"]+)"', source, re.M)
    assert match, "could not find _vis_masks in install.sh -- did the gfx block move?"
    names = tuple(match.group(1).split())
    assert names, "_vis_masks is empty"
    return ("ROCR_VISIBLE_DEVICES", *names)


# Renders whatever showformat it is handed, so this tests how production ASKS.
_DPKG_QUERY_STUB = r"""#!/bin/sh
_entries='__ENTRIES__'
_fmt=''
_requested=''
_has_rocm_core=0
_has_hsa_runtime=0
while [ $# -gt 0 ]; do
    case "$1" in
        -f=*)            _fmt=${1#-f=} ;;
        --showformat=*)  _fmt=${1#--showformat=} ;;
        -f|--showformat) shift; _fmt=$1 ;;
        -*)              : ;;
        rocm-core)
            _requested="$_requested $1"
            _has_rocm_core=1
            ;;
        libhsa-runtime64-1)
            _requested="$_requested $1"
            _has_hsa_runtime=1
            ;;
    esac
    shift
done
[ "$_has_rocm_core" -eq 1 ] && [ "$_has_hsa_runtime" -eq 1 ] || exit 1
[ -n "$_fmt" ] || _fmt='${Package}\t${Version}\n'
# Unrecognised fields render empty, like the real dpkg-query.
_emit() {
    _package=$1
    _status=$2
    _ver=$3
    # dpkg's Status field is "<want> <error-flag> <status>".
    case "$_status" in installed) _want=install ;; *) _want=deinstall ;; esac
    _out=$(printf '%s' "$_fmt" | sed \
        -e "s|\${Package}|$_package|g" \
        -e "s|\${Status}|$_want ok $_status|g" \
        -e "s|\${db:Status-Status}|$_status|g" \
        -e "s|\${db:Status-Want}|$_want|g" \
        -e "s|\${db:Status-Eflag}|ok|g" \
        -e "s|\${Version}|$_ver|g" \
        -e "s|\${[^}]*}||g")
    # Terminate the record explicitly. $(...) strips trailing newlines, so a caller
    # whose showformat ends in a REAL newline (Python's "\n") otherwise gets every
    # package concatenated onto one line, and a multi-package assertion silently
    # tests only the first record. install.sh passes a literal backslash-n, which
    # printf re-expands, so that caller just sees one blank line between records and
    # both parsers skip those.
    printf "$_out"
    printf '\n'
}
_missing=0
for _wanted in $_requested; do
    _found=0
    while IFS='|' read -r _package _status _ver; do
        [ "$_package" = "$_wanted" ] && _found=1
    done <<EOF
$_entries
EOF
    [ "$_found" -eq 1 ] || _missing=1
done
# Entries, not requested-argument order, determine stdout order. This proves the
# parser collects every valid line rather than assuming a particular package order.
while IFS='|' read -r _package _status _ver; do
    case " $_requested " in
        *" $_package "*) _emit "$_package" "$_status" "$_ver" ;;
    esac
done <<EOF
$_entries
EOF
exit "$_missing"
"""


def _write_dpkg_query_stub(
    path: str,
    version: str,
    status: str = "installed",
    package: str = "rocm-core",
    packages: dict[str, tuple[str, str]] | None = None,
) -> None:
    entries = packages or {package: (status, version)}
    rendered_entries = "\n".join(
        f"{name}|{entry_status}|{entry_version}"
        for name, (entry_status, entry_version) in entries.items()
    )
    with open(path, "w", encoding = "utf-8") as f:
        f.write(_DPKG_QUERY_STUB.replace("__ENTRIES__", rendered_entries))
    os.chmod(path, 0o755)


_needs_exec_stub = pytest.mark.skipif(os.name == "nt", reason = "shell stubs need a POSIX exec")


def _run_bash_script(shell: str, script: str, **kwargs) -> subprocess.CompletedProcess:
    # From a file, not -c: Git Bash's argv globbing truncates a quoted -c argument at 8K chars.
    fd, path = tempfile.mkstemp(suffix = ".sh")
    try:
        with os.fdopen(fd, "w", encoding = "utf-8", newline = "\n") as f:
            f.write(script)
        return subprocess.run([shell, path.replace("\\", "/")], **kwargs)
    finally:
        os.unlink(path)


_LINUX_X64 = dict(
    system = "Linux",
    machine = "x86_64",
    is_windows = False,
    is_linux = True,
    is_macos = False,
    is_x86_64 = True,
    is_arm64 = False,
)
_WINDOWS_X64 = dict(
    system = "Windows",
    machine = "amd64",
    is_windows = True,
    is_linux = False,
    is_macos = False,
    is_x86_64 = True,
    is_arm64 = False,
)
_MACOS_ARM64 = dict(
    system = "Darwin",
    machine = "arm64",
    is_windows = False,
    is_linux = False,
    is_macos = True,
    is_x86_64 = False,
    is_arm64 = True,
)
_NO_GPU = dict(
    nvidia_smi = None,
    driver_cuda_version = None,
    compute_caps = [],
    visible_cuda_devices = None,
    has_physical_nvidia = False,
    has_usable_nvidia = False,
    has_rocm = False,
)


def _host(platform: dict, gpu: dict, **overrides) -> HostInfo:
    return HostInfo(**{**platform, **_NO_GPU, **gpu, **overrides})


def nvidia_host(**overrides) -> HostInfo:
    """NVIDIA Linux x86_64 host."""
    gpu = dict(
        nvidia_smi = "/usr/bin/nvidia-smi",
        driver_cuda_version = (12, 6),
        compute_caps = ["89"],
        has_physical_nvidia = True,
        has_usable_nvidia = True,
    )
    return _host(_LINUX_X64, gpu, **overrides)


def rocm_host(**overrides) -> HostInfo:
    """AMD ROCm Linux x86_64 host (no NVIDIA)."""
    return _host(_LINUX_X64, dict(has_rocm = True), **overrides)


def cpu_host(**overrides) -> HostInfo:
    """CPU-only Linux x86_64 host."""
    return _host(_LINUX_X64, {}, **overrides)


def macos_host(**overrides) -> HostInfo:
    """macOS arm64 host."""
    return _host(_MACOS_ARM64, {}, **overrides)


def windows_host(**overrides) -> HostInfo:
    """Windows x86_64 host."""
    return _host(_WINDOWS_X64, {}, **overrides)


def windows_rocm_host(**overrides) -> HostInfo:
    """Windows x86_64 host with ROCm."""
    return _host(_WINDOWS_X64, dict(has_rocm = True), **overrides)


LLAMA_TAG = "b8508"

UPSTREAM_ASSETS = {
    f"llama-{LLAMA_TAG}-bin-ubuntu-x64.tar.gz": f"https://example.com/{LLAMA_TAG}-linux-cpu.tar.gz",
    f"llama-{LLAMA_TAG}-bin-ubuntu-rocm-7.2-x64.tar.gz": f"https://example.com/{LLAMA_TAG}-linux-rocm.tar.gz",
    f"llama-{LLAMA_TAG}-bin-win-cpu-x64.zip": f"https://example.com/{LLAMA_TAG}-win-cpu.zip",
    f"llama-{LLAMA_TAG}-bin-win-cuda-12.4-x64.zip": f"https://example.com/{LLAMA_TAG}-win-cuda.zip",
    f"llama-{LLAMA_TAG}-bin-win-hip-radeon-x64.zip": f"https://example.com/{LLAMA_TAG}-win-hip.zip",
    f"llama-{LLAMA_TAG}-bin-macos-arm64.tar.gz": f"https://example.com/{LLAMA_TAG}-macos-arm64.tar.gz",
    f"llama-{LLAMA_TAG}-bin-macos-x64.tar.gz": f"https://example.com/{LLAMA_TAG}-macos-x64.tar.gz",
}


class TestResolveUpstreamAssetChoice:
    """Verify that the asset selection logic picks the right binary for each platform."""

    # Plain routing cases live in test_selection_logic.py; this keeps ROCm/NVIDIA precedence.

    @patch.object(prebuilt_mod, "github_release_assets", return_value = UPSTREAM_ASSETS)
    def test_nvidia_linux_gets_cpu_asset(self, mock_assets):
        """NVIDIA host should NOT hit the ROCm path -- gets CPU asset (CUDA handled elsewhere)."""
        host = nvidia_host()
        choice = resolve_upstream_asset_choice(host, LLAMA_TAG)
        assert choice.install_kind == "linux-cpu"
        assert "ubuntu-x64" in choice.name
        assert "rocm" not in choice.name

    @patch.object(prebuilt_mod, "github_release_assets", return_value = UPSTREAM_ASSETS)
    def test_rocm_linux_gets_rocm_prebuilt(self, mock_assets):
        """AMD ROCm Linux host should get the ROCm prebuilt."""
        host = rocm_host()
        choice = resolve_upstream_asset_choice(host, LLAMA_TAG)
        assert choice.install_kind == "linux-rocm"
        assert "rocm" in choice.name

    @patch.object(prebuilt_mod, "github_release_assets", return_value = UPSTREAM_ASSETS)
    def test_windows_rocm_gets_hip_asset(self, mock_assets):
        """Windows ROCm host should get Windows HIP asset."""
        host = windows_rocm_host()
        choice = resolve_upstream_asset_choice(host, LLAMA_TAG)
        assert choice.install_kind == "windows-hip"
        assert "hip" in choice.name

    @patch.object(prebuilt_mod, "github_release_assets", return_value = UPSTREAM_ASSETS)
    def test_mixed_nvidia_rocm_prefers_nvidia(self, mock_assets):
        """Host with both NVIDIA and ROCm should use NVIDIA (CPU path here, CUDA elsewhere)."""
        host = nvidia_host(has_rocm = True)
        choice = resolve_upstream_asset_choice(host, LLAMA_TAG)
        assert choice.install_kind == "linux-cpu"
        assert "rocm" not in choice.name

    @patch.object(prebuilt_mod, "github_release_assets")
    def test_rocm_linux_no_prebuilt_falls_back(self, mock_assets):
        """AMD ROCm host should fall back to source build when no ROCm prebuilt exists."""
        assets_without_rocm = {k: v for k, v in UPSTREAM_ASSETS.items() if "rocm" not in k}
        mock_assets.return_value = assets_without_rocm
        host = rocm_host()
        with pytest.raises(PrebuiltFallback, match = "ROCm detected"):
            resolve_upstream_asset_choice(host, LLAMA_TAG)

    @patch.object(prebuilt_mod, "github_release_assets")
    def test_windows_rocm_no_hip_falls_to_cpu(self, mock_assets):
        """Windows+ROCm with HIP prebuilt missing should fall through to CPU."""
        assets_no_hip = {k: v for k, v in UPSTREAM_ASSETS.items() if "hip" not in k}
        mock_assets.return_value = assets_no_hip
        host = windows_rocm_host()
        choice = resolve_upstream_asset_choice(host, LLAMA_TAG)
        assert choice.install_kind == "windows-cpu"

    @patch.object(prebuilt_mod, "github_release_assets", return_value = UPSTREAM_ASSETS)
    def test_macos_rocm_impossible_has_rocm_false(self, mock_assets):
        """macOS host should never have has_rocm=True in practice; verify it gets macOS asset."""
        host = macos_host(has_rocm = True)
        choice = resolve_upstream_asset_choice(host, LLAMA_TAG)
        assert choice.install_kind == "macos-arm64"

    @patch.object(prebuilt_mod, "github_release_assets", return_value = UPSTREAM_ASSETS)
    def test_linux_aarch64_rocm_gets_prebuilt_fallback(self, mock_assets):
        """Linux aarch64 with ROCm -- no x86_64 match, should raise PrebuiltFallback."""
        host = rocm_host(machine = "aarch64", is_x86_64 = False, is_arm64 = True)
        with pytest.raises(PrebuiltFallback):
            resolve_upstream_asset_choice(host, LLAMA_TAG)


class TestRuntimePatterns:
    """Verify runtime file patterns for all install kinds."""

    def test_linux_cpu_patterns(self):
        choice = AssetChoice(
            repo = "", tag = "", name = "", url = "", source_label = "", install_kind = "linux-cpu"
        )
        patterns = runtime_patterns_for_choice(choice)
        assert "llama-server" in patterns
        assert "llama-quantize" in patterns
        # lib*.so* covers libllama/libggml/libmtmd plus the libllama-*-impl.so
        # Split from ggml-org/llama.cpp #23462 (between b9279 and b9283).
        assert "lib*.so*" in patterns

    def test_linux_cuda_patterns(self):
        choice = AssetChoice(
            repo = "", tag = "", name = "", url = "", source_label = "", install_kind = "linux-cuda"
        )
        patterns = runtime_patterns_for_choice(choice)
        assert "lib*.so*" in patterns

    def test_linux_rocm_patterns(self):
        choice = AssetChoice(
            repo = "", tag = "", name = "", url = "", source_label = "", install_kind = "linux-rocm"
        )
        patterns = runtime_patterns_for_choice(choice)
        assert "lib*.so*" in patterns
        assert "llama-server" in patterns

    def test_windows_hip_patterns(self):
        choice = AssetChoice(
            repo = "",
            tag = "",
            name = "",
            url = "",
            source_label = "",
            install_kind = "windows-hip",
        )
        patterns = runtime_patterns_for_choice(choice)
        assert "llama-server.exe" in patterns
        assert "llama-quantize.exe" in patterns
        assert "*.dll" in patterns

    def test_macos_patterns(self):
        choice = AssetChoice(
            repo = "",
            tag = "",
            name = "",
            url = "",
            source_label = "",
            install_kind = "macos-arm64",
        )
        patterns = runtime_patterns_for_choice(choice)
        assert "lib*.dylib" in patterns

    def test_diffusion_visual_server_kept(self):
        # The DiffusionGemma visual server must survive the prune.
        for kind, name in (
            ("linux-cuda", "llama-diffusion-gemma-visual-server"),
            ("macos-arm64", "llama-diffusion-gemma-visual-server"),
            ("windows-cuda", "llama-diffusion-gemma-visual-server.exe"),
        ):
            choice = AssetChoice(
                repo = "", tag = "", name = "", url = "", source_label = "", install_kind = kind
            )
            assert name in runtime_patterns_for_choice(choice)

    def test_fit_params_kept_on_macos(self):
        for kind in ("macos-arm64", "macos-x64"):
            choice = AssetChoice(
                repo = "", tag = "", name = "", url = "", source_label = "", install_kind = kind
            )
            assert "llama-fit-params" in runtime_patterns_for_choice(choice)


class TestHostInfoRocm:
    """Verify has_rocm field does not affect other HostInfo behavior."""

    def test_has_rocm_default_false(self):
        host = HostInfo(
            system = "Linux",
            machine = "x86_64",
            is_windows = False,
            is_linux = True,
            is_macos = False,
            is_x86_64 = True,
            is_arm64 = False,
            nvidia_smi = None,
            driver_cuda_version = None,
            compute_caps = [],
            visible_cuda_devices = None,
            has_physical_nvidia = False,
            has_usable_nvidia = False,
        )
        assert host.has_rocm is False

    def test_has_rocm_explicit_true(self):
        host = rocm_host()
        assert host.has_rocm is True

    def test_nvidia_host_no_rocm(self):
        host = nvidia_host()
        assert host.has_rocm is False
        assert host.has_usable_nvidia is True

    def test_detect_host_has_rocm_detection_logic(self):
        """detect_host() should have ROCm GPU detection logic."""
        import inspect

        source = inspect.getsource(prebuilt_mod.detect_host)
        assert "rocminfo" in source or "amd-smi" in source

    def test_detect_host_windows_rocm_detection(self):
        """detect_host() source should have Windows-specific ROCm GPU detection."""
        import inspect

        source = inspect.getsource(prebuilt_mod.detect_host)
        assert "hipinfo" in source or "amd-smi" in source


class TestDetectRocmVersion:
    """Verify ROCm version detection from various sources."""

    def test_no_rocm_returns_none(self, tmp_path):
        """No ROCm installed should return None."""
        with patch.dict(os.environ, {"ROCM_PATH": str(tmp_path / "nonexistent")}):
            with patch("shutil.which", return_value = None):
                result = _detect_rocm_version()
                assert result is None

    # which() is patched to None: the detector reads every source, including a real ROCm install.
    def test_version_from_file(self, tmp_path):
        """Reads version from /opt/rocm/.info/version."""
        info_dir = tmp_path / ".info"
        info_dir.mkdir()
        (info_dir / "version").write_text("7.1.0-12345\n")
        with patch.dict(os.environ, {"ROCM_PATH": str(tmp_path)}):
            with patch("shutil.which", return_value = None):
                result = _detect_rocm_version()
                assert result == (7, 1)

    def test_version_62(self, tmp_path):
        """Reads ROCm 6.2 version."""
        info_dir = tmp_path / ".info"
        info_dir.mkdir()
        (info_dir / "version").write_text("6.2.0\n")
        with patch.dict(os.environ, {"ROCM_PATH": str(tmp_path)}):
            with patch("shutil.which", return_value = None):
                result = _detect_rocm_version()
                assert result == (6, 2)

    def test_hipconfig_fallback(self, tmp_path):
        """Falls back to hipconfig --version when file not found."""
        with patch.dict(os.environ, {"ROCM_PATH": str(tmp_path / "nonexistent")}):
            mock_result = MagicMock()
            mock_result.returncode = 0
            mock_result.stdout = b"6.3.21234.2\n"
            with patch("shutil.which", return_value = "/usr/bin/hipconfig"):
                with patch("subprocess.run", return_value = mock_result):
                    result = _detect_rocm_version()
                    assert result == (6, 3)

    @_needs_exec_stub
    def test_dpkg_fallback_without_hipconfig(self, tmp_path):
        """dpkg rocm-core fallback works when amd-smi and hipconfig are absent
        (regression: a shadowing local re import raised UnboundLocalError)."""
        dpkg = tmp_path / "dpkg-query"
        _write_dpkg_query_stub(str(dpkg), "1:6.3.0-1")

        def which(cmd):
            return str(dpkg) if cmd == "dpkg-query" else None

        with patch.dict(os.environ, {"ROCM_PATH": str(tmp_path / "nonexistent")}):
            with patch("shutil.which", side_effect = which):
                assert _detect_rocm_version() == (6, 3)

    def test_empty_version_file(self, tmp_path):
        """Empty version file should return None."""
        info_dir = tmp_path / ".info"
        info_dir.mkdir()
        (info_dir / "version").write_text("")
        with patch.dict(os.environ, {"ROCM_PATH": str(tmp_path)}):
            with patch("shutil.which", return_value = None):
                result = _detect_rocm_version()
                assert result is None

    def test_multiple_version_sources_highest_wins(self, tmp_path):
        """When ROCm version sources disagree, the highest valid version wins."""
        info_dir = tmp_path / ".info"
        info_dir.mkdir()
        (info_dir / "version").write_text("5.7.0\n")

        def which(cmd):
            return "/usr/bin/hipconfig" if cmd == "hipconfig" else None

        mock_result = MagicMock()
        mock_result.returncode = 0
        mock_result.stdout = b"6.4.0\n"

        with patch.dict(os.environ, {"ROCM_PATH": str(tmp_path)}):
            with patch("shutil.which", side_effect = which):
                with patch("subprocess.run", return_value = mock_result):
                    assert _detect_rocm_version() == (6, 4)

    @_needs_exec_stub
    def test_removed_dpkg_rocm_core_is_ignored(self, tmp_path):
        """Removed-but-not-purged rocm-core must not report a stale version."""
        dpkg = tmp_path / "dpkg-query"
        _write_dpkg_query_stub(
            str(dpkg),
            "1:7.0.0-1",
            "config-files",
        )

        def which(cmd):
            return str(dpkg) if cmd == "dpkg-query" else None

        with patch.dict(
            os.environ,
            {"ROCM_PATH": str(tmp_path / "nonexistent")},
        ):
            with patch("shutil.which", side_effect = which):
                assert _detect_rocm_version() is None

    @_needs_exec_stub
    def test_debian_split_runtime_uses_installed_hsa_runtime(self, tmp_path):
        """Debian can ship hipconfig 5.7 beside HSA 6.1 with no rocm-core."""
        hipconfig = tmp_path / "hipconfig"
        hipconfig.write_text("#!/bin/sh\necho '5.7.31921-0'\n", encoding = "utf-8")
        hipconfig.chmod(0o755)
        dpkg = tmp_path / "dpkg-query"
        _write_dpkg_query_stub(
            str(dpkg),
            "unused",
            packages = {"libhsa-runtime64-1": ("installed", "1:6.1.2-2")},
        )

        def which(cmd):
            if cmd == "hipconfig":
                return str(hipconfig)
            if cmd == "dpkg-query":
                return str(dpkg)
            return None

        with patch.dict(
            os.environ,
            {"ROCM_PATH": str(tmp_path / "nonexistent")},
        ):
            with patch("shutil.which", side_effect = which):
                assert _detect_rocm_version() == (6, 1)

    @_needs_exec_stub
    def test_installed_rocm_core_outranks_the_distro_hsa_package(self, tmp_path):
        """rocm-core wins over libhsa-runtime64-1 even when the HSA reading is HIGHER.
        HSA is emitted first, so taking the highest reading or the first line fails this."""
        dpkg = tmp_path / "dpkg-query"
        _write_dpkg_query_stub(
            str(dpkg),
            "unused",
            packages = {
                "libhsa-runtime64-1": ("installed", "6.4.3+dfsg-4"),
                "rocm-core": ("installed", "1:6.1.2-2"),
            },
        )

        def which(cmd):
            return str(dpkg) if cmd == "dpkg-query" else None

        with patch.dict(os.environ, {"ROCM_PATH": str(tmp_path / "nonexistent")}):
            with patch("shutil.which", side_effect = which):
                assert _detect_rocm_version() == (6, 1)

    @_needs_exec_stub
    def test_distro_hsa_package_does_not_manufacture_a_disagreement(self, capsys, tmp_path):
        """The real Ubuntu shape must resolve quietly, not warn on every install."""
        dpkg = tmp_path / "dpkg-query"
        _write_dpkg_query_stub(
            str(dpkg),
            "unused",
            packages = {
                "libhsa-runtime64-1": ("installed", "5.7.1-2build1"),
                "rocm-core": ("installed", "7.2.1.70201-81~24.04"),
            },
        )

        def which(cmd):
            return str(dpkg) if cmd == "dpkg-query" else None

        with patch.dict(os.environ, {"ROCM_PATH": str(tmp_path / "nonexistent")}):
            with patch("shutil.which", side_effect = which):
                assert _detect_rocm_version() == (7, 2)
        assert "ROCm version sources disagree" not in capsys.readouterr().err

    @_needs_exec_stub
    def test_removed_dpkg_hsa_runtime_is_ignored(self, tmp_path):
        """Removed-but-not-purged HSA runtime must not report a stale version."""
        dpkg = tmp_path / "dpkg-query"
        _write_dpkg_query_stub(
            str(dpkg),
            "unused",
            packages = {"libhsa-runtime64-1": ("config-files", "1:7.0.0-1")},
        )

        def which(cmd):
            return str(dpkg) if cmd == "dpkg-query" else None

        with patch.dict(os.environ, {"ROCM_PATH": str(tmp_path / "nonexistent")}):
            with patch("shutil.which", side_effect = which):
                assert _detect_rocm_version() is None

    def test_disagreeing_sources_emit_one_warning(self, tmp_path, capsys):
        """A real disagreement warns once and names the highest selected version."""
        info_dir = tmp_path / ".info"
        info_dir.mkdir()
        (info_dir / "version").write_text("5.7.0\n", encoding = "utf-8")

        def which(cmd):
            return "/usr/bin/hipconfig" if cmd == "hipconfig" else None

        hipconfig = MagicMock(returncode = 0, stdout = b"6.1.0\n")
        with patch.dict(os.environ, {"ROCM_PATH": str(tmp_path)}):
            with patch("shutil.which", side_effect = which):
                with patch("subprocess.run", return_value = hipconfig):
                    assert _detect_rocm_version() == (6, 1)
        stderr = capsys.readouterr().err
        assert stderr.count("ROCm version sources disagree") == 1
        assert "using the highest, rocm6.1" in stderr

    def test_agreeing_sources_emit_no_disagreement_warning(self, tmp_path, capsys):
        """Multiple valid sources that agree stay quiet."""
        info_dir = tmp_path / ".info"
        info_dir.mkdir()
        (info_dir / "version").write_text("6.1.0\n", encoding = "utf-8")

        def which(cmd):
            return "/usr/bin/hipconfig" if cmd == "hipconfig" else None

        hipconfig = MagicMock(returncode = 0, stdout = b"6.1.0\n")
        with patch.dict(os.environ, {"ROCM_PATH": str(tmp_path)}):
            with patch("shutil.which", side_effect = which):
                with patch("subprocess.run", return_value = hipconfig):
                    assert _detect_rocm_version() == (6, 1)
        assert "ROCm version sources disagree" not in capsys.readouterr().err

    def test_detection_is_memoized_so_the_warning_prints_once(self, tmp_path, capsys):
        """_ensure_rocm_torch() runs twice on Linux, so the sources must be probed once."""
        info_dir = tmp_path / ".info"
        info_dir.mkdir()
        (info_dir / "version").write_text("5.7.0\n", encoding = "utf-8")
        calls = []

        def which(cmd):
            return "/usr/bin/hipconfig" if cmd == "hipconfig" else None

        def run(cmd, *a, **kw):
            calls.append(cmd)
            return MagicMock(returncode = 0, stdout = b"6.1.0\n")

        with patch.dict(os.environ, {"ROCM_PATH": str(tmp_path)}):
            with patch("shutil.which", side_effect = which):
                with patch("subprocess.run", side_effect = run):
                    assert _detect_rocm_version() == (6, 1)
                    probes_after_first = len(calls)
                    assert _detect_rocm_version() == (6, 1)
                    assert (
                        len(calls) == probes_after_first
                    ), f"second call re-probed: {calls[probes_after_first:]}"
        assert capsys.readouterr().err.count("ROCm version sources disagree") == 1

    def test_memo_reset_reprobes(self, tmp_path):
        """The invalidator must actually clear the memo, or the autouse fixture is a no-op."""
        info_dir = tmp_path / ".info"
        info_dir.mkdir()
        (info_dir / "version").write_text("6.1.0\n", encoding = "utf-8")
        with patch.dict(os.environ, {"ROCM_PATH": str(tmp_path)}):
            with patch("shutil.which", return_value = None):
                assert _detect_rocm_version() == (6, 1)
                (info_dir / "version").write_text("7.0.0\n", encoding = "utf-8")
                assert _detect_rocm_version() == (6, 1)
                stack_mod._invalidate_rocm_version_probe()
                assert _detect_rocm_version() == (7, 0)

    def test_hipconfig_multiline_output(self, tmp_path):
        """hipconfig with multi-line output -- should use first line."""
        with patch.dict(os.environ, {"ROCM_PATH": str(tmp_path / "nonexistent")}):
            mock_result = MagicMock()
            mock_result.returncode = 0
            mock_result.stdout = b"6.3.21234.2\nSome extra info\n"
            with patch("shutil.which", return_value = "/usr/bin/hipconfig"):
                with patch("subprocess.run", return_value = mock_result):
                    result = _detect_rocm_version()
                    assert result == (6, 3)

    def test_hipconfig_timeout(self, tmp_path):
        """hipconfig that times out should return None."""
        with patch.dict(os.environ, {"ROCM_PATH": str(tmp_path / "nonexistent")}):
            with patch("shutil.which", return_value = "/usr/bin/hipconfig"):
                with patch(
                    "subprocess.run",
                    side_effect = subprocess.TimeoutExpired("hipconfig", 5),
                ):
                    result = _detect_rocm_version()
                    assert result is None


def _named_arch_only():
    """Drops product-name inference, so tests see only UNSLOTH_ROCM_GFX_ARCH, never the runner's board."""
    return (os.environ.get("UNSLOTH_ROCM_GFX_ARCH") or "").strip().lower() or None


def run_ensure_rocm_torch(
    probe: str = "",
    *,
    env = None,
    isdir = True,
    timeout = False,
    attrs = None,
    return_result = False,
    **stubs,
):
    """Runs _ensure_rocm_torch() on a mocked host; an empty probe means CPU torch, with no marker
    printed."""
    pip = MagicMock()
    pip_try = MagicMock(return_value = True)
    result = MagicMock(returncode = 0, stdout = (_MARK + probe + "\n") if probe else "\n")
    with contextlib.ExitStack() as stack:
        # Pin Linux x86_64: on macOS or aarch64 the repair is a no-op.
        stack.enter_context(patch("platform.machine", return_value = "x86_64"))
        for name, value in {"IS_WINDOWS": False, "IS_MACOS": False, **(attrs or {})}.items():
            stack.enter_context(patch.object(stack_mod, name, value))
        stack.enter_context(patch.object(stack_mod, "pip_install", pip))
        stack.enter_context(patch.object(stack_mod, "pip_install_try", pip_try))
        for name, value in {"_has_usable_nvidia_gpu": False, **stubs}.items():
            stack.enter_context(patch.object(stack_mod, name, return_value = value))
        if env is not None:
            stack.enter_context(patch.dict(os.environ, env, clear = False))
        stack.enter_context(patch("os.path.isdir", return_value = isdir))
        if timeout:
            stack.enter_context(
                patch("subprocess.run", side_effect = subprocess.TimeoutExpired("python", 30))
            )
        else:
            stack.enter_context(patch("subprocess.run", return_value = result))
        repair_ok = _ensure_rocm_torch()
    return (pip, pip_try, repair_ok) if return_result else (pip, pip_try)


class TestEnsureRocmTorch:
    """Verify ROCm torch reinstall logic."""

    @pytest.fixture(autouse = True)
    def _isolate_host(self, monkeypatch):
        """Hides KFD topology, ambient GPU masks and cpuinfo inference so no case inherits the
        host's GPU."""
        for _mask in ("HIP_VISIBLE_DEVICES", "ROCR_VISIBLE_DEVICES", "CUDA_VISIBLE_DEVICES"):
            monkeypatch.delenv(_mask, raising = False)
        with (
            patch.object(stack_mod, "_kfd_gfx_targets", return_value = []),
            patch.object(stack_mod, "_infer_linux_amd_gfx_arch", _named_arch_only),
        ):
            yield

    # On a real Strix host live /proc/cpuinfo would take the inferred-install path.
    @patch.object(stack_mod, "pip_install")
    @patch.object(stack_mod, "_has_usable_nvidia_gpu", return_value = False)
    @patch.object(stack_mod, "_infer_linux_amd_gfx_arch", return_value = None)
    def test_no_rocm_skips(self, mock_infer, mock_nvidia, mock_pip):
        """No ROCm toolchain should skip entirely."""
        # Pin to None so a real AMD host's WMI fallback cannot defeat the premise.
        with patch.object(stack_mod, "_detect_windows_gfx_arch", return_value = None):
            with patch("os.path.isdir", return_value = False):
                with patch("shutil.which", return_value = None):
                    _ensure_rocm_torch()
        mock_pip.assert_not_called()

    def test_inferred_gfx_without_rocm_runtime_installs_amd_index(self):
        """Strix Halo without /dev/kfd must still get AMD gfx1151 wheels (unslothai#7301)."""
        mock_pip, _ = run_ensure_rocm_torch(
            "2.10.0+cpu||",
            _has_rocm_gpu = False,
            _infer_linux_amd_gfx_arch = "gfx1151",
            _detect_rocm_version = None,
        )
        torch_call = str(mock_pip.call_args_list[0])
        assert "gfx1151" in torch_call
        assert "torch>=2.11.0,<2.12.0" in torch_call

    def test_inferred_gfx_not_overwritten_when_rocm_userland_readable(self):
        """Readable ROCm userland without /dev/kfd must not overwrite inferred per-arch gfx wheels."""
        mock_pip, _ = run_ensure_rocm_torch(
            "2.10.0+cpu||",
            _has_rocm_gpu = False,
            _infer_linux_amd_gfx_arch = "gfx1151",
            _detect_amd_gfx_codes = [],
            _detect_rocm_version = (7, 1),
        )
        assert mock_pip.call_count == 1, mock_pip.call_args_list
        torch_call = str(mock_pip.call_args_list[0])
        assert "gfx1151" in torch_call
        assert "rocm7.1" not in torch_call
        assert "download.pytorch.org" not in torch_call

    def test_inference_yields_to_runtime_visible_gpu(self):
        """When the runtime can enumerate a GPU, cpuinfo inference installs nothing; the visible
        arch decides."""
        mock_pip, mock_pip_try = run_ensure_rocm_torch(
            "2.10.0+cpu||",
            _has_rocm_gpu = True,
            _infer_linux_amd_gfx_arch = "gfx1151",
            _detect_amd_gfx_codes = ["gfx1100"],
            _detect_rocm_version = (7, 1),
        )
        all_calls = str(mock_pip.call_args_list) + str(mock_pip_try.call_args_list)
        assert "gfx1151" not in all_calls, all_calls
        assert "rocm7.1" in all_calls, all_calls

    def test_gfx_override_installs_despite_visible_rocm(self):
        """An explicit UNSLOTH_ROCM_GFX_ARCH skips the no-visible-ROCm gate, so the named arch still
        installs."""
        mock_pip, _ = run_ensure_rocm_torch(
            "2.10.0+cpu||",
            env = {"UNSLOTH_ROCM_GFX_ARCH": "gfx1151"},
            _has_rocm_gpu = True,
            _detect_amd_gfx_codes = [],
            _detect_rocm_version = None,
        )
        assert mock_pip.call_count == 1, mock_pip.call_args_list
        torch_call = str(mock_pip.call_args_list[0])
        assert "gfx1151" in torch_call
        assert "download.pytorch.org" not in torch_call

    def test_cuda_torch_on_amd_host_reinstalls(self):
        """A CUDA-only torch is unusable on an AMD-only host so it must reinstall to ROCm;
        has_hip_torch keys off the empty HIP marker, not the CUDA version string."""
        mock_pip, _ = run_ensure_rocm_torch(
            "2.10.0+cu126||", _has_rocm_gpu = True, _detect_rocm_version = (7, 1)
        )
        assert mock_pip.call_count == 1
        assert "rocm7.1" in str(mock_pip.call_args_list[0])

    @staticmethod
    def _windows_repair(
        installed_family,
        gfx = "gfx1200",
        dists = (),
    ):
        """Run the Windows ROCm repair with an already-ROCm torch on disk. Returns the
        pip_install_try mock so callers can assert on the reinstall."""
        probe = MagicMock(returncode = 0, stdout = _MARK + "2.10.0+rocm7.1|7.1|\n")
        pip_try = MagicMock(return_value = True)
        from importlib.metadata import PackageNotFoundError
        from types import SimpleNamespace

        packages = {d.metadata["Name"].lower().replace("_", "-"): d for d in dists}
        for package in ("torch", "torchvision"):
            packages[package] = SimpleNamespace(
                metadata = {"Name": package},
                requires = [f'amd-{package}-device-{gfx}==2.11.0; extra == "device-{gfx}"'],
            )

        def distribution(name):
            if name not in packages:
                raise PackageNotFoundError(name)
            return packages[name]

        with patch.dict(os.environ, {}, clear = False):
            os.environ.pop("UNSLOTH_ROCM_TORCH_INSTALLED", None)
            with (
                patch.object(stack_mod, "IS_WINDOWS", True),
                patch.object(stack_mod, "IS_MACOS", False),
                patch.object(stack_mod, "_TORCH_BACKEND", ""),
                patch.object(stack_mod, "_explicit_rocm_torch_index_url", return_value = None),
                patch.object(
                    stack_mod, "_explicit_unknown_family_torch_index_url", return_value = None
                ),
                patch.object(stack_mod, "_has_usable_nvidia_gpu", return_value = False),
                patch.object(stack_mod, "_detect_windows_gfx_arch", return_value = gfx),
                patch.object(
                    stack_mod, "_installed_rocm_wheel_family", return_value = installed_family
                ),
                patch.object(stack_mod, "_install_bnb_windows_rocm", return_value = True),
                patch.object(stack_mod, "pip_install_try", pip_try),
                patch("subprocess.run", return_value = probe),
                patch("importlib.metadata.distributions", return_value = list(dists)),
                patch("importlib.metadata.distribution", side_effect = distribution),
            ):
                _ensure_rocm_torch()
        return pip_try

    def test_wheel_family_change_forces_a_reinstall(self):
        # torch.version.hip only says 'ROCm', so after the #7776 repick the wrong family stayed forever.
        pip_try = self._windows_repair("gfx103x-all")
        assert pip_try.call_count == 1
        assert "torch[device-gfx1200]" in str(pip_try.call_args)
        assert "whl-multi-arch" in str(pip_try.call_args)

    def test_matching_wheel_family_is_left_alone(self):
        # Negative control: the right family must not be re-downloaded on every update.
        assert self._windows_repair("gfx103x-all", gfx = "gfx1033").call_count == 0

    def test_unknown_wheel_family_is_left_alone(self):
        # Older wheels predate the split runtime; guessing forces a multi-GB reinstall per update.
        assert self._windows_repair(None, gfx = "gfx1033").call_count == 0

    def test_swapped_card_on_a_multiarch_install_gets_its_device_pack(self):
        dists = self._dists("amd-torch-device-gfx1151", "amd_torchvision_device_gfx1151")
        pip_try = self._windows_repair(None, gfx = "gfx1200", dists = dists)
        assert pip_try.call_count == 1
        assert "torch[device-gfx1200]" in str(pip_try.call_args)
        assert "torchvision[device-gfx1200]" in str(pip_try.call_args)

    def test_multiarch_install_with_this_cards_packs_is_left_alone(self):
        dists = self._dists("amd-torch-device-gfx1200", "amd-torchvision-device-gfx1200")
        assert self._windows_repair(None, gfx = "gfx1200", dists = dists).call_count == 0

    def test_multiarch_install_without_the_torchvision_pack_is_repaired(self):
        dists = self._dists("amd-torch-device-gfx1200")
        assert self._windows_repair(None, gfx = "gfx1200", dists = dists).call_count == 1

    @staticmethod
    def _dists(*names):
        out = []
        for n in names:
            d = MagicMock()
            d.metadata = {"Name": n}
            d.version = "2.11.0"
            d.requires = []
            out.append(d)
        return out

    def test_installed_wheel_family_reads_the_rocm_metapackage(self):
        # AMD's torch requires rocm[libraries], naming the arch runtime: `rocm` meta is authoritative.
        reqs = [
            "rocm-sdk-core==7.13.0",
            'rocm-sdk-libraries-gfx120X-all==7.13.0; extra == "libraries"',
        ]
        with patch("importlib.metadata.requires", return_value = reqs):
            assert stack_mod._installed_rocm_wheel_family() == "gfx120x-all"

    def test_orphaned_runtime_does_not_masquerade_as_the_active_family(self):
        # pip never uninstalls the old rocm-sdk-libraries-<family>, so first-found reports the orphan.
        reqs = ['rocm-sdk-libraries-gfx120X-all==7.13.0; extra == "libraries"']
        dists = self._dists("rocm_sdk_libraries_gfx103X-all", "rocm_sdk_libraries_gfx120X-all")
        with patch("importlib.metadata.requires", return_value = reqs):
            with patch("importlib.metadata.distributions", return_value = dists):
                assert stack_mod._installed_rocm_wheel_family() == "gfx120x-all"

    def test_two_runtimes_without_a_metapackage_are_unknowable(self):
        # No `rocm` to arbitrate between two runtimes on disk, so leave the install alone.
        dists = self._dists("rocm_sdk_libraries_gfx103X-all", "rocm_sdk_libraries_gfx120X-all")
        with patch("importlib.metadata.requires", return_value = None):
            with patch("importlib.metadata.distributions", return_value = dists):
                assert stack_mod._installed_rocm_wheel_family() is None

    def test_single_runtime_without_a_metapackage_still_reads(self):
        dists = self._dists("rocm_sdk_libraries_gfx1151")
        with patch("importlib.metadata.requires", return_value = None):
            with patch("importlib.metadata.distributions", return_value = dists):
                assert stack_mod._installed_rocm_wheel_family() == "gfx1151"
        with patch("importlib.metadata.requires", return_value = None):
            with patch("importlib.metadata.distributions", return_value = []):
                assert stack_mod._installed_rocm_wheel_family() is None

    def test_switched_host_does_not_reinstall_on_every_update(self):
        # A venv migrated to multi-arch packs keeps orphaned family runtimes; do nothing.
        dists = self._dists(
            "rocm_sdk_libraries_gfx103X-all",
            "rocm_sdk_libraries_gfx120X-all",
            "amd-torch-device-gfx1200",
            "amd-torchvision-device-gfx1200",
        )
        assert self._windows_repair("gfx103x-all", dists = dists).call_count == 0

    def test_a_per_family_install_on_a_multiarch_arch_migrates_once(self):
        pip_try = self._windows_repair("gfx120x-all")
        assert pip_try.call_count == 1
        assert "torchvision[device-gfx1200]" in str(pip_try.call_args)

    def test_torch_already_has_hip_skips(self):
        """If torch already has HIP, should skip ROCm reinstall."""
        mock_pip, _ = run_ensure_rocm_torch(
            "2.10.0+rocm7.1|7.1.12345|",
            _has_rocm_gpu = True,
            _detect_rocm_version = (7, 1),
        )
        mock_pip.assert_not_called()

    def test_cpu_torch_probe_line_not_read_as_hip(self):
        """A CPU build's probe line ("|2.10.0+cpu") must not read as HIP: the version
        after the "|" separator is data, not a HIP marker, so has_hip_torch stays False
        and the reinstall fires."""
        mock_pip, _ = run_ensure_rocm_torch(
            "2.10.0+cpu||", _has_rocm_gpu = True, _detect_rocm_version = (7, 1)
        )
        assert mock_pip.call_count == 1
        assert "rocm7.1" in str(mock_pip.call_args_list[0])

    def test_cpu_torch_gets_rocm_reinstall(self):
        """CPU-only torch on ROCm host should trigger reinstall."""
        mock_pip, mock_pip_try = run_ensure_rocm_torch(
            _has_rocm_gpu = True, _detect_rocm_version = (7, 1)
        )
        assert mock_pip.call_count == 1
        assert "rocm7.1" in str(mock_pip.call_args_list[0])
        assert mock_pip_try.call_count >= 1
        assert "bitsandbytes" in str(mock_pip_try.call_args_list[0])
        assert mock_pip_try.call_args.kwargs["force_pip"] is True

    def test_rocm_63_selects_correct_tag(self):
        """Generic ROCm 6.3 uses the bnb-compatible floor, which needs the arch declared as non-gfx906."""
        mock_pip, _ = run_ensure_rocm_torch(
            _has_rocm_gpu = True,
            _detect_rocm_version = (6, 3),
            _detect_amd_gfx_codes = ["gfx1100"],
            _infer_linux_amd_gfx_arch = None,
        )
        torch_call = mock_pip.call_args_list[0]
        assert "rocm6.4" in str(torch_call)

    def test_query_pytorch_mirror_skips_the_generic_rocm_pin(self, monkeypatch):
        """A synthesized rocm leaf after ?token= is neither usable nor a fallback license."""
        monkeypatch.delenv("UNSLOTH_TORCH_INDEX_URL", raising = False)
        monkeypatch.delenv("UNSLOTH_TORCH_INDEX_FAMILY", raising = False)
        attrs = {"_TORCH_BACKEND": "", "_PYTORCH_WHL_BASE": "https://mirror.example/whl?token=abc"}
        mock_pip, _, repair_ok = run_ensure_rocm_torch(
            _has_rocm_gpu = True,
            _detect_rocm_version = (6, 3),
            attrs = attrs,
            return_result = True,
        )
        mock_pip.assert_not_called()
        assert repair_ok is False

        monkeypatch.setenv("UNSLOTH_TORCH_INDEX_FAMILY", "rocm6.3")
        _, _, repair_ok = run_ensure_rocm_torch(
            _detect_rocm_version = (6, 3), attrs = attrs, return_result = True
        )
        assert repair_ok is False

    def test_old_rocm_skips(self):
        """ROCm version too old (below 6.0) should skip."""
        mock_pip, _ = run_ensure_rocm_torch(_has_rocm_gpu = True, _detect_rocm_version = (5, 0))
        mock_pip.assert_not_called()

    @patch.object(stack_mod, "IS_MACOS", False)
    @patch("platform.machine", return_value = "x86_64")
    @patch.object(stack_mod, "IS_WINDOWS", False)
    @patch.object(stack_mod, "pip_install")
    @patch.object(stack_mod, "_has_usable_nvidia_gpu", return_value = False)
    @patch.object(stack_mod, "_has_rocm_gpu", return_value = True)
    @patch.object(stack_mod, "_infer_linux_amd_gfx_arch", return_value = None)
    @patch.object(stack_mod, "_detect_rocm_version", return_value = None)
    @patch.object(stack_mod, "_installed_rocm_wheel_family", return_value = None)
    @patch.object(stack_mod, "_torch_requires_rocm_sdk", return_value = False)
    def test_version_unreadable_prints_warning(
        self,
        _mock_owns_sdk,
        _mock_family,
        mock_ver,
        mock_infer,
        mock_gpu,
        mock_nvidia,
        mock_pip,
        mock_machine,
        capsys,
    ):
        """ROCm detected but version unreadable should print warning and skip."""
        with patch("os.path.isdir", return_value = True):
            _ensure_rocm_torch()
        mock_pip.assert_not_called()
        captured = capsys.readouterr()
        assert "unreadable" in captured.out

    def test_rocm_72_selects_72_tag(self):
        """ROCm 7.2 should select rocm7.2 tag (now in mapping with torch 2.11.0)."""
        mock_pip, _ = run_ensure_rocm_torch(_has_rocm_gpu = True, _detect_rocm_version = (7, 2))
        torch_call = mock_pip.call_args_list[0]
        assert "rocm7.2" in str(torch_call)

    def test_rocm_714_strix_routes_to_amd_arch_index(self):
        """ROCm 7.14 caps to rocm7.2 on pytorch.org; Strix must use AMD gfx index."""
        mock_pip, _ = run_ensure_rocm_torch(
            "2.11.0+rocm7.2|7.14.60850|",
            _has_rocm_gpu = True,
            _detect_rocm_version = (7, 14),
            _detect_amd_gfx_codes = ["gfx1150"],
        )
        torch_call = str(mock_pip.call_args_list[0])
        assert "gfx1150" in torch_call
        assert "torch>=2.11.0,<2.12.0" in torch_call

    @pytest.mark.parametrize("gfx", ("gfx1200", "gfx1201"))
    def test_rocm_714_rdna4_routes_to_amd_arch_index(self, gfx):
        mock_pip, _ = run_ensure_rocm_torch(
            "2.11.0+rocm7.2|7.14.60850|",
            _has_rocm_gpu = True,
            _detect_rocm_version = (7, 14),
            _detect_amd_gfx_codes = [gfx],
        )
        torch_call = str(mock_pip.call_args_list[0])
        assert "gfx120X-all" in torch_call
        assert "torch>=2.11.0,<2.12.0" in torch_call

    def test_rocm_714_rdna3_stays_on_generic_cap(self):
        mock_pip, _ = run_ensure_rocm_torch(
            "",
            _has_rocm_gpu = True,
            _detect_rocm_version = (7, 14),
            _detect_amd_gfx_codes = ["gfx1100"],
        )
        torch_call = str(mock_pip.call_args_list[0])
        assert "rocm7.2" in torch_call
        assert "repo.amd.com" not in torch_call

    @pytest.mark.parametrize(
        ("gfx", "pending"), (("gfx1201", True), ("gfx1200", True), ("gfx1100", False))
    )
    def test_rdna4_generic_wheel_is_a_pending_reroute(self, gfx, pending, monkeypatch):
        monkeypatch.setattr(stack_mod, "_installed_rocm_wheel_family", lambda: None)
        got = stack_mod._rocm_compat_reroute_pending(gfx, (7, 14), "2.11.0+rocm7.2")
        assert bool(got) is pending

    @patch.object(stack_mod, "IS_MACOS", False)
    @patch("platform.machine", return_value = "x86_64")
    @patch.object(stack_mod, "IS_WINDOWS", False)
    @patch.object(stack_mod, "pip_install_try", return_value = True)
    @patch.object(stack_mod, "pip_install")
    @patch.object(stack_mod, "_has_usable_nvidia_gpu", return_value = False)
    @patch.object(stack_mod, "_has_rocm_gpu", return_value = True)
    @patch.object(stack_mod, "_detect_rocm_version", return_value = (7, 14))
    @patch.object(
        stack_mod, "_detect_amd_gfx_codes", return_value = ["gfx1100", "gfx1100", "gfx1151"]
    )
    def test_mask_indexes_devices_not_deduplicated_arches(
        self, mock_gfx, mock_ver, mock_gpu, mock_nvidia, mock_pip, mock_pip_try, mock_machine
    ):
        # The mask names an ordinal; a deduplicated arch list reads index 1 as the Strix.
        mock_probe = MagicMock()
        mock_probe.returncode = 0
        mock_probe.stdout = _MARK + "2.11.0+rocm7.2|7.14.60850|\n"
        buf = io.StringIO()
        with patch.dict(os.environ, {"CUDA_VISIBLE_DEVICES": "1"}, clear = False):
            for _v in ("HIP_VISIBLE_DEVICES", "ROCR_VISIBLE_DEVICES"):
                os.environ.pop(_v, None)
            with contextlib.redirect_stdout(buf):
                with patch("os.path.isdir", return_value = True):
                    with patch("subprocess.run", return_value = mock_probe):
                        _ensure_rocm_torch()
        calls = str(mock_pip.call_args_list) + str(mock_pip_try.call_args_list)
        assert "gfx1151" not in calls
        assert "selects another runtime target (gfx1100)" in buf.getvalue()

    # _already_on_amd_arch_leaf reads this interpreter's metadata, so mock it like the hardware.
    @patch.object(stack_mod, "_installed_rocm_wheel_family", lambda: None)
    @patch.object(stack_mod, "_torch_requires_rocm_sdk", lambda: False)
    @patch.object(stack_mod, "IS_MACOS", False)
    @patch("platform.machine", return_value = "x86_64")
    @patch.object(stack_mod, "IS_WINDOWS", False)
    @patch.object(stack_mod, "pip_install_try", return_value = True)
    @patch.object(stack_mod, "pip_install")
    @patch.object(stack_mod, "_has_usable_nvidia_gpu", return_value = False)
    @patch.object(stack_mod, "_has_rocm_gpu", return_value = True)
    @patch.object(stack_mod, "_detect_rocm_version", return_value = (7, 14))
    @patch.object(
        stack_mod, "_detect_amd_gfx_codes", return_value = ["gfx1100", "gfx1100", "gfx1151"]
    )
    def test_mask_naming_the_strix_device_still_reroutes(
        self, mock_gfx, mock_ver, mock_gpu, mock_nvidia, mock_pip, mock_pip_try, mock_machine
    ):
        # Negative control: device 2 really is the Strix, so the reroute must still fire.
        mock_probe = MagicMock()
        mock_probe.returncode = 0
        mock_probe.stdout = _MARK + "2.11.0+rocm7.2|7.14.60850|\n"
        with patch.dict(os.environ, {"CUDA_VISIBLE_DEVICES": "2"}, clear = False):
            for _v in ("HIP_VISIBLE_DEVICES", "ROCR_VISIBLE_DEVICES"):
                os.environ.pop(_v, None)
            with patch("os.path.isdir", return_value = True):
                with patch("subprocess.run", return_value = mock_probe):
                    _ensure_rocm_torch()
        calls = str(mock_pip.call_args_list) + str(mock_pip_try.call_args_list)
        assert "gfx1151" in calls

    # rocminfo repeats the gfx token per agent, so a flat findall miscounts GPUs.
    _ROCMINFO = """
*******
Agent 1
*******
  Name:                    AMD Ryzen 9 7950X
  Marketing Name:          AMD Ryzen 9 7950X
*******
Agent 2
*******
  Name:                    gfx1100
  Marketing Name:          AMD Radeon RX 7900 XTX
  ISA Info:
    ISA 1
      Name:                amdgcn-amd-amdhsa--gfx1100
*******
Agent 3
*******
  Name:                    gfx1100
  Marketing Name:          AMD Radeon RX 7900 XTX
  ISA Info:
    ISA 1
      Name:                amdgcn-amd-amdhsa--gfx1100
*******
Agent 4
*******
  Name:                    gfx1151
  Marketing Name:          AMD Radeon 8060S
  ISA Info:
    ISA 1
      Name:                amdgcn-amd-amdhsa--gfx1151
"""

    @staticmethod
    def _probe_gfx(out, dedup):
        which = lambda n: "/usr/bin/rocminfo" if n == "rocminfo" else None  # noqa: E731
        with patch("shutil.which", side_effect = which):
            with patch("subprocess.run", return_value = MagicMock(returncode = 0, stdout = out)):
                return stack_mod._detect_amd_gfx_codes(dedup = dedup)

    def test_gfx_probe_returns_one_entry_per_agent_not_per_token(self):
        assert self._probe_gfx(self._ROCMINFO, False) == ["gfx1100", "gfx1100", "gfx1151"]
        assert self._probe_gfx(self._ROCMINFO, True) == ["gfx1100", "gfx1151"]

    def test_gfx_probe_falls_back_for_flat_output(self):
        # amd-smi and test stubs emit no agent headers; those must still work.
        flat = "Name: gfx1100\nName: gfx1100\nName: gfx1151\n"
        assert self._probe_gfx(flat, False) == ["gfx1100", "gfx1151"]
        assert self._probe_gfx(flat, True) == ["gfx1100", "gfx1151"]

    _AMD_SMI_ASIC = """GPU: 0
    ASIC:
        MARKET_NAME: Radeon RX 7900 XTX
        TARGET_GRAPHICS_VERSION: gfx1100
GPU: 1
    ASIC:
        MARKET_NAME: Radeon RX 7900 XTX
        TARGET_GRAPHICS_VERSION: gfx1100
GPU: 2
    ASIC:
        MARKET_NAME: Radeon RX 9060
        TARGET_GRAPHICS_VERSION: gfx1200
"""

    @staticmethod
    def _probe_gfx_amd_smi(out, dedup):
        which = lambda n: "/usr/bin/amd-smi" if n == "amd-smi" else None  # noqa: E731
        with patch("shutil.which", side_effect = which):
            with patch.object(stack_mod, "_amd_smi_allowed", return_value = True):
                with patch("subprocess.run", return_value = MagicMock(returncode = 0, stdout = out)):
                    return stack_mod._detect_amd_gfx_codes(dedup = dedup)

    def test_gfx_probe_splits_amd_smi_gpu_headers(self):
        """amd-smi heads each device with "GPU: N", so duplicate arches stay separate."""
        assert self._probe_gfx_amd_smi(self._AMD_SMI_ASIC, False) == [
            "gfx1100",
            "gfx1100",
            "gfx1200",
        ]
        assert self._probe_gfx_amd_smi(self._AMD_SMI_ASIC, True) == ["gfx1100", "gfx1200"]

    def test_gfx_probe_splits_amd_smi_bracket_headers(self):
        """The "GPU[N] : gfx" one-line form keeps its token on the header line."""
        bracket = "GPU[0]  : gfx1100\nGPU[1]  : gfx1100\nGPU[2]  : gfx1200\n"
        assert self._probe_gfx_amd_smi(bracket, False) == ["gfx1100", "gfx1100", "gfx1200"]

    def test_gfx_probe_records_which_tool_answered(self):
        # Only rocminfo is mask-filtered, and only by ROCR, so the reroute needs this.
        self._probe_gfx(self._ROCMINFO, False)
        assert stack_mod._LAST_AMD_GFX_PROBE == "rocminfo"

    def test_first_set_visible_mask_is_first_set_wins(self):
        with patch.dict(os.environ, {}, clear = False):
            for _v in ("HIP_VISIBLE_DEVICES", "ROCR_VISIBLE_DEVICES", "CUDA_VISIBLE_DEVICES"):
                os.environ.pop(_v, None)
            assert stack_mod._first_set_visible_mask() is None
            os.environ["CUDA_VISIBLE_DEVICES"] = "1"
            assert stack_mod._first_set_visible_mask() == "CUDA_VISIBLE_DEVICES"
            os.environ["ROCR_VISIBLE_DEVICES"] = ""
            assert stack_mod._first_set_visible_mask() == "ROCR_VISIBLE_DEVICES"
            os.environ["HIP_VISIBLE_DEVICES"] = "0"
            assert stack_mod._first_set_visible_mask() == "HIP_VISIBLE_DEVICES"

    @patch.object(stack_mod, "IS_MACOS", False)
    @patch("platform.machine", return_value = "x86_64")
    @patch.object(stack_mod, "IS_WINDOWS", False)
    @patch.object(stack_mod, "pip_install_try", return_value = True)
    @patch.object(stack_mod, "pip_install")
    @patch.object(stack_mod, "_has_usable_nvidia_gpu", return_value = False)
    @patch.object(stack_mod, "_has_rocm_gpu", return_value = True)
    @patch.object(stack_mod, "_detect_rocm_version", return_value = (6, 4))
    def test_explicit_gfx_index_honored_and_skips_strix_reroute(
        self, mock_ver, mock_gpu, mock_nvidia, mock_pip, mock_pip_try, mock_machine
    ):
        """An explicit gfx wheel-index pin is authoritative: installed verbatim with torch 2.11, not
        re-picked from host ROCm 6.4 or rerouted for Strix; the gfx probe may still run for the
        bnb skip."""
        mock_probe = MagicMock()
        mock_probe.returncode = 0
        mock_probe.stdout = "\n"  # cpu torch -> reinstall
        env = {"UNSLOTH_TORCH_INDEX_URL": "https://repo.amd.com/rocm/whl/gfx1151"}
        with patch.dict(stack_mod.os.environ, env, clear = False):
            stack_mod.os.environ.pop("UNSLOTH_TORCH_INDEX_FAMILY", None)
            with patch("os.path.isdir", return_value = True):
                with patch("subprocess.run", return_value = mock_probe):
                    with patch.object(stack_mod, "_detect_amd_gfx_codes", return_value = ["gfx1151"]):
                        _ensure_rocm_torch()
        assert mock_pip.call_count == 1
        torch_call = str(mock_pip.call_args_list[0])
        assert "gfx1151" in torch_call
        assert "torch>=2.11.0,<2.12.0" in torch_call

    # The pin is compared against THIS interpreter's `rocm` meta-package first, so stub it.
    @patch.object(stack_mod, "_installed_rocm_wheel_family", lambda: None)
    def test_rocm_pin_family_mismatch_helper(self):
        """_rocm_pin_family_mismatch: exact rocm compare, else the 2.11 line."""
        f = stack_mod._rocm_pin_family_mismatch
        base = "https://download.pytorch.org/whl"
        amd = "https://repo.amd.com/rocm/whl"
        assert f(f"{base}/rocm7.2", "2.11.0+rocm7.2") is False
        assert f(f"{base}/rocm7.2", "2.10.0+rocm6.4") is True
        assert f(f"{base}/rocm6.4", "2.10.0+rocm6.4") is False
        # A +rocm7.2 wheel whose release drifted off 2.11 shares the tag but violates the spec.
        assert f(f"{base}/rocm7.2", "2.12.0+rocm7.2") is True
        assert f(f"{base}/rocm7.2", "2.13.0+rocm7.2") is True
        assert f(f"{base}/rocm7.2", "2.11.5+rocm7.2") is False  # patch on 2.11 is in-spec
        # An unknown newer rocm is not floored to 2.11.
        assert f(f"{base}/rocm8.0", "2.12.0+rocm8.0") is False
        assert f(f"{amd}/gfx1151", "2.10.0+rocm6.4") is True
        assert f(f"{amd}/gfx1151", "2.11.0+rocm7.13.0") is False
        # A CPU/CUDA build never satisfies a ROCm pin.
        assert f(f"{base}/rocm7.2", "2.10.0") is True
        assert f(f"{base}/rocm7.2", "2.11.0") is True
        assert f(f"{base}/rocm6.4", "2.10.0") is True
        # A 2.11 gfx pin wants AMD's three-part per-arch wheel, not the generic two-part one.
        assert f(f"{amd}/gfx1151", "2.11.0+rocm7.2") is True
        assert f(f"{amd}/gfx120X-all", "2.11.0+rocm7.2") is True
        # No reinstall loop once the correct gfx wheel is present.
        assert f(f"{amd}/gfx120X-all", "2.11.0+rocm7.13.0") is False
        assert f(f"{amd}/gfx1150", "2.11.0+rocm7.13.0") is False
        # Non-2.11 gfx pins track the default <2.11 spec.
        assert f(f"{amd}/gfx90a", "2.10.0+rocm6.3") is False
        assert f(f"{amd}/gfx908", "2.10.0+rocm7.0") is False
        assert f(f"{amd}/gfx90a", "2.11.0+rocm7.2") is True
        # gfx103X-all / gfx110X-all are floored: a 2.10 per-arch build IS a mismatch.
        assert f(f"{amd}/gfx103X-all", "2.10.0+rocm7.13.0") is True
        assert f(f"{amd}/gfx110X-all", "2.10.0+rocm6.4") is True
        assert f(f"{amd}/gfx103X-all", "2.11.0+rocm7.13.0") is False
        assert f(f"{amd}/gfx110X-all", "2.11.0+rocm7.13.0") is False
        assert f(f"{amd}/gfx908", "2.10.0") is True
        assert f(f"{amd}/gfx90a", "2.10.0") is True
        # Major-only rocm pin: rocm6.x mismatches, any rocm7.x satisfies, untagged never, bare +rocm ok.
        assert f(f"{base}/rocm7", "2.10.0+rocm6.4") is True
        assert f(f"{base}/rocm7", "2.11.0+rocm7.2") is False
        assert f(f"{base}/rocm7", "2.11.0+rocm7.13.0") is False
        assert f(f"{base}/rocm7", "2.10.0") is True
        assert f(f"{base}/rocm7", "2.10.0+rocm") is False

    @patch.object(stack_mod, "IS_MACOS", False)
    @patch("platform.machine", return_value = "x86_64")
    @patch.object(stack_mod, "IS_WINDOWS", False)
    @patch.object(stack_mod, "pip_install_try", return_value = True)
    @patch.object(stack_mod, "pip_install")
    @patch.object(stack_mod, "_has_usable_nvidia_gpu", return_value = False)
    @patch.object(stack_mod, "_has_rocm_gpu", return_value = True)
    @patch.object(stack_mod, "_detect_rocm_version", return_value = (7, 2))
    def test_rocm_pin_mismatch_over_installed_rocm_reinstalls(
        self, mock_ver, mock_gpu, mock_nvidia, mock_pip, mock_pip_try, mock_machine
    ):
        """A rocm7.2 pin over an already-installed OLDER +rocm6.4 build must reinstall,
        even though has_hip_torch is True (the ROCm analogue of the CUDA cuXXX mismatch)."""
        mock_probe = MagicMock()
        mock_probe.returncode = 0
        mock_probe.stdout = _MARK + "2.10.0+rocm6.4|6.4.12345|\n"
        env = {"UNSLOTH_TORCH_INDEX_FAMILY": "rocm7.2"}
        with patch.dict(stack_mod.os.environ, env, clear = False):
            stack_mod.os.environ.pop("UNSLOTH_TORCH_INDEX_URL", None)
            with patch("os.path.isdir", return_value = True):
                with patch("subprocess.run", return_value = mock_probe):
                    _ensure_rocm_torch()
        torch_call = str(mock_pip.call_args_list[0])
        assert "rocm7.2" in torch_call
        assert "torch>=2.11.0,<2.12.0" in torch_call

    @patch.object(stack_mod, "IS_MACOS", False)
    @patch("platform.machine", return_value = "x86_64")
    @patch.object(stack_mod, "IS_WINDOWS", False)
    @patch.object(stack_mod, "pip_install_try", return_value = True)
    @patch.object(stack_mod, "pip_install")
    @patch.object(stack_mod, "_has_usable_nvidia_gpu", return_value = False)
    @patch.object(stack_mod, "_has_rocm_gpu", return_value = True)
    @patch.object(stack_mod, "_detect_rocm_version", return_value = (6, 4))
    def test_gfx_pin_over_installed_pre211_rocm_reinstalls(
        self, mock_ver, mock_gpu, mock_nvidia, mock_pip, mock_pip_try, mock_machine
    ):
        """A gfx* pin (2.11 line) over an installed pre-2.11 +rocm6.4 build reinstalls.
        The gfx probe may run for the bnb-skip flag but must not alter the pinned index."""
        mock_probe = MagicMock()
        mock_probe.returncode = 0
        mock_probe.stdout = _MARK + "2.10.0+rocm6.4|6.4.12345|\n"
        env = {"UNSLOTH_TORCH_INDEX_URL": "https://repo.amd.com/rocm/whl/gfx1151"}
        with patch.dict(stack_mod.os.environ, env, clear = False):
            stack_mod.os.environ.pop("UNSLOTH_TORCH_INDEX_FAMILY", None)
            with patch("os.path.isdir", return_value = True):
                with patch("subprocess.run", return_value = mock_probe):
                    with patch.object(stack_mod, "_detect_amd_gfx_codes", return_value = ["gfx1151"]):
                        _ensure_rocm_torch()
        torch_call = str(mock_pip.call_args_list[0])
        assert "gfx1151" in torch_call
        assert "torch>=2.11.0,<2.12.0" in torch_call

    @patch.object(stack_mod, "IS_WINDOWS", False)
    @patch.object(stack_mod, "pip_install_try", return_value = True)
    @patch.object(stack_mod, "pip_install")
    @patch.object(stack_mod, "_has_usable_nvidia_gpu", return_value = False)
    @patch.object(stack_mod, "_has_rocm_gpu", return_value = True)
    @patch.object(stack_mod, "_detect_rocm_version", return_value = (7, 2))
    def test_rocm_pin_matches_installed_no_torch_reinstall(
        self, mock_ver, mock_gpu, mock_nvidia, mock_pip, mock_pip_try
    ):
        """A rocm7.2 pin over an already-matching +rocm7.2 build must NOT reinstall torch
        (no false reinstall of a correct ROCm venv)."""
        mock_probe = MagicMock()
        mock_probe.returncode = 0
        mock_probe.stdout = _MARK + "2.11.0+rocm7.2|7.2.12345|\n"
        env = {"UNSLOTH_TORCH_INDEX_FAMILY": "rocm7.2"}
        with patch.dict(stack_mod.os.environ, env, clear = False):
            stack_mod.os.environ.pop("UNSLOTH_TORCH_INDEX_URL", None)
            with patch("os.path.isdir", return_value = True):
                with patch("subprocess.run", return_value = mock_probe):
                    _ensure_rocm_torch()
        for _call in mock_pip.call_args_list:
            _args = [str(a) for a in _call.args]
            if "--index-url" in _args:
                _url = _args[_args.index("--index-url") + 1]
                assert "rocm7.2" not in _url or "torch" not in " ".join(
                    _args
                ), "torch must not be reinstalled when the pin already matches"
        assert not any(
            any(str(a).startswith("torch") for a in _c.args) for _c in mock_pip.call_args_list
        )

    # Staleness is judged against the installed family first, so stub it.
    @patch.object(stack_mod, "_installed_rocm_wheel_family", lambda: None)
    @patch.object(stack_mod, "_torch_requires_rocm_sdk", lambda: False)
    @patch.object(stack_mod, "IS_WINDOWS", False)
    @patch.object(stack_mod, "pip_install_try", return_value = True)
    @patch.object(stack_mod, "pip_install")
    @patch.object(stack_mod, "_has_usable_nvidia_gpu", return_value = False)
    @patch.object(stack_mod, "_has_rocm_gpu", return_value = True)
    @patch.object(stack_mod, "_detect_rocm_version", return_value = (6, 4))
    def test_non211_gfx_pin_over_210_rocm_no_reinstall(
        self, mock_ver, mock_gpu, mock_nvidia, mock_pip, mock_pip_try
    ):
        """A gfx90a pin, not in the 2.11 allowlist, over a correct 2.10+rocm wheel must not be
        flagged stale; default <2.11 specs apply, so re-flagging would reinstall every update."""
        mock_probe = MagicMock()
        mock_probe.returncode = 0
        mock_probe.stdout = _MARK + "2.10.0+rocm6.4|6.4.12345|\n"
        env = {"UNSLOTH_TORCH_INDEX_URL": "https://repo.amd.com/rocm/whl/gfx90a"}
        with patch.dict(stack_mod.os.environ, env, clear = False):
            stack_mod.os.environ.pop("UNSLOTH_TORCH_INDEX_FAMILY", None)
            with patch("os.path.isdir", return_value = True):
                with patch("subprocess.run", return_value = mock_probe):
                    _ensure_rocm_torch()
        assert not any(
            any(str(a).startswith("torch") for a in _c.args) for _c in mock_pip.call_args_list
        )

    @patch.object(stack_mod, "IS_MACOS", False)
    @patch("platform.machine", return_value = "x86_64")
    @patch.object(stack_mod, "IS_WINDOWS", False)
    @patch.object(stack_mod, "pip_install_try", return_value = True)
    @patch.object(stack_mod, "pip_install")
    @patch.object(stack_mod, "_has_usable_nvidia_gpu", return_value = False)
    @patch.object(stack_mod, "_has_rocm_gpu", return_value = True)
    @patch.object(stack_mod, "_detect_rocm_version", return_value = (7, 2))
    def test_gfx_pin_over_generic_rocm211_reinstalls(
        self, mock_ver, mock_gpu, mock_nvidia, mock_pip, mock_pip_try, mock_machine
    ):
        """A gfx1151 pin over a generic two-part +rocm7.2 torch 2.11 wheel must reinstall the
        per-arch AMD wheel, since the generic build is not the one pinned."""
        mock_probe = MagicMock()
        mock_probe.returncode = 0
        mock_probe.stdout = _MARK + "2.11.0+rocm7.2|7.2.12345|\n"
        env = {"UNSLOTH_TORCH_INDEX_URL": "https://repo.amd.com/rocm/whl/gfx1151"}
        with patch.dict(stack_mod.os.environ, env, clear = False):
            stack_mod.os.environ.pop("UNSLOTH_TORCH_INDEX_FAMILY", None)
            with patch("os.path.isdir", return_value = True):
                with patch("subprocess.run", return_value = mock_probe):
                    # The gfx probe may run for the bnb-skip flag; Strix must not reroute the pin.
                    with patch.object(stack_mod, "_detect_amd_gfx_codes", return_value = ["gfx1151"]):
                        _ensure_rocm_torch()
        torch_call = str(mock_pip.call_args_list[0])
        assert "gfx1151" in torch_call
        assert "torch>=2.11.0,<2.12.0" in torch_call

    def test_radeon_url_not_classified_as_pip_rocm_family(self):
        """A repo.radeon.com find-links dir (leaf rocm-rel-7.2.1) starts with "rocm" but is
        NOT a pip --index-url ROCm family: it must route to the verbatim path, not a
        --index-url reinstall that fails against a find-links listing."""
        leaf_f = stack_mod._is_pip_rocm_family_leaf
        assert leaf_f("rocm7.2") is True
        assert leaf_f("rocm6.4") is True
        assert leaf_f("gfx120x-all") is True
        assert leaf_f("gfx1151") is True
        assert leaf_f("rocm7") is True
        # A Radeon find-links dir leaf, a custom mirror, cpu and cuda are NOT pip rocm.
        assert leaf_f("rocm-rel-7.2.1") is False
        assert leaf_f("simple") is False
        assert leaf_f("current") is False
        assert leaf_f("cpu") is False
        assert leaf_f("cu128") is False
        # A rocm<digit>-SUFFIX private mirror is a custom pin, so match the family EXACTLY.
        assert leaf_f("rocm7.2-private") is False
        assert leaf_f("rocm7-current") is False
        assert leaf_f("rocm7.2.1") is False  # two-part local suffix -> custom, not rocm7.2

        radeon = "https://repo.radeon.com/rocm/manylinux/rocm-rel-7.2.1"
        pip_rocm = "https://download.pytorch.org/whl/rocm7.2"
        amd_gfx = "https://repo.amd.com/rocm/whl/gfx120X-all"

        def _classify(url, fn):
            with patch.dict(stack_mod.os.environ, {"UNSLOTH_TORCH_INDEX_URL": url}, clear = False):
                stack_mod.os.environ.pop("UNSLOTH_TORCH_INDEX_FAMILY", None)
                return fn()

        rocm_fn = stack_mod._explicit_rocm_torch_index_url
        unk_fn = stack_mod._explicit_unknown_family_torch_index_url
        assert _classify(pip_rocm, rocm_fn) == pip_rocm
        assert _classify(amd_gfx, rocm_fn) == amd_gfx
        assert _classify(pip_rocm, unk_fn) is None
        assert _classify(amd_gfx, unk_fn) is None
        # Radeon find-links is not a pip ROCm family and is unknown, so repair helpers leave it.
        assert _classify(radeon, rocm_fn) is None
        assert _classify(radeon, unk_fn) == radeon

        suffixed = "https://co.internal/whl/rocm7.2-private"
        assert _classify(suffixed, rocm_fn) is None
        assert _classify(suffixed, unk_fn) == suffixed

    @patch.object(stack_mod, "pip_install")
    def test_ensure_cpu_torch_broken_probe_reinstalls(self, mock_pip):
        """_ensure_cpu_torch: torch present but unimportable (probe exit != 0) under an
        explicit CPU pin must reinstall from the pin, not return -- the base update does
        not repair a broken installed torch, so returning would strand it (Codex P2)."""
        mock_probe = MagicMock()
        mock_probe.returncode = 1  # torch present but cannot import
        mock_probe.stdout = ""
        env = {"UNSLOTH_TORCH_INDEX_URL": "https://mirror.local/cpu"}
        with patch.dict(stack_mod.os.environ, env, clear = False):
            stack_mod.os.environ.pop("UNSLOTH_TORCH_INDEX_FAMILY", None)
            with patch("subprocess.run", return_value = mock_probe):
                with patch.object(stack_mod, "NO_TORCH", False):
                    stack_mod._ensure_cpu_torch()
        assert mock_pip.call_count == 1
        assert "https://mirror.local/cpu" in str(mock_pip.call_args)

    def test_probe_timeout_triggers_reinstall(self):
        """Probe subprocess timeout should not crash; should proceed to reinstall."""
        mock_pip, mock_pip_try = run_ensure_rocm_torch(
            timeout = True, _has_rocm_gpu = True, _detect_rocm_version = (7, 1)
        )
        # Probe timeout: treat torch as unusable and reinstall torch + bitsandbytes.
        assert mock_pip.call_count == 1
        assert "rocm7.1" in str(mock_pip.call_args_list[0])
        assert mock_pip_try.call_count >= 1
        assert mock_pip_try.call_args.kwargs["force_pip"] is True

    @patch.object(stack_mod, "pip_install")
    @patch.object(stack_mod, "_has_usable_nvidia_gpu", return_value = False)
    @patch.object(stack_mod, "_has_rocm_gpu", return_value = False)
    @patch.object(stack_mod, "_infer_linux_amd_gfx_arch", return_value = None)
    def test_no_gpu_with_rocm_tools_skips(self, mock_infer, mock_gpu, mock_nvidia, mock_pip):
        """ROCm tools present but no actual AMD GPU should skip entirely."""
        # Pin to None so a real AMD host's WMI fallback cannot defeat the premise.
        with patch.object(stack_mod, "_detect_windows_gfx_arch", return_value = None):
            with patch("os.path.isdir", return_value = True):
                _ensure_rocm_torch()
        mock_pip.assert_not_called()

    @patch.object(stack_mod, "pip_install")
    @patch.object(stack_mod, "_has_rocm_gpu", return_value = True)
    @patch.object(stack_mod, "_has_usable_nvidia_gpu", return_value = True)
    def test_torch_backend_cuda_env_skips_entirely(self, mock_nvidia, mock_gpu, mock_pip):
        """UNSLOTH_TORCH_BACKEND=cuda must short-circuit before any GPU probe."""
        with patch.dict(os.environ, {"UNSLOTH_TORCH_BACKEND": "cuda"}):
            with patch.object(stack_mod, "_TORCH_BACKEND", "cuda"):
                _ensure_rocm_torch()
        mock_pip.assert_not_called()

    @patch.object(stack_mod, "pip_install")
    @patch.object(stack_mod, "_has_rocm_gpu", return_value = True)
    @patch.object(stack_mod, "_has_usable_nvidia_gpu", return_value = True)
    def test_torch_backend_cpu_env_skips_entirely(self, mock_nvidia, mock_gpu, mock_pip):
        """UNSLOTH_TORCH_BACKEND=cpu must short-circuit before any GPU probe."""
        with patch.dict(os.environ, {"UNSLOTH_TORCH_BACKEND": "cpu"}):
            with patch.object(stack_mod, "_TORCH_BACKEND", "cpu"):
                _ensure_rocm_torch()
        mock_pip.assert_not_called()


# Generic wheels after rocm6.3 lack gfx906 code objects; torch must come from rocm6.3.


class TestGfx906LegacyReroute:
    """gfx906 hosts on ROCm >= 6.4 must be rerouted to the rocm6.3 torch index;
    hosts already on gfx906-capable wheels are left alone."""

    @pytest.fixture(autouse = True)
    def _isolate_host(self, monkeypatch):
        """Hides KFD topology and ambient GPU masks, which would add or suppress GPUs the case never
        named."""
        for _mask in ("HIP_VISIBLE_DEVICES", "ROCR_VISIBLE_DEVICES", "CUDA_VISIBLE_DEVICES"):
            monkeypatch.delenv(_mask, raising = False)
        with (
            patch.object(stack_mod, "_kfd_gfx_targets", return_value = []),
            patch.object(stack_mod, "_infer_linux_amd_gfx_arch", _named_arch_only),
            # On the gfx1151 runner an unpatched read finds a real per-arch wheel in the venv.
            patch.object(stack_mod, "_installed_rocm_wheel_family", return_value = None),
            patch.object(stack_mod, "_torch_requires_rocm_sdk", return_value = False),
        ):
            yield

    @staticmethod
    def _gfx906_reroute_block(source: str) -> str:
        """The MI50/gfx906 reroute block, bounded on the ';;' that closes its
        rocm[0-9]* case arm -- robust to comment growth (no magic char offset)."""
        start = source.find("MI50 / Radeon VII (gfx906")
        assert start >= 0, "gfx906 reroute block not found in install.sh"
        end = source.find("\n        ;;", start)
        assert end >= 0, "end of gfx906 case arm not found"
        return source[start:end]

    @staticmethod
    def _reroute(gfx_codes, *, probe = ""):
        """A ROCm 7.2 AMD host whose runtime reports `gfx_codes`, with CPU torch by default."""
        return run_ensure_rocm_torch(
            probe,
            _has_rocm_gpu = True,
            _detect_rocm_version = (7, 2),
            _detect_amd_gfx_codes = gfx_codes,
        )

    def test_gfx906_needs_legacy_index_floor(self):
        f = stack_mod._gfx906_needs_legacy_index
        # rocm6.0-6.3 tags still ship gfx906 kernels: no reroute.
        assert f((6, 3)) is False
        assert f((6, 0)) is False
        assert f((5, 0)) is False
        # Anything that picks a tag newer than rocm6.3 must reroute.
        assert f((6, 4)) is True
        assert f((7, 2)) is True
        assert f((7, 14)) is True

    def test_runtime_target_is_gfx906_selection(self, monkeypatch):
        """Env override wins; else gfx906 only when it is the SOLE distinct arch."""
        monkeypatch.delenv("UNSLOTH_ROCM_GFX_ARCH", raising = False)
        with patch.object(stack_mod, "_detect_amd_gfx_codes", return_value = ["gfx906"]):
            assert stack_mod._runtime_target_is_gfx906() is True
        # De-dup loses ordinals, so never downgrade a non-gfx906 selection on a mixed host.
        with patch.object(stack_mod, "_detect_amd_gfx_codes", return_value = ["gfx906", "gfx1100"]):
            assert stack_mod._runtime_target_is_gfx906() is False
        with patch.object(stack_mod, "_detect_amd_gfx_codes", return_value = []):
            assert stack_mod._runtime_target_is_gfx906() is False
        monkeypatch.setenv("UNSLOTH_ROCM_GFX_ARCH", "gfx906")
        with patch.object(stack_mod, "_detect_amd_gfx_codes", return_value = []):
            assert stack_mod._runtime_target_is_gfx906() is True
        monkeypatch.setenv("UNSLOTH_ROCM_GFX_ARCH", "gfx1100")
        with patch.object(stack_mod, "_detect_amd_gfx_codes", return_value = ["gfx906"]):
            assert stack_mod._runtime_target_is_gfx906() is False
        # A copied gcnArchName suffix must not defeat the exact comparison.
        monkeypatch.setenv("UNSLOTH_ROCM_GFX_ARCH", "gfx906:sramecc-:xnack-")
        with patch.object(stack_mod, "_detect_amd_gfx_codes", return_value = []):
            assert stack_mod._runtime_target_is_gfx906() is True

    def test_gfx906_on_rocm72_routes_to_rocm63(self, monkeypatch):
        """CPU torch on a ROCm 7.2 MI50 host installs from rocm6.3, not rocm7.2."""
        monkeypatch.delenv("UNSLOTH_ROCM_GFX_ARCH", raising = False)
        mock_pip, mock_pip_try = self._reroute(["gfx906"])
        torch_call = str(mock_pip.call_args_list[0])
        assert "rocm6.3" in torch_call
        assert "rocm7.2" not in torch_call
        # The rocm6.3 index has torch <= 2.9.x, so the 2.11 floor cannot apply.
        assert "torch>=2.4,<2.11.0" in torch_call
        # gfx906 has no prebuilt bnb -- the generic wheel must not be installed.
        assert not any("bitsandbytes" in str(c).lower() for c in mock_pip_try.call_args_list)

    def test_query_pytorch_mirror_skips_the_gfx906_legacy_pin(self, monkeypatch):
        monkeypatch.delenv("UNSLOTH_TORCH_INDEX_URL", raising = False)
        monkeypatch.delenv("UNSLOTH_TORCH_INDEX_FAMILY", raising = False)
        monkeypatch.delenv("UNSLOTH_ROCM_GFX_ARCH", raising = False)
        mock_pip, mock_pip_try, repair_ok = run_ensure_rocm_torch(
            _has_rocm_gpu = True,
            _detect_rocm_version = (7, 2),
            _detect_amd_gfx_codes = ["gfx906"],
            attrs = {
                "_TORCH_BACKEND": "",
                "_PYTORCH_WHL_BASE": "https://mirror.example/whl?token=abc",
            },
            return_result = True,
        )
        mock_pip.assert_not_called()
        mock_pip_try.assert_not_called()
        assert repair_ok is False

    def test_gfx906_repairs_existing_rocm72_torch(self, monkeypatch):
        """An installed +rocm7.2 torch IS the broken combo: reinstall from rocm6.3
        even though has_hip_torch is True."""
        monkeypatch.delenv("UNSLOTH_ROCM_GFX_ARCH", raising = False)
        mock_pip, mock_pip_try = self._reroute(["gfx906"], probe = "2.11.0+rocm7.2|7.2.12345|")
        torch_call = str(mock_pip.call_args_list[0])
        assert "rocm6.3" in torch_call
        assert "torch>=2.4,<2.11.0" in torch_call

    def test_gfx906_already_on_rocm63_left_alone(self, monkeypatch):
        """torch already on rocm6.3 wheels must not be reinstalled (no update loop),
        and the generic bnb wheel must not clobber a source build."""
        monkeypatch.delenv("UNSLOTH_ROCM_GFX_ARCH", raising = False)
        mock_pip, mock_pip_try = self._reroute(["gfx906"], probe = "2.7.0+rocm6.3|6.3.42131|")
        mock_pip.assert_not_called()
        mock_pip_try.assert_not_called()

    def test_mixed_host_gfx906_not_sole_arch_skips_reroute(self, monkeypatch):
        """Mixed gfx906 plus dGPU host with no explicit override keeps the generic index, since
        gfx906 is not the sole arch (Codex finding 3: never downgrade an ambiguous mixed host)."""
        monkeypatch.delenv("UNSLOTH_ROCM_GFX_ARCH", raising = False)
        mock_pip, _ = self._reroute(["gfx1100", "gfx906"])
        torch_call = str(mock_pip.call_args_list[0])
        assert "rocm7.2" in torch_call
        assert "rocm6.3" not in torch_call

    def test_gfx906_env_override_forces_reroute(self, monkeypatch):
        """UNSLOTH_ROCM_GFX_ARCH=gfx906 reroutes even when probes emit no gfx token
        (Codex #2: runtime-only ROCm hosts where rocminfo/amd-smi are absent)."""
        monkeypatch.setenv("UNSLOTH_ROCM_GFX_ARCH", "gfx906")
        mock_pip, _ = self._reroute([])
        torch_call = str(mock_pip.call_args_list[0])
        assert "rocm6.3" in torch_call
        assert "rocm7.2" not in torch_call

    def test_gfx906_bnb_skipped_even_when_index_pinned(self, monkeypatch):
        """A gfx906 user who pins the ROCm index and sets the arch override still skips the generic
        bnb wheel; the pin suppresses only the torch reroute, not the bnb skip flag."""
        monkeypatch.setenv("UNSLOTH_ROCM_GFX_ARCH", "gfx906")
        monkeypatch.setenv("UNSLOTH_TORCH_INDEX_URL", "https://download.pytorch.org/whl/rocm6.3")
        mock_pip, mock_pip_try = self._reroute(["gfx906"])
        assert any("rocm6.3" in str(c) for c in mock_pip.call_args_list)
        assert not any("bitsandbytes" in str(c).lower() for c in mock_pip_try.call_args_list)

    def test_gfx906_override_wins_over_strix_probe(self, monkeypatch):
        """Mixed Strix + MI50 host: UNSLOTH_ROCM_GFX_ARCH=gfx906 suppresses the Strix
        override (which probe order would otherwise pick) and routes to rocm6.3."""
        monkeypatch.setenv("UNSLOTH_ROCM_GFX_ARCH", "gfx906")
        mock_pip, mock_pip_try = self._reroute(["gfx1151", "gfx906"])
        torch_call = str(mock_pip.call_args_list[0])
        assert "rocm6.3" in torch_call
        assert "gfx1151" not in torch_call
        assert not any("bitsandbytes" in str(c).lower() for c in mock_pip_try.call_args_list)

    def test_gfx906_bnb_skipped_on_pinned_index_without_env_override(self, monkeypatch):
        """A pinned ROCm index suppresses only the torch reroute, not gfx906 probe detection for the
        bnb skip."""
        monkeypatch.delenv("UNSLOTH_ROCM_GFX_ARCH", raising = False)
        monkeypatch.setenv("UNSLOTH_TORCH_INDEX_URL", "https://download.pytorch.org/whl/rocm6.3")
        mock_pip, mock_pip_try = self._reroute(["gfx906"])
        assert any("rocm6.3" in str(c) for c in mock_pip.call_args_list)
        assert not any("bitsandbytes" in str(c).lower() for c in mock_pip_try.call_args_list)

    def test_install_sh_gfx906_env_suppresses_strix(self):
        """install.sh must skip the Strix reroute when UNSLOTH_ROCM_GFX_ARCH=gfx906."""
        source = (PACKAGE_ROOT / "install.sh").read_text(encoding = "utf-8")
        assert 'if [ "$_gfx906_env" != "gfx906" ]; then' in source

    def test_install_sh_gfx906_normalizes_override_and_clears_radeon(self):
        """install.sh strips the gfx906 feature suffix and clears the Radeon flag on every gfx906 target."""
        source = (PACKAGE_ROOT / "install.sh").read_text(encoding = "utf-8")
        # Strip the gfx906:... feature suffix and whitespace, like Python's .strip().
        assert "_gfx906_env=${_gfx906_env%%:*}" in source
        assert "_bnb_gfx_env=${_bnb_gfx_env%%:*}" in source
        assert source.count("tr -d '[:space:]'") >= 2
        block = self._gfx906_reroute_block(source)
        clear_pos = block.find("_amd_gpu_radeon=false")
        leaf_gate_pos = block.find("_rocm_leaf_below")
        assert clear_pos >= 0 and leaf_gate_pos >= 0
        # the unconditional clear must precede the >=6.4 leaf-gated reroute.
        assert clear_pos < leaf_gate_pos

    def test_install_sh_bnb_skip_probes_under_pin(self):
        """install.sh _is_gfx906_bnb_skip must probe gfx906 when the index is pinned
        (Codex #1): a pin skips the reroute block that sets _gfx906_target, so the
        helper falls back to _probe_amd_gfx_arch to catch a real gfx906 host."""
        source = (PACKAGE_ROOT / "install.sh").read_text(encoding = "utf-8")
        start = source.find("_is_gfx906_bnb_skip() {")
        assert start >= 0
        body = source[start : start + 900]
        assert "_torch_index_pinned" in body
        assert "_probe_amd_gfx_arch" in body

    def test_install_sh_has_gfx906_reroute(self):
        """install.sh must mirror the Python reroute: honor UNSLOTH_ROCM_GFX_ARCH,
        gate on a gfx906 target, route to rocm6.3, with the same _default (<2.11)
        trio, and skip the prebuilt bnb wheel."""
        source = (PACKAGE_ROOT / "install.sh").read_text(encoding = "utf-8")
        block = self._gfx906_reroute_block(source)
        assert "_gfx906_target=" in block
        assert "UNSLOTH_ROCM_GFX_ARCH" in block
        assert "/rocm6.3" in block
        for spec in stack_mod._ROCM_TORCH_PKG_SPECS["_default"]:
            assert spec in block
        assert "_is_gfx906_bnb_skip" in source

    def test_device_type_defaults_compile_off_on_gfx906(self):
        """unsloth/device_type.py must default Dynamo/compile off on gfx906
        (user-overridable via setdefault)."""
        source = (PACKAGE_ROOT / "unsloth" / "device_type.py").read_text(encoding = "utf-8")
        gate_start = source.find("gfx906")
        assert gate_start >= 0
        gate_body = source[gate_start : gate_start + 800]
        assert 'setdefault("TORCHDYNAMO_DISABLE", "1")' in gate_body
        assert 'setdefault("TORCH_COMPILE_DISABLE", "1")' in gate_body
        assert 'setdefault("UNSLOTH_COMPILE_DISABLE", "1")' in gate_body


# Per-arch floors from Tensile library names; gfx1200/1201 match _GENERIC_WHEEL_GFX_MIN_ROCM.
_GFX_FLOOR_TAG = {"gfx1102": "rocm6.3", "gfx1200": "rocm6.4", "gfx1201": "rocm6.4"}
# RDNA 4 below 7.13 lands here instead of the rocm6.4 floor.
_RDNA4_LEAF = "gfx120x-all"


class TestGfx1102Rocm64Floor:
    """The leaf is the higher of the per-arch kernel floor and the bitsandbytes ABI floor."""

    @staticmethod
    def _ensure_for_gfx(
        gfx: str,
        monkeypatch,
        *,
        pinned: bool = False,
        rocm_ver: tuple = (6, 1),
        probe_stdout: str = "\n",
        wheel_family = None,
        kfd = None,
    ):
        m = stack_mod
        for name in (
            "HIP_VISIBLE_DEVICES",
            "ROCR_VISIBLE_DEVICES",
            "CUDA_VISIBLE_DEVICES",
            "UNSLOTH_ROCM_GFX_ARCH",
            "UNSLOTH_TORCH_INDEX_FAMILY",
            "UNSLOTH_TORCH_INDEX_URL",
        ):
            monkeypatch.delenv(name, raising = False)
        if pinned:
            monkeypatch.setenv(
                "UNSLOTH_TORCH_INDEX_URL", "https://download.pytorch.org/whl/rocm6.1"
            )
        probe = MagicMock(returncode = 0, stdout = probe_stdout)
        with (
            patch.object(m, "IS_WINDOWS", False),
            patch.object(m, "IS_MACOS", False),
            patch.object(m, "_TORCH_RUNTIME_PROBE", None),
            patch.object(m, "_TORCH_BACKEND", ""),
            patch.object(m, "_LAST_AMD_GFX_PROBE", None),
            patch.object(m, "pip_install") as pip,
            patch.object(m, "pip_install_try", return_value = True),
            patch.object(m, "_has_usable_nvidia_gpu", return_value = False),
            patch.object(m, "_has_rocm_gpu", return_value = True),
            # Generic wheels pull no rocm[libraries]; unpatched, a gfx1151 runner's real torch leaks in.
            patch.object(m, "_torch_requires_rocm_sdk", return_value = wheel_family is not None),
            patch.object(m, "_installed_rocm_wheel_family", return_value = wheel_family),
            patch.object(m, "_infer_linux_amd_gfx_arch", return_value = None),
            patch.object(m, "_detect_rocm_version", return_value = rocm_ver),
            patch.object(m, "_detect_amd_gfx_codes", return_value = [gfx] if gfx else []),
            # None leaves KFD sysfs unpatched, as before; a list stands in for the topology.
            (
                contextlib.nullcontext()
                if kfd is None
                else patch.object(m, "_kfd_gfx_targets", return_value = kfd)
            ),
            patch("platform.machine", return_value = "x86_64"),
            patch("subprocess.run", return_value = probe),
        ):
            _ensure_rocm_torch()
        return pip

    def test_the_shared_kernel_table_knows_gfx1102(self):
        """rocm6.0 to 6.2 wheels have no gfx1102 Tensile kernels, so the table needs a (6, 3) floor."""
        assert stack_mod._GENERIC_WHEEL_GFX_MIN_ROCM["gfx1102"] == (6, 3)
        assert stack_mod._generic_tag_lacks_kernels("gfx1102", (6, 0)) is True
        assert stack_mod._generic_tag_lacks_kernels("gfx1102", (6, 1)) is True
        assert stack_mod._generic_tag_lacks_kernels("gfx1102", (6, 2)) is True
        # rocm6.3 already has them, so a host there must not be moved.
        assert stack_mod._generic_tag_lacks_kernels("gfx1102", (6, 3)) is False
        assert stack_mod._generic_tag_lacks_kernels("gfx1102", (6, 4)) is False
        assert stack_mod._generic_tag_lacks_kernels("gfx1102", (7, 2)) is False

    def test_the_kernel_table_leaves_the_older_arches_alone(self):
        """gfx1030/gfx1100/gfx1101 are in every family measured, so they have no floor."""
        for unaffected in ("gfx1030", "gfx1100", "gfx1101", "gfx906"):
            assert unaffected not in stack_mod._GENERIC_WHEEL_GFX_MIN_ROCM
            assert stack_mod._generic_tag_lacks_kernels(unaffected, (6, 0)) is False

    def test_debian_split_gfx1100_stays_generic_at_the_bnb_floor(self, monkeypatch):
        """The two floors compose as a maximum: gfx1100 stays generic but the leaf rises to rocm6.4
        for bnb."""
        torch_call = str(self._ensure_for_gfx("gfx1100", monkeypatch).call_args_list[0])
        assert "rocm6.4" in torch_call
        assert "repo.amd.com" not in torch_call

    @pytest.mark.parametrize(
        "gfx, index",
        (
            ("gfx1102", "gfx110X-all"),
            ("gfx1200", "gfx120X-all"),
            ("gfx1201", "gfx120X-all"),
        ),
    )
    def test_debian_split_affected_gfx_leaves_the_kernel_less_generic_wheel(
        self, gfx, index, monkeypatch
    ):
        """The per-arch route beats a newer generic family; repo.amd.com ships wheels built for that
        arch."""
        torch_call = str(self._ensure_for_gfx(gfx, monkeypatch).call_args_list[0])
        assert index in torch_call
        assert "whl/rocm6.1" not in torch_call

    def test_a_gfx1102_host_on_rocm63_stays_generic(self, monkeypatch):
        """rocm6.3 carries gfx1102 kernels, so no per-arch index is needed; the bnb floor lifts it
        to 6.4."""
        torch_call = str(
            self._ensure_for_gfx("gfx1102", monkeypatch, rocm_ver = (6, 3)).call_args_list[0]
        )
        assert "whl/rocm6.4" in torch_call
        assert "repo.amd.com" not in torch_call

    def test_explicit_rocm61_pin_remains_authoritative(self, monkeypatch):
        """The narrow auto-floor never overwrites an explicit user index pin."""
        torch_call = str(
            self._ensure_for_gfx("gfx1102", monkeypatch, pinned = True).call_args_list[0]
        )
        assert "whl/rocm6.1" in torch_call
        assert "repo.amd.com" not in torch_call

    def test_installed_64_wheel_left_alone_on_newer_host(self, monkeypatch):
        """A wheel that already has the kernels must not be reinstalled."""
        pip = self._ensure_for_gfx(
            "gfx1102",
            monkeypatch,
            rocm_ver = (6, 4),
            probe_stdout = _MARK + "2.8.0+rocm6.4|6.4.43482|\n",
        )
        pip.assert_not_called()

    def test_rdna4_generic_64_wheel_moves_to_amd_arch_index(self, monkeypatch):
        pip = self._ensure_for_gfx(
            "gfx1200",
            monkeypatch,
            rocm_ver = (6, 4),
            probe_stdout = _MARK + "2.8.0+rocm6.4|6.4.43482|\n",
        )
        torch_call = str(pip.call_args_list[0])
        assert "gfx120X-all" in torch_call
        assert "torch>=2.11.0,<2.12.0" in torch_call

    def test_rdna4_already_on_amd_arch_wheel_is_left_alone(self, monkeypatch):
        pip = self._ensure_for_gfx(
            "gfx1201",
            monkeypatch,
            rocm_ver = (7, 14),
            probe_stdout = _MARK + "2.11.0+rocm7.13.0|7.13.99004|\n",
            wheel_family = "gfx120x-all",
        )
        pip.assert_not_called()

    def test_rdna4_pinned_index_stays_authoritative(self, monkeypatch):
        torch_call = str(
            self._ensure_for_gfx("gfx1201", monkeypatch, pinned = True).call_args_list[0]
        )
        assert "whl/rocm6.1" in torch_call
        assert "repo.amd.com" not in torch_call

    def test_kernel_unaffected_arch_is_still_repaired_below_the_bnb_floor(self, monkeypatch):
        """An installed +rocm6.1 torch is reinstalled on gfx1100 because it sits below the bnb ABI floor."""
        pip = self._ensure_for_gfx(
            "gfx1100",
            monkeypatch,
            rocm_ver = (6, 4),
            probe_stdout = _MARK + "2.8.0+rocm6.1|6.1.40093|\n",
        )
        torch_call = str(pip.call_args_list[0])
        assert "whl/rocm6.4" in torch_call
        assert "repo.amd.com" not in torch_call

    def test_kfd_only_gfx906_keeps_the_literal_generic_tag(self, monkeypatch):
        """A KFD-only gfx906 MI50 must keep the rocm6.1 tag; moving it to rocm6.4 loses the BLAS kernels."""
        torch_call = str(self._ensure_for_gfx("", monkeypatch, kfd = ["gfx906"]).call_args_list[0])
        assert "whl/rocm6.1" in torch_call
        assert "whl/rocm6.4" not in torch_call

    def test_unreadable_arch_keeps_the_literal_generic_tag(self, monkeypatch):
        """An unknown arch might be gfx906, so the bnb floor is skipped and the literal generic tag
        is kept."""
        pip = self._ensure_for_gfx("", monkeypatch, kfd = [])
        for call in pip.call_args_list:
            assert "whl/rocm6.4" not in str(call)

    def test_install_sh_floors_resolved_rocm61_by_runtime_gfx(self):
        """On a Debian rocm6.1 leaf, gfx1100 stays, while gfx1102 and RDNA 4 move to a family with
        kernels."""
        shell = shutil.which("bash")
        if not shell:
            pytest.skip("bash needed to execute install.sh architecture routing")
        source = _INSTALL_SH_PATH.read_text(encoding = "utf-8")
        leaf_helper = _extract_sh_function_body(source, "_rocm_leaf_below")
        start = source.find('case "$_torch_index_leaf" in\n    rocm[0-9]*)')
        end = source.find("\nfi  # _torch_index_pinned guard", start)
        assert leaf_helper and start >= 0 and end >= 0
        routing_block = source[start:end]
        floor_pos = routing_block.find("_amd_rocm64_base")
        pin_start = source.rfind('if [ "$_torch_index_pinned" = false ]; then', 0, start)
        assert floor_pos >= 0 and pin_start >= 0 and pin_start < start

        for gfx, expected_url, expected in (
            ("gfx1100", "rocm6.1", "rocm6.1"),
            ("gfx1102", "rocm6.3", "rocm6.3"),
            ("gfx1200", "gfx120X-all", _RDNA4_LEAF),
            ("gfx1201", "gfx120X-all", _RDNA4_LEAF),
        ):
            script = (
                "set -euo pipefail\n"
                + leaf_helper
                + "\n"
                + 'TORCH_INDEX_URL="https://download.pytorch.org/whl/rocm6.1"\n'
                + '_torch_index_leaf="rocm6.1"\n'
                + "_torch_index_pinned=false\nSKIP_TORCH=false\n_amd_gpu_radeon=false\n"
                + f'UNSLOTH_ROCM_GFX_ARCH="{gfx}"\n'
                + "unset HSA_OVERRIDE_GFX_VERSION HIP_VISIBLE_DEVICES ROCR_VISIBLE_DEVICES UNSLOTH_PYTORCH_MIRROR\n"
                + routing_block
                + '\nprintf "URL:%s LEAF:%s\\n" "$TORCH_INDEX_URL" "$_torch_index_leaf"\n'
            )
            result = _run_bash_script(shell, script, capture_output = True, text = True)
            assert result.returncode == 0, result.stderr
            out = result.stdout.strip()
            assert out.endswith(f"/{expected_url} LEAF:{expected}") or out.endswith(
                f"/{expected_url}/ LEAF:{expected}"
            ), out

    @staticmethod
    def _install_sh_routing_result(
        preamble: str,
        leaf: str = "rocm6.1",
        radeon_in: str = "false",
    ) -> tuple:
        """Run install.sh's architecture-routing block, returning (leaf, floor-target flag)."""
        shell = shutil.which("bash")
        if not shell:
            pytest.skip("bash needed to execute install.sh architecture routing")
        source = _INSTALL_SH_PATH.read_text(encoding = "utf-8")
        leaf_helper = _extract_sh_function_body(source, "_rocm_leaf_below")
        gfx_helper = _gpu_record_helpers(source)
        start = source.find('case "$_torch_index_leaf" in\n    rocm[0-9]*)')
        end = source.find("\nfi  # _torch_index_pinned guard", start)
        assert leaf_helper and gfx_helper and start >= 0 and end >= 0
        script = (
            "set -euo pipefail\n"
            + leaf_helper
            + "\n"
            + gfx_helper
            + "\n"
            + f'TORCH_INDEX_URL="https://download.pytorch.org/whl/{leaf}"\n'
            + f'_torch_index_leaf="{leaf}"\n'
            + f"_torch_index_pinned=false\nSKIP_TORCH=false\n_amd_gpu_radeon={radeon_in}\n"
            + "_gfx_rocm64_target=false\n"
            + "unset HSA_OVERRIDE_GFX_VERSION HIP_VISIBLE_DEVICES ROCR_VISIBLE_DEVICES\n"
            + "unset CUDA_VISIBLE_DEVICES UNSLOTH_ROCM_GFX_ARCH UNSLOTH_PYTORCH_MIRROR\n"
            + preamble
            + "\n"
            + source[start:end]
            + '\nprintf "LEAF:%s TARGET:%s RADEON:%s\\n" '
            '"$_torch_index_leaf" "$_gfx_rocm64_target" "$_amd_gpu_radeon"\n'
        )
        result = _run_bash_script(shell, script, capture_output = True, text = True)
        assert result.returncode == 0, result.stderr
        tail = result.stdout.strip().rsplit("LEAF:", 1)[-1]
        _leaf, _rest = tail.split(" TARGET:")
        _target, _radeon = _rest.split(" RADEON:")
        return _leaf, _target, _radeon

    @classmethod
    def _run_install_sh_routing(cls, preamble: str) -> str:
        return cls._install_sh_routing_result(preamble)[0]

    @pytest.mark.parametrize(
        ("mask", "expected"),
        (
            ("CUDA_VISIBLE_DEVICES=1", _RDNA4_LEAF),
            ("HIP_VISIBLE_DEVICES=1", _RDNA4_LEAF),
            ("CUDA_VISIBLE_DEVICES=0", "rocm6.1"),
            # a set-but-empty hip mask selects no gpu instead of deferring to cuda
            ("HIP_VISIBLE_DEVICES= CUDA_VISIBLE_DEVICES=1", "rocm6.1"),
        ),
    )
    def test_install_sh_runtime_target_honors_cuda_visible_devices(self, mask, expected):
        """HIP exposes devices through CUDA_VISIBLE_DEVICES too, so it must select the target."""
        preamble = "rocminfo() { printf 'Name: gfx1100\\nName: gfx1200\\n'; }\n" + "".join(
            f"export {assignment}\n" for assignment in mask.split()
        )
        assert self._run_install_sh_routing(preamble) == expected

    @pytest.mark.parametrize(
        ("arches", "mask", "expected"),
        (
            # An unmasked iGPU listed first used to pick the wheels and strand the dGPU.
            (("gfx1036", "gfx1201"), "", _RDNA4_LEAF),
            (("gfx1103", "gfx1201"), "", _RDNA4_LEAF),
            (("gfx1201", "gfx1036"), "", _RDNA4_LEAF),
            # A mask names the device, so the iGPU it selects keeps the route.
            (("gfx1036", "gfx1201"), "HIP_VISIBLE_DEVICES=0", "rocm6.1"),
            (("gfx1036", "gfx1201"), "CUDA_VISIBLE_DEVICES=0", "rocm6.1"),
            (("gfx1036",), "", "rocm6.1"),
        ),
    )
    def test_install_sh_prefers_the_discrete_gpu_over_a_leading_igpu(self, arches, mask, expected):
        preamble = (
            "rocminfo() { printf '"
            + "".join(f"Name: {a}\\n" for a in arches)
            + "'; }\n"
            + "".join(f"export {assignment}\n" for assignment in mask.split())
        )
        assert self._run_install_sh_routing(preamble) == expected

    @staticmethod
    def _rocminfo_stub(*arches: str) -> str:
        """A rocminfo stub whose agents repeat their gfx token across Name and ISA."""
        blocks = [
            "Agent 1",
            "  Name:                    AMD Ryzen 9 7950X",
            "  Device Type:             CPU",
        ]
        for n, gfx in enumerate(arches, start = 2):
            blocks += [
                "*******",
                f"Agent {n}",
                "*******",
                f"  Name:                    {gfx}",
                "  Marketing Name:          AMD Radeon Graphics",
                f"      Name:                amdgcn-amd-amdhsa--{gfx}",
            ]
        body = "\\n".join(blocks)
        return f"rocminfo() {{ printf '{body}\\n'; }}\n"

    @staticmethod
    def _amd_smi_stub(*arches, hip_ids = None):
        """Routing declines without a HIP_ID map; hip_ids defaults to the identity, i.e. discovery order."""
        asic = "\\n".join(
            line
            for n, gfx in enumerate(arches)
            for line in (f"GPU: {n}", f"        TARGET_GRAPHICS_VERSION: {gfx}")
        )
        hip_ids = range(len(arches)) if hip_ids is None else hip_ids
        hip_map = "\\n".join(
            line
            for n, hip in enumerate(hip_ids)
            for line in (f"GPU: {n}", f"        HIP_ID: {hip}")
        )
        return (
            "amd-smi() {\n"
            '    case "$*" in\n'
            f"        *'list -e'*) printf '{hip_map}\\n' ;;\n"
            f"        *) printf '{asic}\\n' ;;\n"
            "    esac\n"
            "}\n"
        )

    @pytest.mark.parametrize(
        ("mask", "expected"),
        (
            ("HIP_VISIBLE_DEVICES=2", _RDNA4_LEAF),
            ("CUDA_VISIBLE_DEVICES=2", _RDNA4_LEAF),
            ("HIP_VISIBLE_DEVICES=1", "rocm6.1"),
            ("HIP_VISIBLE_DEVICES=0", "rocm6.1"),
        ),
    )
    def test_install_sh_mask_indexes_devices_not_arches(self, mask, expected):
        """A visible-device mask is a device ordinal, so duplicate arches each count."""
        preamble = self._rocminfo_stub("gfx1100", "gfx1100", "gfx1200") + f"export {mask}"
        assert self._run_install_sh_routing(preamble) == expected

    @pytest.mark.parametrize(
        ("rocr", "arches", "expected"),
        (
            # rocminfo reports the rocr selection in runtime order, so index 0 is the target
            ("1,0", ("gfx1200", "gfx1100"), _RDNA4_LEAF),
            ("1", ("gfx1200",), _RDNA4_LEAF),
            ("0,1", ("gfx1100", "gfx1200"), "rocm6.1"),
        ),
    )
    def test_install_sh_does_not_reapply_a_rocr_mask_rocminfo_already_applied(
        self, rocr, arches, expected
    ):
        """rocminfo output is already filtered and ordered by ROCR_VISIBLE_DEVICES."""
        preamble = self._rocminfo_stub(*arches) + f"export ROCR_VISIBLE_DEVICES={rocr}"
        assert self._run_install_sh_routing(preamble) == expected

    @pytest.mark.parametrize(
        ("mask", "expected"),
        (("HIP_VISIBLE_DEVICES=2", _RDNA4_LEAF), ("HIP_VISIBLE_DEVICES=1", "rocm6.1")),
    )
    def test_install_sh_splits_amd_smi_gpu_headers(self, mask, expected):
        """amd-smi heads each device with "GPU: N", so two gfx1100 cards stay two entries."""
        preamble = (
            "rocminfo() { return 1; }\n"
            + self._amd_smi_stub("gfx1100", "gfx1100", "gfx1200")
            + f"export {mask}"
        )
        assert self._run_install_sh_routing(preamble) == expected

    def test_install_sh_still_indexes_amd_smi_output_under_a_rocr_mask(self):
        """amd-smi ignores ROCR_VISIBLE_DEVICES, so its output still needs indexing."""
        preamble = (
            "rocminfo() { return 1; }\n"
            + self._amd_smi_stub("gfx1200", "gfx1100")
            + "export ROCR_VISIBLE_DEVICES=1"
        )
        assert self._run_install_sh_routing(preamble) == "rocm6.1"

    def test_install_sh_layers_rocr_under_hip_for_amd_smi(self):
        """ROCr decides which devices exist; HIP indexes the survivors, not the whole list."""
        preamble = (
            "rocminfo() { return 1; }\n"
            + self._amd_smi_stub("gfx1100", "gfx1200", "gfx1100")
            + "export ROCR_VISIBLE_DEVICES=1,0; export HIP_VISIBLE_DEVICES=1"
        )
        assert self._run_install_sh_routing(preamble) == "rocm6.1"

    @pytest.mark.parametrize("rocr, hip", (("0,0,1", "2"), ("0,99,1", "1")))
    def test_install_sh_ends_rocr_survivors_at_a_repeated_or_missing_ordinal(self, rocr, hip):
        """Only card 0 survives, so the HIP index falls back to it, never the gfx1200."""
        preamble = (
            "rocminfo() { return 1; }\n"
            + self._amd_smi_stub("gfx1100", "gfx1200")
            + f"export ROCR_VISIBLE_DEVICES={rocr}; export HIP_VISIBLE_DEVICES={hip}"
        )
        assert self._run_install_sh_routing(preamble) == "rocm6.1"

    def test_install_sh_declines_a_rocr_uuid_mask_over_unlike_amd_smi_adapters(self):
        """A UUID names a device but no position, so no survivor is known to be it."""
        preamble = (
            "rocminfo() { return 1; }\n"
            + self._amd_smi_stub("gfx1200", "gfx1100")
            + "export ROCR_VISIBLE_DEVICES=GPU-deadbeefdeadbeef"
        )
        assert self._run_install_sh_routing(preamble) == "rocm6.1"

    def test_install_sh_keeps_routing_on_a_rocr_uuid_mask_over_like_adapters(self):
        """One arch everywhere: whichever device the UUID names, the wheel is the same."""
        preamble = (
            "rocminfo() { return 1; }\n"
            + self._amd_smi_stub("gfx1200", "gfx1200")
            + "export ROCR_VISIBLE_DEVICES=GPU-deadbeefdeadbeef"
        )
        assert self._run_install_sh_routing(preamble) == _RDNA4_LEAF

    def test_install_sh_declines_an_ordinal_when_amd_smi_gives_no_hip_map(self):
        """Without list -e there is no HIP order, so routing must decline instead of indexing by
        discovery."""
        preamble = (
            "rocminfo() { return 1; }\n"
            "amd-smi() { printf 'GPU: 0\\n        TARGET_GRAPHICS_VERSION: gfx1100\\n"
            "GPU: 1\\n        TARGET_GRAPHICS_VERSION: gfx1200\\n'; }\n"
            "export HIP_VISIBLE_DEVICES=1"
        )
        assert self._run_install_sh_routing(preamble) == "rocm6.1"

    @pytest.mark.parametrize(
        ("gfx", "leaf", "expected_leaf", "expected_target"),
        (
            ("gfx1200", "rocm6.4", _RDNA4_LEAF, "true"),
            ("gfx1200", "rocm7.2", _RDNA4_LEAF, "true"),
            ("gfx1200", "rocm6.1", _RDNA4_LEAF, "true"),
            ("gfx1100", "rocm6.4", "rocm6.4", "false"),
            ("gfx1100", "rocm6.1", "rocm6.1", "false"),
        ),
    )
    def test_install_sh_marks_the_floor_target_independently_of_the_leaf(
        self, gfx, leaf, expected_leaf, expected_target
    ):
        """The migrated repair keys off the arch, so the flag cannot depend on a reroute."""
        result = self._install_sh_routing_result(f'export UNSLOTH_ROCM_GFX_ARCH="{gfx}"', leaf)
        assert result[:2] == (expected_leaf, expected_target)

    @pytest.mark.parametrize(
        ("gfx", "leaf", "expected_radeon"),
        (
            # repo.radeon.com rocm-rel-6.4/7.2 trios lack gfx1102 Tensile libraries.
            ("gfx1102", "rocm6.1", "false"),
            ("gfx1200", "rocm6.1", "false"),
            ("gfx1201", "rocm6.1", "false"),
            # Cleared even when the leaf satisfies the floor and the reroute is a no-op.
            ("gfx1102", "rocm6.4", "false"),
            ("gfx1102", "rocm7.2", "false"),
            ("gfx1100", "rocm6.1", "true"),
            ("gfx1101", "rocm6.4", "true"),
        ),
    )
    def test_install_sh_floor_arches_leave_the_radeon_branch(self, gfx, leaf, expected_radeon):
        """A floored arch must not be handed repo.radeon.com wheels lacking its kernels."""
        result = self._install_sh_routing_result(
            f'export UNSLOTH_ROCM_GFX_ARCH="{gfx}"', leaf, radeon_in = "true"
        )
        assert result[2] == expected_radeon, result

    @staticmethod
    def _run_migrated_rocm_repair(
        torch_version: str,
        hip: str,
        gfx_target: str,
        floor: "tuple[int, int]" = (6, 4),
        arch_routed: str = "false",
        venv_family: str = "",
        routed_family: str = "",
    ) -> str:
        """Execute install.sh's migrated-environment ROCm repair with a stubbed venv torch."""
        shell = shutil.which("bash")
        if not shell:
            pytest.skip("bash needed to execute the migrated repair block")
        source = _INSTALL_SH_PATH.read_text(encoding = "utf-8")
        start = source.find("        # Repair ROCm torch if overwritten during migrated install")
        end = source.find("        _gfx906_bnb_prune", start)
        assert start >= 0 and end >= 0
        with tempfile.TemporaryDirectory() as d:
            venv_py = os.path.join(d, "python")
            with open(venv_py, "w", encoding = "utf-8") as fh:
                fh.write(
                    "#!/bin/sh\n"
                    # install.sh probes the venv with `-I -c CODE`; the stub only reads CODE.
                    'if [ "$1" = "-I" ]; then shift; fi\n'
                    f'exec \'{sys.executable.replace(os.sep, "/")}\' -c "\n'
                    "import sys, types\n"
                    "t = types.ModuleType('torch')\n"
                    f"t.__version__ = '{torch_version}'\n"
                    "v = types.ModuleType('torch.version')\n"
                    f"v.hip = '{hip}'\n"
                    "t.version = v\n"
                    "sys.modules['torch'] = t\n"
                    "sys.modules['torch.version'] = v\n"
                    "import importlib.metadata as md\n"
                    f"fam = '{venv_family}'\n"
                    # Always stubbed: an AMD host's own rocm package must not leak in.
                    "md.requires = lambda n: ['rocm-sdk-libraries-' + fam + '==7.13.0'] if fam and n == 'rocm' else []\n"
                    "exec(sys.argv[1])\n"
                    '" "$2"\n'
                )
            os.chmod(venv_py, 0o755)
            script = (
                "set -euo pipefail\n"
                + _extract_sh_function_body(source, "_rocm_leaf_below")
                + "\n"
                + _extract_sh_function_body(source, "_venv_torch_rocm_below")
                + "\n"
                + _extract_sh_function_body(source, "_venv_torch_amd_family")
                + "\n"
                + "substep() { :; }\n"
                + '_install_torch_default_index() { printf "REINSTALL\\n"; }\n'
                + f'_VENV_PY="{venv_py.replace(os.sep, "/")}"\n'
                + f"_gfx_rocm64_target={gfx_target}\n"
                + f"_gfx_rocm64_floor_maj={floor[0]}\n_gfx_rocm64_floor_min={floor[1]}\n"
                + f"_amd_arch_index_routed={arch_routed}\n"
                + f'_amd_arch_index_family="{routed_family}"\n'
                + f'_torch_index_leaf="rocm{floor[0]}.{floor[1]}"\n'
                + source[start:end]
                + '\nprintf "DONE\\n"\n'
            )
            r = _run_bash_script(shell, script, capture_output = True, text = True)
            assert r.returncode == 0, r.stderr
            return r.stdout

    @pytest.mark.parametrize(
        ("torch_version", "hip", "gfx_target", "floor", "reinstalls"),
        (
            # the gap: a migrated venv keeps its hip torch, so an older wheel survived
            ("2.5.1+rocm6.1", "6.1.40093", "true", (6, 4), True),
            ("2.7.0+rocm6.3", "6.3.42131", "true", (6, 4), True),
            ("2.8.0+rocm6.4", "6.4.43482", "true", (6, 4), False),
            ("2.11.0+rocm7.13.0", "7.13.0", "true", (6, 4), False),
            # gfx1102 floors at 6.3, so a 6.3 wheel already has its kernels.
            ("2.5.1+rocm6.1", "6.1.40093", "true", (6, 3), True),
            ("2.7.0+rocm6.3", "6.3.42131", "true", (6, 3), False),
            ("2.8.0+rocm6.4", "6.4.43482", "true", (6, 3), False),
            ("2.5.1+rocm6.1", "6.1.40093", "false", (6, 4), False),
            ("2.5.1+cpu", "", "false", (6, 4), True),
        ),
    )
    def test_install_sh_migrated_repair_honors_the_rocm64_floor(
        self, torch_version, hip, gfx_target, floor, reinstalls
    ):
        out = self._run_migrated_rocm_repair(torch_version, hip, gfx_target, floor)
        assert "DONE" in out, out
        assert ("REINSTALL" in out) is reinstalls, out

    @pytest.mark.parametrize(
        ("torch_version", "arch_routed", "reinstalls"),
        (
            ("2.11.0+rocm7.2", "true", True),
            ("2.9.1+rocm6.4", "true", True),
            ("2.11.0+rocm7.13.0", "true", False),
            ("2.11.0+rocm7.2", "false", False),
        ),
    )
    def test_install_sh_migrated_repair_honors_the_arch_index_route(
        self, torch_version, arch_routed, reinstalls
    ):
        out = self._run_migrated_rocm_repair(
            torch_version, "7.2.0", "true", (6, 4), arch_routed = arch_routed
        )
        assert "DONE" in out, out
        assert ("REINSTALL" in out) is reinstalls, out

    @pytest.mark.parametrize(
        ("venv_family", "routed_family", "reinstalls"),
        (
            # A 7.13 wheel for another family (a gfx1151 venv reused on RDNA 4) has no kernels.
            ("gfx1151", "gfx120x-all", True),
            ("gfx120X-all", "gfx1151", True),
            ("gfx120X-all", "gfx120x-all", False),
            ("gfx1151", "gfx1151", False),
            ("", "gfx120x-all", False),
        ),
    )
    def test_install_sh_migrated_repair_honors_the_arch_index_family(
        self, venv_family, routed_family, reinstalls
    ):
        out = self._run_migrated_rocm_repair(
            "2.11.0+rocm7.13.0",
            "7.13.0",
            "true",
            (6, 4),
            arch_routed = "true",
            venv_family = venv_family,
            routed_family = routed_family,
        )
        assert "DONE" in out, out
        assert ("REINSTALL" in out) is reinstalls, out

    @pytest.mark.parametrize(
        "override",
        ("gfx1200:xnack-", "GFX1200:XNACK-", " gfx1200 ", "gfx1201:sramecc+:xnack-"),
    )
    def test_install_sh_normalizes_gcn_arch_name_override(self, override):
        """UNSLOTH_ROCM_GFX_ARCH copied from a HIP gcnArchName still matches the floor."""
        preamble = f'export UNSLOTH_ROCM_GFX_ARCH="{override}"'
        assert self._run_install_sh_routing(preamble) == _RDNA4_LEAF

    @pytest.mark.parametrize(
        ("gfx", "leaf", "expected"),
        (
            ("gfx1201", "rocm7.2", _RDNA4_LEAF),
            ("gfx1200", "rocm7.2", _RDNA4_LEAF),
            ("gfx1201", "rocm7.13", "rocm7.13"),
            ("gfx1100", "rocm7.2", "rocm7.2"),
        ),
    )
    def test_install_sh_routes_rdna4_below_713_to_amd_arch_index(self, gfx, leaf, expected):
        preamble = f'export UNSLOTH_ROCM_GFX_ARCH="{gfx}"'
        assert self._install_sh_routing_result(preamble, leaf = leaf)[0] == expected

    @pytest.mark.parametrize(("py", "expected"), (("3.9", "rocm6.4"), ("3.12", _RDNA4_LEAF)))
    def test_install_sh_keeps_rdna4_generic_on_python_39(self, tmp_path, py, expected):
        """gfx120X-all has no cp39 wheels, so a 3.9 venv keeps the generic rocm6.4 route."""
        (tmp_path / "bin").mkdir()
        stub = tmp_path / "bin" / "python"
        stub.write_text(f"#!/bin/sh\necho {py}\n")
        stub.chmod(0o755)
        preamble = f'export UNSLOTH_ROCM_GFX_ARCH="gfx1201"\nVENV_DIR="{tmp_path}"'
        assert self._install_sh_routing_result(preamble, leaf = "rocm6.4")[0] == expected


class TestHasRocmGpuKfdVendorGuard:
    """KFD sysfs fallback rejects non-AMD (NVIDIA) KFD nodes (source-level checks)."""

    def _src(self) -> str:
        """Return the source of _has_rocm_gpu from install_python_stack.py."""
        import inspect
        return inspect.getsource(stack_mod._has_rocm_gpu)

    def test_vendor_id_check_present(self):
        """_has_rocm_gpu sysfs fallback must check vendor_id 4098 (AMD 0x1002)."""
        src = self._src()
        assert "vendor_id" in src, (
            "_has_rocm_gpu KFD sysfs fallback must read the properties file "
            "to check vendor_id and exclude NVIDIA KFD nodes"
        )
        assert "4098" in src, (
            "_has_rocm_gpu must require AMD vendor_id 4098 (0x1002) in the "
            "KFD node properties to avoid false positives on NVIDIA systems"
        )

    def test_vendor_regex_pattern_anchored(self):
        """The vendor_id regex must use a word boundary to avoid partial matches."""
        import re as _re

        src = self._src()
        # Word boundary so "vendor_id 41098" doesn't match "vendor_id 4098".
        assert (
            _re.search(r"\\b.*vendor_id.*\\b", src) or "\\bvendor_id" in src
        ), "_has_rocm_gpu vendor_id check should use word boundary anchors"

    def test_sysfs_fallback_guarded_by_non_win32(self):
        """KFD sysfs fallback must be Linux-only (guarded by sys.platform != 'win32')."""
        src = self._src()
        assert "win32" in src, "_has_rocm_gpu sysfs fallback must be guarded by sys.platform check"

    def test_cpu_node_excluded(self):
        """gpu_id == '0' must be excluded (CPU topology nodes)."""
        src = self._src()
        assert (
            '!= "0"' in src or "== '0'" in src or "!= '0'" in src or '"0"' in src
        ), "_has_rocm_gpu must skip gpu_id 0 nodes (CPU nodes)"

    def test_install_sh_has_vendor_check(self):
        """The sysfs fallback must check vendor_id 4098 inside _has_amd_rocm_gpu, which tests lift
        by name."""
        sh_path = PACKAGE_ROOT / "install.sh"
        source = sh_path.read_text(encoding = "utf-8")
        func_body = _extract_sh_function_body(source, "_has_amd_rocm_gpu")
        assert "vendor_id" in func_body, "_has_amd_rocm_gpu sysfs fallback must check vendor_id"
        assert "4098" in func_body, "_has_amd_rocm_gpu must require AMD vendor_id 4098 (0x1002)"

    def test_has_rocm_gpu_returns_false_when_nvidia_present(self):
        """_has_rocm_gpu returns False when _has_usable_nvidia_gpu is True (NVIDIA always wins)."""
        with patch.object(stack_mod, "_has_usable_nvidia_gpu", return_value = True):
            with patch("shutil.which", return_value = "/usr/bin/rocminfo"):
                mock_result = MagicMock()
                mock_result.returncode = 0
                mock_result.stdout = "Name: gfx1100\n"
                with patch("subprocess.run", return_value = mock_result):
                    assert not stack_mod._has_rocm_gpu(), (
                        "_has_rocm_gpu must return False when NVIDIA GPU is detected, "
                        "regardless of what rocminfo reports"
                    )

    def test_install_sh_has_rocm_gpu_nvidia_guard(self):
        """_has_amd_rocm_gpu in install.sh must call _has_usable_nvidia_gpu and return 1 if true."""
        sh_path = PACKAGE_ROOT / "install.sh"
        source = sh_path.read_text(encoding = "utf-8")
        func_start = source.find("_has_amd_rocm_gpu()")
        func_end = source.find("\n}", func_start)
        func_body = source[func_start:func_end]
        assert (
            "_has_usable_nvidia_gpu" in func_body
        ), "_has_amd_rocm_gpu must call _has_usable_nvidia_gpu to block NVIDIA hosts"
        assert (
            "return 1" in func_body
        ), "_has_amd_rocm_gpu must return 1 (false) when NVIDIA GPU is detected"

    def test_has_usable_nvidia_gpu_proc_fallback_present(self):
        """`_has_usable_nvidia_gpu` must have a /proc/driver/nvidia fallback."""
        import inspect

        src = inspect.getsource(stack_mod._has_usable_nvidia_gpu)
        assert "/proc/driver/nvidia" in src, (
            "_has_usable_nvidia_gpu must fall back to /proc/driver/nvidia/gpus when "
            "nvidia-smi subprocess fails, to handle PATH gaps and driver init races"
        )

    def test_install_sh_has_usable_nvidia_gpu_proc_fallback(self):
        """_has_usable_nvidia_gpu in install.sh must also have a /proc/driver/nvidia fallback."""
        sh_path = PACKAGE_ROOT / "install.sh"
        source = sh_path.read_text(encoding = "utf-8")
        func_start = source.find("_has_usable_nvidia_gpu()")
        func_end = source.find("\n}", func_start)
        func_body = source[func_start:func_end]
        assert "/proc/driver/nvidia" in func_body, (
            "_has_usable_nvidia_gpu in install.sh must fall back to "
            "/proc/driver/nvidia/gpus when nvidia-smi fails"
        )


class TestRocmTorchIndex:
    """Verify the ROCm version -> torch index tag mapping."""

    def test_mapping_is_sorted_descending(self):
        """Keys should be in descending order for the next() iteration to work."""
        keys = list(_ROCM_TORCH_INDEX.keys())
        assert keys == sorted(keys, reverse = True)

    def test_rocm_72_in_mapping(self):
        """ROCm 7.2 should be in the active mapping (torch 2.11.0 now supported)."""
        assert (7, 2) in _ROCM_TORCH_INDEX
        assert _ROCM_TORCH_INDEX[(7, 2)] == "rocm7.2"

    def test_rocm_71_maps_correctly(self):
        assert _ROCM_TORCH_INDEX[(7, 1)] == "rocm7.1"

    def test_rocm_63_maps_correctly(self):
        assert _ROCM_TORCH_INDEX[(6, 3)] == "rocm6.3"

    def test_rocm_60_maps_correctly(self):
        assert _ROCM_TORCH_INDEX[(6, 0)] == "rocm6.0"

    def test_all_tags_use_download_pytorch(self):
        """All tags should be for download.pytorch.org, not repo.radeon.com."""
        for tag in _ROCM_TORCH_INDEX.values():
            assert tag.startswith("rocm")
            assert "radeon" not in tag

    def test_newer_rocm_selects_best_match(self):
        """ROCm 7.2 (now in map) should select rocm7.2 directly."""
        ver = (7, 2)
        tag = next(
            (
                t
                for (maj, mn), t in sorted(_ROCM_TORCH_INDEX.items(), reverse = True)
                if ver >= (maj, mn)
            ),
            None,
        )
        assert tag == "rocm7.2"

    def test_rocm_64_selects_64(self):
        ver = (6, 4)
        tag = next(
            (
                t
                for (maj, mn), t in sorted(_ROCM_TORCH_INDEX.items(), reverse = True)
                if ver >= (maj, mn)
            ),
            None,
        )
        assert tag == "rocm6.4"

    @pytest.mark.parametrize(
        "ver, published, automatic",
        [
            ((6, 0), "rocm6.0", "rocm6.4"),
            ((6, 1), "rocm6.1", "rocm6.4"),
            ((6, 2), "rocm6.2", "rocm6.4"),
            ((6, 3), "rocm6.3", "rocm6.4"),
            ((6, 4), "rocm6.4", "rocm6.4"),
            ((7, 0), "rocm7.0", "rocm7.0"),
            ((7, 1), "rocm7.1", "rocm7.1"),
            ((7, 2), "rocm7.2", "rocm7.2"),
        ],
    )
    def test_automatic_generic_tag_adds_only_the_bnb_floor(self, ver, published, automatic):
        assert stack_mod._generic_pytorch_rocm_tag(ver) == published
        assert stack_mod._automatic_generic_pytorch_rocm_tag(ver) == automatic

    def test_shell_and_python_automatic_floor_are_in_parity(self):
        """The install.sh helper must floor exactly the same old generic tags as Python."""
        source = _INSTALL_SH_PATH.read_text(encoding = "utf-8")
        constant = re.search(r"^_ROCM_BNB_GENERIC_FLOOR_TAG=.*$", source, re.M)
        helper = _extract_sh_function_body(source, "_rocm_bnb_compatible_generic_tag")
        assert constant and helper
        shell = shutil.which("sh")
        if not shell:
            pytest.skip("POSIX shell needed for resolver parity")
        tags = tuple(_ROCM_TORCH_INDEX.values())
        script = (
            "set -eu\n"
            + constant.group(0)
            + "\n"
            + helper
            + "\n"
            + "for tag in "
            + " ".join(tags)
            + '; do _rocm_bnb_compatible_generic_tag "$tag"; done\n'
        )
        result = subprocess.run([shell, "-c", script], capture_output = True, text = True)
        assert result.returncode == 0, result.stderr
        shell_tags = result.stdout.splitlines()
        python_tags = [
            stack_mod._automatic_generic_pytorch_rocm_tag(ver)
            for ver, tag in _ROCM_TORCH_INDEX.items()
            if tag in tags
        ]
        # The dict is ordered newest-first, as is the shell input above.
        assert shell_tags == python_tags


class TestHardwareRocmFlag:
    """Verify IS_ROCM flag behavior without importing the full hardware module."""

    def test_hardware_py_has_is_rocm(self):
        """hardware.py should define IS_ROCM."""
        hw_path = PACKAGE_ROOT / "studio" / "backend" / "utils" / "hardware" / "hardware.py"
        source = hw_path.read_text(encoding = "utf-8")
        assert "IS_ROCM: bool" in source and "False" in source

    def test_hardware_py_sets_is_rocm_on_hip(self):
        """detect_hardware() should set IS_ROCM when torch.version.hip is set."""
        hw_path = PACKAGE_ROOT / "studio" / "backend" / "utils" / "hardware" / "hardware.py"
        source = hw_path.read_text(encoding = "utf-8")
        assert 'torch.version, "hip"' in source or "torch.version.hip" in source

    def test_hardware_py_still_returns_cuda_for_rocm(self):
        """DeviceType should remain CUDA even on ROCm -- no DeviceType.ROCM."""
        hw_path = PACKAGE_ROOT / "studio" / "backend" / "utils" / "hardware" / "hardware.py"
        source = hw_path.read_text(encoding = "utf-8")
        enum_section = source.split("class DeviceType")[1].split("\n\n")[0]
        assert "ROCM" not in enum_section

    def test_hardware_py_has_rocm_in_package_versions(self):
        """get_package_versions() should include 'rocm' key."""
        hw_path = PACKAGE_ROOT / "studio" / "backend" / "utils" / "hardware" / "hardware.py"
        source = hw_path.read_text(encoding = "utf-8")
        assert '"rocm"' in source

    def test_hardware_py_device_type_cuda_references_intact(self):
        """All existing DeviceType.CUDA references should still be present."""
        hw_path = PACKAGE_ROOT / "studio" / "backend" / "utils" / "hardware" / "hardware.py"
        source = hw_path.read_text(encoding = "utf-8")
        assert "DeviceType.CUDA" in source
        assert "DEVICE = DeviceType.CUDA" in source

    def test_is_rocm_exported_from_init(self):
        """IS_ROCM should be exported from hardware __init__.py."""
        init_path = PACKAGE_ROOT / "studio" / "backend" / "utils" / "hardware" / "__init__.py"
        source = init_path.read_text(encoding = "utf-8")
        assert "IS_ROCM" in source

    def test_is_rocm_in_all_list(self):
        """IS_ROCM should be in __all__ list in __init__.py."""
        init_path = PACKAGE_ROOT / "studio" / "backend" / "utils" / "hardware" / "__init__.py"
        source = init_path.read_text(encoding = "utf-8")
        assert '"IS_ROCM"' in source

    def test_get_package_versions_returns_rocm_key(self):
        """get_package_versions() source should return both 'cuda' and 'rocm' keys."""
        hw_path = PACKAGE_ROOT / "studio" / "backend" / "utils" / "hardware" / "hardware.py"
        source = hw_path.read_text(encoding = "utf-8")
        func_start = source.find("def get_package_versions")
        func_body = source[func_start : source.find("\ndef ", func_start + 1)]
        assert '"cuda"' in func_body
        assert '"rocm"' in func_body

    def test_distributed_stubs_cover_is_torchelastic_launched(self):
        """Must stub is_torchelastic_launched (Windows ROCm torch.distributed lacks it)."""
        hw_path = PACKAGE_ROOT / "studio" / "backend" / "utils" / "hardware" / "hardware.py"
        source = hw_path.read_text(encoding = "utf-8")
        assert "is_torchelastic_launched" in source

    def test_distributed_stubs_cover_core_helpers(self):
        """_determine_attention_impl_for_gpu_estimate must stub the four core distributed helpers."""
        hw_path = PACKAGE_ROOT / "studio" / "backend" / "utils" / "hardware" / "hardware.py"
        source = hw_path.read_text(encoding = "utf-8")
        for attr in ("is_initialized", "is_available", "get_rank", "get_world_size"):
            assert attr in source, f"distributed stub for '{attr}' missing from hardware.py"


class TestTokenizerErrorMessage:
    """Verify the AMD error message is updated."""

    def test_no_old_amd_message(self):
        """Old 'We do not support AMD' message should be gone."""
        tu_path = PACKAGE_ROOT / "unsloth" / "tokenizer_utils.py"
        source = tu_path.read_text(encoding = "utf-8")
        assert "We do not support AMD" not in source

    def test_new_message_has_docs_link(self):
        """New message should point to Unsloth AMD docs."""
        tu_path = PACKAGE_ROOT / "unsloth" / "tokenizer_utils.py"
        source = tu_path.read_text(encoding = "utf-8")
        assert "docs.unsloth.ai" in source or "No GPU detected" in source


class TestInstallShStructure:
    """Verify install.sh structural properties without running it."""

    def test_no_here_strings(self):
        """install.sh must not use the bash-only `<<<` here-string operator (breaks dash)."""
        import re

        sh_path = PACKAGE_ROOT / "install.sh"
        source = sh_path.read_text(encoding = "utf-8")
        for i, line in enumerate(source.splitlines(), 1):
            stripped = line.lstrip()
            if stripped.startswith("#"):
                continue
            # Strip quoted literals so `<<<` inside them is ignored.
            unquoted = re.sub(r"'[^']*'", "", line)
            unquoted = re.sub(r'"[^"]*"', "", unquoted)
            assert "<<<" not in unquoted, f"install.sh:{i} uses non-POSIX <<< here-string"

    def test_rocm_detection_present(self):
        """install.sh should have ROCm detection in get_torch_index_url."""
        sh_path = PACKAGE_ROOT / "install.sh"
        source = sh_path.read_text(encoding = "utf-8")
        assert "amd-smi" in source

    def test_cpu_index_note_respects_explicit_pin(self):
        """Pinned CPU index is a request, not a failure: the pin arm must open the chain holding the
        note."""
        sh_path = PACKAGE_ROOT / "install.sh"
        source = sh_path.read_text(encoding = "utf-8")
        self._assert_guarded_by_pin_arm(
            source,
            'substep "AMD GPU detected, but no usable ROCm/HIP install',
            'substep "CPU-only PyTorch (index pinned via UNSLOTH_TORCH_INDEX_URL / _FAMILY)."',
            "elif _has_amd_rocm_gpu; then",
            "the */cpu note must check the explicit pin before diagnosing ROCm",
        )
        assert (
            '[ "$OS" = "wsl" ] && [ "$_torch_index_pinned" = false ]' in source
        ), "ROCm-on-WSL guidance is detection advice; skip it for pinned installs"
        self._assert_guarded_by_pin_arm(
            source,
            'step "gpu" "AMD GPU (no usable ROCm -- CPU fallback)"',
            'step "gpu" "AMD GPU (torch index pinned: $_torch_index_leaf)"',
            "else",
            "the gpu summary must not claim no usable ROCm for a pinned index",
        )

    _PIN_ARM = 'if [ "$_torch_index_pinned" = true ]'

    # `fi` after `;` too: install.sh closes chains inline (`else :; fi`).
    _CLOSER = re.compile(r"(?:^|;)\s*fi\b")

    # install.sh indents in 4-space steps; compare against the step, not the exact indent.
    _INDENT_STEP = 4

    @staticmethod
    def _indent(line):
        return len(line) - len(line.lstrip())

    @classmethod
    def _assert_guarded_by_pin_arm(cls, source, needle, pinned_note, arm, message):
        """Indentation check: the pin arm must be an exact line, and the message must sit in its
        open chain."""
        lines = source.splitlines()
        # Skip comments: a commented-out `substep` is not a message users ever see.
        hits = [
            i
            for i, line in enumerate(lines)
            if needle in line and not line.lstrip().startswith("#")
        ]
        assert len(hits) == 1, f"expected exactly one {needle!r} in install.sh, found {len(hits)}"
        message_line = hits[0]

        opens = [i for i in range(message_line) if lines[i].strip() == cls._PIN_ARM + "; then"]
        assert opens, f"{message}\nno pin check appears anywhere above the message"
        pin_line = opens[-1]
        indent = cls._indent(lines[pin_line])
        between = range(pin_line + 1, message_line)

        closed = [
            i + 1
            for i in between
            if cls._CLOSER.search(lines[i].strip())
            and cls._indent(lines[i]) < indent + cls._INDENT_STEP
        ]
        assert not closed, (
            f"{message}\ninstall.sh:{pin_line + 1} opens on the pin check but the chain "
            f"closes at install.sh:{closed[0]}, before the message at "
            f"install.sh:{message_line + 1}, so the two are unrelated"
        )
        later_arms = [
            i
            for i in between
            if re.match(r"^(elif|else)\b", lines[i].strip()) and cls._indent(lines[i]) == indent
        ]
        assert later_arms, (
            f"{message}\nthe message at install.sh:{message_line + 1} is inside the pin "
            f"arm opened at install.sh:{pin_line + 1}, so a pinned host still reaches it"
        )
        enclosing_arm = lines[later_arms[-1]].strip()
        assert enclosing_arm == arm, (
            f"{message}\nthe message at install.sh:{message_line + 1} sits in "
            f"`{enclosing_arm}` at install.sh:{later_arms[-1] + 1}, not in the expected "
            f"`{arm}`, so the chain guarding it is not the one this test describes"
        )
        pin_body = lines[pin_line + 1 : later_arms[0]]
        assert any(
            pinned_note in line and not line.lstrip().startswith("#") for line in pin_body
        ), (
            f"{message}\nthe pin arm at install.sh:{pin_line + 1} no longer says "
            f"{pinned_note!r}, so a pinned host is told nothing at all"
        )

    _ROCM_VERSION_SOURCES = (
        "_rocm_tag_from_amd_smi",
        "_rocm_tag_from_version_file",
        "_rocm_tag_from_hipconfig",
        "_rocm_tag_from_dpkg",
        "_rocm_tag_from_rpm",
    )

    def _rocm_version_detection_script(self, source: str, rocm_prefix: str) -> str:
        """Redirects /opt/rocm to rocm_prefix, so a real ROCm install cannot answer a probe and skew
        the test."""
        parts = []
        # _run_bounded first: _rocm_tag_from_rpm routes through it.
        for name in (
            "_run_bounded",
            *self._ROCM_VERSION_SOURCES,
            "_highest_rocm_tag",
            "_detect_rocm_version_tag",
        ):
            body = _extract_sh_function_body(source, name)
            assert body, f"install.sh no longer defines {name}()"
            parts.append(body)
        assignment = re.search(
            r'^        _rocm_tag=\$\(_detect_rocm_version_tag.*?\|\| _rocm_tag=""\n',
            source,
            re.S | re.M,
        )
        assert assignment, "could not extract the guarded _rocm_tag assignment"
        parts.append(assignment.group(0))
        return "\n".join(parts).replace("/opt/rocm", rocm_prefix.replace("\\", "/"))

    def test_rocm_version_chain_survives_no_source_under_set_e(self):
        """With no version source, detection must return empty and succeed, or set -e aborts before
        the WARN."""
        shell = shutil.which("bash")
        if not shell:
            pytest.skip("bash needed to execute the version chain")
        sh_path = PACKAGE_ROOT / "install.sh"
        source = sh_path.read_text(encoding = "utf-8")
        with tempfile.TemporaryDirectory() as d:
            script_body = self._rocm_version_detection_script(source, os.path.join(d, "rocm"))
            # Tools exist on PATH but yield nothing usable, like a box with the
            # probe tools installed and no rocm-core package.
            for name in ("amd-smi", "hipconfig", "dpkg-query", "rpm"):
                p = os.path.join(d, name)
                with open(p, "w", encoding = "utf-8") as f:
                    f.write("#!/bin/sh\nexit 1\n")
                os.chmod(p, 0o755)
            script = (
                "set -euo pipefail\n" + script_body + '\nprintf "SURVIVED:%s\\n" "$_rocm_tag"\n'
            )
            env = dict(os.environ, PATH = d + os.pathsep + os.environ.get("PATH", ""))
            r = _run_bash_script(shell, script, env = env, capture_output = True, text = True)
            assert r.returncode == 0, f"version detection aborted under set -e: {r.stderr}"
            assert r.stdout.strip() == "SURVIVED:", r.stdout
        assert "rocm" in source.lower()

    def test_rocm_version_detection_takes_the_highest_source(self):
        """Every ROCm version source is read and the highest wins, since the first answer can be an
        older 5.7."""
        shell = shutil.which("bash")
        if not shell:
            pytest.skip("bash needed to execute the version chain")
        source = (PACKAGE_ROOT / "install.sh").read_text(encoding = "utf-8")
        # Whichever position holds the 6.4 reading must win; version-file is /opt/rocm/.info/version.
        outputs = {
            "amd-smi": "AMDSMI Tool: 25.0.1 | ROCm version: {v}",
            "version-file": "{v}.1-98",
            "hipconfig": "{v}.31921-0",
            "dpkg-query": "1:{v}.4-1",
            "rpm": "{v}.4-1",
        }
        for winner in outputs:
            with tempfile.TemporaryDirectory() as d:
                rocm_prefix = os.path.join(d, "rocm")
                script_body = self._rocm_version_detection_script(source, rocm_prefix)
                for name, template in outputs.items():
                    line = template.format(v = "6.4" if name == winner else "5.7")
                    if name == "version-file":
                        os.makedirs(os.path.join(rocm_prefix, ".info"), exist_ok = True)
                        with open(
                            os.path.join(rocm_prefix, ".info", "version"), "w", encoding = "utf-8"
                        ) as f:
                            f.write(line + "\n")
                        continue
                    if name == "dpkg-query":
                        _write_dpkg_query_stub(os.path.join(d, name), line, "installed")
                        continue
                    p = os.path.join(d, name)
                    with open(p, "w", encoding = "utf-8") as f:
                        f.write(f"#!/bin/sh\necho '{line}'\n")
                    os.chmod(p, 0o755)
                script = "set -euo pipefail\n" + script_body + '\nprintf "TAG:%s\\n" "$_rocm_tag"\n'
                env = dict(os.environ, PATH = d + os.pathsep + os.environ.get("PATH", ""))
                r = _run_bash_script(shell, script, env = env, capture_output = True, text = True)
                assert r.returncode == 0, r.stderr
                assert r.stdout.strip() == "TAG:rocm6.4", (
                    f"{winner} reported 6.4 while every other source reported 5.7, "
                    f"but detection resolved {r.stdout.strip()}"
                )

    def test_debian_split_runtime_uses_installed_hsa_runtime_package(self):
        """Debian 13 may have hipconfig 5.7, HSA runtime 6.1, and no rocm-core."""
        shell = shutil.which("bash")
        if not shell:
            pytest.skip("bash needed to execute the version chain")
        source = (PACKAGE_ROOT / "install.sh").read_text(encoding = "utf-8")

        with tempfile.TemporaryDirectory() as d:
            rocm_prefix = os.path.join(d, "rocm")
            script_body = self._rocm_version_detection_script(source, rocm_prefix)

            hipconfig = os.path.join(d, "hipconfig")
            with open(hipconfig, "w", encoding = "utf-8") as f:
                f.write("#!/bin/sh\necho '5.7.31921-0'\n")
            os.chmod(hipconfig, 0o755)

            _write_dpkg_query_stub(
                os.path.join(d, "dpkg-query"),
                "1:6.1.2-2",
                "installed",
                package = "libhsa-runtime64-1",
            )

            for name in ("amd-smi", "rpm"):
                p = os.path.join(d, name)
                with open(p, "w", encoding = "utf-8") as f:
                    f.write("#!/bin/sh\nexit 1\n")
                os.chmod(p, 0o755)

            script = "set -euo pipefail\n" + script_body + '\nprintf "TAG:%s\\n" "$_rocm_tag"\n'
            env = dict(
                os.environ,
                PATH = d + os.pathsep + os.environ.get("PATH", ""),
            )
            r = _run_bash_script(
                shell,
                script,
                env = env,
                capture_output = True,
                text = True,
            )
            assert r.returncode == 0, r.stderr
            assert r.stdout.strip() == "TAG:rocm6.1", r.stdout

    def test_removed_dpkg_rocm_core_cannot_pick_the_wheels(self):
        """Only dpkg state installed counts: a removed rocm-core keeps its old version and would win."""
        shell = shutil.which("bash")
        if not shell:
            pytest.skip("bash needed to execute the version chain")
        source = (PACKAGE_ROOT / "install.sh").read_text(encoding = "utf-8")
        for status, expected in (("config-files", "TAG:rocm6.1"), ("installed", "TAG:rocm7.0")):
            with tempfile.TemporaryDirectory() as d:
                rocm_prefix = os.path.join(d, "rocm")
                script_body = self._rocm_version_detection_script(source, rocm_prefix)
                os.makedirs(os.path.join(rocm_prefix, ".info"), exist_ok = True)
                with open(
                    os.path.join(rocm_prefix, ".info", "version"), "w", encoding = "utf-8"
                ) as f:
                    f.write("6.1.2-98\n")
                _write_dpkg_query_stub(os.path.join(d, "dpkg-query"), "1:7.0.0-1", status)
                # Silence the rest so a real ROCm on the test host cannot answer.
                for name in ("amd-smi", "hipconfig", "rpm"):
                    p = os.path.join(d, name)
                    with open(p, "w", encoding = "utf-8") as f:
                        f.write("#!/bin/sh\nexit 1\n")
                    os.chmod(p, 0o755)
                script = "set -euo pipefail\n" + script_body + '\nprintf "TAG:%s\\n" "$_rocm_tag"\n'
                env = dict(os.environ, PATH = d + os.pathsep + os.environ.get("PATH", ""))
                r = _run_bash_script(shell, script, env = env, capture_output = True, text = True)
                assert r.returncode == 0, r.stderr
                assert r.stdout.strip() == expected, (
                    f"dpkg rocm-core 7.0 in state {status!r} next to a 6.1 version file "
                    f"resolved {r.stdout.strip()}, expected {expected}"
                )

    def test_cuda_precedence(self):
        """AMD probes run only without NVIDIA, or when UNSLOTH_FORCE_ROCM_TORCH explicitly requests ROCm."""
        sh_path = PACKAGE_ROOT / "install.sh"
        source = sh_path.read_text(encoding = "utf-8")
        body = _extract_sh_function_body(source, "get_torch_index_url")
        nvidia_call = body.find("_has_usable_nvidia_gpu")
        # _nvidia_detected, not -z $_smi: proc-only NVIDIA hosts have no nvidia-smi.
        no_nvidia_branch = body.find('if [ "$_nvidia_detected" -eq 0 ]')
        if no_nvidia_branch < 0:
            no_nvidia_branch = body.find('if [ -z "$_smi" ]')
        assert nvidia_call >= 0, "get_torch_index_url should call _has_usable_nvidia_gpu"
        assert no_nvidia_branch >= 0, "get_torch_index_url should gate ROCm on no-nvidia branch"
        assert (
            nvidia_call < no_nvidia_branch
        ), "NVIDIA detection should run before the no-NVIDIA branch"

        assert (
            body.find("_has_amd_rocm_gpu", no_nvidia_branch) > no_nvidia_branch
        ), "ROCm detection should sit inside the 'no NVIDIA' branch"

        # A bare probe before the branch gives AMD precedence over CUDA. Drop comment lines
        # (they name the probe) and join continuations (guard and probe straddle a backslash).
        lines = [
            line
            for line in body[:no_nvidia_branch].splitlines()
            if not line.lstrip().startswith("#")
        ]
        statement, guarded = [], []
        for line in lines:
            statement.append(line)
            if not line.rstrip().endswith("\\"):
                guarded.append(" ".join(statement))
                statement = []
        for stmt in guarded:
            if "_has_amd_rocm_gpu" not in stmt:
                continue
            assert "_rocm_torch_explicitly_requested" in stmt, (
                "an AMD probe before the no-NVIDIA branch must be guarded by "
                f"_rocm_torch_explicitly_requested, got: {stmt.strip()!r}"
            )

    def test_bitsandbytes_amd_install(self):
        """install.sh should install bitsandbytes for AMD when ROCm detected."""
        sh_path = PACKAGE_ROOT / "install.sh"
        source = sh_path.read_text(encoding = "utf-8")
        assert "bitsandbytes" in source
        assert "rocm*)" in source

    def test_cpu_hint_mentions_amd(self):
        """CPU-only hint should mention AMD ROCm."""
        sh_path = PACKAGE_ROOT / "install.sh"
        source = sh_path.read_text(encoding = "utf-8")
        assert "ROCm" in source

    def test_rocm72_supported_future_capped(self):
        """ROCm 7.2 should pass through directly; 7.3+ falls back to rocm7.2."""
        sh_path = PACKAGE_ROOT / "install.sh"
        source = sh_path.read_text(encoding = "utf-8")
        assert 'echo "$_base/rocm7.2"' in source  # fallback for unknown future versions
        assert "_rocm_bnb_compatible_generic_tag" in source
        assert "rocm6.*" in source
        assert "rocm7.0" in source
        assert "rocm7.1" in source
        assert "rocm7.2" in source

    def test_rocm_tag_validation_guard_exists(self):
        """install.sh should validate _rocm_tag with a case guard."""
        sh_path = PACKAGE_ROOT / "install.sh"
        source = sh_path.read_text(encoding = "utf-8")
        assert "rocm[1-9]*.[0-9]*)" in source
        assert '_rocm_tag=""' in source

    def test_dpkg_epoch_handling(self):
        """install.sh should strip Debian epoch prefix from dpkg-query output."""
        sh_path = PACKAGE_ROOT / "install.sh"
        source = sh_path.read_text(encoding = "utf-8")
        assert 'sub(/^[0-9]+:/, "", v)' in source

    def test_no_double_bracket_in_rocm_block(self):
        """ROCm block must not use bash-only [[ ]] (POSIX char classes [[:space:]] are fine)."""
        sh_path = PACKAGE_ROOT / "install.sh"
        source = sh_path.read_text(encoding = "utf-8")
        func_start = source.find("get_torch_index_url()")
        func_end = source.find("\n}", func_start)
        func_body = source[func_start:func_end]
        import re

        for i, line in enumerate(func_body.splitlines(), 1):
            stripped = line.lstrip()
            if stripped.startswith("#"):
                continue
            # Strip POSIX char classes [[:foo:]] before checking for [[ ]].
            cleaned = re.sub(r"\[\[:[a-z]+:\]\]", "", line)
            assert "[[" not in cleaned, f"get_torch_index_url line {i} uses non-POSIX [["

    def test_no_arithmetic_expansion_in_rocm_block(self):
        """ROCm detection block should not use (( )) (bash-only)."""
        sh_path = PACKAGE_ROOT / "install.sh"
        source = sh_path.read_text(encoding = "utf-8")
        func_start = source.find("get_torch_index_url()")
        func_end = source.find("\n}", func_start)
        func_body = source[func_start:func_end]
        for i, line in enumerate(func_body.splitlines(), 1):
            stripped = line.lstrip()
            if stripped.startswith("#"):
                continue
            assert (
                "((" not in line or "))" not in line or "$(()" in line
            ), f"get_torch_index_url line {i} may use non-POSIX (( ))"

    def test_macos_returns_cpu_before_rocm_check(self):
        """macOS should return CPU immediately (before any ROCm check)."""
        sh_path = PACKAGE_ROOT / "install.sh"
        source = sh_path.read_text(encoding = "utf-8")
        func_start = source.find("get_torch_index_url()")
        func_body = source[func_start:]
        darwin_pos = func_body.find("Darwin")
        rocm_pos = func_body.find("amd-smi")
        assert darwin_pos < rocm_pos, "macOS check should come before ROCm detection"

    def test_unsloth_torch_backend_exported_after_get_torch_index_url(self):
        """install.sh exports UNSLOTH_TORCH_BACKEND after TORCH_INDEX_URL (lets the stack skip GPU re-detection)."""
        sh_path = PACKAGE_ROOT / "install.sh"
        source = sh_path.read_text(encoding = "utf-8")
        torch_url_pos = source.find("TORCH_INDEX_URL=$(get_torch_index_url)")
        # The export itself, not the first mention: a comment above the block mentions it too.
        backend_pos = source.find("export UNSLOTH_TORCH_BACKEND=")
        assert backend_pos > 0, "UNSLOTH_TORCH_BACKEND must be set in install.sh"
        assert (
            backend_pos > torch_url_pos
        ), "UNSLOTH_TORCH_BACKEND must be set AFTER TORCH_INDEX_URL is resolved"
        assert '"cuda"' in source[backend_pos : backend_pos + 500]
        assert '"rocm"' in source[backend_pos : backend_pos + 500]
        assert '"cpu"' in source[backend_pos : backend_pos + 500]
        assert "export UNSLOTH_TORCH_BACKEND" in source

    def test_kfd_sysfs_amd_vendor_check_in_has_amd_rocm_gpu(self):
        """The sysfs fallback must require AMD vendor_id 4098 (nvidia-open registers KFD nodes too)."""
        sh_path = PACKAGE_ROOT / "install.sh"
        source = sh_path.read_text(encoding = "utf-8")
        func_body = _extract_sh_function_body(source, "_has_amd_rocm_gpu")
        assert (
            "vendor_id" in func_body
        ), "_has_amd_rocm_gpu sysfs fallback must check vendor_id to exclude NVIDIA KFD nodes"
        assert (
            "4098" in func_body
        ), "_has_amd_rocm_gpu sysfs fallback must require AMD vendor_id 4098 (0x1002)"

    def test_kfd_awk_vendor_check_is_per_line(self):
        """KFD awk must decide on a single vendor_id line with no cross-node state; only AMD GPUs
        report 4098."""
        sh_path = PACKAGE_ROOT / "install.sh"
        source = sh_path.read_text(encoding = "utf-8")
        func_body = _extract_sh_function_body(source, "_has_amd_rocm_gpu")
        assert "$2 == 4098" in func_body, (
            "_has_amd_rocm_gpu KFD awk must match `vendor_id 4098` as a single-line "
            "condition so no per-node state can leak across KFD nodes"
        )
        assert "/gpu_id/" not in func_body, (
            "_has_amd_rocm_gpu KFD awk must not key on a gpu_id line: gpu_id is a "
            "sibling sysfs file, not a line in properties, so it never matches there"
        )

    def test_setup_sh_kfd_awk_matches_install_sh(self):
        """setup.sh KFD awk must match install.sh's per-line vendor_id check, or ROCm hosts get CPU
        llama.cpp."""
        source = (PACKAGE_ROOT / "studio" / "setup.sh").read_text(encoding = "utf-8")
        assert (
            "$2 == 4098" in source
        ), "setup.sh KFD awk must match `vendor_id 4098` as a single-line condition"
        assert (
            "/gpu_id/" not in source
        ), "setup.sh KFD awk must not key on a gpu_id line inside properties"

    def test_kfd_only_torch_falls_back_to_cpu(self):
        """An AMD host with unreadable gfx gets CPU torch, as a generic rocm index has no gfx to
        correct it."""
        source = (PACKAGE_ROOT / "install.sh").read_text(encoding = "utf-8")
        body = _extract_sh_function_body(source, "get_torch_index_url")
        probe = body.find("_amd_gfx_probe=$(_probe_amd_gfx_arch)")
        assert probe >= 0, "get_torch_index_url must probe the gfx arch before picking a rocm index"
        # Reads gfx from rocminfo AND amd-smi, so a non-enumerating probe still falls to CPU.
        helper = _extract_sh_function_body(source, "_probe_amd_gfx_arch")
        assert helper, "install.sh must define the shared _probe_amd_gfx_arch helper"
        assert (
            "rocminfo 2>/dev/null) | grep -oE 'gfx" in helper
        ), "probe must read gfx from rocminfo"
        assert (
            "amd-smi list 2>/dev/null) | grep -oE 'gfx" in helper
        ), "probe must read gfx from amd-smi"
        # Clearing the masks keeps ROCR_VISIBLE_DEVICES=-1 from blinding KFD detection.
        assert (
            '_pg_strip="ROCR_VISIBLE_DEVICES HIP_VISIBLE_DEVICES"' in helper
        ), "the gfx probe must clear the visibility masks so a mask can't force CPU"
        assert "unset $_pg_strip" in helper, "the strip list must reach unset"
        cpu_guard = body.find('if [ -z "$_amd_gfx_probe" ]')
        assert cpu_guard >= 0, "unreadable gfx must fall back to CPU"
        assert cpu_guard < body.find(
            "_rocm_tag="
        ), "the gfx gate must run before the ROCm version/index selection"

    def test_kfd_only_llama_requires_hipcc(self):
        """Without hipcc, a gfx-unknown host must not get --has-rocm, since the source build would
        then fail."""
        source = (PACKAGE_ROOT / "studio" / "setup.sh").read_text(encoding = "utf-8")
        idx = source.find("_PREBUILT_CMD+=(--has-rocm)")
        assert idx >= 0, "setup.sh must still be able to forward --has-rocm"
        window = source[max(0, idx - 900) : idx]
        assert (
            "hipcc" in window
        ), "the gfx-unknown --has-rocm branch must gate on hipcc (a usable HIP toolchain)"
        assert (
            "command -v hipcc" in window or "/opt/rocm/bin/hipcc" in window
        ), "hipcc presence must be checked via command -v or the rocm bin path"
        assert (
            "/opt/rocm-*/bin/hipcc" in window
        ), "the hipcc gate must also accept a versioned /opt/rocm-*/bin/hipcc toolchain"

    def test_gfx_unknown_guard_honors_override(self):
        """A user-set UNSLOTH_ROCM_GFX_ARCH must seed the gfx probe before the CPU
        fallback: an air-gapped/rocminfo-less Strix host that names its arch should
        still reach a rocm index instead of being forced to CPU."""
        source = (PACKAGE_ROOT / "install.sh").read_text(encoding = "utf-8")
        helper = _extract_sh_function_body(source, "_probe_amd_gfx_arch")
        assert helper, "install.sh must define the shared _probe_amd_gfx_arch helper"
        seed = helper.find("$(printf")
        assert seed >= 0, "the gfx probe must seed from UNSLOTH_ROCM_GFX_ARCH"
        assert "UNSLOTH_ROCM_GFX_ARCH" in helper[seed : seed + 80]
        assert seed < helper.find(
            "rocminfo 2>/dev/null) | grep -oE 'gfx"
        ), "the override must be read before probing rocminfo"
        body = _extract_sh_function_body(source, "get_torch_index_url")
        call = body.find("_amd_gfx_probe=$(_probe_amd_gfx_arch)")
        assert call >= 0, "get_torch_index_url must call the shared probe"
        assert call < body.find(
            'if [ -z "$_amd_gfx_probe" ]; then'
        ), "the probe must run before the CPU fallback guard"

    def test_gfx_override_seeds_reroute_without_tools(self):
        """The Strix reroute must honour UNSLOTH_ROCM_GFX_ARCH even when rocminfo and
        amd-smi are absent, so a manual override reaches the arch index; with no
        override and no tools it must stay empty (no false Strix routing)."""
        shell = shutil.which("bash")
        if not shell:
            pytest.skip("bash needed to execute the probe block")
        source = _INSTALL_SH_PATH.read_text(encoding = "utf-8")
        block = re.search(
            r'^        _gfx_all=\$\(printf[^\n]*\n.*?(?=^        _strix_gfx="")',
            source,
            re.S | re.M,
        )
        assert block, "could not extract the gfx-detection block"
        with tempfile.TemporaryDirectory() as d:
            for name in ("rocminfo", "amd-smi"):
                p = os.path.join(d, name)
                with open(p, "w", encoding = "utf-8") as f:
                    f.write("#!/bin/sh\nexit 0\n")
                os.chmod(p, 0o755)
            script = (
                'set -euo pipefail\nHIP_VISIBLE_DEVICES=""\nROCR_VISIBLE_DEVICES=""\n'
                + block.group(0)
                + '\nprintf "OK:%s\\n" "$_gfx_all"\n'
            )

            def run(**extra):
                env = dict(os.environ, PATH = d + os.pathsep + os.environ.get("PATH", ""), **extra)
                return _run_bash_script(shell, script, env = env, capture_output = True, text = True)

            r = run(UNSLOTH_ROCM_GFX_ARCH = "GFX1151")
            assert r.returncode == 0, f"override probe aborted: {r.stderr}"
            assert "OK:gfx1151" in r.stdout, f"override not honoured/lowercased: {r.stdout!r}"
            r2 = run()
            assert r2.returncode == 0, f"empty probe aborted: {r2.stderr}"
            assert (
                "OK:\n" in r2.stdout or r2.stdout.strip() == "OK:"
            ), f"no override + no tools must leave gfx empty: {r2.stdout!r}"

    def test_gfx_probe_ignores_visibility_mask(self):
        """The gfx probe clears visibility masks, since KFD detection ignores env while rocminfo
        honours them."""
        shell = shutil.which("bash")
        if not shell:
            pytest.skip("bash needed to execute the probe block")
        source = _INSTALL_SH_PATH.read_text(encoding = "utf-8")
        probe_fn = _extract_sh_function_body(source, "_probe_amd_gfx_arch")
        assert probe_fn, "could not extract _probe_amd_gfx_arch"
        with tempfile.TemporaryDirectory() as d:
            with open(os.path.join(d, "rocminfo"), "w", encoding = "utf-8") as f:
                f.write(
                    "#!/bin/sh\n"
                    'if [ "${ROCR_VISIBLE_DEVICES:-}" = "-1" ]; then echo "no agents"; exit 0; fi\n'
                    'echo "  Name:  gfx1151"\n'
                )
            os.chmod(os.path.join(d, "rocminfo"), 0o755)
            script = (
                "set -euo pipefail\n"
                "_ensure_rocm_probe_env() { :; }\n"
                + probe_fn
                + '\n_amd_gfx_probe=$(_probe_amd_gfx_arch)\nprintf "OK:%s\\n" "$_amd_gfx_probe"\n'
            )

            def run(**extra):
                env = dict(os.environ, PATH = d + os.pathsep + os.environ.get("PATH", ""), **extra)
                return _run_bash_script(shell, script, env = env, capture_output = True, text = True)

            r = run(ROCR_VISIBLE_DEVICES = "-1")
            assert r.returncode == 0, f"masked probe aborted: {r.stderr}"
            assert (
                "OK:gfx1151" in r.stdout
            ), f"a visibility mask must not blind the gfx probe: {r.stdout!r}"

    def test_kfd_only_inferable_gfx_defers_to_reroute(self):
        """A KFD-only host with an inferable arch defers the CPU warning; the reroute upgrades the
        wheels."""
        shell = shutil.which("bash")
        if not shell:
            pytest.skip("bash needed to execute get_torch_index_url")
        source = _INSTALL_SH_PATH.read_text(encoding = "utf-8")
        fn = _extract_sh_function_body(source, "get_torch_index_url")
        probe_fn = _extract_sh_function_body(source, "_probe_amd_gfx_arch")
        family_fn = _extract_sh_function_body(source, "_amd_arch_index_family_for_gfx")
        arch_fns = "\n".join(
            _extract_sh_function_body(source, _n)
            for _n in (
                "_amd_probe_arches",
                "_amd_agreed_index_family",
                "_amd_sole_index_arch",
            )
        )
        assert fn and probe_fn and family_fn and arch_fns
        with tempfile.TemporaryDirectory() as d:
            with open(os.path.join(d, "uname"), "w", encoding = "utf-8", newline = "\n") as f:
                f.write('#!/bin/sh\ncase "${1:-}" in -m) echo x86_64 ;; *) echo Linux ;; esac\n')
            for name in ("rocminfo", "amd-smi"):
                with open(os.path.join(d, name), "w", encoding = "utf-8", newline = "\n") as f:
                    f.write("#!/bin/sh\nexit 0\n")
            for name in ("uname", "rocminfo", "amd-smi"):
                os.chmod(os.path.join(d, name), 0o755)

            def run(infer_stub):
                script = (
                    "set -euo pipefail\n"
                    "_ensure_rocm_probe_env() { :; }\n"
                    "_trim_index_path_slashes() { printf '%s\\n' \"$1\"; }\n"
                    "_has_usable_nvidia_gpu() { return 1; }\n"
                    "_has_amd_rocm_gpu() { return 0; }\n"
                    + infer_stub
                    + "\n"
                    + probe_fn
                    + "\n"
                    + family_fn
                    + "\n"
                    + arch_fns
                    + "\n"
                    + fn
                    + "\n"
                    "get_torch_index_url\n"
                )
                # Run from a file, not -c: Windows bash mangles multi-KB -c strings.
                sp = os.path.join(d, "gtiu.sh")
                with open(sp, "w", encoding = "utf-8", newline = "\n") as f:
                    f.write(script)
                env = dict(os.environ, PATH = d + os.pathsep + os.environ.get("PATH", ""))
                for var in (
                    "UNSLOTH_ROCM_GFX_ARCH",
                    "UNSLOTH_TORCH_INDEX_URL",
                    "UNSLOTH_TORCH_INDEX_FAMILY",
                    "UNSLOTH_PYTORCH_MIRROR",
                    "ROCR_VISIBLE_DEVICES",
                    "HIP_VISIBLE_DEVICES",
                ):
                    env.pop(var, None)
                return subprocess.run(
                    [shell, sp.replace("\\", "/")], env = env, capture_output = True, text = True
                )

            r = run("_infer_linux_amd_gfx_arch() { echo gfx1100; }")
            assert r.returncode == 0, f"inferable case aborted: {r.stderr}"
            assert r.stdout.strip().endswith(
                "/cpu"
            ), f"must hand */cpu to the reroute: {r.stdout!r}"
            assert (
                "inferring gfx1100" in r.stderr
            ), f"must announce the inference handoff: {r.stderr!r}"
            assert (
                "installing CPU-only PyTorch" not in r.stderr
            ), f"must not promise a CPU-only install the reroute will override: {r.stderr!r}"
            r2 = run("_infer_linux_amd_gfx_arch() { return 1; }")
            assert r2.returncode == 0, f"uninferable case aborted: {r2.stderr}"
            assert r2.stdout.strip().endswith("/cpu")
            assert (
                "installing CPU-only PyTorch" in r2.stderr
            ), f"uninferable gfx must keep the actionable CPU warning: {r2.stderr!r}"
            r3 = run("_infer_linux_amd_gfx_arch() { echo gfx906; }")
            assert r3.returncode == 0, f"unsupported-family case aborted: {r3.stderr}"
            assert r3.stdout.strip().endswith("/cpu")
            assert (
                "installing CPU-only PyTorch" in r3.stderr
            ), f"an inferred arch with no wheel family must keep the CPU warning: {r3.stderr!r}"

    def test_no_version_cpu_warning_respects_gfx_override(self):
        """A supported UNSLOTH_ROCM_GFX_ARCH defers the CPU warning; the reroute installs per-arch
        wheels."""
        shell = shutil.which("bash")
        if not shell:
            pytest.skip("bash needed to execute get_torch_index_url")
        source = _INSTALL_SH_PATH.read_text(encoding = "utf-8")
        fn = _extract_sh_function_body(source, "get_torch_index_url")
        probe_fn = _extract_sh_function_body(source, "_probe_amd_gfx_arch")
        family_fn = _extract_sh_function_body(source, "_amd_arch_index_family_for_gfx")
        bnb_floor_constant = re.search(r"^_ROCM_BNB_GENERIC_FLOOR_TAG=.*$", source, re.M)
        bnb_floor_fn = _extract_sh_function_body(source, "_rocm_bnb_compatible_generic_tag")
        arch_fns = "\n".join(
            _extract_sh_function_body(source, _n)
            for _n in (
                "_amd_probe_arches",
                "_amd_agreed_index_family",
                "_amd_sole_index_arch",
            )
        )
        # Without the version helpers, the guarded assignment swallows 127 and the endpoint is reached
        # for the wrong reason (verified by mutation).
        version_fns = [
            _extract_sh_function_body(source, name)
            for name in (
                "_rocm_tag_from_amd_smi",
                "_rocm_tag_from_version_file",
                "_rocm_tag_from_hipconfig",
                "_rocm_tag_from_dpkg",
                "_rocm_tag_from_rpm",
                "_highest_rocm_tag",
                "_detect_rocm_version_tag",
            )
        ]
        assert fn and probe_fn and family_fn and arch_fns and bnb_floor_constant and bnb_floor_fn
        assert all(version_fns), "ROCm version helpers not found in install.sh"
        with tempfile.TemporaryDirectory() as d:
            # The version file read lives in _rocm_tag_from_version_file, so rewrite the helper text.
            version_fns = [
                body.replace("/opt/rocm/.info/version", Path(d, "no-rocm-version").as_posix())
                for body in version_fns
            ]
            with open(os.path.join(d, "uname"), "w", encoding = "utf-8", newline = "\n") as f:
                f.write('#!/bin/sh\ncase "${1:-}" in -m) echo x86_64 ;; *) echo Linux ;; esac\n')
            # A dev box with real hipconfig/dpkg would resolve a version otherwise.
            with open(os.path.join(d, "amd-smi"), "w", encoding = "utf-8", newline = "\n") as f:
                f.write("#!/bin/sh\nexit 0\n")
            for name in ("hipconfig", "dpkg-query", "rpm"):
                with open(os.path.join(d, name), "w", encoding = "utf-8", newline = "\n") as f:
                    f.write("#!/bin/sh\nexit 1\n")
            for name in ("uname", "amd-smi", "hipconfig", "dpkg-query", "rpm"):
                os.chmod(os.path.join(d, name), 0o755)
            script = (
                "set -euo pipefail\n"
                "_ensure_rocm_probe_env() { :; }\n"
                "_trim_index_path_slashes() { printf '%s\\n' \"$1\"; }\n"
                "_has_usable_nvidia_gpu() { return 1; }\n"
                "_has_amd_rocm_gpu() { return 0; }\n"
                "_infer_linux_amd_gfx_arch() { return 1; }\n"
                + probe_fn
                + "\n"
                + family_fn
                + "\n"
                + arch_fns
                + "\n"
                + "\n".join(version_fns)
                + "\n"
                + bnb_floor_constant.group(0)
                + "\n"
                + bnb_floor_fn
                + "\n"
                + fn
                + "\n"
                "get_torch_index_url\n"
            )
            sp = os.path.join(d, "gtiu.sh")
            with open(sp, "w", encoding = "utf-8", newline = "\n") as f:
                f.write(script)

            def run(rocminfo_body, **extra):
                with open(os.path.join(d, "rocminfo"), "w", encoding = "utf-8", newline = "\n") as f:
                    f.write("#!/bin/sh\n" + rocminfo_body)
                os.chmod(os.path.join(d, "rocminfo"), 0o755)
                env = dict(os.environ, PATH = d + os.pathsep + os.environ.get("PATH", ""), **extra)
                for var in (
                    "UNSLOTH_TORCH_INDEX_URL",
                    "UNSLOTH_TORCH_INDEX_FAMILY",
                    "UNSLOTH_PYTORCH_MIRROR",
                    "ROCR_VISIBLE_DEVICES",
                    "HIP_VISIBLE_DEVICES",
                ):
                    env.pop(var, None)
                if "UNSLOTH_ROCM_GFX_ARCH" not in extra:
                    env.pop("UNSLOTH_ROCM_GFX_ARCH", None)
                return subprocess.run(
                    [shell, sp.replace("\\", "/")], env = env, capture_output = True, text = True
                )

            r = run("exit 0\n", UNSLOTH_ROCM_GFX_ARCH = "gfx1151")
            assert r.returncode == 0, f"override case aborted: {r.stderr}"
            assert r.stdout.strip().endswith("/cpu")
            assert (
                "falling back to CPU-only PyTorch" not in r.stderr
            ), f"a supported override must not get the false CPU warning: {r.stderr!r}"
            assert (
                "UNSLOTH_ROCM_GFX_ARCH=gfx1151 is set" in r.stderr
            ), f"the override deferral must be announced: {r.stderr!r}"
            r2 = run("exit 0\n", UNSLOTH_ROCM_GFX_ARCH = "gfx906")
            assert r2.returncode == 0, f"unsupported-override case aborted: {r2.stderr}"
            assert (
                "falling back to CPU-only PyTorch" in r2.stderr
            ), f"an unmappable override must keep the CPU warning: {r2.stderr!r}"
            # gfx1151 has its own index, so a missing version is a detection miss, not a decision.
            r3 = run('echo "  Name:  gfx1151"\n')
            assert r3.returncode == 0, f"readable-gfx case aborted: {r3.stderr}"
            assert r3.stdout.strip().endswith("/cpu")
            assert (
                "falling back to CPU-only PyTorch" not in r3.stderr
            ), f"a mapped arch must not get the CPU warning: {r3.stderr!r}"
            assert (
                "routing to AMD per-arch wheels" in r3.stderr
            ), f"a mapped arch with no version defers to the reroute: {r3.stderr!r}"
            # No per-arch index: picking a generic leaf needs a version.
            r4 = run('echo "  Name:  gfx906"\n')
            assert r4.returncode == 0, f"unmapped readable-gfx case aborted: {r4.stderr}"
            assert (
                "falling back to CPU-only PyTorch" in r4.stderr
            ), f"an unmapped arch without a version keeps the CPU warning: {r4.stderr!r}"

    def test_reroute_gate_covers_kfd_only(self):
        """The reroute fires on KFD-only hosts but never overrides a */cpu index picked with a
        readable gfx."""
        shell = shutil.which("bash")
        if not shell:
            pytest.skip("bash needed to execute the reroute block")
        source = _INSTALL_SH_PATH.read_text(encoding = "utf-8")
        block = re.search(
            r'^if \[ "\$_torch_index_pinned" = false \] && \[ "\$SKIP_TORCH" = false \] && \\\n'
            r".*?^fi\n",
            source,
            re.S | re.M,
        )
        assert block, "could not extract the runtime-less reroute block"
        family_fn = _extract_sh_function_body(source, "_amd_arch_index_family_for_gfx")
        arch_fns = "\n".join(
            _extract_sh_function_body(source, _n)
            for _n in (
                "_amd_probe_arches",
                "_amd_agreed_index_family",
                "_amd_sole_index_arch",
            )
        )
        assert family_fn and arch_fns
        with tempfile.TemporaryDirectory() as d:
            with open(os.path.join(d, "uname"), "w", encoding = "utf-8", newline = "\n") as f:
                f.write('#!/bin/sh\ncase "${1:-}" in -m) echo x86_64 ;; *) echo Linux ;; esac\n')
            os.chmod(os.path.join(d, "uname"), 0o755)

            def run(gpu_stub, probe_stub):
                script = (
                    "set -euo pipefail\n"
                    "_has_usable_nvidia_gpu() { return 1; }\n"
                    f"_has_amd_rocm_gpu() {{ {gpu_stub}; }}\n"
                    f"_probe_amd_gfx_arch() {{ {probe_stub}; }}\n"
                    "_infer_linux_amd_gfx_arch() { echo gfx1100; }\n"
                    "_strip_index_url_credentials() { printf '%s\\n' \"$1\"; }\n"
                    + family_fn
                    + "\n"
                    + arch_fns
                    + "\n"
                    "_torch_index_pinned=false\nSKIP_TORCH=false\n_ARCH=x86_64\n"
                    "TORCH_INDEX_URL=https://download.pytorch.org/whl/cpu\n"
                    + block.group(0)
                    + 'printf "URL:%s GFX:%s\\n" "$TORCH_INDEX_URL" "${UNSLOTH_ROCM_GFX_ARCH:-}"\n'
                )
                # Run from a file, not -c: Windows bash mangles multi-KB -c strings.
                sp = os.path.join(d, "reroute.sh")
                with open(sp, "w", encoding = "utf-8", newline = "\n") as f:
                    f.write(script)
                env = dict(os.environ, PATH = d + os.pathsep + os.environ.get("PATH", ""))
                for var in ("UNSLOTH_ROCM_GFX_ARCH", "UNSLOTH_AMD_ROCM_MIRROR"):
                    env.pop(var, None)
                return subprocess.run(
                    [shell, sp.replace("\\", "/")], env = env, capture_output = True, text = True
                )

            r = run("return 0", "printf '\\n'")
            assert r.returncode == 0, f"kfd-only reroute aborted: {r.stderr}"
            assert (
                "URL:https://repo.amd.com/rocm/whl/gfx110X-all/ GFX:gfx1100" in r.stdout
            ), f"KFD-only host must reach the AMD arch index: {r.stdout!r}"
            # KFD visibility is what detected this host, so do not claim /dev/kfd is missing.
            assert (
                "ROCm runtime not visible" not in r.stderr
            ), f"KFD-only reroute must not claim /dev/kfd is missing: {r.stderr!r}"
            assert (
                "visible via the kernel driver (KFD)" in r.stderr
            ), f"KFD-only reroute must name the tooling gap: {r.stderr!r}"
            r2 = run("return 0", "echo gfx1151")
            assert r2.returncode == 0, f"readable-gfx case aborted: {r2.stderr}"
            assert (
                "URL:https://download.pytorch.org/whl/cpu GFX:" in r2.stdout
            ), f"a deliberate CPU fallback must not be rerouted: {r2.stdout!r}"
            r3 = run("return 1", "printf '\\n'")
            assert r3.returncode == 0, f"undetected-GPU case aborted: {r3.stderr}"
            assert (
                "URL:https://repo.amd.com/rocm/whl/gfx110X-all/ GFX:gfx1100" in r3.stdout
            ), f"the original undetected-GPU reroute must keep working: {r3.stdout!r}"
            assert (
                "ROCm runtime not visible" in r3.stderr
            ), f"a truly runtime-invisible host keeps the original diagnostic: {r3.stderr!r}"

    @staticmethod
    def _family_for_probed(arch):
        return {"gfx1151": "gfx1151", "gfx1100": "gfx110X-all"}.get(arch, "")

    def test_no_version_reroute_routes_on_the_probed_arch(self):
        """Routes on the arch rocminfo/amd-smi READ, not the lspci marketing-name
        inference: the two disagree on a mixed APU + discrete host (unslothai#8731)."""
        shell = shutil.which("bash")
        if not shell:
            pytest.skip("bash needed to execute the reroute block")
        source = _INSTALL_SH_PATH.read_text(encoding = "utf-8")
        block = re.search(
            r'^if \[ "\$_torch_index_pinned" = false \] && \[ "\$SKIP_TORCH" = false \] && \\\n'
            r".*?^fi\n",
            source,
            re.S | re.M,
        )
        assert block, "could not extract the runtime-less reroute block"
        family_fn = _extract_sh_function_body(source, "_amd_arch_index_family_for_gfx")
        arch_fns = "\n".join(
            _extract_sh_function_body(source, _n)
            for _n in (
                "_amd_probe_arches",
                "_amd_agreed_index_family",
                "_amd_sole_index_arch",
            )
        )
        assert family_fn and arch_fns
        with tempfile.TemporaryDirectory() as d:
            with open(os.path.join(d, "uname"), "w", encoding = "utf-8", newline = "\n") as f:
                f.write('#!/bin/sh\ncase "${1:-}" in -m) echo x86_64 ;; *) echo Linux ;; esac\n')
            os.chmod(os.path.join(d, "uname"), 0o755)

            def run(
                no_version_state,
                probed_first,
                probe_stub = "echo gfx1151",
                probed_family = None,
                inferred = "gfx1100",
                override = "",
            ):
                script = (
                    "set -euo pipefail\n"
                    "_has_usable_nvidia_gpu() { return 1; }\n"
                    "_has_amd_rocm_gpu() { return 0; }\n"
                    f"_probe_amd_gfx_arch() {{ {probe_stub}; }}\n"
                    f"_infer_linux_amd_gfx_arch() {{ echo {inferred}; }}\n"
                    "_strip_index_url_credentials() { printf '%s\\n' \"$1\"; }\n"
                    + family_fn
                    + "\n"
                    + arch_fns
                    + "\n"
                    "_torch_index_pinned=false\nSKIP_TORCH=false\n_ARCH=x86_64\n"
                    f"_amd_no_rocm_version_reroute={no_version_state}\n"
                    f'_amd_probed_gfx_first="{probed_first}"\n'
                    f'_amd_probed_family="{probed_family if probed_family is not None else self._family_for_probed(probed_first)}"\n'
                    "TORCH_INDEX_URL=https://download.pytorch.org/whl/cpu\n"
                    + block.group(0)
                    + 'printf "URL:%s GFX:%s\\n" "$TORCH_INDEX_URL" "${UNSLOTH_ROCM_GFX_ARCH:-}"\n'
                )
                sp = os.path.join(d, "reroute_probe.sh")
                with open(sp, "w", encoding = "utf-8", newline = "\n") as f:
                    f.write(script)
                env = dict(os.environ, PATH = d + os.pathsep + os.environ.get("PATH", ""))
                for var in ("UNSLOTH_ROCM_GFX_ARCH", "UNSLOTH_AMD_ROCM_MIRROR"):
                    env.pop(var, None)
                if override:
                    env["UNSLOTH_ROCM_GFX_ARCH"] = override
                return subprocess.run(
                    [shell, sp.replace("\\", "/")], env = env, capture_output = True, text = True
                )

            r = run("true", "gfx1151")
            assert r.returncode == 0, f"no-version reroute aborted: {r.stderr}"
            assert (
                "URL:https://repo.amd.com/rocm/whl/gfx1151/ GFX:gfx1151" in r.stdout
            ), f"the probed arch must win over lspci inference here: {r.stdout!r}"
            r_bad = run("true", "gfx1033", probed_family = "gfx103X-all")
            assert r_bad.returncode == 0, f"gfx1033 no-version reroute aborted: {r_bad.stderr}"
            assert (
                "URL:https://download.pytorch.org/whl/cpu GFX:\n" in r_bad.stdout
            ), f"the no-version reroute must not bypass the gfx1033 gate: {r_bad.stdout!r}"
            # setup.sh prefers UNSLOTH_ROCM_GFX_ARCH and would build llama.cpp for a guessed card.
            r_pair = run("true", "", probed_family = "gfx120X-all")
            assert r_pair.returncode == 0, f"same-family pair aborted: {r_pair.stderr}"
            assert (
                "URL:https://repo.amd.com/rocm/whl/gfx120X-all/ GFX:\n" in r_pair.stdout
            ), f"a same-family pair routes but must not export a card: {r_pair.stdout!r}"
            r_mixed = run("true", "", probed_family = "gfx103X-all")
            assert r_mixed.returncode == 0, f"mixed gfx103X pair aborted: {r_mixed.stderr}"
            assert (
                "URL:https://repo.amd.com/rocm/whl/gfx103X-all/ GFX:\n" in r_mixed.stdout
            ), f"a healthy same-family GPU beside gfx1033 must keep ROCm: {r_mixed.stdout!r}"
            r2 = run("false", "", probe_stub = "printf '\\n'")
            assert r2.returncode == 0, f"inferred-arch case aborted: {r2.stderr}"
            assert (
                "URL:https://repo.amd.com/rocm/whl/gfx110X-all/ GFX:gfx1100" in r2.stdout
            ), f"an empty-probe reroute keeps the inferred arch: {r2.stdout!r}"
            r3 = run("false", "")
            assert r3.returncode == 0, f"deliberate-fallback case aborted: {r3.stderr}"
            assert (
                "URL:https://download.pytorch.org/whl/cpu GFX:" in r3.stdout
            ), f"a deliberate CPU fallback must stay un-rerouted: {r3.stdout!r}"
            # The index case table has no arm for the :sramecc+:xnack- suffix.
            r4 = run(
                "false",
                "",
                inferred = "gfx1201:sramecc+:xnack-",
                override = "gfx1201:sramecc+:xnack-",
            )
            assert r4.returncode == 0, f"suffixed override aborted: {r4.stderr}"
            assert (
                "URL:https://repo.amd.com/rocm/whl/gfx120X-all/ GFX:gfx1201" in r4.stdout
            ), f"a gcnArchName suffix must not cost the reroute: {r4.stdout!r}"

    def test_get_torch_index_url_uses_nvidia_detected_flag(self):
        """get_torch_index_url must track NVIDIA via _nvidia_detected (proc-only NVIDIA still picks CUDA)."""
        sh_path = PACKAGE_ROOT / "install.sh"
        source = sh_path.read_text(encoding = "utf-8")
        func_start = source.find("get_torch_index_url()")
        func_end = source.find("\n}", func_start)
        func_body = source[func_start:func_end]
        assert "_nvidia_detected" in func_body, (
            "get_torch_index_url must use a _nvidia_detected flag (separate from "
            "_smi) so that proc-only NVIDIA detection still selects CUDA wheels"
        )
        assert (
            '_nvidia_detected" -eq 0' in func_body or "_nvidia_detected" in func_body
        ), "get_torch_index_url AMD branch must be skipped when _nvidia_detected=1"


class TestLiveRegression:
    """Live checks that run on the actual host -- skip if no NVIDIA GPU."""

    def test_get_torch_index_url_returns_cuda_on_nvidia(self):
        """On an NVIDIA machine, get_torch_index_url should return a CUDA URL."""
        import shutil

        if not shutil.which("nvidia-smi"):
            pytest.skip("No nvidia-smi available")
        # Skip if nvidia-smi exists but lists no GPU (binary without driver).
        check = subprocess.run(
            [
                "bash",
                "-c",
                "nvidia-smi -L 2>/dev/null | awk '/^GPU[[:space:]]+[0-9]+:/{f=1} END{exit !f}'",
            ],
            capture_output = True,
        )
        if check.returncode != 0:
            pytest.skip("nvidia-smi is on PATH but no GPU is listed")

        sh_path = PACKAGE_ROOT / "install.sh"
        extract_cmd = (
            f"sed -n '/^_has_amd_rocm_gpu()/,/^}}$/p; "
            f"/^_has_usable_nvidia_gpu()/,/^}}$/p; "
            f"/^get_torch_index_url()/,/^}}$/p' '{sh_path}'"
        )
        result = subprocess.run(
            ["bash", "-c", f'eval "$({extract_cmd})"; get_torch_index_url'],
            capture_output = True,
            text = True,
            timeout = 30,
        )
        if result.returncode != 0:
            pytest.skip("Could not extract get_torch_index_url for live test")
        url = result.stdout.strip()
        assert "cu1" in url or "cuda" in url.lower(), f"Expected CUDA URL, got: {url}"


_WORKER_PATH = PACKAGE_ROOT / "studio" / "backend" / "core" / "training" / "worker.py"
_EXPORT_WORKER_PATH = PACKAGE_ROOT / "studio" / "backend" / "core" / "export" / "worker.py"
_TORCHAO_STUB_PATH = PACKAGE_ROOT / "studio" / "backend" / "core" / "_torchao_stub.py"
_EMBEDDINGS_PATH = PACKAGE_ROOT / "studio" / "backend" / "core" / "rag" / "embeddings.py"
_WHEEL_UTILS_PATH = PACKAGE_ROOT / "studio" / "backend" / "utils" / "wheel_utils.py"


class TestWorkerRocmMambaSsm:
    """Verify worker.py Mamba/SSM install logic on ROCm."""

    def test_probe_returns_hip_version_field(self):
        """The wheel probe should include hip_version, and worker.py consumes it."""
        assert "hip_version" in _WHEEL_UTILS_PATH.read_text(encoding = "utf-8")
        assert "hip_version" in _WORKER_PATH.read_text(encoding = "utf-8")

    def test_probe_script_has_getattr_hip(self):
        """Probe script should use getattr for torch.version.hip (safe on CUDA)."""
        source = _WHEEL_UTILS_PATH.read_text(encoding = "utf-8")
        assert "getattr(torch.version, 'hip', None)" in source

    def test_direct_wheel_url_returns_none_without_cuda_major(self, monkeypatch):
        """direct_wheel_url should return None when cuda_major is empty (ROCm)."""
        _worker_spec = importlib.util.spec_from_file_location("test_worker", _WORKER_PATH)
        assert _worker_spec is not None and _worker_spec.loader is not None
        worker_mod = importlib.util.module_from_spec(_worker_spec)

        # Via monkeypatch so the fake 'utils' is undone for later tests.
        loggers_mock = MagicMock()
        loggers_mock.get_logger = MagicMock(return_value = MagicMock())
        monkeypatch.setitem(sys.modules, "structlog", MagicMock())
        monkeypatch.setitem(sys.modules, "loggers", loggers_mock)
        monkeypatch.setitem(sys.modules, "utils", MagicMock())
        monkeypatch.setitem(sys.modules, "utils.hardware", MagicMock())

        try:
            _worker_spec.loader.exec_module(worker_mod)
        except Exception:
            pytest.skip("Could not load worker module in test environment")

        env_rocm = {
            "python_tag": "cp312",
            "torch_mm": "2.6",
            "cuda_major": "",
            "hip_version": "7.1.12345",
            "cxx11abi": "TRUE",
        }
        result = worker_mod.direct_wheel_url(
            filename_prefix = "causal_conv1d",
            package_version = "1.6.1",
            release_tag = "v1.6.1.post4",
            release_base_url = "https://github.com/Dao-AILab/causal-conv1d/releases/download",
            env = env_rocm,
        )
        assert result is None

    def test_hipcc_check_exists_in_source(self):
        """worker.py should check for hipcc before ROCm source builds."""
        source = _WORKER_PATH.read_text(encoding = "utf-8")
        assert "hipcc" in source

    def test_rocm_source_build_status_message(self):
        """worker.py should send a specific status for ROCm source compilation."""
        source = _WORKER_PATH.read_text(encoding = "utf-8")
        assert "Compiling" in source and "from source for ROCm" in source

    def test_rocm_build_failure_message(self):
        """worker.py should send a clear error on ROCm build failure."""
        source = _WORKER_PATH.read_text(encoding = "utf-8")
        assert "Failed to compile" in source and "for ROCm" in source

    def test_timeout_on_install(self):
        """worker.py should have a timeout on pip install subprocess."""
        source = _WORKER_PATH.read_text(encoding = "utf-8")
        assert "TimeoutExpired" in source
        assert "timeout" in source


class TestAmdGpuMonitoring:
    """Verify amd.py module structure and mock behavior."""

    def test_amd_py_exists(self):
        """amd.py should exist in the hardware directory."""
        amd_path = PACKAGE_ROOT / "studio" / "backend" / "utils" / "hardware" / "amd.py"
        assert amd_path.exists()

    def test_amd_py_has_required_functions(self):
        """amd.py should export the same function signatures as nvidia.py."""
        amd_path = PACKAGE_ROOT / "studio" / "backend" / "utils" / "hardware" / "amd.py"
        source = amd_path.read_text(encoding = "utf-8")
        assert "def get_physical_gpu_count" in source
        assert "def get_primary_gpu_utilization" in source
        assert "def get_visible_gpu_utilization" in source

    def test_amd_smi_json_parsing(self, monkeypatch):
        """Verify _extract_gpu_metrics parses amd-smi JSON correctly."""
        amd_path = PACKAGE_ROOT / "studio" / "backend" / "utils" / "hardware" / "amd.py"
        _amd_spec = importlib.util.spec_from_file_location("test_amd", amd_path)
        assert _amd_spec is not None and _amd_spec.loader is not None
        amd_mod = importlib.util.module_from_spec(_amd_spec)

        loggers_mock = MagicMock()
        loggers_mock.get_logger = MagicMock(return_value = MagicMock())
        monkeypatch.setitem(sys.modules, "loggers", loggers_mock)

        try:
            _amd_spec.loader.exec_module(amd_mod)
        except Exception:
            pytest.skip("Could not load amd module in test environment")

        gpu_data = {
            "usage": {"gfx_activity": "85"},
            "temperature": {"edge": "72"},
            "power": {
                "current_socket_power": "200.5",
                "power_cap": "300",
            },
            "vram": {
                "vram_used": 8192,  # MB
                "vram_total": 16384,
            },
        }
        metrics = amd_mod._extract_gpu_metrics(gpu_data)
        assert metrics["gpu_utilization_pct"] == 85.0
        assert metrics["temperature_c"] == 72.0
        assert metrics["power_draw_w"] == 200.5
        assert metrics["power_limit_w"] == 300.0
        assert metrics["vram_used_gb"] == round(8192 / 1024, 2)
        assert metrics["vram_total_gb"] == round(16384 / 1024, 2)
        assert metrics["vram_utilization_pct"] is not None
        assert metrics["power_utilization_pct"] is not None

    def test_amd_primary_gpu_with_mock(self, monkeypatch):
        """get_primary_gpu_utilization returns correct dict with mocked amd-smi."""
        amd_path = PACKAGE_ROOT / "studio" / "backend" / "utils" / "hardware" / "amd.py"
        _amd_spec = importlib.util.spec_from_file_location("test_amd2", amd_path)
        assert _amd_spec is not None and _amd_spec.loader is not None
        amd_mod = importlib.util.module_from_spec(_amd_spec)

        loggers_mock = MagicMock()
        loggers_mock.get_logger = MagicMock(return_value = MagicMock())
        monkeypatch.setitem(sys.modules, "loggers", loggers_mock)

        try:
            _amd_spec.loader.exec_module(amd_mod)
        except Exception:
            pytest.skip("Could not load amd module")

        # CI often sets CUDA_VISIBLE_DEVICES='', which hides every GPU.
        for var in (
            "HIP_VISIBLE_DEVICES",
            "ROCR_VISIBLE_DEVICES",
            "CUDA_VISIBLE_DEVICES",
        ):
            monkeypatch.delenv(var, raising = False)

        # amd-smi is gated off on Windows without a HIP SDK.
        monkeypatch.setenv("UNSLOTH_ENABLE_AMD_SMI", "1")

        mock_json = json.dumps(
            [
                {
                    "usage": {"gfx_activity": "50"},
                    "temperature": {"edge": "65"},
                    "power": {"current_socket_power": "150", "power_cap": "250"},
                    "vram": {"vram_used": 4096, "vram_total": 16384},
                }
            ]
        )
        mock_result = MagicMock()
        mock_result.returncode = 0
        mock_result.stdout = mock_json

        # The guard which()-checks before spawning.
        with patch.object(amd_mod.shutil, "which", return_value = "/usr/bin/amd-smi"):
            with patch.object(subprocess, "run", return_value = mock_result):
                result = amd_mod.get_primary_gpu_utilization()
        assert result["available"] is True
        assert result["gpu_utilization_pct"] == 50.0
        assert result["temperature_c"] == 65.0

    def test_amd_smi_not_found_returns_unavailable(self, monkeypatch):
        """get_primary_gpu_utilization returns available=False when amd-smi is missing."""
        amd_path = PACKAGE_ROOT / "studio" / "backend" / "utils" / "hardware" / "amd.py"
        _amd_spec = importlib.util.spec_from_file_location("test_amd3", amd_path)
        assert _amd_spec is not None and _amd_spec.loader is not None
        amd_mod = importlib.util.module_from_spec(_amd_spec)

        loggers_mock = MagicMock()
        loggers_mock.get_logger = MagicMock(return_value = MagicMock())
        monkeypatch.setitem(sys.modules, "loggers", loggers_mock)

        try:
            _amd_spec.loader.exec_module(amd_mod)
        except Exception:
            pytest.skip("Could not load amd module")

        with (
            patch.dict(os.environ, {"UNSLOTH_ENABLE_AMD_SMI": "1"}),
            patch.object(subprocess, "run", side_effect = OSError("amd-smi not found")),
        ):
            result = amd_mod.get_primary_gpu_utilization()
        assert result["available"] is False

    def test_amd_timeout_returns_unavailable(self, monkeypatch):
        """get_primary_gpu_utilization handles timeout gracefully."""
        amd_path = PACKAGE_ROOT / "studio" / "backend" / "utils" / "hardware" / "amd.py"
        _amd_spec = importlib.util.spec_from_file_location("test_amd4", amd_path)
        assert _amd_spec is not None and _amd_spec.loader is not None
        amd_mod = importlib.util.module_from_spec(_amd_spec)

        loggers_mock = MagicMock()
        loggers_mock.get_logger = MagicMock(return_value = MagicMock())
        monkeypatch.setitem(sys.modules, "loggers", loggers_mock)

        try:
            _amd_spec.loader.exec_module(amd_mod)
        except Exception:
            pytest.skip("Could not load amd module")

        with (
            patch.dict(os.environ, {"UNSLOTH_ENABLE_AMD_SMI": "1"}),
            patch.object(
                subprocess,
                "run",
                side_effect = subprocess.TimeoutExpired("amd-smi", 5),
            ),
        ):
            result = amd_mod.get_primary_gpu_utilization()
        assert result["available"] is False


class TestHardwareAmdBranching:
    """Verify hardware.py branches to amd.py when IS_ROCM is True."""

    def test_hardware_imports_amd_module(self):
        """hardware.py should import from amd module when IS_ROCM."""
        hw_path = PACKAGE_ROOT / "studio" / "backend" / "utils" / "hardware" / "hardware.py"
        source = hw_path.read_text(encoding = "utf-8")
        assert "from . import amd" in source

    def test_hardware_branches_on_is_rocm_for_utilization(self):
        """get_gpu_utilization dispatches visible metrics through amd.py on ROCm."""
        hw_path = PACKAGE_ROOT / "studio" / "backend" / "utils" / "hardware" / "hardware.py"
        source = hw_path.read_text(encoding = "utf-8")
        func_start = source.find("def get_gpu_utilization")
        func_body = source[func_start : source.find("\ndef ", func_start + 1)]
        assert "_smi_visible_utilization(" in func_body
        assert "_reconcile_rocm_unified_memory" in func_body
        helper_start = source.find("def _smi_visible_utilization")
        helper = source[helper_start : source.find("\ndef ", helper_start + 1)]
        assert re.search(r'_smi_query\(\s*"get_visible_gpu_utilization"', helper)
        smi = source[
            source.find("def _smi_query") : source.find("\ndef ", source.find("def _smi_query") + 1)
        ]
        assert "IS_ROCM" in smi
        assert "from . import amd" in smi

    def test_hardware_branches_on_is_rocm_for_visible(self):
        """get_visible_gpu_utilization dispatches to amd.py via _smi_query when IS_ROCM."""
        hw_path = PACKAGE_ROOT / "studio" / "backend" / "utils" / "hardware" / "hardware.py"
        source = hw_path.read_text(encoding = "utf-8")
        func_start = source.find("def get_visible_gpu_utilization")
        func_body = source[func_start : source.find("\ndef ", func_start + 1)]
        assert "_smi_visible_utilization(" in func_body
        helper_start = source.find("def _smi_visible_utilization")
        helper = source[helper_start : source.find("\ndef ", helper_start + 1)]
        assert re.search(r'_smi_query\(\s*"get_visible_gpu_utilization"', helper)
        smi = source[
            source.find("def _smi_query") : source.find("\ndef ", source.find("def _smi_query") + 1)
        ]
        assert "IS_ROCM" in smi
        assert "from . import amd" in smi

    def test_hardware_branches_on_is_rocm_for_physical_count(self):
        """get_physical_gpu_count should try amd.py when IS_ROCM."""
        hw_path = PACKAGE_ROOT / "studio" / "backend" / "utils" / "hardware" / "hardware.py"
        source = hw_path.read_text(encoding = "utf-8")
        func_start = source.find("def get_physical_gpu_count")
        func_body = source[func_start : source.find("\ndef ", func_start + 1)]
        assert "IS_ROCM" in func_body
        assert "from . import amd" in func_body


class TestApplyGpuIdsRocmFallback:
    """apply_gpu_ids sets HIP_VISIBLE_DEVICES on ROCm hosts even when IS_ROCM is still False (issue #5180)."""

    def test_apply_gpu_ids_falls_back_to_torch_version_hip(self):
        """apply_gpu_ids probes torch.version.hip when IS_ROCM is False and no ROCm env vars set."""
        hw_path = PACKAGE_ROOT / "studio" / "backend" / "utils" / "hardware" / "hardware.py"
        source = hw_path.read_text(encoding = "utf-8")
        func_start = source.find("def apply_gpu_ids")
        func_body = source[func_start : source.find("\ndef ", func_start + 1)]
        assert 'getattr(_torch.version, "hip", None)' in func_body

    def test_apply_gpu_ids_sets_hip_but_not_rocr_visible_devices(self):
        """apply_gpu_ids sets HIP_VISIBLE_DEVICES but leaves ROCR_VISIBLE_DEVICES inherited (HSA indexing; issue #6118)."""
        hw_path = PACKAGE_ROOT / "studio" / "backend" / "utils" / "hardware" / "hardware.py"
        source = hw_path.read_text(encoding = "utf-8")
        func_start = source.find("def apply_gpu_ids")
        func_body = source[func_start : source.find("\ndef ", func_start + 1)]
        assert 'os.environ["HIP_VISIBLE_DEVICES"] = value' in func_body
        assert 'os.environ["ROCR_VISIBLE_DEVICES"] = value' not in func_body

    def test_apply_gpu_ids_rocm_fallback_is_guarded_by_try_except(self):
        """torch import in apply_gpu_ids must be wrapped in try/except so a missing torch never crashes."""
        hw_path = PACKAGE_ROOT / "studio" / "backend" / "utils" / "hardware" / "hardware.py"
        source = hw_path.read_text(encoding = "utf-8")
        func_start = source.find("def apply_gpu_ids")
        func_body = source[func_start : source.find("\ndef ", func_start + 1)]
        assert "import torch as _torch" in func_body
        assert "except Exception" in func_body


class TestWindowsRocmWarning:
    """Verify Windows AMD GPU detection and warning message."""

    def test_windows_amd_warning_in_source(self):
        """install_python_stack.py should warn Windows AMD users."""
        source = _STACK_PATH.read_text(encoding = "utf-8")
        assert "AMD GPU detected" in source

    def test_windows_amd_warning_checks_hipinfo_or_amdsmi(self):
        """Warning should check for hipinfo or amd-smi."""
        source = _STACK_PATH.read_text(encoding = "utf-8")
        assert "hipinfo" in source
        assert "amd-smi" in source

    def test_windows_amd_warning_has_docs_link(self):
        """Warning should include AMD docs link."""
        source = _STACK_PATH.read_text(encoding = "utf-8")
        assert "unsloth.ai/docs/get-started/install/amd" in source


class TestIsRdnaExpansion:
    """Verify is_rdna() covers RDNA2, RDNA3, RDNA3.5, RDNA4 architectures."""

    def test_is_rdna_source_has_rdna2(self):
        """is_rdna() should include RDNA2 architectures."""
        utils_path = PACKAGE_ROOT / "unsloth" / "kernels" / "utils.py"
        source = utils_path.read_text(encoding = "utf-8")
        func_start = source.find("def is_rdna()")
        func_body = source[func_start : source.find("\ndef ", func_start + 1)]
        assert "gfx1030" in func_body
        assert "gfx1031" in func_body
        assert "gfx1032" in func_body
        assert "gfx1033" in func_body
        assert "gfx1034" in func_body
        assert "gfx1035" in func_body
        assert "gfx1036" in func_body

    def test_is_rdna_source_has_rdna3(self):
        """is_rdna() should include RDNA3 architectures."""
        utils_path = PACKAGE_ROOT / "unsloth" / "kernels" / "utils.py"
        source = utils_path.read_text(encoding = "utf-8")
        func_start = source.find("def is_rdna()")
        func_body = source[func_start : source.find("\ndef ", func_start + 1)]
        assert "gfx1100" in func_body
        assert "gfx1101" in func_body
        assert "gfx1102" in func_body
        assert "gfx1103" in func_body

    def test_is_rdna_source_has_rdna35(self):
        """is_rdna() should include RDNA3.5 architectures."""
        utils_path = PACKAGE_ROOT / "unsloth" / "kernels" / "utils.py"
        source = utils_path.read_text(encoding = "utf-8")
        func_start = source.find("def is_rdna()")
        func_body = source[func_start : source.find("\ndef ", func_start + 1)]
        assert "gfx1150" in func_body
        assert "gfx1151" in func_body
        assert "gfx1152" in func_body

    def test_is_rdna_source_has_rdna4(self):
        """is_rdna() should include RDNA4 architectures."""
        utils_path = PACKAGE_ROOT / "unsloth" / "kernels" / "utils.py"
        source = utils_path.read_text(encoding = "utf-8")
        func_start = source.find("def is_rdna()")
        func_body = source[func_start : source.find("\ndef ", func_start + 1)]
        assert "gfx1200" in func_body
        assert "gfx1201" in func_body

    def test_is_cdna_not_changed(self):
        """is_cdna() should remain unchanged (no RDNA architectures added)."""
        utils_path = PACKAGE_ROOT / "unsloth" / "kernels" / "utils.py"
        source = utils_path.read_text(encoding = "utf-8")
        func_start = source.find("def is_cdna()")
        func_body = source[func_start : source.find("\ndef ", func_start + 1)]
        assert "gfx940" in func_body
        assert "gfx941" in func_body
        assert "gfx942" in func_body
        assert "gfx950" in func_body
        assert "gfx1030" not in func_body
        assert "gfx1100" not in func_body


class TestWindowsRocmIndexUrl:
    """Verify GPU arch → AMD pip index URL mapping."""

    @pytest.fixture(autouse = True)
    def _no_mirror(self, monkeypatch):
        monkeypatch.delenv("UNSLOTH_ROCM_WINDOWS_MIRROR", raising = False)
        monkeypatch.delenv("UNSLOTH_ROCM_WINDOWS_MULTIARCH_MIRROR", raising = False)
        monkeypatch.setattr(stack_mod, "_ROCM_WINDOWS_INDEX_BASE", "https://repo.amd.com/rocm/whl")

    @pytest.mark.parametrize("gfx", ["gfx1200", "gfx1201", "gfx1151", "gfx1150", "gfx1100"])
    def test_rdna_maps_to_the_multiarch_index(self, gfx):
        url = stack_mod._windows_rocm_index_url(gfx)
        assert url == "https://repo.amd.com/rocm/whl-multi-arch/"

    @pytest.mark.parametrize(
        "gfx,leaf",
        [
            ("gfx1200", "gfx120X-all"),
            ("gfx1201", "gfx120X-all"),
            ("gfx1151", "gfx1151"),
            ("gfx1150", "gfx1150"),
            ("gfx1100", "gfx110X-all"),
        ],
    )
    def test_family_mirror_keeps_the_family_leaf(self, gfx, leaf, monkeypatch):
        monkeypatch.setenv("UNSLOTH_ROCM_WINDOWS_MIRROR", "https://mirror.example/whl")
        monkeypatch.setattr(stack_mod, "_ROCM_WINDOWS_INDEX_BASE", "https://mirror.example/whl")
        assert stack_mod._windows_rocm_index_url(gfx) == f"https://mirror.example/whl/{leaf}/"

    def test_cdna_keeps_its_family(self):
        assert (
            stack_mod._windows_rocm_index_url("gfx90a") == "https://repo.amd.com/rocm/whl/gfx90a/"
        )

    def test_unknown_arch_returns_none(self):
        assert stack_mod._windows_rocm_index_url("gfx9999") is None

    def test_none_arch_returns_none(self):
        assert stack_mod._windows_rocm_index_url(None) is None

    def test_url_ends_with_slash(self):
        """AMD pip index URLs must end with / for --index-url compatibility."""
        url = stack_mod._windows_rocm_index_url("gfx1200")
        assert url is not None
        assert url.endswith("/")

    def test_base_url_uses_repo_amd_com_by_default(self):
        url = stack_mod._windows_rocm_index_url("gfx1200")
        assert url is not None
        assert "repo.amd.com" in url

    def test_mirror_env_var_overrides_base(self, monkeypatch):
        monkeypatch.setenv("UNSLOTH_ROCM_WINDOWS_MIRROR", "https://my-mirror.example.com/rocm/whl")
        url = stack_mod._windows_rocm_index_url("gfx1200")
        # The env var is read at import for _ROCM_WINDOWS_INDEX_BASE.
        assert url is not None


class TestDetectWindowsGfxArch:
    """Verify hipinfo parsing for GPU arch detection on Windows."""

    def test_returns_none_when_hipinfo_not_on_path(self):
        # The suite may run on a real AMD host where WMI would answer.
        with patch("shutil.which", return_value = None):
            with patch("os.path.isfile", return_value = False):
                with patch("subprocess.run", side_effect = FileNotFoundError):
                    result = stack_mod._detect_windows_gfx_arch()
        assert result is None

    def test_parses_gcnarchname_from_hipinfo_output(self):
        mock_result = MagicMock()
        mock_result.returncode = 0
        mock_result.stdout = b"gcnArchName : gfx1200\nsome other line\n"
        with patch("shutil.which", return_value = "/usr/bin/hipinfo"):
            with patch("subprocess.run", return_value = mock_result):
                result = stack_mod._detect_windows_gfx_arch()
        assert result == "gfx1200"

    def test_returns_arch_on_crash_with_gcnarchname_in_output(self):
        # hipinfo may crash (0xC0000005 on RDNA 4) after gcnArchName; accept on any exit code.
        mock_result = MagicMock()
        mock_result.returncode = -1073741819  # 0xC0000005 STATUS_ACCESS_VIOLATION
        mock_result.stdout = b"gcnArchName : gfx1200\nsome other line\n"
        with patch("shutil.which", return_value = "/usr/bin/hipinfo"):
            with patch("subprocess.run", return_value = mock_result):
                result = stack_mod._detect_windows_gfx_arch()
        assert result == "gfx1200"

    def test_mixed_igpu_dgpu_prefers_discrete(self, monkeypatch):
        # HIP enumerates the Raphael iGPU (gfx1036) before the dGPU, so index 0 picked the iGPU.
        monkeypatch.delenv("HIP_VISIBLE_DEVICES", raising = False)
        monkeypatch.delenv("ROCR_VISIBLE_DEVICES", raising = False)
        monkeypatch.delenv("CUDA_VISIBLE_DEVICES", raising = False)
        mock_result = MagicMock()
        mock_result.returncode = 0
        mock_result.stdout = b"gcnArchName : gfx1036\ngcnArchName : gfx1200\n"
        with patch("shutil.which", return_value = "/usr/bin/hipinfo"):
            with patch("subprocess.run", return_value = mock_result):
                result = stack_mod._detect_windows_gfx_arch()
        assert result == "gfx1200"

    def test_mixed_igpu_dgpu_reports_the_override_env_var(self, monkeypatch, capsys):
        monkeypatch.delenv("HIP_VISIBLE_DEVICES", raising = False)
        monkeypatch.delenv("ROCR_VISIBLE_DEVICES", raising = False)
        monkeypatch.delenv("CUDA_VISIBLE_DEVICES", raising = False)
        mock_result = MagicMock()
        mock_result.returncode = 0
        mock_result.stdout = b"gcnArchName : gfx1036\ngcnArchName : gfx1200\n"
        with patch("shutil.which", return_value = "/usr/bin/hipinfo"):
            with patch("subprocess.run", return_value = mock_result):
                stack_mod._detect_windows_gfx_arch()
        out = capsys.readouterr().out
        assert "HIP_VISIBLE_DEVICES" in out
        assert "gfx1036" in out and "gfx1200" in out

    def test_explicit_visible_devices_still_wins_over_igpu_preference(self, monkeypatch):
        # A user who pinned the iGPU on purpose must keep getting the iGPU.
        monkeypatch.setenv("HIP_VISIBLE_DEVICES", "0")
        monkeypatch.delenv("ROCR_VISIBLE_DEVICES", raising = False)
        monkeypatch.delenv("CUDA_VISIBLE_DEVICES", raising = False)
        mock_result = MagicMock()
        mock_result.returncode = 0
        mock_result.stdout = b"gcnArchName : gfx1036\ngcnArchName : gfx1200\n"
        with patch("shutil.which", return_value = "/usr/bin/hipinfo"):
            with patch("subprocess.run", return_value = mock_result):
                result = stack_mod._detect_windows_gfx_arch()
        assert result == "gfx1036"

    def test_strix_igpu_selection_is_unchanged(self, monkeypatch):
        # gfx1151 is a training target, not a shadowing APU.
        monkeypatch.delenv("HIP_VISIBLE_DEVICES", raising = False)
        monkeypatch.delenv("ROCR_VISIBLE_DEVICES", raising = False)
        monkeypatch.delenv("CUDA_VISIBLE_DEVICES", raising = False)
        mock_result = MagicMock()
        mock_result.returncode = 0
        mock_result.stdout = b"gcnArchName : gfx1151\ngcnArchName : gfx1200\n"
        with patch("shutil.which", return_value = "/usr/bin/hipinfo"):
            with patch("subprocess.run", return_value = mock_result):
                result = stack_mod._detect_windows_gfx_arch()
        assert result == "gfx1151"

    def test_single_igpu_host_is_unchanged(self, monkeypatch):
        monkeypatch.delenv("HIP_VISIBLE_DEVICES", raising = False)
        monkeypatch.delenv("ROCR_VISIBLE_DEVICES", raising = False)
        monkeypatch.delenv("CUDA_VISIBLE_DEVICES", raising = False)
        mock_result = MagicMock()
        mock_result.returncode = 0
        mock_result.stdout = b"gcnArchName : gfx1036\n"
        with patch("shutil.which", return_value = "/usr/bin/hipinfo"):
            with patch("subprocess.run", return_value = mock_result):
                result = stack_mod._detect_windows_gfx_arch()
        assert result == "gfx1036"

    def test_unsupported_discrete_does_not_depose_a_supported_igpu(self, monkeypatch):
        # Preferring a wheel-less dGPU resolves to no index and falls back to CPU.
        monkeypatch.delenv("HIP_VISIBLE_DEVICES", raising = False)
        monkeypatch.delenv("ROCR_VISIBLE_DEVICES", raising = False)
        monkeypatch.delenv("CUDA_VISIBLE_DEVICES", raising = False)
        assert stack_mod._windows_rocm_index_url("gfx803") is None
        mock_result = MagicMock()
        mock_result.returncode = 0
        mock_result.stdout = b"gcnArchName : gfx1036\ngcnArchName : gfx803\n"
        with patch("shutil.which", return_value = "/usr/bin/hipinfo"):
            with patch("subprocess.run", return_value = mock_result):
                result = stack_mod._detect_windows_gfx_arch()
        assert result == "gfx1036"
        assert stack_mod._windows_rocm_index_url(result) is not None

    def test_unsupported_igpu_still_yields_to_the_discrete_card(self, monkeypatch):
        # With neither pick wheel-backed the swap costs nothing, so the discrete card wins.
        monkeypatch.delenv("HIP_VISIBLE_DEVICES", raising = False)
        monkeypatch.delenv("ROCR_VISIBLE_DEVICES", raising = False)
        monkeypatch.delenv("CUDA_VISIBLE_DEVICES", raising = False)
        assert stack_mod._windows_rocm_index_url("gfx1013") is None
        assert stack_mod._windows_rocm_index_url("gfx803") is None
        mock_result = MagicMock()
        mock_result.returncode = 0
        mock_result.stdout = b"gcnArchName : gfx1013\ngcnArchName : gfx803\n"
        with patch("shutil.which", return_value = "/usr/bin/hipinfo"):
            with patch("subprocess.run", return_value = mock_result):
                result = stack_mod._detect_windows_gfx_arch()
        assert result == "gfx803"

    def test_cuda_visible_devices_also_pins_the_igpu(self, monkeypatch):
        # HIP honours CUDA_VISIBLE_DEVICES like its own masks.
        monkeypatch.delenv("HIP_VISIBLE_DEVICES", raising = False)
        monkeypatch.delenv("ROCR_VISIBLE_DEVICES", raising = False)
        monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "0")
        mock_result = MagicMock()
        mock_result.returncode = 0
        mock_result.stdout = b"gcnArchName : gfx1036\ngcnArchName : gfx1200\n"
        with patch("shutil.which", return_value = "/usr/bin/hipinfo"):
            with patch("subprocess.run", return_value = mock_result):
                result = stack_mod._detect_windows_gfx_arch()
        assert result == "gfx1036"

    def test_cuda_visible_devices_indexes_the_enumeration(self, monkeypatch):
        # hipinfo output is already mask-filtered and renumbered; re-indexing applies the mask twice.
        monkeypatch.delenv("HIP_VISIBLE_DEVICES", raising = False)
        monkeypatch.delenv("ROCR_VISIBLE_DEVICES", raising = False)
        monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "1")
        mock_result = MagicMock()
        mock_result.returncode = 0
        mock_result.stdout = b"gcnArchName : gfx1200\n"
        with patch("shutil.which", return_value = "/usr/bin/hipinfo"):
            with patch("subprocess.run", return_value = mock_result):
                result = stack_mod._detect_windows_gfx_arch()
        assert result == "gfx1200"

    def test_a_reordering_mask_is_not_applied_twice(self, monkeypatch):
        # CUDA_VISIBLE_DEVICES=1,0 makes hipinfo print the dGPU first.
        monkeypatch.delenv("HIP_VISIBLE_DEVICES", raising = False)
        monkeypatch.delenv("ROCR_VISIBLE_DEVICES", raising = False)
        monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "1,0")
        mock_result = MagicMock()
        mock_result.returncode = 0
        mock_result.stdout = b"gcnArchName : gfx1200\ngcnArchName : gfx1036\n"
        with patch("shutil.which", return_value = "/usr/bin/hipinfo"):
            with patch("subprocess.run", return_value = mock_result):
                result = stack_mod._detect_windows_gfx_arch()
        assert result == "gfx1200"

    def test_cuda_visible_devices_index_matches_the_other_masks(self, monkeypatch):
        for _var in ("HIP_VISIBLE_DEVICES", "ROCR_VISIBLE_DEVICES", "CUDA_VISIBLE_DEVICES"):
            monkeypatch.delenv(_var, raising = False)
        for _value, _expected in (
            ("1", 1),
            ("1,0", 1),
            ("7", 0),
            ("-1", 0),
            ("", 0),
            ("GPU-abc", 0),
        ):
            monkeypatch.setenv("CUDA_VISIBLE_DEVICES", _value)
            assert stack_mod._pick_visible_index(2) == _expected, _value

    @pytest.mark.parametrize("spelling", ["", "-1"])
    def test_all_hiding_masks_count_as_a_selection(self, monkeypatch, spelling):
        # '' and '-1' select NO GPU (clr stores empty as ' '), a deliberate choice.
        monkeypatch.delenv("HIP_VISIBLE_DEVICES", raising = False)
        monkeypatch.delenv("ROCR_VISIBLE_DEVICES", raising = False)
        monkeypatch.setenv("CUDA_VISIBLE_DEVICES", spelling)
        assert stack_mod._visible_devices_pinned() is True
        mock_result = MagicMock()
        mock_result.returncode = 0
        mock_result.stdout = b"gcnArchName : gfx1036\ngcnArchName : gfx1200\n"
        with patch("shutil.which", return_value = "/usr/bin/hipinfo"):
            with patch("subprocess.run", return_value = mock_result):
                result = stack_mod._detect_windows_gfx_arch()
        assert result == "gfx1036"

    def _hipinfo_pick(self, arches):
        mock_result = MagicMock()
        mock_result.returncode = 0
        mock_result.stdout = "".join(f"gcnArchName : {a}\n" for a in arches).encode()
        with patch("shutil.which", return_value = "/usr/bin/hipinfo"):
            with patch("subprocess.run", return_value = mock_result):
                return stack_mod._detect_windows_gfx_arch()

    def test_an_all_hiding_hip_mask_shadows_a_later_cuda_mask(self, monkeypatch):
        # clr uses the HIP mask whenever its first byte is not NUL, so an empty one shadows CUDA's.
        monkeypatch.delenv("ROCR_VISIBLE_DEVICES", raising = False)
        for _spelling in ("", "-1"):
            monkeypatch.setenv("HIP_VISIBLE_DEVICES", _spelling)
            monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "1")
            assert stack_mod._pick_visible_index(2) == 0
            assert self._hipinfo_pick(["gfx1036", "gfx1200"]) == "gfx1036"

    def test_gcn_arch_feature_suffix_is_stripped(self, monkeypatch):
        # hipinfo can print 'gfx90a:sramecc+:xnack-'; setup.ps1 splits on ':'.
        for _m in ("HIP_VISIBLE_DEVICES", "ROCR_VISIBLE_DEVICES", "CUDA_VISIBLE_DEVICES"):
            monkeypatch.delenv(_m, raising = False)
        assert self._hipinfo_pick(["gfx1036:xnack-", "gfx1200:xnack-"]) == "gfx1200"

    def test_prefers_a_wheel_backed_discrete_over_an_unsupported_one(self, monkeypatch):
        # gfx803 has no Windows wheel index, so do not stop at the first non-integrated token.
        for _m in ("HIP_VISIBLE_DEVICES", "ROCR_VISIBLE_DEVICES", "CUDA_VISIBLE_DEVICES"):
            monkeypatch.delenv(_m, raising = False)
        assert stack_mod._windows_rocm_index_url("gfx803") is None
        assert self._hipinfo_pick(["gfx1036", "gfx803", "gfx1200"]) == "gfx1200"
        assert self._hipinfo_pick(["gfx1013", "gfx803", "gfx1200"]) == "gfx1200"

    def test_pinning_a_wheelless_gpu_says_why_torch_will_be_cpu(self, monkeypatch, capsys):
        # Honour the pin, but warn: silently installing CPU torch hides that the mask caused it.
        monkeypatch.delenv("HIP_VISIBLE_DEVICES", raising = False)
        monkeypatch.delenv("ROCR_VISIBLE_DEVICES", raising = False)
        monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "1,0")
        assert self._hipinfo_pick(["gfx803", "gfx1036"]) == "gfx803"
        out = capsys.readouterr().out
        assert "no AMD Windows wheels" in out
        assert "gfx1036" in out

    def test_deduplicated_callers_do_not_get_a_bogus_range_warning(self, monkeypatch, capsys):
        # _detect_amd_gfx_codes() dedupes, so a valid index can read as out of range.
        monkeypatch.delenv("ROCR_VISIBLE_DEVICES", raising = False)
        monkeypatch.delenv("CUDA_VISIBLE_DEVICES", raising = False)
        monkeypatch.setenv("HIP_VISIBLE_DEVICES", "1")
        assert stack_mod._pick_visible_index(1, warn = False) == 0
        assert capsys.readouterr().out == ""
        assert stack_mod._pick_visible_index(1) == 0
        assert "out of range" in capsys.readouterr().out

    def test_advisory_names_the_selected_gpus_real_index(self, monkeypatch, capsys):
        # Not always device 1: naming 1 here would expose the wheel-less gfx803.
        for _m in ("HIP_VISIBLE_DEVICES", "ROCR_VISIBLE_DEVICES", "CUDA_VISIBLE_DEVICES"):
            monkeypatch.delenv(_m, raising = False)
        assert self._hipinfo_pick(["gfx1036", "gfx803", "gfx1200"]) == "gfx1200"
        out = capsys.readouterr().out
        assert "HIP_VISIBLE_DEVICES 2" in out
        assert "HIP_VISIBLE_DEVICES 1" not in out

    def test_shadowing_set_holds_only_apu_arches(self):
        # gfx1037 is not an AMDGPU target in LLVM, so no Windows tool emits it.
        assert "gfx1037" not in stack_mod._SHADOWING_INTEGRATED_GFX
        # gfx1033 (Van Gogh) has a wheel family; omitted, it posed as 'discrete'.
        assert "gfx1033" in stack_mod._SHADOWING_INTEGRATED_GFX
        assert self._hipinfo_pick(["gfx1036", "gfx1033"]) == "gfx1036"

    def test_returns_none_on_nonzero_returncode_without_gcnarchname(self):
        mock_result = MagicMock()
        mock_result.returncode = 1
        mock_result.stdout = b"HIP runtime error: no device detected\n"
        with patch("shutil.which", return_value = "/usr/bin/hipinfo"):
            with patch("subprocess.run", return_value = mock_result):
                result = stack_mod._detect_windows_gfx_arch()
        assert result is None

    def test_returns_none_when_no_gcnarchname_in_output(self):
        # The WMI fallback must get nothing so the mocked name cannot resolve via the table.
        mock_result = MagicMock()
        mock_result.returncode = 0
        mock_result.stdout = b"deviceName : SomeUnknownDevice\n"

        def _run(cmd, **kwargs):
            if cmd and "powershell" in str(cmd[0]).lower():
                raise FileNotFoundError(cmd[0])
            return mock_result

        with patch("shutil.which", return_value = "/usr/bin/hipinfo"):
            with patch("subprocess.run", side_effect = _run):
                result = stack_mod._detect_windows_gfx_arch()
        assert result is None

    def test_returns_none_on_timeout(self):
        with patch("shutil.which", return_value = "/usr/bin/hipinfo"):
            with patch(
                "subprocess.run",
                side_effect = subprocess.TimeoutExpired("hipinfo", 10),
            ):
                result = stack_mod._detect_windows_gfx_arch()
        assert result is None

    def test_strips_whitespace_from_arch(self):
        mock_result = MagicMock()
        mock_result.returncode = 0
        mock_result.stdout = b"  gcnArchName :   gfx1201  \n"
        with patch("shutil.which", return_value = "/usr/bin/hipinfo"):
            with patch("subprocess.run", return_value = mock_result):
                result = stack_mod._detect_windows_gfx_arch()
        assert result == "gfx1201"


class TestGfxArchNameFallback:
    """With no amd-smi/hipinfo on Windows, arch must resolve from the GPU name via WMI (mirrors setup.ps1)."""

    @pytest.mark.parametrize(
        "name, expected",
        [
            ("AMD Radeon(TM) 8060S Graphics", "gfx1151"),
            ("AMD Radeon(TM) 8065S Graphics", "gfx1151"),
            ("AMD Ryzen AI MAX+ 395 w/ Radeon 8060S", "gfx1151"),
            ("AMD Radeon(TM) 890M", "gfx1150"),
            ("AMD Ryzen AI 9 HX 370 w/ Radeon 890M", "gfx1150"),
            ("AMD Radeon RX 9070 XT", "gfx1201"),
            ("AMD Radeon RX 9070", "gfx1201"),  # Navi 48 like the XT, not Navi 44
            # Navi 48 workstation card; its name has neither 9070 nor 9080.
            ("AMD Radeon AI PRO R9700", "gfx1201"),
            ("ATI Radeon 9700 PRO", None),  # a bare "9700" arm would claim this 2002 card
            ("AMD Radeon RX 9060 XT", "gfx1200"),
            ("AMD Radeon RX 7700S", "gfx1102"),  # (?!S) lookahead must not hit gfx1101
            ("AMD Radeon RX 7700 XT", "gfx1101"),
            ("AMD Radeon RX 7900 XTX", "gfx1100"),
            ("AMD Radeon(TM) 780M", "gfx1103"),
            ("NVIDIA GeForce RTX 4090", None),
            ("Microsoft Basic Display Adapter", None),
            ("", None),
        ],
    )
    def test_name_to_arch_mapping(self, name, expected):
        assert stack_mod._gfx_arch_from_gpu_name(name) == expected

    def test_wmi_fallback_resolves_arch_without_any_tools(self):
        """hipinfo absent everywhere + amd-smi absent -> WMI name fallback."""
        ps_result = MagicMock()
        ps_result.returncode = 0
        ps_result.stdout = b"AMD Radeon(TM) 8060S Graphics\r\nMicrosoft Basic Display Adapter\r\n"

        def _run(cmd, **kwargs):
            if cmd and "powershell.exe" in str(cmd[0]).lower():
                return ps_result
            raise FileNotFoundError(cmd[0])

        with patch.dict(os.environ, {}, clear = False):
            for _v in (
                "HIP_PATH",
                "ROCM_PATH",
                "UNSLOTH_ROCM_GFX_ARCH",
                "UNSLOTH_ENABLE_AMD_SMI",
            ):
                os.environ.pop(_v, None)
            with patch("shutil.which", return_value = None):
                with patch("os.path.isfile", return_value = False):
                    with patch("subprocess.run", side_effect = _run):
                        result = stack_mod._detect_windows_gfx_arch()
        assert result == "gfx1151"

    def test_wmi_fallback_returns_none_for_non_amd_hosts(self):
        ps_result = MagicMock()
        ps_result.returncode = 0
        ps_result.stdout = b"NVIDIA GeForce RTX 4090\r\n"

        def _run(cmd, **kwargs):
            if cmd and "powershell.exe" in str(cmd[0]).lower():
                return ps_result
            raise FileNotFoundError(cmd[0])

        with patch.dict(os.environ, {}, clear = False):
            for _v in ("HIP_PATH", "ROCM_PATH", "UNSLOTH_ROCM_GFX_ARCH"):
                os.environ.pop(_v, None)
            with patch("shutil.which", return_value = None):
                with patch("os.path.isfile", return_value = False):
                    with patch("subprocess.run", side_effect = _run):
                        result = stack_mod._detect_windows_gfx_arch()
        assert result is None

    @staticmethod
    def _wmi_arch(names, env):
        """Drive the WMI fallback over `names`, emulating the probe's AMD filter."""
        ps_result = MagicMock()
        ps_result.returncode = 0
        amd = [n for n in names if re.search(r"AMD|Radeon", n, re.IGNORECASE)]
        ps_result.stdout = ("\r\n".join(amd) + "\r\n").encode()

        def _run(cmd, **kwargs):
            if cmd and "powershell.exe" in str(cmd[0]).lower():
                return ps_result
            raise FileNotFoundError(cmd[0])

        buf = io.StringIO()
        with patch.dict(os.environ, {}, clear = False):
            for _v in (
                "HIP_PATH",
                "ROCM_PATH",
                "UNSLOTH_ROCM_GFX_ARCH",
                "UNSLOTH_ENABLE_AMD_SMI",
                "HIP_VISIBLE_DEVICES",
                "ROCR_VISIBLE_DEVICES",
                "CUDA_VISIBLE_DEVICES",
            ):
                os.environ.pop(_v, None)
            os.environ.update(env)
            with contextlib.redirect_stdout(buf):
                with patch("shutil.which", return_value = None):
                    with patch("os.path.isfile", return_value = False):
                        with patch("subprocess.run", side_effect = _run):
                            result = stack_mod._detect_windows_gfx_arch()
        return result, buf.getvalue()

    @pytest.mark.parametrize(
        "name",
        [
            "NVIDIA GeForce RTX 4090",
            "Intel(R) Arc(TM) A770 Graphics",
            "Microsoft Basic Display Adapter",
            "Apple M3 Max",
        ],
    )
    def test_non_amd_adapters_produce_no_amd_output(self, name):
        # The vendor filter is a PowerShell string; a non-AMD adapter would shift the mask index.
        arch, out = self._wmi_arch([name], {"CUDA_VISIBLE_DEVICES": "0"})
        assert arch is None
        assert out == ""

    @pytest.mark.parametrize("code", ["22", "45", "43", "1"])
    def test_sole_unhealthy_adapter_still_yields_its_arch(self, code):
        # With one adapter there is nothing to depose; code 45 is routine on muxless laptops.
        arch, _ = self._wmi_arch([f"AMD Radeon RX 9060 XT|{code}"], {})
        assert arch == "gfx1200"

    def test_unhealthy_adapter_still_cannot_depose_a_healthy_one(self):
        # Negative control: with a healthy iGPU present the disabled dGPU stays out, so
        # the shadowing skip cannot install wheels for a GPU Windows never exposes.
        arch, _ = self._wmi_arch(["AMD Radeon(TM) 780M Graphics|0", "AMD Radeon RX 9060 XT|22"], {})
        assert arch == "gfx1103"

    def test_mask_out_of_range_is_reported_once(self):
        # _dedup_pick re-resolves the masks, so without warn=False warnings print twice.
        _, out = self._wmi_arch(["AMD Radeon RX 9060 XT|0"], {"HIP_VISIBLE_DEVICES": "1"})
        assert out.count("is out of range") == 1

    def test_wmi_probe_lists_only_amd_adapters(self):
        # The masks index AMD devices only; same -match filter as setup.ps1's $wmiGpus.
        source = _STACK_PATH.read_text(encoding = "utf-8")
        assert "$_.Name -match 'AMD|Radeon'" in source
        arch, _ = self._wmi_arch(
            ["Intel(R) UHD Graphics", "AMD Radeon RX 9060 XT"], {"HIP_VISIBLE_DEVICES": "0"}
        )
        assert arch == "gfx1200"

    def test_wmi_unmappable_adapter_suppresses_the_repick(self):
        # An unmapped name drops out of the arch list; like setup.ps1, repick only when all mapped.
        arch, out = self._wmi_arch(
            ["AMD Radeon Graphics", "AMD Radeon 780M Graphics", "AMD Radeon RX 9060 XT"], {}
        )
        assert arch == "gfx1103"
        assert "setx HIP_VISIBLE_DEVICES" not in out

    def test_wmi_repicks_the_dgpu_when_every_adapter_maps(self):
        arch, out = self._wmi_arch(["AMD Radeon 780M Graphics", "AMD Radeon RX 9060 XT"], {})
        assert arch == "gfx1200"
        assert "setx HIP_VISIBLE_DEVICES 1" in out

    def test_wmi_masked_unmappable_adapter_warns_instead_of_substituting(self):
        # Under a mask another card's arch must not stand in (same rule as setup.ps1).
        arch, out = self._wmi_arch(
            ["AMD Radeon Graphics", "AMD Radeon RX 9060 XT"], {"HIP_VISIBLE_DEVICES": "0"}
        )
        assert arch is None
        assert "UNSLOTH_ROCM_GFX_ARCH" in out and "AMD Radeon Graphics" in out

    def test_wmi_unpinned_unmappable_adapter_still_infers(self):
        # Negative control: unpinned there is no choice to respect, so the adapter that
        # did map still decides and the host keeps its wheels.
        arch, _ = self._wmi_arch(["AMD Radeon Graphics", "AMD Radeon RX 9060 XT"], {})
        assert arch == "gfx1200"

    def test_wmi_mask_selects_the_adapter_it_names(self):
        arch, _ = self._wmi_arch(
            ["AMD Radeon 780M Graphics", "AMD Radeon RX 9060 XT"], {"HIP_VISIBLE_DEVICES": "0"}
        )
        assert arch == "gfx1103"

    def test_wmi_fallback_skips_disabled_adapters(self):
        # WMI lists disabled/errored Radeons; filter on ConfigManagerErrorCode like setup.ps1.
        captured = {}

        ps_result = MagicMock()
        ps_result.returncode = 0
        ps_result.stdout = b"AMD Radeon(TM) 780M Graphics\r\n"

        def _run(cmd, **kwargs):
            if cmd and "powershell.exe" in str(cmd[0]).lower():
                captured["cmd"] = cmd[-1]
                return ps_result
            raise FileNotFoundError(cmd[0])

        with patch.dict(os.environ, {}, clear = False):
            for _v in (
                "HIP_PATH",
                "ROCM_PATH",
                "UNSLOTH_ROCM_GFX_ARCH",
                "UNSLOTH_ENABLE_AMD_SMI",
                "HIP_VISIBLE_DEVICES",
                "ROCR_VISIBLE_DEVICES",
                "CUDA_VISIBLE_DEVICES",
            ):
                os.environ.pop(_v, None)
            with patch("shutil.which", return_value = None):
                with patch("os.path.isfile", return_value = False):
                    with patch("subprocess.run", side_effect = _run):
                        result = stack_mod._detect_windows_gfx_arch()
        assert result == "gfx1103"
        assert "ConfigManagerErrorCode" in captured["cmd"]

    @pytest.mark.skipif(shutil.which("pwsh") is None, reason = "pwsh not installed")
    def test_wmi_adapter_filter_is_valid_powershell(self):
        # Run the real filter: a null error code (some drivers) and 0 stay, the rest go.
        source = _STACK_PATH.read_text(encoding = "utf-8")
        assert "ConfigManagerErrorCode" in source
        script = """
class FakeAdapter { [string]$Name; [object]$ConfigManagerErrorCode }
$adapters = @(
    [FakeAdapter]@{ Name = 'AMD Radeon(TM) 780M Graphics'; ConfigManagerErrorCode = 0 }
    [FakeAdapter]@{ Name = 'AMD Radeon RX 9060 XT';        ConfigManagerErrorCode = 22 }
    [FakeAdapter]@{ Name = 'AMD Radeon RX 7900 XTX';       ConfigManagerErrorCode = $null }
)
($adapters | Where-Object { ($null -eq $_.ConfigManagerErrorCode) -or (
    $_.ConfigManagerErrorCode -eq 0) }).Name
"""
        # run_pwsh: a crashed interpreter would read as a malformed filter.
        out = run_pwsh(
            ["pwsh", "-NoProfile", "-NonInteractive", "-Command", script],
            check = True,
            capture_output = True,
            text = True,
        ).stdout.split()
        assert "9060" not in " ".join(out)
        assert "780M" in " ".join(out) and "7900" in " ".join(out)

    def test_stack_probes_venv_hipinfo(self):
        """venv Scripts hipInfo.exe (from AMD torch wheels) must be a probe candidate for driver-only hosts."""
        source = _STACK_PATH.read_text(encoding = "utf-8")
        assert 'os.path.join(os.path.dirname(sys.executable), "hipInfo.exe")' in source

    def test_prebuilt_resolve_exe_probes_venv_dir(self):
        """_resolve_exe must include the venv Scripts candidate for driver-only standalone reruns."""
        source = _PREBUILT_PATH.read_text(encoding = "utf-8")
        assert "_venv_candidate" in source

    def test_runtime_monitor_guards_amd_smi_absence(self):
        """amd.py must which()-check amd-smi before spawning (absence disables the poller)."""
        amd_path = PACKAGE_ROOT / "studio" / "backend" / "utils" / "hardware" / "amd.py"
        source = amd_path.read_text(encoding = "utf-8")
        assert 'shutil.which("amd-smi") is None' in source


class TestSetupPs1ShadowingParity:
    """setup.ps1 builds $ROCmIndexUrl itself before the Python installer runs, so both halves of
    #7776 must exist in its PowerShell copy or fresh Windows installs pick the iGPU."""

    @staticmethod
    def _setup_ps1() -> str:
        return (PACKAGE_ROOT / "studio" / "setup.ps1").read_text(encoding = "utf-8")

    def test_repick_is_wheel_aware(self):
        # Deposing a supported APU for a wheel-less dGPU drops the host to CPU.
        source = self._setup_ps1()
        body = source[source.index("function Resolve-ShadowingGfxPick") :]
        body = body[: body.index("\n}\n")]
        assert "archFamilyMap" in body, "repick must consult the wheel index map"
        assert "pickedHasWheels" in body

    def test_arch_family_map_precedes_the_repick(self):
        # Consumed during detection, so it must be defined before the reader.
        source = self._setup_ps1()
        assert source.index("$archFamilyMap = @{") < source.index(
            "function Resolve-ShadowingGfxPick"
        )

    def test_wmi_fallback_keeps_every_amd_adapter(self):
        # WMI order follows driver enumeration, so first-AMD-match reintroduced #7776.
        source = self._setup_ps1()
        block = source[source.index("$amdGpus = @(Get-CimInstance Win32_VideoController") :]
        block = block[: block.index("$ROCmGpuLabel = $script:ROCmGpuLabels[0]")]
        assert "Select-Object -First 1" not in block
        # A disabled or driver-errored Radeon must not depose a working iGPU.
        assert "ConfigManagerErrorCode" in block
        # @() around the WHOLE if: a one-element CimInstance .Count is $null on PS 5.1 (#8335).
        assert "@(if ($healthyGpus.Count -gt 0) { $healthyGpus } else { $amdGpus })" in block

    def test_index_picks_read_the_same_masks_as_the_pin_check(self):
        # Every mask-to-index site must read CUDA_VISIBLE_DEVICES too; one helper keeps them agreeing.
        source = self._setup_ps1()
        helper = source[source.index("function Resolve-VisibleGpuIndex") :]
        helper = helper[: helper.index("\n}\n")]
        for _mask in ("HIP_VISIBLE_DEVICES", "ROCR_VISIBLE_DEVICES", "CUDA_VISIBLE_DEVICES"):
            assert _mask in helper
        # hipinfo, amd-smi list, amd-smi static --asic, WMI name inference.
        assert (
            source.count("Resolve-VisibleGpuIndex $") + source.count("Resolve-VisibleGpuIndex ")
            >= 4
        )
        assert "$_hipVisIdx = if (" not in source
        assert "$visGpu = if (" not in source

    def test_hipinfo_is_not_reindexed(self):
        # hipinfo already applied the mask; only amd-smi and WMI may call the resolver.
        source = self._setup_ps1()
        hipinfo = source[source.index("$_hipAllArches.Count -gt 0") :]
        hipinfo = hipinfo[: hipinfo.index("Resolve-ShadowingGfxPick")]
        assert "Resolve-VisibleGpuIndex" not in hipinfo
        assert "$_hipAllArches[0]" in hipinfo

    def test_unknown_selected_wmi_adapter_is_not_substituted(self):
        # Under a mask, borrowing another adapter's arch installs for a masked-away GPU.
        source = self._setup_ps1()
        block = source[source.index("$pickedName = Get-GfxArchFromGpuName") :]
        block = block[: block.index("if ($pickedName) {")]
        assert "Test-VisibleDevicesPinned" in block

    def test_name_inference_runs_the_shadowing_pick(self):
        source = self._setup_ps1()
        assert "Get-GfxArchFromGpuName" in source
        infer = source[source.index("$nameArches = @()") :]
        infer = infer[: infer.index("Tip: set UNSLOTH_ROCM_GFX_ARCH")]
        assert "Resolve-ShadowingGfxPick" in infer


@pytest.mark.skipif(shutil.which("pwsh") is None, reason = "pwsh not installed")
class TestSetupPs1ShadowingBehaviour:
    """Runs the real Resolve-ShadowingGfxPick / Resolve-VisibleGpuIndex. The parity
    class above only greps text, so a rename fails it while a semantic bug passes.
    Both helpers are sliced out by AST -- setup.ps1 is never dot-sourced."""

    _HARNESS = r"""
$ErrorActionPreference = "Stop"
$errors = $null; $tokens = $null
$ast = [System.Management.Automation.Language.Parser]::ParseFile($env:SETUP_PS1, [ref]$tokens, [ref]$errors)
if ($errors -and $errors.Count -gt 0) { throw "setup.ps1 has $($errors.Count) parse errors" }
function substep { param([string]$Message, [string]$Color = "DarkGray") }
foreach ($n in @("Test-VisibleDevicesPinned", "Resolve-VisibleGpuIndex", "Resolve-ShadowingGfxPick")) {
    $f = $ast.Find({ param($x) $x -is [System.Management.Automation.Language.FunctionDefinitionAst] -and $x.Name -eq $n }, $true)
    if (-not $f) { throw "$n not found" }
    Invoke-Expression $f.Extent.Text
}
foreach ($v in @('$script:ShadowingIntegratedGfx', '$archFamilyMap')) {
    $a = $ast.FindAll({ param($x) $x -is [System.Management.Automation.Language.AssignmentStatementAst] }, $true) |
        Where-Object { $_.Left.Extent.Text -eq $v } | Select-Object -First 1
    if (-not $a) { throw "$v not found" }
    Invoke-Expression $a.Extent.Text
}
"""

    def _run(
        self,
        body: str,
        env: "dict[str, str] | None" = None,
    ) -> str:
        merged = {k: v for k, v in os.environ.items() if not k.endswith("_VISIBLE_DEVICES")}
        merged["SETUP_PS1"] = str(PACKAGE_ROOT / "studio" / "setup.ps1")
        merged.update(env or {})
        # run_pwsh: a crashed interpreter returns no stdout, read as a wrong pick.
        out = run_pwsh(
            ["pwsh", "-NoProfile", "-NonInteractive", "-Command", self._HARNESS + body],
            check = True,
            capture_output = True,
            text = True,
            env = merged,
        )
        return out.stdout.strip()

    _MIXED = 'Resolve-ShadowingGfxPick -Picked "gfx1036" -AllArches @("gfx1036","gfx1200")'

    def test_unpinned_mixed_host_picks_the_discrete_gpu(self):
        assert self._run(self._MIXED) == "gfx1200"

    def test_explicit_pin_is_honoured(self):
        assert self._run(self._MIXED, {"HIP_VISIBLE_DEVICES": "0"}) == "gfx1036"

    def test_strix_is_not_deposed(self):
        body = 'Resolve-ShadowingGfxPick -Picked "gfx1151" -AllArches @("gfx1151","gfx1200")'
        assert self._run(body) == "gfx1151"

    def test_all_hiding_spellings_count_as_a_selection(self):
        # "" / "-1" surface zero devices, so they are a deliberate choice and suppress
        # the repick rather than invite it.
        for _val in ("", "-1"):
            assert self._run(self._MIXED, {"HIP_VISIBLE_DEVICES": _val}) == "gfx1036"

    @pytest.mark.parametrize("mask", ["1", "1,0", " 1 "])
    def test_index_resolves_every_mask_spelling(self, mask):
        for _var in ("HIP_VISIBLE_DEVICES", "ROCR_VISIBLE_DEVICES", "CUDA_VISIBLE_DEVICES"):
            assert self._run("Resolve-VisibleGpuIndex 2", {_var: mask}) == "1"

    def test_out_of_range_and_unparseable_masks_fall_back_to_zero(self):
        for _val in ("7", "GPU-abc123", "-1", ""):
            assert self._run("Resolve-VisibleGpuIndex 2", {"HIP_VISIBLE_DEVICES": _val}) == "0"

    def test_an_all_hiding_hip_mask_shadows_a_later_cuda_mask(self):
        # First-set-wins, as in Python: an empty HIP mask shadows CUDA_VISIBLE_DEVICES
        # in the runtime rather than deferring to it.
        env = {"HIP_VISIBLE_DEVICES": "", "CUDA_VISIBLE_DEVICES": "1"}
        assert self._run("Resolve-VisibleGpuIndex 2", env) == "0"

    def test_wheelless_discrete_does_not_depose_a_supported_igpu(self):
        body = 'Resolve-ShadowingGfxPick -Picked "gfx1036" -AllArches @("gfx1036","gfx1010")'
        assert self._run(body) == "gfx1036"

    def test_wheelless_igpu_still_prefers_a_wheel_backed_discrete(self):
        # gfx90c has no wheel index, so the guard is inactive; taking the first discrete went CPU-only.
        for _igpu in ("gfx90c", "gfx1013", "gfx1153"):
            body = (
                f'Resolve-ShadowingGfxPick -Picked "{_igpu}" '
                f'-AllArches @("{_igpu}","gfx1010","gfx1200")'
            )
            assert self._run(body) == "gfx1200", _igpu
        # Not over-tightened: with nothing wheel-backed to move to, a wheel-less iGPU
        # still yields to the discrete card, as _dedup_pick() does.
        body = 'Resolve-ShadowingGfxPick -Picked "gfx90c" -AllArches @("gfx90c","gfx1010")'
        assert self._run(body) == "gfx1010"

    def test_advisory_names_the_selected_gpus_real_index(self):
        body = (
            '$null = Resolve-ShadowingGfxPick -Picked "gfx1036" '
            '-AllArches @("gfx1036","gfx1010","gfx1200"); '
            '($script:Msgs -join " ")'
        )
        harness = self._HARNESS.replace(
            'function substep { param([string]$Message, [string]$Color = "DarkGray") }',
            'function substep { param([string]$Message, [string]$Color = "DarkGray") '
            "$script:Msgs += $Message }\n$script:Msgs = @()",
        )
        merged = {k: v for k, v in os.environ.items() if not k.endswith("_VISIBLE_DEVICES")}
        merged["SETUP_PS1"] = str(PACKAGE_ROOT / "studio" / "setup.ps1")
        # run_pwsh: a pwsh killed at startup looks like a lost index.
        out = run_pwsh(
            ["pwsh", "-NoProfile", "-NonInteractive", "-Command", harness + body],
            check = True,
            capture_output = True,
            text = True,
            env = merged,
        ).stdout
        assert "HIP_VISIBLE_DEVICES 2" in out
        assert "HIP_VISIBLE_DEVICES 1" not in out

    def test_degenerate_inputs_do_not_throw(self):
        for _all in ("$null", "@()", '@("gfx1036")'):
            body = f'Resolve-ShadowingGfxPick -Picked "gfx1036" -AllArches {_all}'
            assert self._run(body) == "gfx1036"


class TestInstallBnbWindowsRocm:
    """Verify AMD Windows BNB wheel install helper."""

    @pytest.fixture(autouse = True)
    def _isolate_sitecustomize_persistence(self, monkeypatch, request):
        """Keep helper tests from writing to the active interpreter site-packages."""
        if request.node.name.startswith("test_persist"):
            return
        monkeypatch.setattr(
            stack_mod,
            "_persist_bnb_rocm_version",
            lambda version: True,
        )

    def test_calls_pip_install_try_with_win_amd64_url(self):
        """Should call pip_install_try with the win_amd64 wheel URL via plain pip."""
        with patch.object(stack_mod, "pip_install_try", return_value = True) as mock_pip:
            stack_mod._install_bnb_windows_rocm()
        assert mock_pip.call_count == 1
        call_args = str(mock_pip.call_args_list[0])
        assert "bitsandbytes" in call_args
        assert "win_amd64" in call_args
        # Plain pip: uv mangles the bitsandbytes wheel.
        assert mock_pip.call_args.kwargs.get("force_pip") is True

    def test_forces_plain_pip_not_uv(self):
        """The bnb wheel must be installed with plain pip, never uv."""
        with patch.object(stack_mod, "pip_install_try", return_value = True) as mock_pip:
            stack_mod._install_bnb_windows_rocm()
        assert mock_pip.call_args.kwargs.get("force_pip") is True

    def test_does_not_touch_uv_skip_env_var(self):
        """The UV_SKIP_WHEEL_FILENAME_CHECK hack is gone; the env must be untouched."""
        observed = {}

        def _capture(*args, **kwargs):
            observed["during"] = os.environ.get("UV_SKIP_WHEEL_FILENAME_CHECK")
            return True

        with patch.dict(os.environ, {}, clear = False):
            os.environ.pop("UV_SKIP_WHEEL_FILENAME_CHECK", None)
            with patch.object(stack_mod, "pip_install_try", side_effect = _capture):
                stack_mod._install_bnb_windows_rocm()
            assert observed.get("during") is None
            assert "UV_SKIP_WHEEL_FILENAME_CHECK" not in os.environ

    def test_returns_false_on_pip_failure(self):
        """A failed pip_install_try must surface as a False return, not BNB_ROCM_VERSION."""
        with patch.dict(os.environ, {}, clear = False):
            os.environ.pop("BNB_ROCM_VERSION", None)
            with patch.object(stack_mod, "pip_install_try", return_value = False):
                result = stack_mod._install_bnb_windows_rocm()
            assert result is False
            assert "BNB_ROCM_VERSION" not in os.environ

    def test_falls_back_to_pypi_when_win_amd64_url_missing(self):
        """No win_amd64 pre-release wheel must not mean no bitsandbytes: PyPI
        >=0.50.0 ships libbitsandbytes_rocm{714,72}.dll, so it is a real ROCm build."""
        with patch.object(stack_mod, "_BNB_ROCM_PRERELEASE_URLS", {}):
            with patch.object(stack_mod, "pip_install_try", return_value = True) as mock_pip:
                stack_mod._install_bnb_windows_rocm()
        assert mock_pip.call_count == 1
        assert stack_mod._BNB_ROCM_PYPI_FALLBACK in mock_pip.call_args.args

    def test_falls_back_to_pypi_when_prerelease_install_fails(self):
        """A blocked GitHub pre-release URL must fall through to the PyPI floor rather
        than leaving Windows ROCm with no working bitsandbytes."""
        with patch.object(stack_mod, "pip_install_try", side_effect = [False, True]) as mock_pip:
            with patch.object(stack_mod, "_detect_bnb_rocm_dll_ver", return_value = "72"):
                result = stack_mod._install_bnb_windows_rocm()
        assert result is True
        assert mock_pip.call_count == 2
        assert "win_amd64" in str(mock_pip.call_args_list[0])
        assert stack_mod._BNB_ROCM_PYPI_FALLBACK in mock_pip.call_args_list[1].args

    def test_returns_false_only_when_both_paths_fail(self):
        """Both the pre-release wheel and the PyPI fallback must fail before the helper reports failure."""
        with patch.dict(os.environ, {}, clear = False):
            os.environ.pop("BNB_ROCM_VERSION", None)
            with patch.object(stack_mod, "pip_install_try", return_value = False) as mock_pip:
                result = stack_mod._install_bnb_windows_rocm()
        assert result is False
        assert mock_pip.call_count == 2

    def test_sets_bnb_rocm_version_from_detected_dll(self):
        """BNB_ROCM_VERSION is set from the DLL detected after install."""
        with patch.dict(os.environ, {}, clear = False):
            os.environ.pop("BNB_ROCM_VERSION", None)
            os.environ.pop(stack_mod._BNB_ROCM_VERSION_SOURCE_ENV, None)
            with patch.object(stack_mod, "pip_install_try", return_value = True):
                with patch.object(stack_mod, "_detect_bnb_rocm_dll_ver", return_value = "72"):
                    stack_mod._install_bnb_windows_rocm()
            assert os.environ.get("BNB_ROCM_VERSION") == "72"

    def test_sets_bnb_rocm_version_from_newer_dll(self):
        """If AMD ships a newer DLL (e.g. rocm713.dll), that version is used."""
        with patch.dict(os.environ, {}, clear = False):
            os.environ.pop("BNB_ROCM_VERSION", None)
            os.environ.pop(stack_mod._BNB_ROCM_VERSION_SOURCE_ENV, None)
            with patch.object(stack_mod, "pip_install_try", return_value = True):
                with patch.object(stack_mod, "_detect_bnb_rocm_dll_ver", return_value = "713"):
                    stack_mod._install_bnb_windows_rocm()
            assert os.environ.get("BNB_ROCM_VERSION") == "713"

    def test_falls_back_to_72_when_detection_fails(self):
        """Falls back to '72' when DLL detection returns None."""
        with patch.dict(os.environ, {}, clear = False):
            os.environ.pop("BNB_ROCM_VERSION", None)
            os.environ.pop(stack_mod._BNB_ROCM_VERSION_SOURCE_ENV, None)
            with patch.object(stack_mod, "pip_install_try", return_value = True):
                with patch.object(stack_mod, "_detect_bnb_rocm_dll_ver", return_value = None):
                    stack_mod._install_bnb_windows_rocm()
            assert os.environ.get("BNB_ROCM_VERSION") == "72"

    def test_does_not_override_existing_bnb_rocm_version(self):
        """An explicit BNB_ROCM_VERSION in the caller's env must not be clobbered."""
        with patch.dict(os.environ, {"BNB_ROCM_VERSION": "60"}):
            os.environ.pop(stack_mod._BNB_ROCM_VERSION_SOURCE_ENV, None)
            with patch.object(stack_mod, "pip_install_try", return_value = True):
                stack_mod._install_bnb_windows_rocm()
            assert os.environ.get("BNB_ROCM_VERSION") == "60"

    def test_does_not_persist_existing_bnb_rocm_version(self):
        """A caller override must not become the venv's managed default."""
        with patch.dict(os.environ, {"BNB_ROCM_VERSION": "60"}):
            os.environ.pop(stack_mod._BNB_ROCM_VERSION_SOURCE_ENV, None)
            with patch.object(stack_mod, "pip_install_try", return_value = True):
                with patch.object(stack_mod, "_detect_bnb_rocm_dll_ver") as mock_detect:
                    with patch.object(
                        stack_mod, "_persist_bnb_rocm_version", return_value = True
                    ) as mock_persist:
                        stack_mod._install_bnb_windows_rocm()

            assert os.environ.get("BNB_ROCM_VERSION") == "60"
            mock_detect.assert_not_called()
            mock_persist.assert_not_called()

    def test_redetects_when_bnb_rocm_version_came_from_sitecustomize(self):
        """Persisted defaults should not mask a newer DLL suffix after reinstall."""
        with patch.dict(
            os.environ,
            {
                "BNB_ROCM_VERSION": "72",
                stack_mod._BNB_ROCM_VERSION_SOURCE_ENV: (
                    stack_mod._BNB_ROCM_VERSION_SOURCE_SITECUSTOMIZE
                ),
            },
        ):
            with patch.object(stack_mod, "pip_install_try", return_value = True):
                with patch.object(stack_mod, "_detect_bnb_rocm_dll_ver", return_value = "713"):
                    with patch.object(
                        stack_mod, "_persist_bnb_rocm_version", return_value = True
                    ) as mock_persist:
                        stack_mod._install_bnb_windows_rocm()

            assert os.environ.get("BNB_ROCM_VERSION") == "713"
            assert (
                os.environ.get(stack_mod._BNB_ROCM_VERSION_SOURCE_ENV)
                == stack_mod._BNB_ROCM_VERSION_SOURCE_DETECTED
            )
            mock_persist.assert_called_once_with("713")

    def test_persists_bnb_rocm_version_for_direct_venv_python(self, tmp_path):
        """BNB_ROCM_VERSION must apply to a fresh Python process in the venv."""
        site_packages = tmp_path / "site-packages"

        with patch.dict(os.environ, {}, clear = False):
            os.environ.pop("BNB_ROCM_VERSION", None)
            os.environ.pop(stack_mod._BNB_ROCM_VERSION_SOURCE_ENV, None)
            with patch.object(stack_mod, "pip_install_try", return_value = True):
                with patch.object(stack_mod, "_detect_bnb_rocm_dll_ver", return_value = "72"):
                    with patch.object(
                        stack_mod.sysconfig, "get_path", return_value = str(site_packages)
                    ):
                        stack_mod._install_bnb_windows_rocm()

        sitecustomize = site_packages / "sitecustomize.py"
        source = sitecustomize.read_text(encoding = "utf-8")
        assert "BNB_ROCM_VERSION" in source
        assert stack_mod._BNB_ROCM_VERSION_SOURCE_ENV in source
        assert "'72'" in source

        probe_env = os.environ.copy()
        probe_env.pop("BNB_ROCM_VERSION", None)
        probe_env.pop(stack_mod._BNB_ROCM_VERSION_SOURCE_ENV, None)
        probe_env["PYTHONPATH"] = str(site_packages)
        result = subprocess.run(
            [
                sys.executable,
                "-c",
                (
                    "import os; "
                    "print(os.environ.get('BNB_ROCM_VERSION', ''), "
                    "os.environ.get('UNSLOTH_BNB_ROCM_VERSION_SOURCE', ''))"
                ),
            ],
            env = probe_env,
            stdout = subprocess.PIPE,
            stderr = subprocess.PIPE,
            text = True,
            check = True,
        )
        assert result.stdout.strip() == "72 sitecustomize"

    def test_persist_bnb_rocm_version_replaces_existing_managed_block(self, tmp_path):
        """Updating sitecustomize.py must not duplicate the managed BNB block."""
        site_packages = tmp_path / "site-packages"
        site_packages.mkdir()
        sitecustomize = site_packages / "sitecustomize.py"
        sitecustomize.write_text(
            "EXISTING = True\n"
            "# BEGIN Unsloth BNB_ROCM_VERSION\n"
            "import os as _unsloth_os\n"
            "_unsloth_os.environ.setdefault('BNB_ROCM_VERSION', '72')\n"
            "# END Unsloth BNB_ROCM_VERSION\n",
            encoding = "utf-8",
        )

        with patch.object(stack_mod.sysconfig, "get_path", return_value = str(site_packages)):
            assert stack_mod._persist_bnb_rocm_version("713") is True

        source = sitecustomize.read_text(encoding = "utf-8")
        assert source.count("# BEGIN Unsloth BNB_ROCM_VERSION") == 1
        assert "EXISTING = True" in source
        assert "'713'" in source
        assert "'72'" not in source

    def test_persist_bnb_rocm_version_handles_non_utf8_sitecustomize(self, tmp_path):
        """A legacy non-UTF-8 sitecustomize.py should not abort installation."""
        site_packages = tmp_path / "site-packages"
        site_packages.mkdir()
        sitecustomize = site_packages / "sitecustomize.py"
        sitecustomize.write_bytes(b"\xff\xfe\x00")

        with patch.object(stack_mod.sysconfig, "get_path", return_value = str(site_packages)):
            assert stack_mod._persist_bnb_rocm_version("72") is False

    def test_persist_bnb_rocm_version_repairs_truncated_block(self, tmp_path):
        """A managed block missing its END marker is replaced, not duplicated."""
        site_packages = tmp_path / "site-packages"
        site_packages.mkdir()
        sitecustomize = site_packages / "sitecustomize.py"
        sitecustomize.write_text(
            "EXISTING = True\n"
            "# BEGIN Unsloth BNB_ROCM_VERSION\n"
            "import os as _unsloth_os\n"
            "_unsloth_os.environ.setdefault('BNB_ROCM_VERSION', '72')\n",
            encoding = "utf-8",
        )

        with patch.object(stack_mod.sysconfig, "get_path", return_value = str(site_packages)):
            assert stack_mod._persist_bnb_rocm_version("713") is True

        source = sitecustomize.read_text(encoding = "utf-8")
        assert source.count("# BEGIN Unsloth BNB_ROCM_VERSION") == 1
        assert source.count("# END Unsloth BNB_ROCM_VERSION") == 1
        assert "EXISTING = True" in source
        assert "'713'" in source
        assert "'72'" not in source

    def test_persist_bnb_rocm_version_dedupes_duplicate_blocks(self, tmp_path):
        """Multiple managed blocks collapse to one while preserving user content."""
        site_packages = tmp_path / "site-packages"
        site_packages.mkdir()
        sitecustomize = site_packages / "sitecustomize.py"
        block = (
            "# BEGIN Unsloth BNB_ROCM_VERSION\n"
            "import os as _unsloth_os\n"
            "_unsloth_os.environ.setdefault('BNB_ROCM_VERSION', '72')\n"
            "# END Unsloth BNB_ROCM_VERSION\n"
        )
        sitecustomize.write_text(block + "USER_MID = 1\n" + block, encoding = "utf-8")

        with patch.object(stack_mod.sysconfig, "get_path", return_value = str(site_packages)):
            assert stack_mod._persist_bnb_rocm_version("713") is True

        source = sitecustomize.read_text(encoding = "utf-8")
        assert source.count("# BEGIN Unsloth BNB_ROCM_VERSION") == 1
        assert source.count("# END Unsloth BNB_ROCM_VERSION") == 1
        assert "USER_MID = 1" in source
        assert "'713'" in source
        assert "'72'" not in source

    def test_persist_bnb_rocm_version_atomic_no_leftover_tmp(self, tmp_path):
        """The write-then-rename path must not leave its temp file behind."""
        site_packages = tmp_path / "site-packages"
        site_packages.mkdir()

        with patch.object(stack_mod.sysconfig, "get_path", return_value = str(site_packages)):
            assert stack_mod._persist_bnb_rocm_version("72") is True

        leftovers = [p.name for p in site_packages.iterdir() if "unsloth-tmp" in p.name]
        assert leftovers == []
        assert (site_packages / "sitecustomize.py").exists()


class TestRuntimeBnbRocmSourceGuards:
    """Runtime entrypoints redetect managed defaults but keep caller overrides."""

    _MAIN_PATH = PACKAGE_ROOT / "studio" / "backend" / "main.py"
    _TRAINING_WORKER_PATH = PACKAGE_ROOT / "studio" / "backend" / "core" / "training" / "worker.py"

    def test_main_gate_redetects_persisted_default(self):
        source = self._MAIN_PATH.read_text(encoding = "utf-8")
        assert 'os.environ.get("UNSLOTH_BNB_ROCM_VERSION_SOURCE") == "sitecustomize"' in source
        assert 'os.environ["UNSLOTH_BNB_ROCM_VERSION_SOURCE"] = "detected"' in source

    def test_worker_gate_redetects_persisted_default(self):
        source = self._TRAINING_WORKER_PATH.read_text(encoding = "utf-8")
        assert 'os.environ.get("UNSLOTH_BNB_ROCM_VERSION_SOURCE") == "sitecustomize"' in source
        assert 'os.environ["UNSLOTH_BNB_ROCM_VERSION_SOURCE"] = "detected"' in source

    def test_fallback_prefers_seeded_value_over_hardcoded_72(self):
        """A failed redetect must not downgrade a persisted suffix to '72'."""
        for path in (self._MAIN_PATH, self._TRAINING_WORKER_PATH):
            source = path.read_text(encoding = "utf-8")
            assert (
                '_bnb_rocm_ver or os.environ.get("BNB_ROCM_VERSION") or "72"' in source
            ), path.name

    def test_main_requires_found_rocm_dll(self):
        """HIP_PATH/ROCM_PATH alone (HIP SDK on a CUDA/CPU box) must not force
        a ROCm backend onto a non-ROCm bitsandbytes."""
        source = self._MAIN_PATH.read_text(encoding = "utf-8")
        assert "if _found_rocm_bnb:" in source
        assert "_hip_env" not in source

    def test_worker_requires_found_rocm_dll(self):
        """No DLL found: the worker must not write any override or touch the
        seeded marker (later import fixes must still see sitecustomize)."""
        source = self._TRAINING_WORKER_PATH.read_text(encoding = "utf-8")
        assert "if _found_rocm_bnb:" in source


class TestDetectBnbRocmDllVer:
    """Unit tests for _detect_bnb_rocm_dll_ver()."""

    def test_returns_none_when_bnb_not_installed(self):
        """Returns None if bitsandbytes is not importable."""
        import importlib.util
        with patch.object(importlib.util, "find_spec", return_value = None):
            assert stack_mod._detect_bnb_rocm_dll_ver() is None

    def test_detects_rocm72_dll(self, tmp_path):
        """Returns '72' when libbitsandbytes_rocm72.dll is present."""
        (tmp_path / "libbitsandbytes_rocm72.dll").write_text("")
        mock_spec = MagicMock()
        mock_spec.submodule_search_locations = [str(tmp_path)]
        import importlib.util

        with patch.object(importlib.util, "find_spec", return_value = mock_spec):
            assert stack_mod._detect_bnb_rocm_dll_ver() == "72"

    def test_detects_rocm713_dll(self, tmp_path):
        """Returns '713' when libbitsandbytes_rocm713.dll is present."""
        (tmp_path / "libbitsandbytes_rocm713.dll").write_text("")
        mock_spec = MagicMock()
        mock_spec.submodule_search_locations = [str(tmp_path)]
        import importlib.util

        with patch.object(importlib.util, "find_spec", return_value = mock_spec):
            assert stack_mod._detect_bnb_rocm_dll_ver() == "713"

    def test_returns_none_when_only_cuda_dlls(self, tmp_path):
        """Returns None when only CUDA DLLs are present (no ROCm DLL)."""
        (tmp_path / "libbitsandbytes_cuda121.dll").write_text("")
        mock_spec = MagicMock()
        mock_spec.submodule_search_locations = [str(tmp_path)]
        import importlib.util

        with patch.object(importlib.util, "find_spec", return_value = mock_spec):
            assert stack_mod._detect_bnb_rocm_dll_ver() is None

    def test_picks_highest_suffix_when_multiple_dlls(self, tmp_path):
        """Returns the highest numeric suffix across ROCm DLL variants (glob order is not guaranteed)."""
        (tmp_path / "libbitsandbytes_rocm72.dll").write_text("")
        (tmp_path / "libbitsandbytes_rocm713.dll").write_text("")
        mock_spec = MagicMock()
        mock_spec.submodule_search_locations = [str(tmp_path)]
        import importlib.util

        with patch.object(importlib.util, "find_spec", return_value = mock_spec):
            assert stack_mod._detect_bnb_rocm_dll_ver() == "713"


class TestRocmTorchInstalledEnvVar:
    """Verify UNSLOTH_ROCM_TORCH_INSTALLED=1 skips main install but still installs BNB."""

    @staticmethod
    def _ok_torch_probe(*a, **kw):
        # A non-empty HIP version marks the build as ROCm.
        rv = MagicMock()
        rv.returncode = 0
        rv.stdout = _MARK + "2.10.0+rocm7.1|7.1|\n"
        return rv

    @patch.object(stack_mod, "_install_bnb_windows_rocm")
    @patch.object(stack_mod, "pip_install")
    def test_env_var_skips_main_pip_install(self, mock_pip, mock_bnb):
        """UNSLOTH_ROCM_TORCH_INSTALLED=1 should not trigger torch pip_install."""
        with (
            patch.dict(os.environ, {"UNSLOTH_ROCM_TORCH_INSTALLED": "1"}),
            patch.object(stack_mod.subprocess, "run", side_effect = self._ok_torch_probe),
        ):
            stack_mod._ensure_rocm_torch()
        mock_pip.assert_not_called()

    @patch.object(stack_mod, "_install_bnb_windows_rocm")
    @patch.object(stack_mod, "pip_install")
    def test_env_var_calls_bnb_install(self, mock_pip, mock_bnb):
        """UNSLOTH_ROCM_TORCH_INSTALLED=1 should still call _install_bnb_windows_rocm."""
        with (
            patch.dict(os.environ, {"UNSLOTH_ROCM_TORCH_INSTALLED": "1"}),
            patch.object(stack_mod.subprocess, "run", side_effect = self._ok_torch_probe),
        ):
            stack_mod._ensure_rocm_torch()
        mock_bnb.assert_called_once()

    @patch.object(stack_mod, "_install_bnb_windows_rocm")
    @patch.object(stack_mod, "pip_install")
    def test_env_var_sets_rocm_windows_flag(self, mock_pip, mock_bnb):
        """UNSLOTH_ROCM_TORCH_INSTALLED=1 should set _rocm_windows_torch_installed."""
        stack_mod._rocm_windows_torch_installed = False
        with (
            patch.dict(os.environ, {"UNSLOTH_ROCM_TORCH_INSTALLED": "1"}),
            patch.object(stack_mod.subprocess, "run", side_effect = self._ok_torch_probe),
        ):
            stack_mod._ensure_rocm_torch()
        assert stack_mod._rocm_windows_torch_installed is True

    @patch.object(stack_mod, "_install_bnb_windows_rocm")
    @patch.object(stack_mod, "pip_install")
    def test_env_var_falls_through_when_torch_missing(self, mock_pip, mock_bnb):
        """If the venv was wiped between runs, the stale env-var must not suppress reinstall."""
        stack_mod._rocm_windows_torch_installed = False

        def _bad_probe(*a, **kw):
            rv = MagicMock()
            rv.returncode = 1
            return rv

        with (
            patch.dict(os.environ, {"UNSLOTH_ROCM_TORCH_INSTALLED": "1"}),
            patch.object(stack_mod.subprocess, "run", side_effect = _bad_probe),
            patch.object(stack_mod, "IS_WINDOWS", False),
            patch.object(stack_mod, "IS_MACOS", True),
        ):
            stack_mod._ensure_rocm_torch()
        # macOS branch is the next exit; the point is the early-return did NOT fire.
        mock_bnb.assert_not_called()


class TestWindowsRocmTorchaoGuard:
    """Verify the torchao skip can detect an installed Windows ROCm torch build."""

    def test_installed_torch_is_windows_rocm_accepts_rocm_probe(self):
        rv = MagicMock()
        rv.returncode = 0
        rv.stdout = _MARK + "2.10.0+rocm7.1|7.1|"
        with (
            patch.object(stack_mod, "IS_WINDOWS", True),
            patch.object(stack_mod.subprocess, "run", return_value = rv),
        ):
            assert stack_mod._installed_torch_is_windows_rocm() is True

    def test_installed_torch_is_windows_rocm_rejects_non_rocm_probe(self):
        rv = MagicMock()
        rv.returncode = 0
        rv.stdout = ""
        with (
            patch.object(stack_mod, "IS_WINDOWS", True),
            patch.object(stack_mod.subprocess, "run", return_value = rv),
        ):
            assert stack_mod._installed_torch_is_windows_rocm() is False

    def test_installed_torch_is_windows_rocm_is_non_windows_noop(self):
        with patch.object(stack_mod, "IS_WINDOWS", False):
            assert stack_mod._installed_torch_is_windows_rocm() is False

    @patch.object(stack_mod, "_repair_bad_accelerate")
    @patch.object(stack_mod, "_repair_bad_anyio")
    @patch.object(stack_mod, "_ensure_rocm_torch")
    @patch.object(stack_mod, "_ensure_cuda_torch")
    @patch.object(stack_mod, "_has_usable_nvidia_gpu", return_value = True)
    @patch.object(stack_mod, "run")
    @patch.object(stack_mod, "pip_install")
    def test_install_python_stack_installs_pypi_torchao_when_windows_rocm_torch_is_installed(
        self,
        mock_pip,
        mock_run,
        mock_has_nvidia,
        mock_cuda,
        mock_rocm,
        mock_anyio,
        mock_accelerate,
        tmp_path,
    ):
        unstructured_plugin = tmp_path / "unstructured"
        github_plugin = tmp_path / "github"
        unstructured_plugin.mkdir()
        github_plugin.mkdir()

        subprocess_result = MagicMock()
        subprocess_result.returncode = 0
        subprocess_result.stdout = ""

        with (
            patch.dict(os.environ, {"SKIP_STUDIO_BASE": "1"}),
            patch.object(stack_mod, "IS_WINDOWS", True),
            patch.object(stack_mod, "IS_MACOS", False),
            patch.object(stack_mod, "IS_MAC_ARM", False),
            patch.object(stack_mod, "NO_TORCH", False),
            patch.object(stack_mod, "_rocm_windows_torch_installed", False),
            patch.object(stack_mod, "_bootstrap_uv", return_value = False),
            patch.object(stack_mod, "_installed_torch_is_windows_rocm", return_value = True),
            patch.object(stack_mod, "LOCAL_DD_UNSTRUCTURED_PLUGIN", unstructured_plugin),
            patch.object(stack_mod, "LOCAL_DD_GITHUB_PLUGIN", github_plugin),
            # The final core-payload check reads the real env; report unsloth installed and intact.
            patch.object(stack_mod.install_manifest, "installed_versions", return_value = ["0"]),
            patch.object(stack_mod.install_manifest, "damaged_payload_files", return_value = []),
            patch.object(stack_mod.subprocess, "run", return_value = subprocess_result),
            patch.object(stack_mod, "pip_install_try", return_value = True) as mock_try,
        ):
            assert stack_mod.install_python_stack() == 0

        torchao_calls = [
            [str(arg) for arg in call.args]
            for call in mock_pip.call_args_list + mock_try.call_args_list
            if any(str(arg).startswith("torchao") for arg in call.args)
        ]
        # From PyPI: download.pytorch.org's rocm leaves serve Linux only.
        assert torchao_calls
        assert all("--index-url" not in c for c in torchao_calls)


class TestProgressStepCountMatchesTotal:
    """Progress must reach exactly _TOTAL, so every _progress() step must be counted in base_total;
    a repair step once overran _STEP past _TOTAL (Codex P2)."""

    def _run_stack(
        self,
        tmp_path,
        *,
        is_windows,
        is_macos,
        is_mac_arm,
        skip_base = True,
        no_torch = False,
    ):
        unstructured_plugin = tmp_path / "unstructured"
        github_plugin = tmp_path / "github"
        unstructured_plugin.mkdir()
        github_plugin.mkdir()
        sub = MagicMock()
        sub.returncode = 0
        sub.stdout = ""
        with (
            patch.dict(os.environ, {"SKIP_STUDIO_BASE": "1" if skip_base else "0"}),
            patch.object(stack_mod, "_report_mlx_stack_health"),
            patch.object(stack_mod, "_bitsandbytes_installed", return_value = False),
            patch.object(stack_mod, "IS_WINDOWS", is_windows),
            patch.object(stack_mod, "IS_MACOS", is_macos),
            patch.object(stack_mod, "IS_MAC_ARM", is_mac_arm),
            # The mocked subprocess writes no wheel metadata, so the no-torch pair probe (#16157)
            # would find nothing; model a feed that serves a compatible Unsloth / Zoo pair.
            # One patch for both: this with statement is at Python's 20 nested block limit.
            patch.multiple(
                stack_mod,
                NO_TORCH = no_torch,
                _no_torch_pair = MagicMock(return_value = ("2026.11.1", "2026.11.1", 0)),
            ),
            patch.object(stack_mod, "_rocm_windows_torch_installed", False),
            patch.object(stack_mod, "_bootstrap_uv", return_value = True),
            patch.object(stack_mod, "_installed_torch_is_windows_rocm", return_value = False),
            patch.object(stack_mod, "_has_usable_nvidia_gpu", return_value = True),
            patch.object(stack_mod, "_repair_bad_anyio"),
            patch.object(stack_mod, "_repair_bad_accelerate"),
            patch.object(stack_mod, "_ensure_cuda_torch"),
            patch.object(stack_mod, "_ensure_rocm_torch"),
            patch.object(stack_mod, "_ensure_cpu_torch"),
            patch.object(stack_mod, "LOCAL_DD_UNSTRUCTURED_PLUGIN", unstructured_plugin),
            patch.object(stack_mod, "LOCAL_DD_GITHUB_PLUGIN", github_plugin),
            # The final core-payload check reads the real env; report unsloth installed and intact.
            patch.object(stack_mod.install_manifest, "installed_versions", return_value = ["0"]),
            patch.object(stack_mod.install_manifest, "damaged_payload_files", return_value = []),
            patch.object(stack_mod.subprocess, "run", return_value = sub),
        ):
            assert stack_mod.install_python_stack() == 0
            return stack_mod._STEP, stack_mod._TOTAL

    def test_windows_progress_reaches_total(self, tmp_path):
        step, total = self._run_stack(tmp_path, is_windows = True, is_macos = False, is_mac_arm = False)
        assert step == total, f"Windows progress {step} != total {total} (final step uncounted)"

    def test_linux_progress_reaches_total(self, tmp_path):
        step, total = self._run_stack(tmp_path, is_windows = False, is_macos = False, is_mac_arm = False)
        assert step == total, f"Linux progress {step} != total {total}"

    @pytest.mark.parametrize("skip_base", [True, False])
    @pytest.mark.parametrize(
        "is_windows, is_macos, is_mac_arm",
        [(False, False, False), (True, False, False), (False, True, True), (False, True, False)],
        ids = ["linux", "windows", "macos_arm", "macos_intel"],
    )
    def test_progress_reaches_total_on_both_core_paths(
        self, tmp_path, is_windows, is_macos, is_mac_arm, skip_base
    ):
        """Both the installer handoff and `studio update`, on every platform. The update
        path on Apple Silicon runs the MLX step, which the earlier SKIP_STUDIO_BASE=1-only
        cases never reached, so its slot went uncounted."""
        step, total = self._run_stack(
            tmp_path,
            is_windows = is_windows,
            is_macos = is_macos,
            is_mac_arm = is_mac_arm,
            skip_base = skip_base,
        )
        assert step == total, f"progress {step} != total {total}"

    @pytest.mark.parametrize("skip_base", [True, False])
    @pytest.mark.parametrize(
        "is_windows, is_macos, is_mac_arm",
        [(False, False, False), (True, False, False), (False, True, True)],
        ids = ["linux", "windows", "macos_arm"],
    )
    def test_progress_reaches_total_without_torch(
        self, tmp_path, is_windows, is_macos, is_mac_arm, skip_base
    ):
        """--no-torch, on both core paths. The no-torch runtime deps step announces its own
        progress slot, which the denominator has to count: an update printed 15/14 without it."""
        step, total = self._run_stack(
            tmp_path,
            is_windows = is_windows,
            is_macos = is_macos,
            is_mac_arm = is_mac_arm,
            skip_base = skip_base,
            no_torch = True,
        )
        assert step == total, f"progress {step} != total {total}"


class TestAccelerateRepair:
    """accelerate 1.15 reaches torch._C._distributed_c10d, absent from AMD's Windows ROCm
    wheels (#4249). The constraints cap misses a fresh install, which sets SKIP_STUDIO_BASE=1."""

    def _repair(
        self,
        installed,
        *,
        is_windows = True,
        succeeds = True,
    ):
        with (
            patch.object(stack_mod, "IS_WINDOWS", is_windows),
            patch.object(stack_mod, "_installed_version", return_value = installed),
            patch.object(stack_mod, "pip_install_try", return_value = succeeds) as mock_pip,
        ):
            stack_mod._repair_bad_accelerate()
        return mock_pip

    def test_a_failed_repair_does_not_abort_the_install(self):
        """pip_install exits, and this fires on nearly every fresh Windows install, so one
        unreachable index would fail an install that used to finish."""
        mock_pip = self._repair((1, 15), succeeds = False)
        assert mock_pip.call_count == 1

    def test_the_repair_is_never_the_fatal_variant(self):
        """At source level: swapping the call back changes nothing the mock above sees."""
        src = (PACKAGE_ROOT / "studio" / "install_python_stack.py").read_text(encoding = "utf-8")
        body = src.split("def _repair_bad_accelerate()")[1].split("\ndef ")[0]
        assert "pip_install_try(" in body
        assert "\n    pip_install(" not in body

    @pytest.mark.parametrize("installed", [(1, 15), (1, 16), (2, 0)])
    def test_repairs_a_capped_out_accelerate_on_windows(self, installed):
        mock_pip = self._repair(installed)
        args = [str(a) for a in mock_pip.call_args.args]
        assert "accelerate<1.15.0" in args

    @pytest.mark.parametrize("installed", [None, (1, 14), (1, 9), (0, 34)])
    def test_leaves_a_good_or_missing_accelerate_alone(self, installed):
        """(1, 9) guards a string compare, where "1.9" sorts above "1.15"."""
        assert self._repair(installed).call_count == 0

    @pytest.mark.parametrize("is_windows", [False, True])
    def test_only_windows_is_touched(self, is_windows):
        """WSL reports sys.platform "linux", and every non-Windows build ships c10d."""
        assert self._repair((1, 15), is_windows = is_windows).call_count == int(is_windows)

    def test_repair_is_no_deps(self):
        """accelerate requires torch>=2.0.0: with deps, the reinstall replaces the ROCm wheel."""
        args = [str(a) for a in self._repair((1, 15)).call_args.args]
        assert "--no-deps" in args
        assert "--force-reinstall" not in args

    def test_repair_survives_flag_translation_to_uv(self):
        """pip_install prefers uv, so --no-deps must reach uv too, not just pip."""
        args = ("--no-cache-dir", "--no-deps", "accelerate<1.15.0")
        assert "--no-deps" in stack_mod._translate_pip_args_for_uv(args)
        assert "--no-deps" in stack_mod._build_pip_cmd(args)

    @pytest.mark.parametrize(
        "raw, expected",
        [
            ("1.15.0", (1, 15)),
            ("1.15", (1, 15)),
            ("1.15.0.dev0", (1, 15)),
            ("1.14.0", (1, 14)),
            ("1.9.0", (1, 9)),
            ("unknown", None),
        ],
    )
    def test_version_parsing(self, raw, expected):
        with patch.object(stack_mod, "_installed_version", wraps = stack_mod._installed_version):
            with patch("importlib.metadata.version", return_value = raw):
                assert stack_mod._installed_version("accelerate") == expected

    def test_constraints_file_caps_accelerate_on_win32_only(self):
        """The repair and the constraint must agree, or install and update disagree."""
        text = (
            PACKAGE_ROOT / "studio" / "backend" / "requirements" / "single-env" / "constraints.txt"
        ).read_text(encoding = "utf-8")
        line = next(
            ln
            for ln in text.splitlines()
            if ln.strip().startswith("accelerate") and not ln.strip().startswith("#")
        )
        assert line.strip() == 'accelerate<1.15.0; sys_platform == "win32"'


class TestWorkerWindowsRocmPatches:
    """Verify worker.py contains the required Windows ROCm runtime patches."""

    def test_grouped_mm_dispatch_patch_present(self):
        """worker.py must register a _grouped_mm CUDA dispatch override."""
        source = _WORKER_PATH.read_text(encoding = "utf-8")
        assert '_gm_lib.impl("_grouped_mm"' in source

    def test_grouped_mm_patch_targets_cuda_dispatch_key(self):
        """The dispatch override must target the CUDA key (not CompositeImplicitAutograd)."""
        source = _WORKER_PATH.read_text(encoding = "utf-8")
        assert '"_grouped_mm", _grouped_mm_safe_impl, "CUDA"' in source

    def test_grouped_mm_lib_kept_alive(self):
        """The Library object must be stored to prevent GC clearing the registration."""
        source = _WORKER_PATH.read_text(encoding = "utf-8")
        assert "_WINDOWS_ROCM_GROUPED_MM_LIB" in source

    def test_grouped_mm_handles_offs_grouped_case(self):
        """_grouped_mm fallback must handle the grouped (offs!=None) variant."""
        source = _WORKER_PATH.read_text(encoding = "utf-8")
        assert "offs_list" in source
        assert "offs.tolist()" in source

    def test_worker_calls_shared_torchao_stub(self):
        """worker.py must invoke the shared torchao stub entrypoint."""
        source = _WORKER_PATH.read_text(encoding = "utf-8")
        assert "install_torchao_windows_rocm_stub()" in source

    def test_export_worker_calls_shared_torchao_stub(self):
        """export/worker.py loads real torchao when it can and otherwise the same shared stub."""
        source = _EXPORT_WORKER_PATH.read_text(encoding = "utf-8")
        assert "install_torchao_windows_rocm_real_or_stub()" in source

    def test_embedder_calls_shared_torchao_stub(self):
        """embeddings.py must install the stub before importing sentence-transformers in the main
        process, or torchao crashes on Windows ROCm and the embedder drops to llama-server."""
        source = _EMBEDDINGS_PATH.read_text(encoding = "utf-8")
        assert "install_torchao_windows_rocm_stub()" in source

    def test_torchao_stub_uses_stub_type_meta(self):
        """Torchao stub must use _StubTypeMeta so isinstance() returns False not TypeError."""
        source = _TORCHAO_STUB_PATH.read_text(encoding = "utf-8")
        assert "_StubTypeMeta" in source

    def test_stub_type_meta_has_instancecheck(self):
        """_StubTypeMeta must define __instancecheck__ returning False."""
        source = _TORCHAO_STUB_PATH.read_text(encoding = "utf-8")
        assert "__instancecheck__" in source

    def test_stub_subpackage_finder_registered(self):
        """_StubSubpackageFinder must be appended to sys.meta_path."""
        source = _TORCHAO_STUB_PATH.read_text(encoding = "utf-8")
        assert "sys.meta_path.append(_StubSubpackageFinder())" in source

    def test_torchao_key_submodules_pre_stubbed(self):
        """Key torchao submodules (dtypes, quantization) must be pre-stubbed."""
        source = _TORCHAO_STUB_PATH.read_text(encoding = "utf-8")
        assert "torchao.dtypes" in source
        assert "torchao.quantization" in source

    def test_torchdynamo_disabled_on_windows_rocm(self):
        """worker.py should disable dynamo on Windows ROCm as belt-and-suspenders."""
        source = _WORKER_PATH.read_text(encoding = "utf-8")
        assert "TORCHDYNAMO_DISABLE" in source

    def test_bnb_rocm_version_set_on_windows_rocm(self):
        """worker.py must set BNB_ROCM_VERSION from the detected DLL suffix (BNB's auto-detect can mismatch)."""
        source = _WORKER_PATH.read_text(encoding = "utf-8")
        assert "BNB_ROCM_VERSION" in source
        assert "_detect_bnb_rocm_dll_ver" in source or "libbitsandbytes_rocm" in source
        # Falls back to the seeded value, never a blind "72".
        assert '_bnb_rocm_ver or os.environ.get("BNB_ROCM_VERSION")' in source

    def test_bnb_rocm_version_set_before_ml_imports(self):
        """BNB_ROCM_VERSION must appear in section 1f, before section 2 ML imports."""
        source = _WORKER_PATH.read_text(encoding = "utf-8")
        idx_bnb = source.find("BNB_ROCM_VERSION")
        # Use the entry-point section-2 marker (not the trainer helper's own "# ── 2.").
        idx_sec2 = source.find("# ── 2. Now import ML libraries")
        assert idx_bnb != -1, "BNB_ROCM_VERSION not found in worker.py"
        assert idx_sec2 != -1, "'# ── 2. Now import ML libraries' marker not found in worker.py"
        assert idx_bnb < idx_sec2, (
            "BNB_ROCM_VERSION must be set before section 2 ML imports "
            f"(found at {idx_bnb}, section 2 at {idx_sec2})"
        )

    def test_grouped_mm_patch_guarded_by_windows_and_hip_check(self):
        """_grouped_mm patch must only apply on Windows + HIP torch."""
        source = _WORKER_PATH.read_text(encoding = "utf-8")
        assert 'sys.platform == "win32"' in source
        assert '"version"' in source and '"hip"' in source

    def test_hip_ver_at_least_helper_defined(self):
        """_hip_ver_at_least helper must be defined inside the Windows ROCm block."""
        source = _WORKER_PATH.read_text(encoding = "utf-8")
        assert "def _hip_ver_at_least(major: int, minor: int)" in source

    def test_grouped_mm_patch_gated_on_hip_lt_713(self):
        """_grouped_mm patch must be skipped on HIP >= 7.13 (AMD fixed the bug in ROCm 7.13)."""
        source = _WORKER_PATH.read_text(encoding = "utf-8")
        assert "_hip_ver_at_least(7, 13)" in source
        assert "if not _hip_ver_at_least(7, 13):" in source

    def test_grouped_mm_hip_713_skip_message_present(self):
        """worker.py must log a message when skipping the patch on HIP >= 7.13."""
        source = _WORKER_PATH.read_text(encoding = "utf-8")
        assert "HIP >= 7.13" in source
        assert "7.13" in source

    def test_grouped_mm_patch_else_branch_present(self):
        """An else branch must follow the _hip_ver_at_least gate (skip path for 7.13+)."""
        source = _WORKER_PATH.read_text(encoding = "utf-8")
        gate_idx = source.find("if not _hip_ver_at_least(7, 13):")
        assert gate_idx != -1, "Version gate not found in worker.py"
        else_idx = source.find("else:", gate_idx)
        assert else_idx != -1, "else: branch after _hip_ver_at_least gate not found"

    def test_hip_ver_at_least_handles_amd_version_format(self):
        """_hip_ver_at_least must split on '.' and compare only major.minor (handles '7.13.99004')."""
        source = _WORKER_PATH.read_text(encoding = "utf-8")
        assert 'split(".")[:2]' in source or ".split('.')[:2]" in source


class TestRocmTorchPkgSpecs:
    """Verify per-tag torch version specs are correct."""

    def test_rocm72_has_torch_211(self):
        """rocm7.2 should specify torch 2.11.x."""
        specs = stack_mod._ROCM_TORCH_PKG_SPECS.get("rocm7.2")
        assert specs is not None
        torch_spec = specs[0]
        assert "2.11" in torch_spec

    def test_default_caps_below_211(self):
        """Default spec (rocm7.0 and earlier) should cap below 2.11."""
        specs = stack_mod._ROCM_TORCH_PKG_SPECS.get("_default")
        assert specs is not None
        torch_spec = specs[0]
        assert "<2.11" in torch_spec

    def test_rocm71_repair_matches_install_sh_default_range(self):
        """rocm7.1 serves a paired 2.11 trio, so its repair spec must allow 2.11, matching
        install.sh's range."""
        specs = stack_mod._ROCM_TORCH_PKG_SPECS.get("rocm7.1")
        assert specs is not None, "rocm7.1 must have its own repair spec"
        assert specs == (
            "torch>=2.4,<2.12.0",
            "torchvision>=0.19,<0.27.0",
            "torchaudio>=2.4,<2.12.0",
        )
        # Not the rocm7.2 spec: no 2.11 floor applies to rocm7.1.
        assert specs != stack_mod._ROCM_TORCH_PKG_SPECS["rocm7.2"]

    def test_rocm71_is_not_a_known_211_floor_version(self):
        """The widened rocm7.1 range must NOT promote it to a floored 2.11 line."""
        assert (7, 1) not in stack_mod._ROCM_KNOWN_TORCH211_VERSIONS
        assert (7, 2) in stack_mod._ROCM_KNOWN_TORCH211_VERSIONS

    def test_specs_have_torch_vision_audio(self):
        """Each entry should be a 3-tuple: torch, torchvision, torchaudio."""
        for tag, specs in stack_mod._ROCM_TORCH_PKG_SPECS.items():
            assert len(specs) == 3, f"{tag}: expected (torch, torchvision, torchaudio)"
            assert "torch" in specs[0]
            assert "torchvision" in specs[1]
            assert "torchaudio" in specs[2]

    def test_gfx_to_amd_index_covers_rdna4(self):
        """_GFX_TO_AMD_INDEX_ARCH must cover gfx1200 and gfx1201 (RDNA 4)."""
        mapping = stack_mod._GFX_TO_AMD_INDEX_ARCH
        assert mapping.get("gfx1200") == "gfx120X-all"
        assert mapping.get("gfx1201") == "gfx120X-all"

    def test_gfx_to_amd_index_covers_strix_halo(self):
        """_GFX_TO_AMD_INDEX_ARCH must cover gfx1151 and gfx1150 (RDNA 3.5)."""
        mapping = stack_mod._GFX_TO_AMD_INDEX_ARCH
        assert mapping.get("gfx1151") == "gfx1151"
        assert mapping.get("gfx1150") == "gfx1150"

    def test_gfx_to_amd_index_covers_rdna3(self):
        """_GFX_TO_AMD_INDEX_ARCH must cover gfx1100-gfx1103 (RDNA 3)."""
        mapping = stack_mod._GFX_TO_AMD_INDEX_ARCH
        for arch in ("gfx1100", "gfx1101", "gfx1102", "gfx1103"):
            assert mapping.get(arch) == "gfx110X-all", f"{arch} missing from mapping"


_SETUP_PS1_PATH = PACKAGE_ROOT / "studio" / "setup.ps1"
_INSTALL_PS1_PATH = PACKAGE_ROOT / "install.ps1"


class TestStrixHaloGfxArchDetection:
    """setup.ps1 / install.ps1 gfx arch detection for Strix Halo / iGPU (HIP runtime only, no hipinfo)."""

    def test_amd_smi_static_asic_attempted_in_setup(self):
        """setup.ps1 must try 'amd-smi static --asic' when list output lacks gfx arch."""
        source = _SETUP_PS1_PATH.read_text(encoding = "utf-8")
        assert "static --asic" in source

    def test_amd_smi_static_asic_attempted_in_install(self):
        """install.ps1 must try 'amd-smi static --asic' when list output lacks gfx arch."""
        source = _INSTALL_PS1_PATH.read_text(encoding = "utf-8")
        assert "static --asic" in source

    def test_env_var_override_in_setup(self):
        """setup.ps1 must honour UNSLOTH_ROCM_GFX_ARCH as a manual arch override."""
        source = _SETUP_PS1_PATH.read_text(encoding = "utf-8")
        assert "UNSLOTH_ROCM_GFX_ARCH" in source

    def test_env_var_override_in_install(self):
        """install.ps1 must honour UNSLOTH_ROCM_GFX_ARCH as a manual arch override."""
        source = _INSTALL_PS1_PATH.read_text(encoding = "utf-8")
        assert "UNSLOTH_ROCM_GFX_ARCH" in source

    def test_name_arch_table_covers_strix_halo_in_setup(self):
        """setup.ps1 name→arch table must map 890M / Strix Halo to gfx1151."""
        source = _SETUP_PS1_PATH.read_text(encoding = "utf-8")
        assert "gfx1151" in source
        assert "890M" in source or "Strix Halo" in source

    def test_name_arch_table_covers_strix_halo_in_install(self):
        """install.ps1 name→arch table must map 890M / Strix Halo to gfx1151."""
        source = _INSTALL_PS1_PATH.read_text(encoding = "utf-8")
        assert "gfx1151" in source
        assert "890M" in source or "Strix Halo" in source

    def test_name_arch_table_covers_strix_point_in_setup(self):
        """setup.ps1 name→arch table must map 880M / Strix Point to gfx1150."""
        source = _SETUP_PS1_PATH.read_text(encoding = "utf-8")
        assert "gfx1150" in source
        assert "880M" in source or "Strix Point" in source

    def test_name_arch_table_covers_strix_point_in_install(self):
        """install.ps1 name→arch table must map 880M / Strix Point to gfx1150."""
        source = _INSTALL_PS1_PATH.read_text(encoding = "utf-8")
        assert "gfx1150" in source
        assert "880M" in source or "Strix Point" in source

    def test_name_arch_table_covers_rdna3_phoenix_in_setup(self):
        """setup.ps1 name→arch table must map 780M / Phoenix to gfx1103."""
        source = _SETUP_PS1_PATH.read_text(encoding = "utf-8")
        assert "gfx1103" in source
        assert "780M" in source or "Phoenix" in source

    def test_wmi_does_not_set_hasrocm_in_setup(self):
        """WMI block in setup.ps1 must NOT set $HasROCm = $true (no runtime confirmation)."""
        source = _SETUP_PS1_PATH.read_text(encoding = "utf-8")
        wmi_idx = source.find("Win32_VideoController")
        assert wmi_idx != -1, "WMI block not found in setup.ps1"
        # $HasROCm = $true must not appear within 300 chars of the WMI call.
        wmi_context = source[wmi_idx : wmi_idx + 300]
        assert "$HasROCm = $true" not in wmi_context

    def test_gfx_arch_regex_parses_from_amd_smi_output(self):
        """Both files must use the gfx\\d+[a-z]? regex to parse arch from amd-smi output."""
        for path in (_SETUP_PS1_PATH, _INSTALL_PS1_PATH):
            source = path.read_text(encoding = "utf-8")
            assert (
                "gfx\\d+" in source or r"gfx\d+" in source
            ), f"gfx arch regex not found in {path.name}"


class TestHipSdkEnvPathResolution:
    """Both install scripts resolve hipinfo/hipconfig via HIP_PATH/ROCM_PATH off $PATH, and warn."""

    @staticmethod
    def _assert_accepts_partial_hipinfo_output(source: str):
        hipout_idx = source.find("$hipOut = & $hipinfoExe.Source")
        assert hipout_idx != -1
        # Bound at the probe's own `catch`, not a char count, which drifts as the block grows.
        end_idx = source.find("} catch {", hipout_idx)
        assert end_idx != -1
        hipinfo_block = source[hipout_idx:end_idx]
        assert 'if ($hipOut -match "(?i)gcnArchName")' in hipinfo_block
        assert "$LASTEXITCODE -eq 0 -and $hipOut -match" not in hipinfo_block
        assert "but reported gcnArchName" in hipinfo_block

    def test_setup_checks_hip_path_for_hipinfo(self):
        """setup.ps1 must reference HIP_PATH when resolving hipinfo."""
        source = _SETUP_PS1_PATH.read_text(encoding = "utf-8")
        assert "HIP_PATH" in source
        assert "hipinfo" in source

    def test_install_checks_hip_path_for_hipinfo(self):
        """install.ps1 must reference HIP_PATH when resolving hipinfo."""
        source = _INSTALL_PS1_PATH.read_text(encoding = "utf-8")
        assert "HIP_PATH" in source
        assert "hipinfo" in source

    def test_setup_checks_rocm_path_as_hipinfo_fallback(self):
        """setup.ps1 must also check ROCM_PATH as a secondary hipinfo fallback."""
        source = _SETUP_PS1_PATH.read_text(encoding = "utf-8")
        assert "ROCM_PATH" in source
        assert "ROCM_PATH" in source and "HIP_PATH" in source

    def test_install_checks_rocm_path_as_hipinfo_fallback(self):
        """install.ps1 must also check ROCM_PATH as a secondary hipinfo fallback."""
        source = _INSTALL_PS1_PATH.read_text(encoding = "utf-8")
        assert "ROCM_PATH" in source
        assert "ROCM_PATH" in source and "HIP_PATH" in source

    def test_setup_resolves_hipinfo_via_bin_subdir(self):
        """setup.ps1 must join the env var root with 'bin\\hipinfo.exe'."""
        source = _SETUP_PS1_PATH.read_text(encoding = "utf-8")
        assert r"bin\hipinfo.exe" in source

    def test_install_resolves_hipinfo_via_bin_subdir(self):
        """install.ps1 must join the env var root with 'bin\\hipinfo.exe'."""
        source = _INSTALL_PS1_PATH.read_text(encoding = "utf-8")
        assert r"bin\hipinfo.exe" in source

    def test_setup_warns_when_hipinfo_not_on_path(self):
        """setup.ps1 must warn when hipinfo is found via env var but not on PATH."""
        source = _SETUP_PS1_PATH.read_text(encoding = "utf-8")
        assert "hipinfo not on PATH" in source

    def test_install_warns_when_hipinfo_not_on_path(self):
        """install.ps1 must warn when hipinfo is found via env var but not on PATH."""
        source = _INSTALL_PS1_PATH.read_text(encoding = "utf-8")
        assert "hipinfo not on PATH" in source

    def test_setup_warns_when_hip_path_set_but_exe_missing(self):
        """setup.ps1 must warn when HIP_PATH is set but hipinfo.exe is not present."""
        source = _SETUP_PS1_PATH.read_text(encoding = "utf-8")
        assert "incomplete" in source or "not found at" in source

    def test_install_warns_when_hip_path_set_but_exe_missing(self):
        """install.ps1 must warn when HIP_PATH is set but hipinfo.exe is not present."""
        source = _INSTALL_PS1_PATH.read_text(encoding = "utf-8")
        assert "incomplete" in source or "not found at" in source

    def test_setup_warns_on_hipinfo_nonzero_exit(self):
        """setup.ps1 must warn when hipinfo runs but returns a non-zero exit code."""
        source = _SETUP_PS1_PATH.read_text(encoding = "utf-8")
        assert "HIP runtime error" in source or "runtime error" in source.lower()

    def test_install_warns_on_hipinfo_nonzero_exit(self):
        """install.ps1 must warn when hipinfo runs but returns a non-zero exit code."""
        source = _INSTALL_PS1_PATH.read_text(encoding = "utf-8")
        assert "HIP runtime error" in source or "runtime error" in source.lower()

    def test_setup_accepts_hipinfo_gcnarchname_on_nonzero_exit(self):
        """setup.ps1 must accept partial hipinfo output from the #6043 crash path."""
        source = _SETUP_PS1_PATH.read_text(encoding = "utf-8")
        self._assert_accepts_partial_hipinfo_output(source)

    def test_install_accepts_hipinfo_gcnarchname_on_nonzero_exit(self):
        """install.ps1 must accept partial hipinfo output from the #6043 crash path."""
        source = _INSTALL_PS1_PATH.read_text(encoding = "utf-8")
        self._assert_accepts_partial_hipinfo_output(source)

    def test_setup_resolves_hipconfig_via_bin_subdir(self):
        """setup.ps1 must also fall back to HIP_PATH/bin/hipconfig.exe for version detection."""
        source = _SETUP_PS1_PATH.read_text(encoding = "utf-8")
        assert r"bin\hipconfig.exe" in source

    def test_install_resolves_hipconfig_via_bin_subdir(self):
        """install.ps1 must also fall back to HIP_PATH/bin/hipconfig.exe for version detection."""
        source = _INSTALL_PS1_PATH.read_text(encoding = "utf-8")
        assert r"bin\hipconfig.exe" in source

    def test_setup_warns_when_hipconfig_not_on_path(self):
        """setup.ps1 must warn when hipconfig is found via env var but not on PATH."""
        source = _SETUP_PS1_PATH.read_text(encoding = "utf-8")
        assert "hipconfig not on PATH" in source

    def test_install_warns_when_hipconfig_not_on_path(self):
        """install.ps1 must warn when hipconfig is found via env var but not on PATH."""
        source = _INSTALL_PS1_PATH.read_text(encoding = "utf-8")
        assert "hipconfig not on PATH" in source

    def test_setup_provides_path_fix_hint(self):
        """setup.ps1 must tell the user how to add the HIP bin dir to PATH."""
        source = _SETUP_PS1_PATH.read_text(encoding = "utf-8")
        assert "PATH" in source and ("SetEnvironmentVariable" in source or "Add" in source)

    def test_install_provides_path_fix_hint(self):
        """install.ps1 must tell the user how to add the HIP bin dir to PATH."""
        source = _INSTALL_PS1_PATH.read_text(encoding = "utf-8")
        assert "PATH" in source and ("SetEnvironmentVariable" in source or "Add" in source)


class TestHipSdkDetectedSubstep:
    """Both scripts print HIP SDK path and full hipconfig version as substeps when ROCm is detected."""

    def test_setup_prints_hip_sdk_path_substep(self):
        """setup.ps1 must print an 'HIP SDK:' substep showing the resolved path."""
        source = _SETUP_PS1_PATH.read_text(encoding = "utf-8")
        assert "HIP SDK:" in source

    def test_install_prints_hip_sdk_path_substep(self):
        """install.ps1 must print an 'HIP SDK:' substep showing the resolved path."""
        source = _INSTALL_PS1_PATH.read_text(encoding = "utf-8")
        assert "HIP SDK:" in source

    def test_setup_shows_hipconfig_full_version(self):
        """setup.ps1 must capture and display the full hipconfig version string."""
        source = _SETUP_PS1_PATH.read_text(encoding = "utf-8")
        assert "ROCmVersionFull" in source or "hipconfig:" in source

    def test_install_shows_hipconfig_full_version(self):
        """install.ps1 must capture and display the full hipconfig version string."""
        source = _INSTALL_PS1_PATH.read_text(encoding = "utf-8")
        assert "ROCmVersionFull" in source or "hipconfig:" in source

    def test_setup_captures_full_version_not_just_major_minor(self):
        """setup.ps1 must store the raw hipconfig output line, not just major.minor."""
        source = _SETUP_PS1_PATH.read_text(encoding = "utf-8")
        assert "ROCmVersionFull" in source

    def test_install_captures_full_version_not_just_major_minor(self):
        """install.ps1 must store the raw hipconfig output line, not just major.minor."""
        source = _INSTALL_PS1_PATH.read_text(encoding = "utf-8")
        assert "ROCmVersionFull" in source

    def test_setup_uses_hip_path_or_rocm_path_for_sdk_display(self):
        """setup.ps1 HIP SDK path substep must check HIP_PATH then ROCM_PATH."""
        source = _SETUP_PS1_PATH.read_text(encoding = "utf-8")
        assert "HIP_PATH" in source and "ROCM_PATH" in source

    def test_install_uses_hip_path_or_rocm_path_for_sdk_display(self):
        """install.ps1 HIP SDK path substep must check HIP_PATH then ROCM_PATH."""
        source = _INSTALL_PS1_PATH.read_text(encoding = "utf-8")
        assert "HIP_PATH" in source and "ROCM_PATH" in source

    def test_setup_rocm_step_uses_full_version(self):
        """setup.ps1 'rocm' step label must prefer the full version string."""
        source = _SETUP_PS1_PATH.read_text(encoding = "utf-8")
        assert "ROCmVersionFull" in source and "rocm" in source


_INSTALL_SH_PATH = PACKAGE_ROOT / "install.sh"
_SETUP_SH_PATH = PACKAGE_ROOT / "studio" / "setup.sh"


class TestStrixRocm71Override:
    """install.sh routes gfx1151/gfx1150 to AMD's arch index instead of ROCm 7.1 (_grouped_mm segfault)."""

    def test_linux_gfx_inference_helpers_present(self):
        source = _INSTALL_SH_PATH.read_text(encoding = "utf-8")
        assert "_infer_linux_amd_gfx_arch" in source
        assert "_amd_arch_index_family_for_gfx" in source
        assert "_amd_gpu_present_via_pci" in source
        assert "unslothai#7301" in source

    def test_infer_linux_amd_gfx_from_cpuinfo(self):
        assert stack_mod._linux_amd_gfx_from_cpuinfo is not None
        with patch.object(
            Path,
            "read_text",
            return_value = "model name : AMD Ryzen AI Max+ 395 w/ Radeon 8060S\n",
        ):
            assert stack_mod._linux_amd_gfx_from_cpuinfo() == "gfx1151"
        # 8065S (Gorgon Halo) must match on the Radeon name alone, like setup.sh/setup.ps1.
        with patch.object(Path, "read_text", return_value = "model name : AMD Radeon 8065S\n"):
            assert stack_mod._linux_amd_gfx_from_cpuinfo() == "gfx1151"

    def test_infer_gfx_gated_out_of_wsl_without_runtime(self):
        """On WSL, cpuinfo inference is skipped without librocdxg; an explicit UNSLOTH_ROCM_GFX_ARCH
        wins."""
        m = stack_mod
        with (
            patch.object(m, "_linux_amd_gfx_from_cpuinfo", return_value = "gfx1151"),
            patch.object(m, "_linux_amd_gfx_from_lspci", return_value = None),
            patch.object(m, "_linux_amd_display_device_present", return_value = True),
            patch.dict(os.environ, {"UNSLOTH_ROCM_GFX_ARCH": ""}),
        ):
            # WSL + no runtime -> inference suppressed (CPU torch stays).
            with (
                patch.object(m, "_is_wsl", return_value = True),
                patch.object(m, "_wsl_rocm_runtime_present", return_value = False),
            ):
                assert m._infer_linux_amd_gfx_arch() is None
            with (
                patch.object(m, "_is_wsl", return_value = True),
                patch.object(m, "_wsl_rocm_runtime_present", return_value = True),
            ):
                assert m._infer_linux_amd_gfx_arch() == "gfx1151"
            with (
                patch.object(m, "_is_wsl", return_value = False),
                patch.object(m, "_wsl_rocm_runtime_present", return_value = False),
            ):
                assert m._infer_linux_amd_gfx_arch() == "gfx1151"
        # Explicit override wins even on a bare WSL box (no runtime).
        with (
            patch.object(m, "_is_wsl", return_value = True),
            patch.object(m, "_wsl_rocm_runtime_present", return_value = False),
            patch.dict(os.environ, {"UNSLOTH_ROCM_GFX_ARCH": "gfx1151"}),
        ):
            assert m._infer_linux_amd_gfx_arch() == "gfx1151"

    def test_infer_gfx_requires_amd_display_device_on_native_linux(self):
        """Native Linux: CPU-model inference needs an AMD PCI display device, since a VM shows the
        host CPU."""
        m = stack_mod
        with (
            patch.object(m, "_linux_amd_gfx_from_cpuinfo", return_value = "gfx1151"),
            patch.object(m, "_linux_amd_gfx_from_lspci", return_value = None),
            patch.object(m, "_is_wsl", return_value = False),
            patch.dict(os.environ, {"UNSLOTH_ROCM_GFX_ARCH": ""}),
        ):
            # No AMD display device -> the CPU-model text alone must not infer.
            with patch.object(m, "_linux_amd_display_device_present", return_value = False):
                assert m._infer_linux_amd_gfx_arch() is None
            with patch.object(m, "_linux_amd_display_device_present", return_value = True):
                assert m._infer_linux_amd_gfx_arch() == "gfx1151"
        # Explicit override needs no device evidence (headless/cross-install).
        with (
            patch.object(m, "_is_wsl", return_value = False),
            patch.object(m, "_linux_amd_display_device_present", return_value = False),
            patch.dict(os.environ, {"UNSLOTH_ROCM_GFX_ARCH": "GFX1151"}),
        ):
            assert m._infer_linux_amd_gfx_arch() == "gfx1151"

    def test_install_sh_cpuinfo_inference_requires_pci_evidence(self):
        """install.sh mirror of the VM/container guard: every cpuinfo grep must be
        gated on _gpu_evidence (AMD PCI display device via _amd_gpu_present_via_pci,
        or the WSL librocdxg gate), and the gate must sit before the first grep."""
        source = _INSTALL_SH_PATH.read_text(encoding = "utf-8")
        body = _extract_sh_function_body(source, "_infer_linux_amd_gfx_arch")
        assert body, "could not extract _infer_linux_amd_gfx_arch"
        pci = body.find("_amd_gpu_present_via_pci")
        infer = body.find("grep -qiE 'Ryzen AI Max")
        assert pci >= 0 and infer >= 0
        assert pci < infer, "the PCI evidence check must run before the cpuinfo inference"
        assert body.count("grep -qiE") == body.count(
            '[ -n "$_gpu_evidence" ] && grep -qiE'
        ), "every cpuinfo grep (gfx1151/gfx1150/gfx1152) must be gated on _gpu_evidence"

    def test_lspci_scan_covers_all_display_controllers(self):
        """Scan every display line, since a non-AMD controller often enumerates first; NVIDIA must
        not map."""
        m = stack_mod

        def fake_lspci(stdout):
            result = SimpleNamespace(returncode = 0, stdout = stdout)
            return (
                patch.object(m.shutil, "which", return_value = "/usr/bin/lspci"),
                patch.object(m.subprocess, "run", return_value = result),
            )

        intel_then_amd = (
            "00:02.0 VGA compatible controller [0300]: Intel Corporation Raptor Lake-S GT1 [8086:a780]\n"
            "03:00.0 VGA compatible controller [0300]: Advanced Micro Devices, Inc. [AMD/ATI]"
            " Navi 31 [Radeon RX 7900 XT] [1002:744c]\n"
        )
        nvidia_only = "01:00.0 3D controller [0302]: NVIDIA Corporation GM107M [GeForce GTX 860M] [10de:1392]\n"
        domain_prefixed = (
            "0000:c5:00.0 VGA compatible controller [0300]: Advanced Micro Devices, Inc. [AMD/ATI]"
            " Strix Halo [Radeon Graphics / Radeon 8060S] [1002:150e]\n"
        )
        unmapped_then_mapped = (
            "03:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI]"
            " Cape Verde [FirePro W600] [1002:6821]\n"
            "04:00.0 VGA compatible controller [0300]: Advanced Micro Devices, Inc. [AMD/ATI]"
            " Navi 33 [Radeon RX 7600] [1002:7480]\n"
        )
        for stdout, expected in (
            (intel_then_amd, "gfx1100"),
            (nvidia_only, None),
            (domain_prefixed, "gfx1151"),
            (unmapped_then_mapped, "gfx1102"),
        ):
            w, r = fake_lspci(stdout)
            with w, r:
                assert m._linux_amd_gfx_from_lspci() == expected, stdout

    def test_install_sh_lspci_scan_covers_all_display_controllers(self):
        """install.sh mirrors the scan-all lspci behaviour: Intel-first still finds the AMD dGPU,
        NVIDIA-only maps nothing (860M collision), and domain-prefixed AMD lines still map."""
        shell = shutil.which("bash")
        if not shell:
            pytest.skip("bash needed to execute the probe block")
        source = _INSTALL_SH_PATH.read_text(encoding = "utf-8")
        name_fn = re.search(
            r"^_infer_amd_gfx_arch_from_gpu_name\(\) \{\n.*?\n\}\n", source, re.S | re.M
        )
        scan = re.search(
            r"^    if command -v lspci[^\n]*\n.*?\nEOF\n    fi\n.*?    return 1\n",
            source,
            re.S | re.M,
        )
        assert name_fn and scan, "could not extract the lspci scan block"
        cases = (
            (
                "00:02.0 VGA compatible controller [0300]: Intel Corporation UHD [8086:a780]\n"
                "03:00.0 VGA compatible controller [0300]: Advanced Micro Devices, Inc. [AMD/ATI]"
                " Navi 31 [Radeon RX 7900 XT] [1002:744c]",
                "OK:gfx1100",
            ),
            (
                "01:00.0 3D controller [0302]: NVIDIA Corporation GM107M [GeForce GTX 860M] [10de:1392]",
                "OK:",
            ),
            (
                "0000:c5:00.0 VGA compatible controller [0300]: Advanced Micro Devices, Inc."
                " [AMD/ATI] Strix Halo [Radeon 8060S] [1002:150e]",
                "OK:gfx1151",
            ),
        )
        for lspci_out, expected in cases:
            with tempfile.TemporaryDirectory() as d:
                p = os.path.join(d, "lspci")
                with open(p, "w", encoding = "utf-8") as f:
                    f.write(f'#!/bin/sh\ncat <<"EOT"\n{lspci_out}\nEOT\n')
                os.chmod(p, 0o755)
                script = (
                    "set -euo pipefail\n"
                    + name_fn.group(0)
                    + "probe() {\n"
                    + scan.group(0)
                    + "}\nprintf 'OK:%s\\n' \"$(probe || true)\"\n"
                )
                env = dict(os.environ, PATH = d + os.pathsep + os.environ.get("PATH", ""))
                r = _run_bash_script(shell, script, env = env, capture_output = True, text = True)
                assert r.returncode == 0, f"scan aborted: {r.stderr}"
                assert (
                    r.stdout.splitlines()[-1] == expected
                ), f"lspci scan wrong for {lspci_out!r}: {r.stdout!r}"

    def test_install_sh_infer_gfx_gated_on_wsl_runtime(self):
        """install.sh's _infer_linux_amd_gfx_arch must, like the Python side, skip
        the cpuinfo/lspci inference on WSL unless librocdxg is present -- the
        override still returns first, so it stays authoritative."""
        source = _INSTALL_SH_PATH.read_text(encoding = "utf-8")
        body = _extract_sh_function_body(source, "_infer_linux_amd_gfx_arch")
        assert body, "could not extract _infer_linux_amd_gfx_arch"
        override = body.find("UNSLOTH_ROCM_GFX_ARCH")
        dxg = body.find("/dev/dxg")
        rocdxg = body.find("librocdxg")
        # Anchor on the first cpuinfo *inference* (the grep), not a comment mention.
        infer = body.find("grep -qiE 'Ryzen AI Max")
        assert override >= 0 and dxg >= 0 and rocdxg >= 0 and infer >= 0
        assert "microsoft" in body, "WSL gate must also detect WSL via /proc/version"
        assert override < dxg, "the explicit override must return before the WSL gate"
        assert (
            dxg < infer and rocdxg < infer
        ), "the WSL/librocdxg gate must run before the cpuinfo/lspci inference"

    def test_install_sh_reroute_is_x86_64_only(self):
        """The Linux inferred-gfx reroute must be x86_64-only: ROCm torch wheels are
        not published for arm64, so an inferred/overridden gfx must not push an
        arm64 host to the AMD arch index (get_torch_index_url returns CPU there)."""
        source = _INSTALL_SH_PATH.read_text(encoding = "utf-8")
        idx = source.find("_linux_inferred_gfx=$(_infer_linux_amd_gfx_arch")
        assert idx >= 0, "reroute consumer not found"
        window = source[max(0, idx - 400) : idx]
        assert (
            'case "$_ARCH" in x86_64|amd64)' in window
        ), "the inferred-gfx reroute must guard on x86_64|amd64 arch"

    def test_install_sh_reroute_skips_visible_rocm_gpu(self):
        """A */cpu index on a host with visible ROCm GPUs is deliberate; the reroute must not
        override it."""
        source = _INSTALL_SH_PATH.read_text(encoding = "utf-8")
        idx = source.find("_linux_inferred_gfx=$(_infer_linux_amd_gfx_arch")
        assert idx >= 0, "reroute consumer not found"
        window = source[max(0, idx - 700) : idx]
        assert (
            "! _has_amd_rocm_gpu" in window
        ), "the reroute must be gated on _has_amd_rocm_gpu being false"
        assert (
            '[ -n "${UNSLOTH_ROCM_GFX_ARCH:-}" ] || ! _has_amd_rocm_gpu' in window
        ), "an explicit UNSLOTH_ROCM_GFX_ARCH override must bypass the visible-GPU gate"

    def test_install_sh_reroute_exports_gfx_for_setup_sh(self):
        """Exports the inferred arch as UNSLOTH_ROCM_GFX_ARCH so setup.sh picks the ROCm llama.cpp
        prebuilt."""
        source = _INSTALL_SH_PATH.read_text(encoding = "utf-8")
        assign = source.find('TORCH_INDEX_URL="${_amd_mirror}/${_amd_family}/"')
        assert assign >= 0, "inferred-gfx index assignment not found"
        block_end = source.find("esac", assign)
        assert (
            'export UNSLOTH_ROCM_GFX_ARCH="$_linux_inferred_gfx"' in source[assign:block_end]
        ), "the reroute must export the inferred gfx for the setup.sh handoff"
        setup_source = (PACKAGE_ROOT / "studio" / "setup.sh").read_text(encoding = "utf-8")
        assert "UNSLOTH_ROCM_GFX_ARCH" in setup_source

    def test_amd_arch_index_url_linux_honors_amd_mirror(self):
        """Linux repairs read UNSLOTH_AMD_ROCM_MIRROR as install.sh does, not the Windows mirror
        variable."""
        m = stack_mod
        with (
            patch.object(m, "IS_WINDOWS", False),
            patch.dict(os.environ, {"UNSLOTH_AMD_ROCM_MIRROR": "https://mirror.local/rocm"}),
        ):
            assert m._amd_arch_index_url("gfx1151") == "https://mirror.local/rocm/gfx1151/"
        with (
            patch.object(m, "IS_WINDOWS", False),
            patch.dict(os.environ, {"UNSLOTH_AMD_ROCM_MIRROR": ""}),
        ):
            assert m._amd_arch_index_url("gfx1151") == "https://repo.amd.com/rocm/whl/gfx1151/"
            assert m._amd_arch_index_url("gfx9999") is None
        with patch.object(m, "IS_WINDOWS", True):
            assert m._amd_arch_index_url("gfx1151") == m._windows_rocm_index_url("gfx1151")

    def test_amd_arch_index_url_refuses_miscomputing_arch(self):
        """gfx1033 miscomputes, so no ROCm index is returned even when the override bypasses the
        runtime gate."""
        m = stack_mod
        with (
            patch.object(m, "IS_WINDOWS", False),
            patch.dict(os.environ, {"UNSLOTH_AMD_ROCM_MIRROR": ""}),
        ):
            assert m._amd_arch_index_url("gfx1033") is None
            assert m._amd_arch_index_url("GFX1033") is None
            for _gfx in ("gfx1030", "gfx1031", "gfx1032", "gfx1034", "gfx1035", "gfx1036"):
                assert m._amd_arch_index_url(_gfx) == "https://repo.amd.com/rocm/whl/gfx103X-all/"

    def test_a_windows_mirror_token_ending_in_a_slash_survives_the_join(self):
        """A Windows mirror token ending in / must survive: trimming the raw env value first would
        eat it."""
        _mirror = "https://mirror.example/whl?token=YWJj/"
        # The Windows base is read once at import, so the env has to be set before the load.
        with patch.dict(os.environ, {"UNSLOTH_ROCM_WINDOWS_MIRROR": _mirror}):
            _spec = importlib.util.spec_from_file_location("stack_mirror_query", _STACK_PATH)
            m = importlib.util.module_from_spec(_spec)
            _spec.loader.exec_module(m)
        with patch.object(m, "IS_WINDOWS", True):
            assert m._windows_rocm_index_url("gfx1151") == (
                "https://mirror.example/whl/gfx1151/?token=YWJj/"
            )
        with (
            patch.object(m, "IS_WINDOWS", False),
            patch.dict(os.environ, {"UNSLOTH_AMD_ROCM_MIRROR": _mirror}),
        ):
            assert m._amd_arch_index_url("gfx1151") == (
                "https://mirror.example/whl/gfx1151/?token=YWJj/"
            )
        with patch.object(stack_mod, "_ROCM_WINDOWS_INDEX_BASE", "https://mirror.example/whl/"):
            assert stack_mod._windows_rocm_index_url("gfx1151") == (
                "https://mirror.example/whl/gfx1151/"
            )

    def test_a_query_authenticated_mirror_says_pip_cannot_use_it(self):
        """pip joins package names as path text, so a query or fragment mirror URL cannot resolve
        anything."""
        m = stack_mod
        for _base in ("https://mirror.example/whl?token=x", "https://mirror.example/whl#tok"):
            m._WARNED_QUERY_INDEX_BASES.discard(_base)
            _said = []
            with patch.object(m, "_safe_print", side_effect = _said.append):
                m._index_url_join(_base, "gfx110X-all")
                m._index_url_join(_base, "gfx110X-all")
            assert len(_said) == 1, _said
            assert "netrc" in _said[0] and "credential" in _said[0], _said
        # Userinfo lives in the netloc, which the join never touches.
        _plain = "https://user:[email protected]/whl"
        _quiet = []
        with patch.object(m, "_safe_print", side_effect = _quiet.append):
            assert m._index_url_join(_plain, "gfx110X-all") == (
                "https://user:[email protected]/whl/gfx110X-all/"
            )
        assert _quiet == [], _quiet

    def test_strix_gfx_detection_in_install_sh(self):
        """install.sh must detect gfx1151 and gfx1150 for the override."""
        source = _INSTALL_SH_PATH.read_text(encoding = "utf-8")
        assert "gfx1151" in source and "gfx1150" in source

    def test_rocm71_override_to_amd_arch_index_in_install_sh(self):
        """install.sh must override TORCH_INDEX_URL to AMD arch-specific index for Strix."""
        source = _INSTALL_SH_PATH.read_text(encoding = "utf-8")
        assert "repo.amd.com/rocm/whl" in source
        assert "_strix_gfx" in source
        strix_idx = source.find("_amd_strix_base")
        assert strix_idx != -1
        ctx = source[strix_idx : strix_idx + 500]
        assert "_strix_gfx" in ctx

    def test_radeon_repo_bypassed_for_strix_in_install_sh(self):
        """install.sh must set _amd_gpu_radeon=false when Strix + ROCm 7.1 detected."""
        source = _INSTALL_SH_PATH.read_text(encoding = "utf-8")
        assert "_amd_gpu_radeon=false" in source

    def test_strix_override_warns_with_moe_utils_reference(self):
        """install.sh must emit a [WARN] mentioning the moe_utils segfault."""
        source = _INSTALL_SH_PATH.read_text(encoding = "utf-8")
        assert "moe_utils" in source or "_grouped_mm" in source

    def test_strix_override_scoped_below_arch_floor(self):
        """Strix reroute must fire for rocm leaves BELOW the arch floor (7.13) and
        NOT at/above it. Executed via _rocm_leaf_below so it verifies the actual
        version comparison, not a text match that a comment could satisfy."""
        source = _INSTALL_SH_PATH.read_text(encoding = "utf-8")
        # Switch on the index LEAF: a mirror base path may contain its own rocm token.
        assert 'case "$_torch_index_leaf" in' in source
        assert '_rocm_leaf_below "$_torch_index_leaf" 7 13' in source
        shell = shutil.which("sh") or shutil.which("bash")
        if not shell:
            pytest.skip("no POSIX shell to execute _rocm_leaf_below")
        match = re.search(r"^_rocm_leaf_below\(\) \{.*?^\}", source, re.S | re.M)
        assert match, "could not extract _rocm_leaf_below from install.sh"
        fn = match.group(0)

        def below(leaf):
            return (
                subprocess.run(
                    [shell, "-c", f'{fn}\n_rocm_leaf_below "$1" 7 13', "_", leaf]
                ).returncode
                == 0
            )

        for leaf in ("rocm6.0", "rocm7.0", "rocm7.1", "rocm7.2", "rocm7.12"):
            assert below(leaf), f"{leaf} must reroute (below arch floor 7.13)"
        for leaf in ("rocm7.13", "rocm7.14", "rocm8.0", "gfx1151", "cu128", "cpu"):
            assert not below(leaf), f"{leaf} must NOT reroute (>= floor or non-rocm)"

    def test_gfx_probe_survives_no_match_under_set_e(self):
        """A gfx probe with no match must not abort under set -euo pipefail before the amd-smi fallback."""
        shell = shutil.which("bash")
        if not shell:
            pytest.skip("bash needed to execute the probe block")
        source = _INSTALL_SH_PATH.read_text(encoding = "utf-8")
        block = re.search(
            r'^        _gfx_all=\$\(printf[^\n]*\n.*?(?=^        _strix_gfx="")',
            source,
            re.S | re.M,
        )
        assert block, "could not extract the gfx-detection block"
        with tempfile.TemporaryDirectory() as d:
            # Real `amd-smi list` carries no arch; the shim proves `static --asic` answers.
            amd_smi = (
                '#!/bin/sh\ncase "$1" in\n'
                '  list) printf "GPU: 0\\n    BDF: 0000:03:00.0\\n" ;;\n'
                '  *) printf "GPU: 0\\n    TARGET_GRAPHICS_VERSION: gfx1151\\n" ;;\n'
                "esac\n"
            )
            for name, body in (
                ("rocminfo", '#!/bin/sh\ncat <<"EOT"\nno gpu here\nEOT\n'),
                ("amd-smi", amd_smi),
            ):
                p = os.path.join(d, name)
                with open(p, "w", encoding = "utf-8") as f:
                    f.write(body)
                os.chmod(p, 0o755)
            script = (
                'set -euo pipefail\nHIP_VISIBLE_DEVICES=""\nROCR_VISIBLE_DEVICES=""\n'
                + _gpu_record_helpers(source)
                + "\n"
                + block.group(0)
                + '\nprintf "OK:%s\\n" "$_gfx_all"\n'
            )
            env = dict(os.environ, PATH = d + os.pathsep + os.environ.get("PATH", ""))
            r = _run_bash_script(shell, script, env = env, capture_output = True, text = True)
            assert r.returncode == 0, f"probe aborted under set -e: {r.stderr}"
            assert "OK:gfx1151" in r.stdout, f"amd-smi fallback not reached: {r.stdout!r}"

    def test_strix_reroute_reprobes_when_mask_hides_all(self):
        """A mask hiding all agents must not skip the Strix reroute; the re-probe must run unmasked."""
        shell = shutil.which("bash")
        if not shell:
            pytest.skip("bash needed to execute the probe block")
        source = _INSTALL_SH_PATH.read_text(encoding = "utf-8")
        block = re.search(
            r'^        _gfx_all=\$\(printf[^\n]*\n.*?(?=^        _strix_gfx="")',
            source,
            re.S | re.M,
        )
        assert block, "could not extract the gfx-detection block"
        with tempfile.TemporaryDirectory() as d:
            # rocminfo honours ROCR_VISIBLE_DEVICES like the real tool: -1 and set-but-empty hide
            # both agents, 1 renumbers to the dGPU only, unset shows both.
            rocminfo = (
                "#!/bin/sh\n"
                'case "${ROCR_VISIBLE_DEVICES-__unset__}" in\n'
                '  __unset__) printf "Name: gfx1151\\nName: gfx1201\\n" ;;\n'
                '  ""|-1) echo "no visible agents" ;;\n'
                '  1) printf "Name: gfx1201\\n" ;;\n'
                '  *) printf "Name: gfx1151\\nName: gfx1201\\n" ;;\n'
                "esac\n"
            )
            for name, body in (("rocminfo", rocminfo), ("amd-smi", "#!/bin/sh\nexit 0\n")):
                p = os.path.join(d, name)
                with open(p, "w", encoding = "utf-8") as f:
                    f.write(body)
                os.chmod(p, 0o755)
            script = (
                "set -euo pipefail\n"
                + _gpu_record_helpers(source)
                + "\n"
                + block.group(0)
                + '\nprintf "OK:%s\\n" "$_runtime_gfx"\n'
            )

            def run(**extra):
                env = dict(os.environ, PATH = d + os.pathsep + os.environ.get("PATH", ""))
                env.pop("UNSLOTH_ROCM_GFX_ARCH", None)
                # CUDA_VISIBLE_DEVICES is HIP's alias; leaving it set makes the test depend on the host.
                for name in _visibility_mask_names(source):
                    env.pop(name, None)
                env.update(extra)
                return _run_bash_script(shell, script, env = env, capture_output = True, text = True)

            r = run(ROCR_VISIBLE_DEVICES = "-1")
            assert r.returncode == 0, f"masked probe aborted: {r.stderr}"
            assert "OK:gfx1151" in r.stdout, f"reroute blinded by full mask: {r.stdout!r}"
            # A set-but-empty mask hides every agent too (${VAR+x}, not ${VAR:-}).
            r0 = run(ROCR_VISIBLE_DEVICES = "")
            assert r0.returncode == 0, f"empty-mask probe aborted: {r0.stderr}"
            assert "OK:gfx1151" in r0.stdout, f"reroute blinded by empty mask: {r0.stdout!r}"
            # A partial mask is already reflected; no unmasked re-probe may override the pick.
            r2 = run(ROCR_VISIBLE_DEVICES = "1")
            assert r2.returncode == 0, f"partial-mask probe aborted: {r2.stderr}"
            assert "OK:gfx1201" in r2.stdout, f"partial mask selection lost: {r2.stdout!r}"

    def test_the_probe_clears_every_visibility_mask_production_reads(self):
        """The probe must clear every mask install.sh reads, so none is silently inherited from the host."""
        source = _INSTALL_SH_PATH.read_text(encoding = "utf-8")
        names = _visibility_mask_names(source)
        assert set(names) == {
            "ROCR_VISIBLE_DEVICES",
            "HIP_VISIBLE_DEVICES",
            "CUDA_VISIBLE_DEVICES",
        }, names
        for name in names:
            assert name in source, name

    def test_strix_routing_helpers_cover_rocm714(self):
        # Mirrors install.sh _rocm_leaf_below: reroute only below the 7.13 arch floor.
        assert stack_mod._generic_pytorch_rocm_tag((7, 14)) == "rocm7.2"
        assert stack_mod._strix_needs_amd_arch_index((7, 14)) is True
        assert stack_mod._strix_needs_amd_arch_index((7, 0)) is True
        assert stack_mod._strix_needs_amd_arch_index((6, 0)) is True
        # No generic tag below 6.0, so the per-arch index is the only route.
        assert stack_mod._strix_needs_amd_arch_index((5, 0)) is True

    def test_torch_constraint_updated_for_strix_amd_index(self):
        """install.sh must set TORCH_CONSTRAINT>=2.11 when routing Strix to AMD index."""
        source = _INSTALL_SH_PATH.read_text(encoding = "utf-8")
        assert "TORCH_CONSTRAINT" in source and "2.11" in source

    def test_torch_constraint_211_matches_leaf_not_whole_url(self):
        """Matches the 2.11 pin on the index leaf only, so a mirror path containing gfx or rocm7.2
        is ignored."""
        source = _INSTALL_SH_PATH.read_text(encoding = "utf-8")
        # Switch on $_torch_index_leaf; only _grouped_mm-bug families get 2.11; gfx90a/908 stay bare.
        assert (
            'case "$_torch_index_leaf" in\n    rocm7.2|gfx120x-all|gfx1151|gfx1150|gfx1152|gfx103x-all|gfx110x-all)'
            in source
        ), (
            "the torch>=2.11 constraint must match the specific gfx leaves that need "
            "it (rocm7.2|gfx120x-all|gfx1151|gfx1150|gfx1152|gfx103x-all|gfx110x-all), not a bare gfx* or the URL"
        )

    def test_amd_rocm_mirror_env_var_respected(self):
        """install.sh must honour UNSLOTH_AMD_ROCM_MIRROR for air-gapped installs."""
        source = _INSTALL_SH_PATH.read_text(encoding = "utf-8")
        assert "UNSLOTH_AMD_ROCM_MIRROR" in source

    def test_tauri_family_recognises_amd_arch_url(self):
        """_tauri_torch_index_family must return a rocm* family for AMD arch-specific URLs."""
        source = _INSTALL_SH_PATH.read_text(encoding = "utf-8")
        assert "rocm/whl/gfx" in source


class TestSetupShGccInstallDir:
    """setup.sh applies --gcc-install-dir for HIP builds on Ubuntu 24.04+ (ROCm 7.x clang-20 header bug)."""

    def test_gcc_install_dir_search_loop_present(self):
        """setup.sh must iterate gcc versions 14→11 to find one with C++ headers."""
        source = _SETUP_SH_PATH.read_text(encoding = "utf-8")
        assert "_GCC_INSTALL_DIR" in source
        assert "/usr/lib/gcc/x86_64-linux-gnu" in source

    def test_gcc_install_dir_checks_include_dir(self):
        """setup.sh must check that the gcc dir has an 'include' subdirectory."""
        source = _SETUP_SH_PATH.read_text(encoding = "utf-8")
        assert "include" in source and "_GCC_INSTALL_DIR" in source

    def test_gcc_install_dir_appended_to_cmake_hip_flags(self):
        """setup.sh must pass --gcc-install-dir via CMAKE_HIP_FLAGS."""
        source = _SETUP_SH_PATH.read_text(encoding = "utf-8")
        assert "CMAKE_HIP_FLAGS" in source
        assert "gcc-install-dir" in source

    def test_gcc_install_dir_only_applied_in_hip_build_block(self):
        """The --gcc-install-dir fix must only apply in the HIP/ROCm build branch."""
        source = _SETUP_SH_PATH.read_text(encoding = "utf-8")
        hip_idx = source.find("GGML_HIP=ON")
        gcc_idx = source.find("gcc-install-dir")
        assert hip_idx != -1 and gcc_idx != -1
        assert hip_idx < gcc_idx

    def test_gcc_install_dir_logs_substep(self):
        """setup.sh must print a substep when the gcc install dir is resolved."""
        source = _SETUP_SH_PATH.read_text(encoding = "utf-8")
        assert "gcc install dir" in source or "GCC_INSTALL_DIR" in source


_MAIN_PY_PATH = PACKAGE_ROOT / "studio" / "backend" / "main.py"
_HARDWARE_PY_PATH = PACKAGE_ROOT / "studio" / "backend" / "utils" / "hardware" / "hardware.py"


class TestServerStartupRocmFixes:
    """main.py sets BNB_ROCM_VERSION pre-bnb-import; hardware.py stubs _distributed_c10d pre-torch.distributed."""

    def test_main_py_sets_bnb_rocm_version(self):
        """main.py must set BNB_ROCM_VERSION in the server process before imports."""
        source = _MAIN_PY_PATH.read_text(encoding = "utf-8")
        assert "BNB_ROCM_VERSION" in source

    def test_main_py_bnb_detection_scoped_to_win32(self):
        """main.py BNB_ROCM_VERSION logic must be inside the win32 platform guard."""
        source = _MAIN_PY_PATH.read_text(encoding = "utf-8")
        win32_idx = source.find('sys.platform == "win32"')
        bnb_idx = source.find("BNB_ROCM_VERSION")
        assert win32_idx != -1 and bnb_idx != -1
        assert win32_idx < bnb_idx

    def test_main_py_bnb_dll_detection_uses_glob(self):
        """main.py must scan for libbitsandbytes_rocm*.dll to find the right version."""
        source = _MAIN_PY_PATH.read_text(encoding = "utf-8")
        assert "libbitsandbytes_rocm" in source

    def test_main_py_bnb_falls_back_to_72(self):
        """main.py must fall back to BNB_ROCM_VERSION='72' when no DLL is found."""
        source = _MAIN_PY_PATH.read_text(encoding = "utf-8")
        assert '"72"' in source or "'72'" in source

    def test_main_py_bnb_only_set_when_not_already_in_env(self):
        """main.py must not override an existing BNB_ROCM_VERSION env var."""
        source = _MAIN_PY_PATH.read_text(encoding = "utf-8")
        assert '"BNB_ROCM_VERSION" not in os.environ' in source

    # bnb runs hipinfo.exe via PATH at import; the venv Scripts dir is on PATH only when activated.

    def test_main_py_prepends_hipinfo_dir_to_path(self):
        """main.py must make hipInfo.exe resolvable before bnb imports."""
        source = _MAIN_PY_PATH.read_text(encoding = "utf-8")
        assert "hipInfo.exe" in source
        # Prepend must precede the BNB_ROCM_VERSION block so bnb sees the fixed PATH.
        assert source.find("hipInfo.exe") < source.find("BNB_ROCM_VERSION")

    def test_main_py_hipinfo_prepend_gated_on_file_presence(self):
        """Prepend must check hipInfo.exe exists first (only AMD wheels ship it; leave NVIDIA/CPU untouched)."""
        source = _MAIN_PY_PATH.read_text(encoding = "utf-8")
        assert 'os.path.isfile(os.path.join(_scripts_dir, "hipInfo.exe"))' in source

    def test_worker_py_prepends_hipinfo_dir_to_path(self):
        """worker.py must mirror the prepend for standalone-spawned workers."""
        source = _WORKER_PATH.read_text(encoding = "utf-8")
        assert "hipInfo.exe" in source

    def test_install_stack_prepends_hipinfo_dir_to_path(self):
        """install_python_stack.py must prepend so child import checks inherit a PATH where bnb's probe works."""
        source = _STACK_PATH.read_text(encoding = "utf-8")
        assert "hipInfo.exe" in source

    def test_hardware_py_injects_distributed_c10d_stub(self):
        """hardware.py must inject torch._C._distributed_c10d into sys.modules."""
        source = _HARDWARE_PY_PATH.read_text(encoding = "utf-8")
        assert "_distributed_c10d" in source

    def test_hardware_py_stub_injected_before_distributed_import(self):
        """The sys.modules stub must be injected BEFORE import torch.distributed."""
        source = _HARDWARE_PY_PATH.read_text(encoding = "utf-8")
        c10d_idx = source.find("_distributed_c10d")
        dist_idx = source.find("import torch.distributed")
        assert c10d_idx != -1 and dist_idx != -1
        assert c10d_idx < dist_idx

    def test_hardware_py_stub_uses_types_moduletype(self):
        """hardware.py must create the stub with types.ModuleType."""
        source = _HARDWARE_PY_PATH.read_text(encoding = "utf-8")
        assert "ModuleType" in source

    def test_hardware_py_stub_scoped_to_win32(self):
        """hardware.py distributed stub injection must be gated on win32."""
        source = _HARDWARE_PY_PATH.read_text(encoding = "utf-8")
        assert 'platform == "win32"' in source or "win32" in source

    def test_hardware_py_stub_exposes_fake_process_group(self):
        """hardware.py stub must set FakeProcessGroup so torch.distributed doesn't raise AttributeError."""
        source = _HARDWARE_PY_PATH.read_text(encoding = "utf-8")
        assert "FakeProcessGroup" in source

    def test_hardware_py_stub_exposes_process_group(self):
        """hardware.py stub must set ProcessGroup on the c10d stub."""
        source = _HARDWARE_PY_PATH.read_text(encoding = "utf-8")
        assert "ProcessGroup" in source

    def test_hardware_py_stub_uses_setattr_for_symbols(self):
        """hardware.py must use setattr to populate stub symbols dynamically."""
        source = _HARDWARE_PY_PATH.read_text(encoding = "utf-8")
        assert "setattr" in source

    def test_hardware_py_stub_all_c10d_siblings_covered(self):
        """hardware.py must stub all three torch._C._distributed_* submodules."""
        source = _HARDWARE_PY_PATH.read_text(encoding = "utf-8")
        assert "_distributed_c10d" in source
        assert "_distributed_autograd" in source
        assert "_distributed_rpc" in source


class TestHipSdkInstalledButDeviceInaccessible:
    """When hipinfo is found but exits non-zero, both scripts distinguish device-inaccessible from SDK-not-found."""

    def test_install_ps1_has_hip_sdk_installed_flag(self):
        """install.ps1 must track HipSdkInstalled separately from HasROCm."""
        source = _INSTALL_PS1_PATH.read_text(encoding = "utf-8")
        assert "HipSdkInstalled" in source

    def test_setup_ps1_has_hip_sdk_installed_flag(self):
        """setup.ps1 must track HipSdkInstalled separately from HasROCm."""
        source = _SETUP_PS1_PATH.read_text(encoding = "utf-8")
        assert "HipSdkInstalled" in source

    def test_install_ps1_sets_flag_when_hipinfo_binary_found(self):
        """install.ps1 must set HipSdkInstalled=true inside the 'if ($hipinfoExe)' block."""
        source = _INSTALL_PS1_PATH.read_text(encoding = "utf-8")
        hipinfo_block_idx = source.find("if ($hipinfoExe)")
        sdk_flag_idx = source.find("$HipSdkInstalled = $true", hipinfo_block_idx)
        assert hipinfo_block_idx != -1 and sdk_flag_idx != -1
        assert sdk_flag_idx > hipinfo_block_idx

    def test_setup_ps1_sets_flag_when_hipinfo_binary_found(self):
        """setup.ps1 must set HipSdkInstalled=true inside the 'if ($hipinfoExe)' block."""
        source = _SETUP_PS1_PATH.read_text(encoding = "utf-8")
        hipinfo_block_idx = source.find("if ($hipinfoExe)")
        sdk_flag_idx = source.find("$HipSdkInstalled = $true", hipinfo_block_idx)
        assert hipinfo_block_idx != -1 and sdk_flag_idx != -1
        assert sdk_flag_idx > hipinfo_block_idx

    def test_install_ps1_version_capture_runs_when_sdk_installed(self):
        """install.ps1 must capture hipconfig version when HipSdkInstalled even if HasROCm is false."""
        source = _INSTALL_PS1_PATH.read_text(encoding = "utf-8")
        assert "HasROCm -or $HipSdkInstalled" in source or "$HipSdkInstalled" in source

    def test_setup_ps1_version_capture_runs_when_sdk_installed(self):
        """setup.ps1 must capture hipconfig version when HipSdkInstalled even if HasROCm is false."""
        source = _SETUP_PS1_PATH.read_text(encoding = "utf-8")
        assert "HasROCm -or $HipSdkInstalled" in source or "$HipSdkInstalled" in source

    def test_install_ps1_distinct_message_for_sdk_found_but_device_inaccessible(self):
        """install.ps1 must show 'not ROCm-accessible' message (not 'HIP SDK not found') when SDK present."""
        source = _INSTALL_PS1_PATH.read_text(encoding = "utf-8")
        assert "not ROCm-accessible" in source

    def test_setup_ps1_distinct_message_for_sdk_found_but_device_inaccessible(self):
        """setup.ps1 must show 'not ROCm-accessible' message (not 'HIP SDK not found') when SDK present."""
        source = _SETUP_PS1_PATH.read_text(encoding = "utf-8")
        assert "not ROCm-accessible" in source

    def test_install_ps1_driver_guidance_in_sdk_found_branch(self):
        """install.ps1 must tell user this is a driver issue, not an SDK issue."""
        source = _INSTALL_PS1_PATH.read_text(encoding = "utf-8")
        assert "driver issue" in source

    def test_setup_ps1_driver_guidance_in_sdk_found_branch(self):
        """setup.ps1 must tell user this is a driver issue, not an SDK issue."""
        source = _SETUP_PS1_PATH.read_text(encoding = "utf-8")
        assert "driver issue" in source

    def test_install_ps1_cpu_hint_distinguishes_driver_vs_no_sdk(self):
        """install.ps1 CPU-only hint must say 'GPU not ROCm-accessible' not 'require the HIP SDK' when SDK found."""
        source = _INSTALL_PS1_PATH.read_text(encoding = "utf-8")
        assert "GPU not ROCm-accessible" in source


# TEST: --rocm-gfx forwarding -- setup.sh/setup.ps1 forward their resolved gfx
# arch to install_llama_prebuilt.py so the per-gfx prebuilt is picked.


class TestNormalizeForwardedGfx:
    """A forwarded gfx string is reduced to a single clean gfx token."""

    def test_plain_token(self):
        assert _normalize_forwarded_gfx("gfx1151") == "gfx1151"

    def test_uppercase_normalized(self):
        assert _normalize_forwarded_gfx("GFX1151") == "gfx1151"

    def test_extracts_from_noise(self):
        assert _normalize_forwarded_gfx("gcnArchName: gfx942") == "gfx942"

    def test_malformed_is_ignored(self):
        assert _normalize_forwarded_gfx("not-a-gpu") is None

    def test_empty_and_none(self):
        assert _normalize_forwarded_gfx("") is None
        assert _normalize_forwarded_gfx(None) is None


class TestApplyHostOverrides:
    """Forwarded ROCm detection is folded into the host profile correctly."""

    def test_forwarded_gfx_fills_empty_probe(self):
        host = rocm_host(rocm_gfx_target = None)
        out = _apply_host_overrides(host, override_rocm_gfx = "gfx1151")
        assert out.has_rocm is True
        assert out.rocm_gfx_target == "gfx1151"

    def test_forwarded_gfx_implies_rocm(self):
        out = _apply_host_overrides(cpu_host(), override_rocm_gfx = "gfx1200")
        assert out.has_rocm is True
        assert out.rocm_gfx_target == "gfx1200"

    def test_forwarded_gfx_does_not_clobber_probed_arch(self, monkeypatch):
        # setup.ps1's amd-smi branch drops comma masks, so its pick may be another physical GPU.
        monkeypatch.delenv("UNSLOTH_ROCM_GFX_ARCH", raising = False)
        host = rocm_host(rocm_gfx_target = "gfx1010", rocm_gfx_targets = ["gfx1100", "gfx1010"])
        out = _apply_host_overrides(host, override_rocm_gfx = "gfx1100")
        assert out.rocm_gfx_target == "gfx1010"
        assert out.rocm_gfx_targets == ["gfx1100", "gfx1010"]
        assert out.has_rocm is True

    def test_shadowing_igpu_does_not_veto_the_forwarded_dgpu(self, monkeypatch):
        # Setup skips a leading APU but _pick_rocm_gfx_target() reads it as device 0 (#7776).
        monkeypatch.delenv("UNSLOTH_ROCM_GFX_ARCH", raising = False)
        for _v in ("HIP_VISIBLE_DEVICES", "ROCR_VISIBLE_DEVICES", "CUDA_VISIBLE_DEVICES"):
            monkeypatch.delenv(_v, raising = False)
        host = rocm_host(rocm_gfx_target = "gfx1103", rocm_gfx_targets = ["gfx1103", "gfx1200"])
        out = _apply_host_overrides(host, override_rocm_gfx = "gfx1200")
        assert out.rocm_gfx_target == "gfx1200"
        assert out.rocm_gfx_targets == ["gfx1103", "gfx1200"]

    def test_masked_host_still_keeps_the_probed_arch(self, monkeypatch):
        monkeypatch.delenv("UNSLOTH_ROCM_GFX_ARCH", raising = False)
        monkeypatch.setenv("HIP_VISIBLE_DEVICES", "0")
        host = rocm_host(rocm_gfx_target = "gfx1103", rocm_gfx_targets = ["gfx1103", "gfx1200"])
        out = _apply_host_overrides(host, override_rocm_gfx = "gfx1200")
        assert out.rocm_gfx_target == "gfx1103"

    def test_shadowing_exception_needs_a_probed_dgpu(self, monkeypatch):
        # A family label is a bundle name, so gfx103X over a gfx1033 APU stays advisory.
        monkeypatch.delenv("UNSLOTH_ROCM_GFX_ARCH", raising = False)
        for _v in ("HIP_VISIBLE_DEVICES", "ROCR_VISIBLE_DEVICES", "CUDA_VISIBLE_DEVICES"):
            monkeypatch.delenv(_v, raising = False)
        host = rocm_host(rocm_gfx_target = "gfx1033", rocm_gfx_targets = ["gfx1033"])
        out = _apply_host_overrides(host, override_rocm_gfx = "gfx103X")
        assert out.rocm_gfx_target == "gfx1033"

    def test_forwarded_gfx_absent_from_host_stays_authoritative(self, monkeypatch):
        # An arch no probe reported is an explicit override for a stale probe, so it wins.
        monkeypatch.delenv("UNSLOTH_ROCM_GFX_ARCH", raising = False)
        host = rocm_host(rocm_gfx_target = "gfx1100", rocm_gfx_targets = ["gfx1100"])
        out = _apply_host_overrides(host, override_rocm_gfx = "gfx1151")
        assert out.rocm_gfx_target == "gfx1151"
        # ... but it says which arch HIP targets, not which cards exist, so the probed
        # gfx1100 is still in the box and stays in the per-GPU list.
        assert out.rocm_gfx_targets == ["gfx1100", "gfx1151"]
        assert out.has_rocm is True

    def test_forwarded_family_label_never_overrides_a_probed_arch(self, monkeypatch):
        # The update path derives --rocm-gfx from a family-named asset, so it stays advisory.
        monkeypatch.delenv("UNSLOTH_ROCM_GFX_ARCH", raising = False)
        host = rocm_host(rocm_gfx_target = "gfx1033", rocm_gfx_targets = ["gfx1033"])
        out = _apply_host_overrides(host, override_rocm_gfx = "gfx103X")
        assert out.rocm_gfx_target == "gfx1033"
        assert out.has_rocm is True

    def test_forwarded_family_label_still_fills_an_unprobed_arch(self, monkeypatch):
        monkeypatch.delenv("UNSLOTH_ROCM_GFX_ARCH", raising = False)
        out = _apply_host_overrides(cpu_host(), override_rocm_gfx = "gfx110X")
        assert out.rocm_gfx_target == "gfx110x"
        assert out.has_rocm is True

    def test_forwarded_gfx_matching_active_keeps_physical_gfx_list(self, monkeypatch):
        # Collapsing the list would hide other AMD cards from the auto-Vulkan floor check.
        monkeypatch.delenv("UNSLOTH_ROCM_GFX_ARCH", raising = False)
        host = rocm_host(rocm_gfx_target = "gfx1010", rocm_gfx_targets = ["gfx1100", "gfx1010"])
        out = _apply_host_overrides(host, override_rocm_gfx = "gfx1010")
        assert out.rocm_gfx_target == "gfx1010"
        assert out.rocm_gfx_targets == ["gfx1100", "gfx1010"]

    def test_forwarded_gfx_never_drops_a_probed_physical_gpu(self, monkeypatch):
        # The per-GPU list is the physical inventory; an unseen forwarded arch is ADDED, not replacing.
        monkeypatch.delenv("UNSLOTH_ROCM_GFX_ARCH", raising = False)
        host = rocm_host(rocm_gfx_target = "gfx803", rocm_gfx_targets = ["gfx1100", "gfx803"])
        out = _apply_host_overrides(host, override_rocm_gfx = "gfx900")
        assert out.rocm_gfx_target == "gfx900"
        assert out.rocm_gfx_targets == ["gfx1100", "gfx803", "gfx900"]

    def test_forwarded_gfx_not_duplicated_when_already_probed(self, monkeypatch):
        monkeypatch.setenv("UNSLOTH_ROCM_GFX_ARCH", "gfx803")
        host = rocm_host(rocm_gfx_target = "gfx1100", rocm_gfx_targets = ["gfx1100", "gfx803"])
        out = _apply_host_overrides(host, override_rocm_gfx = "gfx803")
        assert out.rocm_gfx_target == "gfx803"
        assert out.rocm_gfx_targets == ["gfx1100", "gfx803"]

    def test_forwarded_gfx_on_an_unprobed_host_lists_only_itself(self, monkeypatch):
        # Negative control for the two above: nothing probed means no inventory to
        # preserve, so the driver-only host keeps a single-entry list and auto-Vulkan.
        monkeypatch.delenv("UNSLOTH_ROCM_GFX_ARCH", raising = False)
        out = _apply_host_overrides(cpu_host(), override_rocm_gfx = "gfx803")
        assert out.rocm_gfx_target == "gfx803"
        assert out.rocm_gfx_targets == ["gfx803"]

    def test_manual_env_override_still_wins_over_probe(self, monkeypatch):
        # UNSLOTH_ROCM_GFX_ARCH is the manual escape hatch for hosts whose arch the probes
        # get wrong, so it stays authoritative.
        monkeypatch.setenv("UNSLOTH_ROCM_GFX_ARCH", "gfx1151")
        host = rocm_host(rocm_gfx_target = "gfx1100", rocm_gfx_targets = ["gfx1100"])
        out = _apply_host_overrides(host, override_rocm_gfx = "gfx1151")
        assert out.rocm_gfx_target == "gfx1151"
        assert out.rocm_gfx_targets == ["gfx1100", "gfx1151"]

    def test_has_rocm_only_keeps_probe_gfx(self):
        out = _apply_host_overrides(cpu_host(), override_has_rocm = True)
        assert out.has_rocm is True
        assert out.rocm_gfx_target is None

    def test_malformed_forwarded_gfx_falls_back_to_has_rocm(self):
        out = _apply_host_overrides(cpu_host(), override_has_rocm = True, override_rocm_gfx = "junk")
        assert out.has_rocm is True
        assert out.rocm_gfx_target is None

    def test_no_overrides_leaves_host_unchanged(self):
        host = nvidia_host()
        assert _apply_host_overrides(host) is host


class TestRocmGfxForwarding:
    """setup.sh / setup.ps1 forward their resolved gfx; the installer accepts it."""

    def test_installer_exposes_rocm_gfx_arg(self):
        source = _PREBUILT_PATH.read_text(encoding = "utf-8")
        assert '"--rocm-gfx"' in source
        assert 'os.environ.get("UNSLOTH_ROCM_GFX_ARCH")' in source

    def test_setup_sh_forwards_rocm_gfx(self):
        source = _SETUP_SH_PATH.read_text(encoding = "utf-8")
        assert "--rocm-gfx" in source
        assert '"$_setup_gfx"' in source

    def test_setup_sh_forwards_has_rocm(self):
        # If AMD is detected but gfx resolution fails, --has-rocm is still forwarded.
        source = _SETUP_SH_PATH.read_text(encoding = "utf-8")
        assert "--has-rocm" in source
        assert "_setup_amd_detected" in source

    def test_setup_ps1_forwards_rocm_gfx(self):
        source = _SETUP_PS1_PATH.read_text(encoding = "utf-8")
        assert "--rocm-gfx" in source
        assert "$script:ROCmGfxArch" in source

    def test_setup_sh_routes_unconditionally_to_fork(self):
        # The release repo is an unconditional fork assignment now.
        source = _SETUP_SH_PATH.read_text(encoding = "utf-8")
        assert '_HELPER_RELEASE_REPO="unslothai/llama.cpp"' in source
        assert '_HELPER_RELEASE_REPO="ggml-org/llama.cpp"' not in source

    def test_setup_ps1_routes_unconditionally_to_fork(self):
        source = _SETUP_PS1_PATH.read_text(encoding = "utf-8")
        assert '$HelperReleaseRepo = "unslothai/llama.cpp"' in source
        assert "$HelperReleaseRepo = if (" not in source

    # Execute the real routing line too, so a reintroduced conditional is caught.

    @staticmethod
    def _resolve_setup_sh_repo(
        host_machine,
        nvidia_usable,
        setup_gfx,
        rocm_gfx_arch_env = "",
    ):
        """PATH is emptied so tooling probes miss; no GPU input may reroute the release repo to ggml-org."""
        import shutil

        bash = shutil.which("bash")
        if bash is None:
            pytest.skip("bash not available")
        source = _SETUP_SH_PATH.read_text(encoding = "utf-8")
        start = source.index('\n_HELPER_RELEASE_REPO="unslothai/llama.cpp"\n') + 1
        end = source.index("\n_LLAMA_PR=", start)
        block = source[start:end]
        assert "_HELPER_RELEASE_REPO" in block, "setup.sh routing anchors not found"
        env = {
            "PATH": "",  # no ROCm tooling discoverable
            "ROUTING_BLOCK": block,
            "_HOST_SYSTEM": "Linux",
            "_HOST_MACHINE": host_machine,
            "_setup_nvidia_usable": "true" if nvidia_usable else "false",
            "_setup_gfx": setup_gfx,
            "UNSLOTH_ROCM_GFX_ARCH": rocm_gfx_arch_env,
        }
        result = subprocess.run(
            [bash, "-c", 'eval "$ROUTING_BLOCK"; printf "%s" "$_HELPER_RELEASE_REPO"'],
            capture_output = True,
            text = True,
            timeout = 30,
            env = env,
        )
        assert result.returncode == 0, result.stderr
        return result.stdout.strip()

    @pytest.mark.parametrize(
        "machine, nvidia_usable, setup_gfx, env_gfx",
        [
            ("x86_64", False, "", ""),
            ("aarch64", False, "", ""),
            ("x86_64", False, "gfx1100", ""),
            ("x86_64", False, "", "gfx1100"),
            ("x86_64", True, "", ""),
        ],
    )
    def test_setup_sh_routing_block_always_resolves_to_fork(
        self, machine, nvidia_usable, setup_gfx, env_gfx
    ):
        assert (
            self._resolve_setup_sh_repo(
                machine, nvidia_usable, setup_gfx, rocm_gfx_arch_env = env_gfx
            )
            == "unslothai/llama.cpp"
        )

    @staticmethod
    def _resolve_setup_ps1_repo():
        """Run setup.ps1's $HelperReleaseRepo assignment under pwsh and return the resolved repo; it
        is unconditional, so no host inputs vary."""
        import shutil

        pwsh = shutil.which("pwsh")
        if pwsh is None:
            pytest.skip("pwsh not available")
        source = _SETUP_PS1_PATH.read_text(encoding = "utf-8")
        line = next(
            (ln for ln in source.splitlines() if ln.strip().startswith("$HelperReleaseRepo =")),
            None,
        )
        assert line is not None, "$HelperReleaseRepo selection not found in setup.ps1"
        harness = f"{line}\nWrite-Output $HelperReleaseRepo"
        # run_pwsh: a killed pwsh would read as $HelperReleaseRepo failing.
        result = run_pwsh(
            [pwsh, "-NoProfile", "-Command", harness],
            capture_output = True,
            text = True,
            timeout = 60,
        )
        assert result.returncode == 0, result.stderr
        return result.stdout.strip()

    def test_setup_ps1_routing_resolves_to_fork(self):
        assert self._resolve_setup_ps1_repo() == "unslothai/llama.cpp"


# _pick_rocm_gfx_target honours CUDA/HIP_VISIBLE_DEVICES on mixed-arch hosts.

_pick_rocm_gfx_target = prebuilt_mod._pick_rocm_gfx_target


def test_pick_rocm_gfx_target_honors_cuda_visible_devices(monkeypatch):
    """CUDA_VISIBLE_DEVICES=1 must select gfx1100 on a gfx1151 + gfx1100 host (HIP honours CUDA var)."""
    # rocminfo reports each token twice (as in the real tool output).
    probe_out = "gfx1151\ngfx1151\ngfx1100\ngfx1100"
    monkeypatch.delenv("HIP_VISIBLE_DEVICES", raising = False)
    monkeypatch.delenv("ROCR_VISIBLE_DEVICES", raising = False)
    monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "1")
    assert _pick_rocm_gfx_target(probe_out) == "gfx1100"


def test_pick_rocm_gfx_target_cuda_visible_devices_minus_one_returns_none(monkeypatch):
    """CUDA_VISIBLE_DEVICES=-1 means no GPU visible; resolver must return None."""
    probe_out = "gfx1151\ngfx1100"
    monkeypatch.delenv("HIP_VISIBLE_DEVICES", raising = False)
    monkeypatch.delenv("ROCR_VISIBLE_DEVICES", raising = False)
    monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "-1")
    assert _pick_rocm_gfx_target(probe_out) is None


def test_pick_rocm_gfx_target_same_arch_multi_gpu(monkeypatch):
    """Regression: [gfx1100, gfx1100, gfx1151] with HIP_VISIBLE_DEVICES=2 must return gfx1151 (no dict.fromkeys collapse)."""
    probe_out = (
        "***\nAgent 1\n***\n  gfx1100 some info\n  gfx1100\n"
        "***\nAgent 2\n***\n  gfx1100 some info\n  gfx1100\n"
        "***\nAgent 3\n***\n  gfx1151 some info\n  gfx1151\n"
    )
    monkeypatch.delenv("ROCR_VISIBLE_DEVICES", raising = False)
    monkeypatch.delenv("CUDA_VISIBLE_DEVICES", raising = False)
    monkeypatch.setenv("HIP_VISIBLE_DEVICES", "2")
    assert _pick_rocm_gfx_target(probe_out) == "gfx1151"


# TEST: WSL ROCDXG fixes -- drop-in persistence + system-HIP-before-bundle


_LLAMA_CPP_PATH = PACKAGE_ROOT / "studio" / "backend" / "core" / "inference" / "llama_cpp.py"


class TestWslSystemRocmLibDirs:
    """_wsl_system_rocm_lib_dirs: no-op off a ROCDXG WSL host; else returns the system ROCm lib dir for binary_env."""

    def test_empty_without_dev_dxg(self):
        with patch("os.path.exists", return_value = False):
            assert prebuilt_mod._wsl_system_rocm_lib_dirs() == []

    def test_empty_on_bare_metal_linux(self):
        with patch("os.path.exists", lambda p: p == "/dev/dxg"):
            with patch(
                "builtins.open",
                mock_open(read_data = "Linux version 6.8.0-generic"),
            ):
                assert prebuilt_mod._wsl_system_rocm_lib_dirs() == []

    def test_returns_system_lib_on_wsl_with_librocdxg(self):
        # Normalize separators: os.path.join uses "\" on the Windows test host.
        def _exists(p):
            p = str(p).replace("\\", "/")
            return p in ("/dev/dxg", "/opt/rocm/lib/librocdxg.so")

        with patch("os.path.exists", _exists):
            with patch(
                "builtins.open",
                mock_open(read_data = "Linux version 5.15.0-microsoft-standard-WSL2"),
            ):
                assert prebuilt_mod._wsl_system_rocm_lib_dirs() == ["/opt/rocm/lib"]

    def test_empty_on_wsl_without_librocdxg(self):
        with patch("os.path.exists", lambda p: p == "/dev/dxg"):
            with patch(
                "builtins.open",
                mock_open(read_data = "microsoft-standard-WSL2"),
            ):
                assert prebuilt_mod._wsl_system_rocm_lib_dirs() == []


class TestBinaryEnvWslOrdering:
    """binary_env puts system ROCm lib ahead of the bundle dir + sets HSA_ENABLE_DXG_DETECTION on WSL; no-op bare-metal."""

    @staticmethod
    def _linux_host():
        return HostInfo(
            system = "Linux",
            machine = "x86_64",
            is_windows = False,
            is_linux = True,
            is_macos = False,
            is_x86_64 = True,
            is_arm64 = False,
            nvidia_smi = None,
            driver_cuda_version = None,
            compute_caps = [],
            visible_cuda_devices = None,
            has_physical_nvidia = False,
            has_usable_nvidia = False,
            has_rocm = True,
        )

    def test_wsl_prepends_system_rocm_and_sets_hsa(self, tmp_path):
        binary = tmp_path / "bundle" / "llama-server"
        binary.parent.mkdir(parents = True)
        binary.write_text("")
        # dedupe_existing_dirs drops non-existent dirs, so use a real dir.
        sys_rocm = tmp_path / "sysrocm"
        sys_rocm.mkdir()
        with patch.object(prebuilt_mod, "_wsl_system_rocm_lib_dirs", return_value = [str(sys_rocm)]):
            with patch.dict(os.environ, {}, clear = True):
                env = prebuilt_mod.binary_env(binary, tmp_path, self._linux_host())
        ld = env["LD_LIBRARY_PATH"].split(os.pathsep)
        # Compare resolved paths (dedupe_existing_dirs calls Path.resolve()).
        ld_resolved = [str(Path(p).resolve()) for p in ld]
        assert ld_resolved[0] == str(sys_rocm.resolve())
        assert str(binary.parent.resolve()) in ld_resolved
        assert ld_resolved.index(str(sys_rocm.resolve())) < ld_resolved.index(
            str(binary.parent.resolve())
        )
        assert env.get("HSA_ENABLE_DXG_DETECTION") == "1"

    def test_bare_metal_linux_unchanged(self, tmp_path):
        binary = tmp_path / "bundle" / "llama-server"
        binary.parent.mkdir(parents = True)
        binary.write_text("")
        with patch.object(prebuilt_mod, "_wsl_system_rocm_lib_dirs", return_value = []):
            with patch.dict(os.environ, {}, clear = True):
                env = prebuilt_mod.binary_env(binary, tmp_path, self._linux_host())
        ld = env["LD_LIBRARY_PATH"].split(os.pathsep)
        assert ld[0] == str(binary.parent)  # bundle dir first, as before
        assert "HSA_ENABLE_DXG_DETECTION" not in env


class TestInstallShDropinPersistence:
    """install.sh persists the ROCm-on-WSL drop-in even when rocminfo already enumerates the GPU (reinstall safety)."""

    def test_has_persist_helper(self):
        source = _INSTALL_SH_PATH.read_text(encoding = "utf-8")
        assert "_persist_rocm_wsl_dropin()" in source

    def test_gate5_early_return_persists_dropin(self):
        """The rocminfo-already-works early return must call the persist helper before returning."""
        source = _INSTALL_SH_PATH.read_text(encoding = "utf-8")
        # The persist call must precede return 0 at the rocminfo gate with the !/generic/ clause.
        gate = source.find("Name:[[:space:]]*gfx[1-9]/ && !/generic/")
        assert gate != -1
        window = source[gate : gate + 900]
        assert "_persist_rocm_wsl_dropin" in window
        assert window.find("_persist_rocm_wsl_dropin") < window.find("return 0")

    def test_persist_helper_gated_on_librocdxg(self):
        source = _INSTALL_SH_PATH.read_text(encoding = "utf-8")
        body_start = source.find("_persist_rocm_wsl_dropin()")
        body = source[body_start : body_start + 1200]
        assert "librocdxg.so" in body
        assert "profile.d/unsloth-rocm-wsl.sh" in body


_STRIXHALO_WSL_PATH = PACKAGE_ROOT / "scripts" / "install_rocm_wsl_strixhalo.sh"


class TestWslRerouteNvidiaGuard:
    """_maybe_reroute_strixhalo_to_2404 must skip the AMD reroute on hybrid AMD+NVIDIA hosts by
    reusing _has_usable_nvidia_gpu (CUDA_VISIBLE_DEVICES-aware + /proc/driver/nvidia fallback),
    which must be defined before the reroute's call site so it is actually available."""

    def test_reroute_calls_nvidia_helper_before_amd_signal(self):
        source = _INSTALL_SH_PATH.read_text(encoding = "utf-8")
        start = source.find("_maybe_reroute_strixhalo_to_2404()")
        assert start != -1
        # Slice the whole function body, not a fixed window.
        end = source.find("\n}", start)
        assert end != -1
        body = source[start:end]
        nv = body.find("_has_usable_nvidia_gpu")
        wmi = body.find("_wsl_amd_gpu_name")
        assert nv != -1, "reroute must consult _has_usable_nvidia_gpu before deciding to reroute"
        assert wmi != -1
        # The NVIDIA guard must precede the AMD/WMI signal and return early.
        assert nv < wmi
        assert body.find("return 0", nv) < wmi

    def test_nvidia_helper_and_deps_defined_before_reroute_callsite(self):
        source = _INSTALL_SH_PATH.read_text(encoding = "utf-8")
        call = source.find("\n_maybe_reroute_strixhalo_to_2404 || true")
        assert call != -1
        for fn in ("_run_bounded() {", "_cvd_hides_nvidia() {", "_has_usable_nvidia_gpu() {"):
            idx = source.find(fn)
            assert idx != -1 and idx < call, f"{fn} must be defined before the reroute call"


class TestStrixhaloGfxOverridePipefail:
    """The UNSLOTH_WSL_GFX override check must use a consuming grep, not grep -q: under
    `set -o pipefail` an early -q exit SIGPIPEs printf and misreports the arch on large output."""

    def test_gfx_override_uses_consuming_grep(self):
        source = _STRIXHALO_WSL_PATH.read_text(encoding = "utf-8")
        idx = source.find('grep -E "Name:[[:space:]]*${GFX}')
        assert idx != -1, "GFX override must use a consuming grep -E (not grep -q)"
        line = source[idx : source.find("\n", idx)]
        assert ">/dev/null" in line
        assert 'grep -qE "Name:[[:space:]]*${GFX}' not in source


class TestLlamaCppRuntimeWslOrdering:
    """The serve-time launcher mirrors binary_env: system HIP before the bundle dir on WSL."""

    def test_has_wsl_helper(self):
        source = _LLAMA_CPP_PATH.read_text(encoding = "utf-8")
        assert "_wsl_system_rocm_lib_dirs" in source

    def test_prepends_before_binary_dir(self):
        source = _LLAMA_CPP_PATH.read_text(encoding = "utf-8")
        idx_helper = source.find("lib_dirs.extend(_wsl_system_rocm_lib_dirs())")
        idx_binary = source.find("lib_dirs.append(binary_dir)")
        assert idx_helper != -1 and idx_binary != -1
        assert idx_helper < idx_binary


class TestRocmWslSupplyChainPins:
    """The ROCm-on-WSL bootstrap runs unattended with sudo, so every fetched piece of code must come
    from an immutable ref, since a moving branch means root code execution."""

    def test_helper_is_fetched_from_a_pinned_commit_not_a_branch(self):
        source = _INSTALL_SH_PATH.read_text(encoding = "utf-8")
        start = source.find("_maybe_bootstrap_rocm_wsl()")
        assert start != -1
        body = source[start : source.find("\n}", start)]
        assert (
            "unsloth/main/scripts/install_rocm_wsl_strixhalo.sh" not in body
        ), "the helper must not be fetched from the mutable main branch"
        ref = re.search(r'_ROCM_WSL_HELPER_REF="([0-9a-f]+)"', body)
        assert ref is not None, "the helper fetch must pin a full commit SHA"
        assert len(ref.group(1)) == 40
        assert "${_ROCM_WSL_HELPER_REF}/scripts/install_rocm_wsl_strixhalo.sh" in body
        # A helper without the pinned-source check is refused.
        assert 'grep -q "^UNSLOTH_ROCM_WSL_HELPER_CONTRACT=2$" "$_rw_helper"' in body
        assert "predates the pinned-source check" in body

    def test_librocdxg_ref_is_pinned_and_verified_before_build(self):
        source = _STRIXHALO_WSL_PATH.read_text(encoding = "utf-8")
        assert (
            "UNSLOTH_LIBROCDXG_REF:-develop" not in source
        ), "librocdxg must not default to a moving branch"
        sha = re.search(r'LIBROCDXG_SHA="\$\{UNSLOTH_LIBROCDXG_SHA:-([0-9a-f]{40})\}"', source)
        assert sha is not None, "librocdxg must default to a pinned commit SHA"
        # The ref is that commit, so even a helper with no SHA check gets an immutable revision.
        ref = re.search(r'LIBROCDXG_REF="\$\{UNSLOTH_LIBROCDXG_REF:-([0-9a-f]{40})\}"', source)
        assert ref is not None and ref.group(1) == sha.group(1)
        # A failed checkout must not fall through to the default branch.
        assert "_co_failed=1" in source
        assert "Refusing to build the repository's default branch instead" in source
        idx_check = source.find("git rev-parse HEAD")
        idx_cmake = source.find("cmake .. -DWIN_SDK")
        idx_install = source.find("$SUDO make install")
        assert idx_check != -1 and idx_check < idx_cmake < idx_install

    def test_install_sh_forwards_the_same_librocdxg_pin_as_the_helper(self):
        # The pin copies must never disagree, or a bumped helper builds a stale rev.
        install_sh = _INSTALL_SH_PATH.read_text(encoding = "utf-8")
        helper = _STRIXHALO_WSL_PATH.read_text(encoding = "utf-8")
        fwd_ref = re.search(r'_rw_dxg_ref="([0-9a-f]{40})"', install_sh)
        fwd_sha = re.search(r'_rw_dxg_sha="\$_rw_dxg_ref"', install_sh)
        own_ref = re.search(r'LIBROCDXG_REF="\$\{UNSLOTH_LIBROCDXG_REF:-([0-9a-f]{40})\}"', helper)
        own_sha = re.search(r'LIBROCDXG_SHA="\$\{UNSLOTH_LIBROCDXG_SHA:-([0-9a-f]{40})\}"', helper)
        assert None not in (fwd_ref, fwd_sha, own_ref, own_sha)
        assert fwd_ref.group(1) == own_ref.group(1) == own_sha.group(1)
        # A SHA is forwarded AS the ref, so a helper ignoring UNSLOTH_LIBROCDXG_SHA gets that commit.
        assert 'if [ -n "$_rw_dxg_sha" ]; then\n        _rw_dxg_ref="$_rw_dxg_sha"' in install_sh
        assert '_rw_dxg_ref="${UNSLOTH_LIBROCDXG_REF:-}"' in install_sh
        assert (
            'UNSLOTH_LIBROCDXG_REF="$_rw_dxg_ref" UNSLOTH_LIBROCDXG_SHA="$_rw_dxg_sha"'
            in install_sh
        )

    def test_helper_contract_matches_what_install_sh_requires(self):
        # Required and declared contract levels must move together.
        install_sh = _INSTALL_SH_PATH.read_text(encoding = "utf-8")
        helper = _STRIXHALO_WSL_PATH.read_text(encoding = "utf-8")
        required = re.search(r'grep -q "\^UNSLOTH_ROCM_WSL_HELPER_CONTRACT=(\d+)\$"', install_sh)
        declared = re.search(r"^UNSLOTH_ROCM_WSL_HELPER_CONTRACT=(\d+)$", helper, re.M)
        assert required is not None and declared is not None
        assert required.group(1) == declared.group(1)
        assert "git rev-parse HEAD" in helper
        assert "_co_failed=1" in helper


class TestRocmMiscomputingArchDemotion:
    """Declining is not enough: an existing ROCm torch on a miscomputing arch must be demoted to CPU."""

    def _demotion_calls(
        self,
        monkeypatch,
        label,
        archs,
        env = None,
        codes_fn = None,
        kfd = (),
        return_result = False,
    ):
        calls = []
        monkeypatch.setattr(stack_mod, "_kfd_gfx_targets", lambda: list(kfd))
        monkeypatch.setattr(stack_mod, "NO_TORCH", False)
        monkeypatch.setattr(stack_mod, "IS_WINDOWS", False)
        monkeypatch.setattr(stack_mod, "IS_MACOS", False)
        monkeypatch.setattr(stack_mod, "_installed_torch_label_on_disk", lambda: label)
        monkeypatch.setattr(
            stack_mod, "_detect_amd_gfx_codes", codes_fn or (lambda **_kw: list(archs))
        )
        monkeypatch.setattr(stack_mod, "_infer_linux_amd_gfx_arch", lambda: None)
        monkeypatch.setattr(
            stack_mod,
            "_probe_torch_runtime",
            lambda: (True, True, label, "7.1" if "+rocm" in label else "", ""),
        )
        monkeypatch.setattr(stack_mod, "pip_install", lambda *a, **k: calls.append((a, k)))
        for _var in (
            "UNSLOTH_TORCH_INDEX_URL",
            "UNSLOTH_TORCH_INDEX_FAMILY",
            "UNSLOTH_ROCM_GFX_ARCH",
        ):
            monkeypatch.delenv(_var, raising = False)
        for _var, _val in (env or {}).items():
            monkeypatch.setenv(_var, _val)
        result = stack_mod._ensure_cpu_torch()
        return (calls, result) if return_result else calls

    def test_installed_rocm_torch_on_gfx1033_is_demoted(self, monkeypatch):
        calls = self._demotion_calls(monkeypatch, "2.10.0+rocm7.1", ["gfx1033"])
        assert len(calls) == 1, "gfx1033 kept its ROCm torch across the upgrade"
        assert "--force-reinstall" in calls[0][0]
        assert "download.pytorch.org/whl/cpu" in str(calls[0])

    @pytest.mark.parametrize("family", ["cpu", "current"])
    def test_an_unusable_non_rocm_family_still_requires_the_demotion(self, monkeypatch, family):
        monkeypatch.setattr(stack_mod, "_PYTORCH_WHL_BASE", "https://mirror.example/whl?token=abc")
        calls, repair_ok = self._demotion_calls(
            monkeypatch,
            "2.10.0+rocm7.1",
            ["gfx1033"],
            env = {"UNSLOTH_TORCH_INDEX_FAMILY": family},
            return_result = True,
        )
        assert calls == []
        assert repair_ok is False

    def test_an_unusable_rocm_family_may_explicitly_retain_rocm(self, monkeypatch):
        monkeypatch.setattr(stack_mod, "_PYTORCH_WHL_BASE", "https://mirror.example/whl?token=abc")
        calls, repair_ok = self._demotion_calls(
            monkeypatch,
            "2.10.0+rocm7.1",
            ["gfx1033"],
            env = {"UNSLOTH_TORCH_INDEX_FAMILY": "rocm7.2"},
            return_result = True,
        )
        assert calls == []
        assert repair_ok is not False

    def test_hsa_spoofed_gfx1033_is_still_demoted(self, monkeypatch):
        """HSA_OVERRIDE_GFX_VERSION=10.3.0 is the usual Van Gogh workaround and makes
        rocminfo answer gfx1030, so the probe must ignore it or the arch hides itself."""
        seen = {}

        def _codes(**kw):
            seen.update(kw)
            return ["gfx1030"] if not kw.get("ignore_hsa_override") else ["gfx1033"]

        calls = self._demotion_calls(monkeypatch, "2.10.0+rocm7.1", [], codes_fn = _codes)
        assert seen.get("ignore_hsa_override") is True
        assert len(calls) == 1

    def test_a_mask_cannot_shrink_the_host_to_its_bad_gpu(self, monkeypatch):
        """Checks every arch on the whole host: a mask hiding the healthy GPU must not leave only
        the bad one."""
        seen = {}

        def _codes(**kw):
            seen.update(kw)
            return ["gfx1033"] if not kw.get("ignore_visible_masks") else ["gfx1033", "gfx1100"]

        calls = self._demotion_calls(
            monkeypatch,
            "2.10.0+rocm7.1",
            [],
            env = {"ROCR_VISIBLE_DEVICES": "0"},
            codes_fn = _codes,
        )
        assert seen.get("ignore_visible_masks") is True
        assert calls == [], "a mask shrank the host to its miscomputing GPU"

    def test_kfd_topology_names_the_arch_when_no_runtime_does(self, monkeypatch):
        """With no runtime to name the arch, KFD topology supplies it so the ROCm torch is still demoted."""
        calls = self._demotion_calls(monkeypatch, "2.10.0+rocm7.1", [], kfd = ["gfx1033"])
        assert len(calls) == 1, "gfx1033 kept its ROCm torch with only KFD to name it"
        assert "--force-reinstall" in calls[0][0]
        assert "download.pytorch.org/whl/cpu" in str(calls[0])

    def test_kfd_reports_every_node_so_a_mixed_host_keeps_rocm(self, monkeypatch):
        """_kfd_gfx_targets() returns one entry per GPU node, so the all-archs rule reads
        the same mixed host through this source as through the runtime probes."""
        assert (
            self._demotion_calls(monkeypatch, "2.10.0+rocm7.1", [], kfd = ["gfx1033", "gfx1100"])
            == []
        )

    def test_the_runtime_probe_still_wins_over_kfd(self, monkeypatch):
        """KFD is the last resort, not an override: a runtime that answered keeps its
        answer, so HSA_OVERRIDE handling and the mask semantics above are untouched."""
        assert (
            self._demotion_calls(monkeypatch, "2.10.0+rocm7.1", ["gfx1100"], kfd = ["gfx1033"]) == []
        )

    def test_a_declared_arch_does_not_outrank_the_silicon(self, monkeypatch):
        """The declared arch is a hint; hardware probes are asked first, so a stale value cannot
        outrank them."""
        calls = self._demotion_calls(
            monkeypatch,
            "2.10.0+rocm7.1",
            ["gfx1033"],
            env = {"UNSLOTH_ROCM_GFX_ARCH": "gfx1030"},
        )
        assert len(calls) == 1, "a declared arch hid the silicon from the demotion"

    def test_kfd_outranks_a_declared_arch(self, monkeypatch):
        """_infer_linux_amd_gfx_arch() returns the declared arch first, so KFD has to be
        consulted before it or the kernel never gets to answer."""
        calls = self._demotion_calls(
            monkeypatch,
            "2.10.0+rocm7.1",
            [],
            kfd = ["gfx1033"],
            env = {"UNSLOTH_ROCM_GFX_ARCH": "gfx1030"},
        )
        assert len(calls) == 1, "the declared arch outranked the kernel"

    def test_env_override_names_the_arch(self, monkeypatch):
        calls = self._demotion_calls(
            monkeypatch, "2.10.0+rocm7.1", [], env = {"UNSLOTH_ROCM_GFX_ARCH": "GFX1033"}
        )
        assert len(calls) == 1

    @pytest.mark.parametrize(
        "label, archs, env, why",
        [
            # A supported arch keeps its ROCm torch: the gate is scoped to what was measured.
            ("2.10.0+rocm7.1", ["gfx1030"], None, "gfx1030 demoted"),
            ("2.10.0+rocm7.1", ["gfx906"], None, "gfx906 demoted"),
            ("2.10.0+rocm7.1", ["gfx1033", "gfx1100"], None, "mixed host demoted"),
            # Nothing to demote -- and no ROCm probe should be paid for on these hosts.
            ("2.10.0+cu128", ["gfx1033"], None, "CUDA torch demoted"),
            ("2.10.0", ["gfx1033"], None, "CPU torch reinstalled for nothing"),
            # The documented escape hatch stays authoritative.
            (
                "2.10.0+rocm7.1",
                ["gfx1033"],
                {"UNSLOTH_TORCH_INDEX_URL": "https://download.pytorch.org/whl/rocm7.2"},
                "explicit ROCm pin overridden",
            ),
            (
                "2.10.0+rocm7.1",
                ["gfx1033"],
                {"UNSLOTH_TORCH_INDEX_FAMILY": "rocm7.2"},
                "explicit family pin overridden",
            ),
        ],
    )
    def test_no_demotion_outside_the_measured_case(self, monkeypatch, label, archs, env, why):
        assert self._demotion_calls(monkeypatch, label, archs, env) == [], why

    @pytest.mark.parametrize(
        "codes, expect_install",
        [
            (["gfx1033"], False),
            (["gfx1100"], True),
        ],
    )
    def test_ensure_rocm_torch_declines_a_miscomputing_target(
        self, monkeypatch, codes, expect_install
    ):
        """A None index for a miscomputing arch must decline the reroute, not fall back to generic
        wheels."""
        calls = []
        monkeypatch.setattr(stack_mod, "IS_WINDOWS", False)
        monkeypatch.setattr(stack_mod, "IS_MACOS", False)
        monkeypatch.setattr("platform.machine", lambda: "x86_64")
        monkeypatch.setattr(stack_mod, "_TORCH_BACKEND", "")
        monkeypatch.setattr(stack_mod, "_has_usable_nvidia_gpu", lambda: False)
        monkeypatch.setattr(stack_mod, "_has_rocm_gpu", lambda: True)
        monkeypatch.setattr(stack_mod, "_detect_amd_gfx_codes", lambda **_kw: list(codes))
        monkeypatch.setattr(stack_mod, "_kfd_gfx_targets", lambda: [])
        monkeypatch.setattr(stack_mod, "_infer_linux_amd_gfx_arch", lambda: None)
        monkeypatch.setattr(stack_mod, "_detect_rocm_version", lambda: (7, 1))
        monkeypatch.setattr(stack_mod, "pip_install_try", lambda *a, **k: True)
        monkeypatch.setattr(stack_mod, "pip_install", lambda *a, **k: calls.append((a, k)))
        for _var in (
            "UNSLOTH_TORCH_INDEX_URL",
            "UNSLOTH_TORCH_INDEX_FAMILY",
            "UNSLOTH_ROCM_GFX_ARCH",
            "HIP_VISIBLE_DEVICES",
            "ROCR_VISIBLE_DEVICES",
            "CUDA_VISIBLE_DEVICES",
        ):
            monkeypatch.delenv(_var, raising = False)
        _probe = MagicMock(returncode = 0, stdout = _MARK + "2.10.0+cpu||\n")
        with patch("os.path.isdir", return_value = True):
            with patch("subprocess.run", return_value = _probe):
                stack_mod._ensure_rocm_torch()
        assert bool(calls) is expect_install, calls

    def test_a_stale_override_does_not_start_a_rocm_to_cpu_cycle(self, monkeypatch):
        """A stale override must not win the inferred-arch path, or updates cycle between ROCm and
        CPU torch."""
        calls = []
        monkeypatch.setattr(stack_mod, "IS_WINDOWS", False)
        monkeypatch.setattr(stack_mod, "IS_MACOS", False)
        monkeypatch.setattr(stack_mod, "NO_TORCH", False)
        monkeypatch.setattr(stack_mod, "_TORCH_BACKEND", "")
        monkeypatch.setattr(stack_mod, "_has_usable_nvidia_gpu", lambda: False)
        monkeypatch.setattr(stack_mod, "_has_rocm_gpu", lambda: False)
        # The real silicon, from a source the override cannot move.
        monkeypatch.setattr(stack_mod, "_detect_amd_gfx_codes", lambda **_kw: ["gfx1033"])
        monkeypatch.setattr(stack_mod, "_kfd_gfx_targets", lambda: [])
        monkeypatch.setattr(stack_mod, "pip_install", lambda *a, **k: calls.append(a))
        monkeypatch.setattr(stack_mod, "pip_install_try", lambda *a, **k: True)
        monkeypatch.setenv("UNSLOTH_ROCM_GFX_ARCH", "gfx1030")
        for _var in ("UNSLOTH_TORCH_INDEX_URL", "UNSLOTH_TORCH_INDEX_FAMILY"):
            monkeypatch.delenv(_var, raising = False)
        _probe = MagicMock(returncode = 0, stdout = _MARK + "2.10.0+cpu||\n")
        with patch("os.path.isdir", return_value = True):
            with patch("subprocess.run", return_value = _probe):
                stack_mod._ensure_rocm_torch()
        assert calls == [], f"a stale override still bought the ROCm stack: {calls}"

    def test_probe_is_skipped_when_no_rocm_torch_is_installed(self, monkeypatch):
        """The arch probes run subprocesses; they must not run on every ordinary install."""
        monkeypatch.setattr(stack_mod, "IS_WINDOWS", False)
        monkeypatch.setattr(stack_mod, "IS_MACOS", False)
        monkeypatch.setattr(stack_mod, "_installed_torch_label_on_disk", lambda: "2.10.0+cu128")

        def _boom(**_kw):
            raise AssertionError("probed the AMD arch on a host with no ROCm torch")

        monkeypatch.setattr(stack_mod, "_detect_amd_gfx_codes", _boom)
        monkeypatch.setattr(stack_mod, "_infer_linux_amd_gfx_arch", _boom)
        monkeypatch.setattr(stack_mod, "_kfd_gfx_targets", _boom)
        for _var in ("UNSLOTH_TORCH_INDEX_URL", "UNSLOTH_TORCH_INDEX_FAMILY"):
            monkeypatch.delenv(_var, raising = False)
        assert stack_mod._rocm_miscomputing_host() is False


if __name__ == "__main__":
    pytest.main([__file__, "-v"])


def test_the_shell_extractor_says_so_when_a_function_is_gone():
    """A missing shell function must fail the extractor; an empty body falsely reports a missing check."""
    source = (PACKAGE_ROOT / "install.sh").read_text(encoding = "utf-8")
    with pytest.raises(AssertionError, match = "_amd_rocm_gpu_visible"):
        _extract_sh_function_body(source, "_amd_rocm_gpu_visible")


def test_the_extractor_still_returns_the_probe_it_does_define():
    """The control: the surviving name must still come back with a body, or the check
    above would pass on a helper that refuses everything."""
    source = (PACKAGE_ROOT / "install.sh").read_text(encoding = "utf-8")
    body = _extract_sh_function_body(source, "_has_amd_rocm_gpu")
    assert body.startswith("_has_amd_rocm_gpu() {") and "vendor_id" in body


class TestBnbRocmProvenance:
    """A no-op update must not refetch the bnb wheel, because --force-reinstall downloads before
    checking."""

    URL = "https://example.invalid/bitsandbytes-1.33.7.preview-py3-none-manylinux.whl"
    # What the release URL served when the last run looked: an ETag and a size.
    ASSET = '"3f2a"|43000000'

    @pytest.fixture(autouse = True)
    def _healthy_payload(self, monkeypatch):
        monkeypatch.setattr(stack_mod.install_manifest, "damaged_payload_files", lambda *a, **k: [])
        monkeypatch.setattr(stack_mod, "_bnb_asset_identity", lambda _url: self.ASSET)
        monkeypatch.setattr(stack_mod, "_BNB_ROCM_PASS_ASSET", None)

    def _evidence(self, monkeypatch, **extra):
        monkeypatch.setattr(
            stack_mod,
            "_PASS_EVIDENCE",
            {"bnb_rocm": f"url:{self.URL}", "bnb_rocm_asset": self.ASSET, **extra},
        )

    def test_a_republished_wheel_at_the_same_url_is_fetched_again(self, monkeypatch):
        """continuous-release_main replaces the bytes under a fixed asset path, so the URL
        pip recorded says where the wheel came from, not which one it is."""
        self._installed(monkeypatch, "1.33.7rc0", self.URL)
        self._evidence(monkeypatch)
        monkeypatch.setattr(stack_mod, "_bnb_asset_identity", lambda _url: '"9c1d"|43100000')
        assert stack_mod._bnb_rocm_install_is_current(self.URL) is False

    def test_an_unreachable_release_page_keeps_the_recorded_build(self, monkeypatch):
        """Offline, the reinstall could not fetch anything either; the build on disk is one
        the last run recorded landing deliberately."""
        self._installed(monkeypatch, "1.33.7rc0", self.URL)
        self._evidence(monkeypatch)
        monkeypatch.setattr(stack_mod, "_bnb_asset_identity", lambda _url: None)
        assert stack_mod._bnb_rocm_install_is_current(self.URL) is True
        assert stack_mod._BNB_ROCM_PASS_ASSET == self.ASSET

    def test_a_run_that_recorded_no_asset_identity_reinstalls_once(self, monkeypatch):
        self._installed(monkeypatch, "1.33.7rc0", self.URL)
        monkeypatch.setattr(stack_mod, "_PASS_EVIDENCE", {"bnb_rocm": f"url:{self.URL}"})
        assert stack_mod._bnb_rocm_install_is_current(self.URL) is False

    def test_the_kept_asset_identity_is_what_the_pass_records(self, monkeypatch):
        self._installed(monkeypatch, "1.33.7rc0", self.URL)
        self._evidence(monkeypatch)
        assert stack_mod._bnb_rocm_install_is_current(self.URL) is True
        assert stack_mod._BNB_ROCM_PASS_ASSET == self.ASSET
        # A second HEAD would only repeat the first.
        monkeypatch.setattr(
            stack_mod,
            "_bnb_asset_identity",
            lambda _url: (_ for _ in ()).throw(AssertionError("second HEAD")),
        )
        stack_mod._record_bnb_rocm_provenance()
        assert stack_mod._BNB_ROCM_PASS_ASSET == self.ASSET

    def _installed(self, monkeypatch, version, direct_url):
        monkeypatch.setattr(stack_mod, "_installed_distribution_version", lambda _n: version)
        monkeypatch.setattr(stack_mod, "_installed_direct_url", lambda _n: direct_url)

    def test_a_matching_direct_url_recorded_last_run_is_kept(self, monkeypatch):
        self._installed(monkeypatch, "1.33.7rc0", self.URL)
        self._evidence(monkeypatch)
        assert stack_mod._bnb_rocm_install_is_current(self.URL) is True

    def test_a_forced_pass_reinstalls(self, monkeypatch):
        self._installed(monkeypatch, "1.33.7rc0", self.URL)
        monkeypatch.setattr(stack_mod, "_PASS_EVIDENCE", None)
        assert stack_mod._bnb_rocm_install_is_current(self.URL) is False

    def test_the_escape_hatch_reinstalls(self, monkeypatch):
        self._installed(monkeypatch, "1.33.7rc0", self.URL)
        self._evidence(monkeypatch)
        monkeypatch.setenv(stack_mod._FULL_DEPS_ENV, "1")
        assert stack_mod._bnb_rocm_install_is_current(self.URL) is False

    def test_a_damaged_payload_reinstalls(self, monkeypatch):
        """Metadata survives a quarantined payload, and the reinstall this would skip is
        the only thing that puts the shared objects back."""
        self._installed(monkeypatch, "1.33.7rc0", self.URL)
        monkeypatch.setattr(
            stack_mod.install_manifest,
            "damaged_payload_files",
            lambda *a, **k: ["bitsandbytes/libbitsandbytes_rocm.so"],
        )
        self._evidence(monkeypatch)
        assert stack_mod._bnb_rocm_install_is_current(self.URL) is False

    def test_an_absent_bitsandbytes_reinstalls(self, monkeypatch):
        self._installed(monkeypatch, None, None)
        self._evidence(monkeypatch)
        assert stack_mod._bnb_rocm_install_is_current(self.URL) is False

    def test_a_wheel_url_that_moved_reinstalls(self, monkeypatch):
        """A new installer release naming a newer wheel must fetch it exactly once."""
        self._installed(monkeypatch, "1.33.7rc0", self.URL)
        self._evidence(monkeypatch)
        assert stack_mod._bnb_rocm_install_is_current(self.URL + ".new") is False

    def test_a_wheel_nobody_recorded_reinstalls(self, monkeypatch):
        """The PyPI wheel another step pulled in reads identically on disk to the one
        this path installs on purpose, so only the manifest can tell them apart."""
        self._installed(monkeypatch, "0.50.0", None)
        monkeypatch.setattr(stack_mod, "_PASS_EVIDENCE", {"bnb_rocm": None})
        assert stack_mod._bnb_rocm_install_is_current(self.URL) is False

    def test_a_recorded_pypi_fallback_is_kept(self, monkeypatch):
        """The fallback this path takes when the release page is unreachable. Recorded
        deliberately last run, so reinstalling it every update buys nothing."""
        self._installed(monkeypatch, "0.50.0", None)
        monkeypatch.setattr(stack_mod, "_PASS_EVIDENCE", {"bnb_rocm": "version:0.50.0"})
        assert stack_mod._bnb_rocm_install_is_current(self.URL) is True

    def test_a_filename_version_match_stands_in_for_a_missing_direct_url(self, monkeypatch):
        """1.33.7.preview in the filename is 1.33.7rc0 in the metadata."""
        assert stack_mod._bnb_wheel_version(self.URL) == "1.33.7.preview"
        assert stack_mod._versions_are_same_release("1.33.7rc0", "1.33.7.preview") is True
        assert stack_mod._bnb_provenance_matches_request("version:1.33.7rc0", self.URL) is True

    def test_the_second_call_of_one_pass_does_not_repeat_the_install(self, monkeypatch):
        """_ensure_rocm_torch runs twice per dependency pass. The first call settles it."""
        self._installed(monkeypatch, "1.33.7rc0", self.URL)
        monkeypatch.setattr(stack_mod, "_PASS_EVIDENCE", None)
        monkeypatch.setattr(stack_mod, "_BNB_ROCM_PASS_PROVENANCE", None)
        assert stack_mod._bnb_rocm_install_is_current(self.URL) is False
        stack_mod._record_bnb_rocm_provenance()
        assert stack_mod._BNB_ROCM_PASS_PROVENANCE == f"url:{self.URL}"
        assert stack_mod._bnb_rocm_install_is_current(self.URL) is True

    def test_a_clobber_between_the_two_calls_is_still_repaired(self, monkeypatch):
        """The with-deps steps run BETWEEN the two calls and can re-resolve bnb to a
        generic wheel, which is the whole reason the torch repair runs twice."""
        self._installed(monkeypatch, "1.33.7rc0", self.URL)
        monkeypatch.setattr(stack_mod, "_PASS_EVIDENCE", None)
        monkeypatch.setattr(stack_mod, "_BNB_ROCM_PASS_PROVENANCE", None)
        stack_mod._record_bnb_rocm_provenance()
        self._installed(monkeypatch, "0.50.0", None)
        assert stack_mod._bnb_rocm_install_is_current(self.URL) is False

    def test_a_kept_wheel_is_still_recorded_for_the_next_update(self, monkeypatch):
        """Without this the manifest this pass writes forgets what is installed and the
        next update fetches the wheel again."""
        import ast

        source = _STACK_PATH.read_text(encoding = "utf-8")
        calls = [
            node
            for node in ast.walk(ast.parse(source))
            if isinstance(node, ast.Call)
            and isinstance(node.func, ast.Name)
            and node.func.id == "_record_bnb_rocm_provenance"
        ]
        # Two per platform path: the install arm and the keep arm.
        assert len(calls) == 4
        assert '"bnb_rocm": _BNB_ROCM_PASS_PROVENANCE,' in source

    def test_a_current_wheel_makes_the_linux_path_install_nothing(self, monkeypatch):
        self._installed(monkeypatch, "1.33.7rc0", self.URL)
        monkeypatch.setattr(stack_mod, "_bnb_rocm_prerelease_url", lambda: self.URL)
        self._evidence(monkeypatch)
        pip, pip_try = run_ensure_rocm_torch(
            probe = json.dumps({"version": "2.10.0+rocm7.1", "hip": "7.1", "cuda": None}),
            _has_rocm_gpu = True,
            _detect_rocm_version = (7, 1),
        )
        assert pip.call_count == 0
        assert pip_try.call_count == 0

    def test_a_current_wheel_makes_the_windows_path_install_nothing(self, monkeypatch):
        self._installed(monkeypatch, "1.33.7rc0", self.URL)
        monkeypatch.setitem(stack_mod._BNB_ROCM_PRERELEASE_URLS, "win_amd64", self.URL)
        self._evidence(monkeypatch)
        monkeypatch.setattr(stack_mod, "_persist_bnb_rocm_version", lambda _v: True)
        monkeypatch.setattr(stack_mod, "_detect_bnb_rocm_dll_ver", lambda: "72")
        with patch.object(stack_mod, "pip_install_try", return_value = True) as mock_pip:
            assert stack_mod._install_bnb_windows_rocm() is True
        assert mock_pip.call_count == 0


class _Py39(tuple):
    major, minor, micro = 3, 9, 18


def test_python_39_keeps_rdna4_off_the_amd_arch_index(monkeypatch):
    monkeypatch.setattr(sys, "version_info", _Py39((3, 9, 18, "final", 0)))
    spec = importlib.util.spec_from_file_location("studio_install_python_stack_py39", _STACK_PATH)
    mod = importlib.util.module_from_spec(spec)
    spec.loader.exec_module(mod)
    assert not {"gfx1200", "gfx1201"} & mod._AMD_ARCH_INDEX_FLOOR_GFX
    assert {"gfx1150", "gfx1151", "gfx1152"} <= mod._AMD_ARCH_INDEX_FLOOR_GFX
    assert {"gfx1200", "gfx1201"} <= stack_mod._AMD_ARCH_INDEX_FLOOR_GFX