Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

24.9 KB
Raw Blame
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0

"""On ROCm, auto GPU selection must not pick a device the llama.cpp prebuilt has no kernels for."""

import json
import subprocess
import sys
import types

import pytest

# Import eagerly so the fake torch is not in place when the real module graph loads.
from core.inference.llama_cpp import _IGPU_HOST_RESERVE_MIB, LlamaCppBackend
from core.training.worker import _rocm_classify_unified_memory  # noqa: F401


def _binary_with_marker(tmp_path, payload):
    """Lay out <root>/UNSLOTH_PREBUILT_INFO.json with a binary path below it,
    matching the managed install layout the marker walk-up covers."""
    (tmp_path / "UNSLOTH_PREBUILT_INFO.json").write_text(json.dumps(payload), encoding = "utf-8")
    return str(tmp_path / "build" / "bin" / "llama-server")


class TestInstalledLlamaGfxArchs:
    def test_reads_mapped_targets(self, tmp_path):
        binary = _binary_with_marker(
            tmp_path, {"mapped_targets": ["gfx1100", "GFX1101", "gfx1102:xnack-"]}
        )
        archs = LlamaCppBackend._installed_llama_gfx_archs(binary)
        assert archs == frozenset({"gfx1100", "gfx1101", "gfx1102"})

    def test_no_marker_is_unknown(self, tmp_path):
        assert LlamaCppBackend._installed_llama_gfx_archs(str(tmp_path / "llama-server")) is None

    def test_no_binary_is_unknown(self, monkeypatch):
        monkeypatch.setattr(
            LlamaCppBackend, "_find_llama_server_binary", staticmethod(lambda: None)
        )
        assert LlamaCppBackend._installed_llama_gfx_archs() is None

    def test_pre_7669_install_is_unknown(self, tmp_path):
        # Older installs lack mapped_targets: fail open.
        binary = _binary_with_marker(tmp_path, {"asset": "app-windows-x64-rocm-gfx110X.zip"})
        assert LlamaCppBackend._installed_llama_gfx_archs(binary) is None

    def test_empty_targets_is_unknown(self, tmp_path):
        # Non-ROCm bundles record []: fail open rather than drop every GPU.
        binary = _binary_with_marker(tmp_path, {"mapped_targets": []})
        assert LlamaCppBackend._installed_llama_gfx_archs(binary) is None

    def test_unreadable_marker_is_unknown(self, tmp_path, monkeypatch):
        # A raising marker read must answer unknown, never propagate: it runs inside the GPU probe.
        import utils.llama_cpp_freshness as freshness

        def _boom(_binary):
            raise OSError("marker read failed")

        monkeypatch.setattr(freshness, "read_install_marker", _boom)
        assert LlamaCppBackend._installed_llama_gfx_archs(str(tmp_path / "llama-server")) is None


# The marker is untrusted input (upgrades, partial writes, hand edits); callables lay out non-JSON shapes.
_MARKER_CORPUS = [
    ("absent_file", None),
    ("empty_file", ""),
    ("invalid_json", "{not json"),
    ("truncated_json", '{"mapped_targets": ["gfx1030"'),
    ("marker_is_a_list", '["gfx1030"]'),
    ("marker_is_null", "null"),
    ("marker_is_a_string", '"hello"'),
    ("marker_is_empty_dict", "{}"),
    ("targets_null", '{"mapped_targets": null}'),
    ("targets_empty_string", '{"mapped_targets": ""}'),
    ("targets_bare_string", '{"mapped_targets": "gfx1030"}'),
    ("targets_empty_list", '{"mapped_targets": []}'),
    ("targets_list_of_null", '{"mapped_targets": [null]}'),
    ("targets_list_of_int", '{"mapped_targets": [123]}'),
    ("targets_blank_strings", '{"mapped_targets": ["", "  "]}'),
    ("targets_dict", '{"mapped_targets": {"a": "gfx1030"}}'),
    ("targets_huge_list", json.dumps({"mapped_targets": [f"gfx{i:04d}" for i in range(5000)]})),
    ("cuda_bundle", '{"asset": "app-linux-x64-cuda12.tar.gz"}'),
    ("vulkan_bundle", '{"asset": "app-linux-x64-vulkan.tar.gz", "mapped_targets": []}'),
    ("non_utf8_bytes", lambda p: p.write_bytes(b"\xff\xfe\x00garbage")),
    ("directory_in_place_of_file", lambda p: p.mkdir()),
    (
        "permission_denied",
        lambda p: (p.write_text(json.dumps({"mapped_targets": ["gfx1030"]})), p.chmod(0o000)),
    ),
    # mapped_targets comes from the release manifest and may hold tokens no device reports.
    ("future_generic_target", '{"mapped_targets": ["gfx11-generic"]}'),
    ("future_generic_target_dashed", '{"mapped_targets": ["gfx10-3-generic"]}'),
    ("future_family_label", '{"mapped_targets": ["gfx110X"]}'),
    ("future_mixed_concrete_and_generic", '{"mapped_targets": ["gfx1100", "gfx11-generic"]}'),
]


class TestInstalledLlamaGfxArchsCorpus:
    """The gate must degrade to "unknown" on every malformed marker. Raising is fatal
    (this runs inside the GPU probe); a non-None set no device can match is quieter
    and just as fatal, dropping every GPU. None or concrete archs, nothing between."""

    @pytest.mark.parametrize("label,payload", _MARKER_CORPUS, ids = [c[0] for c in _MARKER_CORPUS])
    def test_never_raises_and_never_fails_closed(self, label, payload, tmp_path):
        import utils.llama_cpp_freshness as freshness

        root = tmp_path / label
        root.mkdir()
        marker = root / "UNSLOTH_PREBUILT_INFO.json"
        if callable(payload):
            payload(marker)
        elif payload is not None:
            marker.write_text(payload, encoding = "utf-8")
        # The walk-up memoizes per binary path.
        freshness._marker_cache.clear()

        archs = LlamaCppBackend._installed_llama_gfx_archs(
            str(root / "build" / "bin" / "llama-server")
        )

        if archs is None:
            return
        assert isinstance(archs, frozenset)
        # Every token must be device-reportable or the set gates every GPU off.
        assert archs, f"{label} returned an empty set, which would drop every GPU"
        assert all(
            LlamaCppBackend._CONCRETE_GFX_ARCH.match(a) for a in archs
        ), f"{label} returned unmatchable tokens {sorted(archs)}"

    def test_empty_set_is_converted_to_none(self, tmp_path):
        # Pinned because a frozenset() return would type-check but fail closed.
        binary = _binary_with_marker(tmp_path, {"mapped_targets": ["", "   ", ":"]})
        assert LlamaCppBackend._installed_llama_gfx_archs(binary) is None

    def test_non_list_targets_are_rejected_before_iteration(self, tmp_path, monkeypatch):
        # A tuple is not JSON-reachable, so it alone isolates the isinstance(list) guard.
        import utils.llama_cpp_freshness as freshness
        monkeypatch.setattr(
            freshness, "read_install_marker", lambda _b: {"mapped_targets": ("gfx1030",)}
        )
        assert LlamaCppBackend._installed_llama_gfx_archs(str(tmp_path / "llama-server")) is None

    def test_concrete_targets_still_gate(self, tmp_path):
        # The normal case stays gated, so 'never fails closed' is not met by gutting the feature.
        binary = _binary_with_marker(tmp_path, {"mapped_targets": ["gfx1030", "gfx90a"]})
        assert LlamaCppBackend._installed_llama_gfx_archs(binary) == frozenset(
            {"gfx1030", "gfx90a"}
        )


class TestForwardsCompatibleArchTokens:
    """Generic targets must fail open: devices report concrete archs, so exact matching drops every GPU."""

    @pytest.mark.parametrize(
        "token",
        [
            "gfx11-generic",
            "gfx10-3-generic",
            "gfx9-4-generic",
            "gfx110X",
            "gfx120X",
            "gfx103X",
            "generic",
            "all",
            "native",
        ],
    )
    def test_non_concrete_token_disables_the_gate(self, token, tmp_path):
        binary = _binary_with_marker(tmp_path, {"mapped_targets": [token]})
        assert LlamaCppBackend._installed_llama_gfx_archs(binary) is None

    def test_one_bad_token_disables_the_whole_gate(self, tmp_path):
        # All-or-nothing: keeping only gfx1100 would gate off a gfx1101 the generic object covers.
        binary = _binary_with_marker(tmp_path, {"mapped_targets": ["gfx1100", "gfx11-generic"]})
        assert LlamaCppBackend._installed_llama_gfx_archs(binary) is None

    @pytest.mark.parametrize(
        "token", ["gfx803", "gfx900", "gfx906", "gfx908", "gfx90a", "gfx90c", "gfx942", "gfx950"]
    )
    def test_real_concrete_archs_are_accepted(self, token, tmp_path):
        # CDNA parts end in a hex letter that a digits-only pattern would reject.
        binary = _binary_with_marker(tmp_path, {"mapped_targets": [token]})
        assert LlamaCppBackend._installed_llama_gfx_archs(binary) == frozenset({token})

    def test_every_published_mapped_target_is_accepted(self, tmp_path):
        # Verbatim from llama-prebuilt-manifest.json (b10360-mix-87da1a2): every ROCm bundle's mapped_targets.
        published = [
            "gfx908", "gfx90a",
            "gfx1030", "gfx1031", "gfx1032", "gfx1034",
            "gfx1100", "gfx1101", "gfx1102", "gfx1103",
            "gfx1150", "gfx1151",
            "gfx1200", "gfx1201",
        ]  # fmt: skip
        binary = _binary_with_marker(tmp_path, {"mapped_targets": published})
        assert LlamaCppBackend._installed_llama_gfx_archs(binary) == frozenset(published)

    def test_generic_target_keeps_every_gpu(self, tmp_path, monkeypatch, rocm_probe_env):
        _binary_with_marker(tmp_path, {"mapped_targets": ["gfx11-generic"]})
        monkeypatch.setitem(
            sys.modules, "torch", _fake_torch(["gfx1100", "gfx1101"], [12000, 13000])
        )
        assert LlamaCppBackend._get_gpu_free_memory(for_llama_server = True) == [
            (0, 12000),
            (1, 13000),
        ]


class TestKernelImageInvalidMarker:
    def test_detects_rocm_arch_mismatch(self):
        tail = (
            "load_model: loading model 'x.gguf'\n"
            "E ROCm error: device kernel image is invalid\n"
            "E   current device: 0, in function ggml_cuda_kernel_launch"
        )
        assert LlamaCppBackend._kernel_image_invalid(tail)

    def test_detects_the_no_binary_for_gpu_spelling(self):
        # hipErrorNoBinaryForGpu is the same arch mismatch and must reach the retry too.
        tail = (
            "ggml-cuda.cu:76: ROCm error\n"
            "  no kernel image is available for execution on the device\n"
            "  current device: 1"
        )
        assert LlamaCppBackend._kernel_image_invalid(tail)

    def test_ignores_other_crashes(self):
        assert not LlamaCppBackend._kernel_image_invalid("out of memory")
        assert not LlamaCppBackend._kernel_image_invalid("")
        assert not LlamaCppBackend._kernel_image_invalid(None)


class _FakeProps:
    """hipDeviceProp_t stand-in exposing the canonical arch attribute."""

    _ATTR = "gcnArchName"

    def __init__(self, arch):
        setattr(self, self._ATTR, arch)


def _fake_torch(
    archs,
    free_mib,
    *,
    props_cls = _FakeProps,
    hip = "7.1.0",
    version_str = "",
):
    torch = types.ModuleType("torch")
    torch.version = types.SimpleNamespace() if hip is None else types.SimpleNamespace(hip = hip)
    torch.__version__ = version_str
    torch.cuda = types.SimpleNamespace(
        is_available = lambda: True,
        device_count = lambda: len(archs),
        mem_get_info = lambda o: (free_mib[o] * 1024 * 1024, 32 * 1024**3),
        get_device_properties = lambda o: props_cls(archs[o]),
    )
    return torch


@pytest.fixture
def rocm_probe_env(tmp_path, monkeypatch):
    """Forces _get_gpu_memory onto the torch/ROCm fallback; the fake binary lives under tmp_path."""

    def _no_nvidia_smi(*args, **kwargs):
        raise FileNotFoundError("nvidia-smi")

    monkeypatch.setattr(subprocess, "run", _no_nvidia_smi)
    fake_binary = str(tmp_path / "build" / "bin" / "llama-server")
    monkeypatch.setattr(
        LlamaCppBackend, "_find_llama_server_binary", staticmethod(lambda: fake_binary)
    )
    for var in ("HIP_VISIBLE_DEVICES", "ROCR_VISIBLE_DEVICES", "CUDA_VISIBLE_DEVICES"):
        monkeypatch.delenv(var, raising = False)


class TestGpuArchGate:
    def test_igpu_dropped_when_arch_unsupported(self, tmp_path, monkeypatch, rocm_probe_env):
        # The iGPU's shared-RAM free memory outranks the dGPU's VRAM; only the dGPU may survive.
        _binary_with_marker(
            tmp_path, {"mapped_targets": ["gfx1100", "gfx1101", "gfx1102", "gfx1103"]}
        )
        monkeypatch.setitem(
            sys.modules, "torch", _fake_torch(["gfx1101", "gfx1036"], [12049, 12176])
        )
        assert LlamaCppBackend._get_gpu_free_memory(for_llama_server = True) == [(0, 12049)]

    def test_unknown_coverage_keeps_all_devices(self, tmp_path, monkeypatch, rocm_probe_env):
        monkeypatch.setitem(
            sys.modules, "torch", _fake_torch(["gfx1101", "gfx1036"], [12049, 12176])
        )
        assert LlamaCppBackend._get_gpu_free_memory(for_llama_server = True) == [
            (0, 12049),
            (1, 12176),
        ]

    def test_unknown_device_arch_fails_open(self, tmp_path, monkeypatch, rocm_probe_env):
        _binary_with_marker(tmp_path, {"mapped_targets": ["gfx1101"]})
        monkeypatch.setitem(sys.modules, "torch", _fake_torch(["gfx1101", ""], [12049, 12176]))
        assert LlamaCppBackend._get_gpu_free_memory(for_llama_server = True) == [
            (0, 12049),
            (1, 12176),
        ]

    @pytest.mark.parametrize("attr", ["gcn_arch_name", "arch_name", "gfx_arch_name"])
    def test_alternate_arch_attribute_spellings(self, attr, tmp_path, monkeypatch, rocm_probe_env):
        # AMD SDK / Radeon wheels may omit gcnArchName; reading only it would fail the gate open.
        props_cls = type("_AltProps", (_FakeProps,), {"_ATTR": attr})
        _binary_with_marker(tmp_path, {"mapped_targets": ["gfx1101"]})
        monkeypatch.setitem(
            sys.modules,
            "torch",
            _fake_torch(["gfx1101", "gfx1036"], [12049, 12176], props_cls = props_cls),
        )
        assert LlamaCppBackend._rocm_arch_by_physical_id() == {0: "gfx1101", 1: "gfx1036"}
        assert LlamaCppBackend._get_gpu_free_memory(for_llama_server = True) == [(0, 12049)]

    def test_amd_sdk_wheel_without_version_hip_is_gated(
        self, tmp_path, monkeypatch, rocm_probe_env
    ):
        # AMD SDK / Radeon wheels may leave torch.version.hip unset; use _torch_is_rocm.
        _binary_with_marker(tmp_path, {"mapped_targets": ["gfx1101"]})
        monkeypatch.setitem(
            sys.modules,
            "torch",
            _fake_torch(
                ["gfx1101", "gfx1036"], [12049, 12176], hip = None, version_str = "2.6.0+rocm6.4"
            ),
        )
        assert LlamaCppBackend._get_gpu_free_memory(for_llama_server = True) == [(0, 12049)]

    def test_non_rocm_wheel_is_never_gated(self, tmp_path, monkeypatch, rocm_probe_env):
        _binary_with_marker(tmp_path, {"mapped_targets": ["gfx1101"]})
        monkeypatch.setitem(
            sys.modules,
            "torch",
            _fake_torch(
                ["gfx1101", "gfx1036"], [12049, 12176], hip = None, version_str = "2.6.0+cu124"
            ),
        )
        assert LlamaCppBackend._get_gpu_free_memory(for_llama_server = True) == [
            (0, 12049),
            (1, 12176),
        ]


class TestTorchCallersStayUnfiltered:
    """The gate answers what llama-server can run; torch callers like RAG embeddings stay unfiltered."""

    def test_probe_is_unfiltered_by_default(self, tmp_path, monkeypatch, rocm_probe_env):
        _binary_with_marker(tmp_path, {"mapped_targets": ["gfx1101"]})
        monkeypatch.setitem(
            sys.modules, "torch", _fake_torch(["gfx1101", "gfx1036"], [12049, 12176])
        )
        assert LlamaCppBackend._get_gpu_free_memory() == [(0, 12049), (1, 12176)]
        assert LlamaCppBackend._get_gpu_memory() == [(0, 12049, 32768), (1, 12176, 32768)]

    def test_rag_auto_still_picks_sentence_transformers(
        self, tmp_path, monkeypatch, rocm_probe_env
    ):
        # Unsupported by the prebuilt only; torch can still use the GPU.
        from core.rag import embeddings

        _binary_with_marker(tmp_path, {"mapped_targets": ["gfx1101"]})
        monkeypatch.setitem(sys.modules, "torch", _fake_torch(["gfx1036"], [12176]))
        assert embeddings._resolve_auto() == "sentence-transformers"

    def test_embed_llama_server_probe_opts_in(self, tmp_path, monkeypatch, rocm_probe_env):
        # The GGUF embedding backend is llama-server, so the unsupported device is no GPU there.
        import utils.hardware as uh
        from core.rag.embed_llama_server import LlamaServerBackend

        monkeypatch.setattr(uh, "is_apple_silicon", lambda: False)
        _binary_with_marker(tmp_path, {"mapped_targets": ["gfx1101"]})
        monkeypatch.setitem(sys.modules, "torch", _fake_torch(["gfx1036"], [12176]))
        assert LlamaServerBackend._gpu_available() is False
        monkeypatch.setitem(sys.modules, "torch", _fake_torch(["gfx1101"], [12176]))
        assert LlamaServerBackend._gpu_available() is True


class TestArchGateCoexistsWithUnifiedMemory:
    """APU reserve and the unsupported-device drop share one torch loop; neither may displace the other."""

    def test_apu_reserve_and_arch_gate_together(self, tmp_path, monkeypatch, rocm_probe_env):
        _binary_with_marker(tmp_path, {"mapped_targets": ["gfx1101", "gfx1151"]})
        monkeypatch.setitem(
            sys.modules,
            "torch",
            _fake_torch(["gfx1151", "gfx1101", "gfx1036"], [20000, 12049, 12176]),
        )
        monkeypatch.setattr(
            LlamaCppBackend, "_available_system_memory_mib", staticmethod(lambda: 30000)
        )
        assert LlamaCppBackend._get_gpu_memory(for_llama_server = True) == [
            # Shared pool: host reserve held back, total reported 0.
            (0, 20000 - _IGPU_HOST_RESERVE_MIB, 0),
            (1, 12049, 32768),
        ]

    def test_apu_reserve_survives_unknown_coverage(self, tmp_path, monkeypatch, rocm_probe_env):
        monkeypatch.setitem(sys.modules, "torch", _fake_torch(["gfx1151"], [20000]))
        monkeypatch.setattr(
            LlamaCppBackend, "_available_system_memory_mib", staticmethod(lambda: 30000)
        )
        assert LlamaCppBackend._get_gpu_memory(for_llama_server = True) == [
            (0, 20000 - _IGPU_HOST_RESERVE_MIB, 0)
        ]


# The tail of a ROCm llama-server that split a manual ratio onto a card the build lacks.
_UNPINNED_CRASH = (
    "llama_kv_cache:      ROCm2 KV buffer size =  4248.00 MiB\n"
    "ROCm error: invalid kernel file\n"
    "  current device: 2, in function ggml_cuda_kernel_launch at common.cuh:1740\n"
)


class TestUnpinnedArchCrashRetry:
    """A manual split, or a gate that could not read the build's coverage, launches with
    no mask, so the child's own "current device" is the only name for the bad card."""

    @pytest.fixture(autouse = True)
    def _rocm_host(self, monkeypatch):
        monkeypatch.setattr(LlamaCppBackend, "_host_torch_is_rocm", staticmethod(lambda: True))
        monkeypatch.setattr(
            LlamaCppBackend, "_resolve_visible_physical_ids", staticmethod(lambda: None)
        )
        monkeypatch.setattr(
            LlamaCppBackend, "_rocm_unified_memory_gpu_ids", staticmethod(lambda: set())
        )
        monkeypatch.setattr(
            LlamaCppBackend, "_visibility_mask_is_unmappable", staticmethod(lambda: False)
        )
        monkeypatch.setattr(
            LlamaCppBackend, "_rocm_visibility_masks_are_stacked", staticmethod(lambda: False)
        )
        monkeypatch.delenv("GPU_DEVICE_ORDINAL", raising = False)

    def test_drops_the_crashed_card(self):
        assert LlamaCppBackend._unpinned_arch_crash_retry(_UNPINNED_CRASH, [0, 1, 2]) == (
            [2],
            [0, 1],
        )

    def test_also_drops_a_shared_memory_igpu(self, monkeypatch):
        # 7900 XTX + Ryzen iGPU + R9700 on a gfx110X build: the iGPU is uncovered too.
        monkeypatch.setattr(
            LlamaCppBackend, "_rocm_unified_memory_gpu_ids", staticmethod(lambda: {1})
        )
        assert LlamaCppBackend._unpinned_arch_crash_retry(_UNPINNED_CRASH, [0, 1, 2]) == (
            [2],
            [0],
        )

    def test_keeps_the_igpu_when_it_is_all_that_is_left(self, monkeypatch):
        monkeypatch.setattr(
            LlamaCppBackend, "_rocm_unified_memory_gpu_ids", staticmethod(lambda: {0})
        )
        assert LlamaCppBackend._unpinned_arch_crash_retry(_UNPINNED_CRASH, [0, 2]) == ([2], [0])

    def test_maps_the_child_ordinal_through_an_inherited_mask(self, monkeypatch):
        monkeypatch.setattr(
            LlamaCppBackend, "_resolve_visible_physical_ids", staticmethod(lambda: [1, 3, 5])
        )
        assert LlamaCppBackend._unpinned_arch_crash_retry(_UNPINNED_CRASH, [1, 3, 5]) == (
            [5],
            [1, 3],
        )

    def test_the_childs_own_mask_wins_over_the_parents(self):
        # A manual split pins the child's order (or the gate pins survivors) without gpu_indices.
        assert LlamaCppBackend._unpinned_arch_crash_retry(
            _UNPINNED_CRASH, [0, 1, 2], child_ids = (0, 2, 1)
        ) == ([1], [0, 2])

    def test_ordinal_outside_the_childs_mask_is_not_guessed(self):
        assert LlamaCppBackend._unpinned_arch_crash_retry(
            _UNPINNED_CRASH, [0, 1, 2], child_ids = (0, 1)
        ) == ([], [])

    def test_an_unmappable_mask_is_left_alone(self, monkeypatch):
        # A UUID/MIG mask resolves to no ids; reading the ordinal as physical could expose a hidden card.
        monkeypatch.setattr(
            LlamaCppBackend, "_visibility_mask_is_unmappable", staticmethod(lambda: True)
        )
        assert LlamaCppBackend._unpinned_arch_crash_retry(_UNPINNED_CRASH, [0, 1, 2]) == ([], [])

    def test_gpu_device_ordinal_is_left_alone(self, monkeypatch):
        monkeypatch.setenv("GPU_DEVICE_ORDINAL", "2,0")
        assert LlamaCppBackend._unpinned_arch_crash_retry(_UNPINNED_CRASH, [0, 1, 2]) == ([], [])
        assert LlamaCppBackend._unpinned_arch_crash_retry(
            _UNPINNED_CRASH, [0, 1, 2], child_ids = (0, 2, 1)
        ) == ([], [])

    def test_stacked_rocr_and_hip_masks_are_left_alone(self, monkeypatch):
        # HIP ordinals index the ROCr-filtered set, so the resolved ids are not physical.
        monkeypatch.setattr(
            LlamaCppBackend, "_rocm_visibility_masks_are_stacked", staticmethod(lambda: True)
        )
        assert LlamaCppBackend._unpinned_arch_crash_retry(_UNPINNED_CRASH, [0, 1, 2]) == ([], [])
        assert LlamaCppBackend._unpinned_arch_crash_retry(
            _UNPINNED_CRASH, [0, 1, 2], child_ids = (0, 2, 1)
        ) == ([1], [0, 2])

    def test_the_childs_own_mask_still_maps_under_an_unmappable_parent(self, monkeypatch):
        monkeypatch.setattr(
            LlamaCppBackend, "_visibility_mask_is_unmappable", staticmethod(lambda: True)
        )
        assert LlamaCppBackend._unpinned_arch_crash_retry(
            _UNPINNED_CRASH, [0, 1, 2], child_ids = (0, 2, 1)
        ) == ([1], [0, 2])

    def test_ordinal_outside_the_mask_is_not_guessed(self, monkeypatch):
        monkeypatch.setattr(
            LlamaCppBackend, "_resolve_visible_physical_ids", staticmethod(lambda: [0, 1])
        )
        assert LlamaCppBackend._unpinned_arch_crash_retry(_UNPINNED_CRASH, [0, 1]) == ([], [])

    def test_no_device_line_means_no_retry(self):
        out = "ROCm error: invalid kernel file\n"
        assert LlamaCppBackend._unpinned_arch_crash_retry(out, [0, 1]) == ([], [])

    def test_single_gpu_has_nowhere_to_go(self):
        assert LlamaCppBackend._unpinned_arch_crash_retry(_UNPINNED_CRASH, [2]) == ([2], [])

    def test_cuda_host_is_left_alone(self, monkeypatch):
        # An unmasked CUDA child orders devices by speed, so its ordinal is not our id.
        monkeypatch.setattr(LlamaCppBackend, "_host_torch_is_rocm", staticmethod(lambda: False))
        assert LlamaCppBackend._unpinned_arch_crash_retry(_UNPINNED_CRASH, [0, 1, 2]) == ([], [])


class TestArchCrashRetrySet:
    """Reactive recovery for markerless or custom-linked builds the proactive gate cannot see."""

    def test_prefers_never_selected_devices(self):
        assert LlamaCppBackend._arch_crash_retry_gpu_ids([0], [0, 1, 2]) == [1, 2]

    def test_narrows_to_discrete_when_every_gpu_was_selected(self, monkeypatch):
        monkeypatch.setattr(
            LlamaCppBackend, "_rocm_unified_memory_gpu_ids", staticmethod(lambda: {0})
        )
        assert LlamaCppBackend._arch_crash_retry_gpu_ids([0, 1], [0, 1]) == [1]

    def test_no_retry_when_nothing_would_change(self, monkeypatch):
        monkeypatch.setattr(
            LlamaCppBackend, "_rocm_unified_memory_gpu_ids", staticmethod(lambda: set())
        )
        assert LlamaCppBackend._arch_crash_retry_gpu_ids([0, 1], [0, 1]) == []

    def test_no_retry_when_narrowing_empties_the_selection(self, monkeypatch):
        monkeypatch.setattr(
            LlamaCppBackend, "_rocm_unified_memory_gpu_ids", staticmethod(lambda: {0, 1})
        )
        assert LlamaCppBackend._arch_crash_retry_gpu_ids([0, 1], [0, 1]) == []

    def test_empty_selection_is_a_no_op(self):
        assert LlamaCppBackend._arch_crash_retry_gpu_ids([], [0, 1]) == []

    def test_classifier_failure_is_not_fatal(self, monkeypatch):
        def _boom():
            raise RuntimeError("hip enumeration failed")

        monkeypatch.setattr(LlamaCppBackend, "_rocm_unified_memory_gpu_ids", staticmethod(_boom))
        assert LlamaCppBackend._arch_crash_retry_gpu_ids([0, 1], [0, 1]) == []