Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: do not reapply ROCR_VISIBLE_DEVICES when picking the AMD card in setup.sh and the llama.cpp prebuilt probe (#11965)

Daniel Han5b28886e0ad8
diff --git a/install.sh b/install.sh
index 189a45d566c..cc7b970eb52 100755
--- a/install.sh
+++ b/install.sh
@@ -6671,10 +6671,10 @@ case "$_torch_index_leaf" in
# indexing by ROCR again shadows CUDA, its HIP alias: ROCR=2,1 + CUDA=1 is survivor 2.
_vis_masks="HIP_VISIBLE_DEVICES CUDA_VISIBLE_DEVICES"
if [ "$_gfx_probe" != rocminfo ] && [ -n "${ROCR_VISIBLE_DEVICES:-}" ] && [ "$ROCR_VISIBLE_DEVICES" != "-1" ]; then
- # amd-smi is not ROCr-filtered: ROCr decides which devices exist, then HIP indexes the survivors (_rocr_visible_subset). Ordinals in mask order; none in range keeps the whole list, as _pick_visible_index does.
+ # amd-smi is not ROCr-filtered: keep ROCr's survivors (prefix up to the first out-of-range or repeated ordinal, as _rocr_visible_subset; none keeps all), then HIP indexes them.
_rocr_kept=$(printf '%s\n' "$_gfx_all" | awk -v m="$ROCR_VISIBLE_DEVICES" '
NF { v[n++] = $0 }
- END { k = split(m, t, ","); for (i = 1; i <= k; i++) { gsub(/[[:space:]]/, "", t[i]); if (t[i] ~ /^[0-9]+$/ && t[i] + 0 < n) print v[t[i] + 0] } }')
+ END { k = split(m, t, ","); for (i = 1; i <= k; i++) { gsub(/[[:space:]]/, "", t[i]); if (t[i] !~ /^[0-9]+$/) continue; x = t[i] + 0; if (x >= n || (x in s)) break; s[x] = 1; print v[x] } }')
[ -n "$_rocr_kept" ] && _gfx_all="$_rocr_kept"
# A UUID token names a device but no position here, so with unlike adapters no survivor is known to be the one selected: decline, as _rocr_visible_subset does.
_rocr_unresolved=$(printf '%s' "$ROCR_VISIBLE_DEVICES" | tr -d '0-9, \t')
diff --git a/studio/install_llama_prebuilt.py b/studio/install_llama_prebuilt.py
index 648c6a1ceb9..e80dae5a48c 100644
--- a/studio/install_llama_prebuilt.py
+++ b/studio/install_llama_prebuilt.py
@@ -2765,7 +2765,7 @@ def _list_rocm_gfx_targets(out: str) -> list[str]:
return _tokens
-def _pick_rocm_gfx_target(out: str) -> str | None:
+def _pick_rocm_gfx_target(out: str, rocr_filtered: bool = False) -> str | None:
"""Choose the gfx target rocminfo / hipinfo report for the active GPU.
A bare first-match picked the wrong device on mixed APU + dGPU hosts (Strix Halo gfx1151
@@ -2784,12 +2784,20 @@ def _pick_rocm_gfx_target(out: str) -> str | None:
return None
_vis_raw = None
- # AMD's HIP runtime honours all three env vars with identical semantics.
- for _env in ("HIP_VISIBLE_DEVICES", "ROCR_VISIBLE_DEVICES", "CUDA_VISIBLE_DEVICES"):
+ # rocminfo output is already ROCr-filtered and renumbered: only HIP-layer masks index it.
+ _masks = (
+ ("HIP_VISIBLE_DEVICES", "CUDA_VISIBLE_DEVICES")
+ if rocr_filtered
+ else ("HIP_VISIBLE_DEVICES", "ROCR_VISIBLE_DEVICES", "CUDA_VISIBLE_DEVICES")
+ )
+ for _env in _masks:
_val = os.environ.get(_env)
if _val is not None:
_vis_raw = _val
break
+ # Still an explicit selection: survivor 0, so the discrete repick below must not override it.
+ if _vis_raw is None and rocr_filtered and os.environ.get("ROCR_VISIBLE_DEVICES") is not None:
+ _vis_raw = "0" if os.environ["ROCR_VISIBLE_DEVICES"].strip() not in ("", "-1") else ""
if _vis_raw is not None:
_vis = _vis_raw.strip()
# Empty or "-1" means "no AMD GPU visible" (matches the rest of Unsloth).
@@ -3104,7 +3112,9 @@ def detect_host(*, probe_rocm_with_nvidia: bool = False) -> HostInfo:
if _check(_result.stdout):
has_rocm = True
rocm_gfx_targets = _list_rocm_gfx_targets(_result.stdout)
- rocm_gfx_target = _pick_rocm_gfx_target(_result.stdout)
+ rocm_gfx_target = _pick_rocm_gfx_target(
+ _result.stdout, rocr_filtered = _cmd[0] == "rocminfo"
+ )
break
elif is_windows and (probe_rocm_with_nvidia or not has_usable_nvidia):
# Windows: prefer active probes that validate GPU presence.
diff --git a/studio/setup.sh b/studio/setup.sh
index 45de6e9126b..da9b4268606 100755
--- a/studio/setup.sh
+++ b/studio/setup.sh
@@ -3736,6 +3736,8 @@ _setup_nvidia_physical=false
_setup_gfx_all=""
_setup_gfx=""
_setup_hip_map_missing=0
+_setup_amd_probe=""
+_setup_rocr_uuid_declined=0
_setup_mkt=""
_setup_amd_records=""
@@ -3967,6 +3969,7 @@ if [ "$_setup_nvidia_usable" != true ]; then
fi
if [ -n "$_setup_gfx_all" ]; then
_setup_amd_detected=true
+ _setup_amd_probe=rocminfo
elif command -v amd-smi >/dev/null 2>&1 && \
_setup_run_smi amd-smi list 2>/dev/null | awk '/^GPU[[:space:]]*[:\[][[:space:]]*[0-9]/{ found=1 } END{ exit !found }'; then
_setup_amd_detected=true
@@ -4024,7 +4027,31 @@ if [ "$_setup_nvidia_usable" = true ]; then
# behind on the common path where there is no driver string to print.
if [ -n "$_setup_nv_driver" ]; then substep "Driver: $_setup_nv_driver"; fi
elif [ "$_setup_amd_detected" = true ]; then
- _setup_vis="${HIP_VISIBLE_DEVICES:-${ROCR_VISIBLE_DEVICES:-}}"
+ # As install.sh: ROCr picks survivors (rocminfo is already filtered, amd-smi is not), then the
+ # first SET HIP-layer mask (HIP, then CUDA; empty still shadows) indexes them.
+ if [ "$_setup_amd_probe" != rocminfo ] && [ -n "${ROCR_VISIBLE_DEVICES:-}" ] && [ "$ROCR_VISIBLE_DEVICES" != "-1" ]; then
+ _setup_rocr_keep() {
+ _setup_kept=$(printf '%s\n' "$1" | awk -v m="$ROCR_VISIBLE_DEVICES" '
+ NF { v[n++] = $0 }
+ END { k = split(m, t, ","); for (i = 1; i <= k; i++) { gsub(/[[:space:]]/, "", t[i]); if (t[i] !~ /^[0-9]+$/) continue; x = t[i] + 0; if (x >= n || (x in s)) break; s[x] = 1; print v[x] } }')
+ if [ -n "$_setup_kept" ]; then printf '%s\n' "$_setup_kept"; else printf '%s\n' "$1"; fi
+ }
+ # A UUID has no position in amd-smi's list: with unlike adapters, decline as install.sh does.
+ if [ -n "$(printf '%s' "$ROCR_VISIBLE_DEVICES" | tr -d '0-9, \t')" ] && \
+ [ "$(printf '%s\n' "${_setup_amd_records:-$_setup_gfx_all}" | awk -F'|' \
+ 'NF { k = ($1 != "" ? $1 : "name:" $2); if (!(k in seen)) { seen[k]; n++ } } END { print n + 0 }')" -gt 1 ]; then
+ _setup_amd_records=""
+ _setup_gfx_all=""
+ _setup_rocr_uuid_declined=1
+ fi
+ [ -n "$_setup_amd_records" ] && _setup_amd_records=$(_setup_rocr_keep "$_setup_amd_records")
+ [ -n "$_setup_gfx_all" ] && _setup_gfx_all=$(_setup_rocr_keep "$_setup_gfx_all")
+ fi
+ if [ -n "${HIP_VISIBLE_DEVICES+x}" ]; then
+ _setup_vis="$HIP_VISIBLE_DEVICES"
+ else
+ _setup_vis="${CUDA_VISIBLE_DEVICES:-}"
+ fi
_setup_vis_idx=0
if [ -n "$_setup_vis" ] && [ "$_setup_vis" != "-1" ]; then
_setup_first="${_setup_vis%%,*}"
@@ -4080,6 +4107,10 @@ elif [ "$_setup_amd_detected" = true ]; then
substep "Unlike AMD adapters and no HIP id map (amd-smi list -e needs ROCm 6.4+):"
substep "cannot tell which one this session selects. Set UNSLOTH_ROCM_GFX_ARCH to pick."
fi
+ if [ -z "$_setup_gfx" ] && [ "$_setup_rocr_uuid_declined" = 1 ]; then
+ substep "ROCR_VISIBLE_DEVICES names a GPU by UUID, which amd-smi cannot place, and the"
+ substep "adapters differ. Set UNSLOTH_ROCM_GFX_ARCH to pick."
+ fi
# ROCm version via hipconfig, then amd-smi
_setup_rocm_ver=""
if command -v hipconfig >/dev/null 2>&1; then
diff --git a/tests/sh/test_setup_gpu_summary_probe_sources.sh b/tests/sh/test_setup_gpu_summary_probe_sources.sh
index e097b776b26..c2e0d0e9b3d 100755
--- a/tests/sh/test_setup_gpu_summary_probe_sources.sh
+++ b/tests/sh/test_setup_gpu_summary_probe_sources.sh
@@ -141,6 +141,9 @@ summary() {
${STUB_AMDSMI_MUTE_STATIC:+STUB_AMDSMI_MUTE_STATIC=1} \
${STUB_AMDSMI_E:+STUB_AMDSMI_E="$STUB_AMDSMI_E"} \
${3:+HIP_VISIBLE_DEVICES="$3"} \
+ ${STUB_ROCR:+ROCR_VISIBLE_DEVICES="$STUB_ROCR"} \
+ ${STUB_CUDA:+CUDA_VISIBLE_DEVICES="$STUB_CUDA"} \
+ ${STUB_HIP_EMPTY:+HIP_VISIBLE_DEVICES=} \
/bin/bash -c 'set -euo pipefail; . "$1"; printf "%s|%s\n" "$_setup_gfx" "$_setup_mkt"' \
_ "$WORK/block.sh"
}
@@ -450,13 +453,84 @@ assert_eq "one adapter resolves without a map" \
assert_eq "the rocminfo path is not reordered by a HIP map" "gfx1100|AMD Radeon RX 7900 XTX" \
"$(STUB_AMDSMI_E="$WORK/smi_e_reversed" summary "$WORK/roc_three_dup" "$WORK/empty" 1)"
+# Post-ROCR_VISIBLE_DEVICES rocminfo lists: ROCr already filtered and renumbered them.
+cat > "$WORK/roc_rocr_1_0" <<'EOF'
+Agent 1
+*******
+ Name: AMD Ryzen 9 7950X 16-Core Processor
+ Device Type: CPU
+*******
+Agent 2
+*******
+ Name: gfx1201
+ Marketing Name: AMD Radeon AI PRO R9700
+ Device Type: GPU
+*******
+Agent 3
+*******
+ Name: gfx1036
+ Marketing Name: AMD Radeon Graphics
+ Device Type: GPU
+EOF
+cat > "$WORK/roc_rocr_1_2" <<'EOF'
+Agent 1
+*******
+ Name: AMD Ryzen 9 7950X 16-Core Processor
+ Device Type: CPU
+*******
+Agent 2
+*******
+ Name: gfx1100
+ Marketing Name: AMD Radeon RX 7900 XTX
+ Device Type: GPU
+*******
+Agent 3
+*******
+ Name: gfx1201
+ Marketing Name: AMD Radeon AI PRO R9700
+ Device Type: GPU
+EOF
+echo "=== ROCR_VISIBLE_DEVICES over rocminfo ==="
+assert_eq "ROCR=1,0 selects the first rocminfo survivor, not the iGPU at ordinal 1" \
+ "gfx1201|AMD Radeon AI PRO R9700" "$(STUB_ROCR=1,0 summary "$WORK/roc_rocr_1_0" "$WORK/empty")"
+assert_eq "ROCR=1,2 on three cards selects survivor 0 (physical 1)" \
+ "gfx1100|AMD Radeon RX 7900 XTX" "$(STUB_ROCR=1,2 summary "$WORK/roc_rocr_1_2" "$WORK/empty")"
+assert_eq "HIP still indexes the ROCr survivors" \
+ "gfx1201|AMD Radeon AI PRO R9700" "$(STUB_ROCR=1,2 summary "$WORK/roc_rocr_1_2" "$WORK/empty" 1)"
+assert_eq "CUDA_VISIBLE_DEVICES, HIP's alias, also indexes the ROCr survivors" \
+ "gfx1036|AMD Radeon Graphics" "$(STUB_ROCR=1,0 STUB_CUDA=1 summary "$WORK/roc_rocr_1_0" "$WORK/empty")"
+assert_eq "a set-but-empty HIP mask shadows CUDA, as install.sh" \
+ "gfx1201|AMD Radeon AI PRO R9700" \
+ "$(STUB_ROCR=1,0 STUB_CUDA=1 STUB_HIP_EMPTY=1 summary "$WORK/roc_rocr_1_0" "$WORK/empty")"
+assert_eq "amd-smi is not ROCr-filtered, so its list is still indexed by the ROCr ordinal" \
+ "gfx1100|AMD Radeon RX 7900 XTX" \
+ "$(STUB_ROCR=1 STUB_AMDSMI_E="$WORK/smi_e_identity" summary "$WORK/empty" "$WORK/smi_three")"
+assert_eq "amd-smi: HIP=1 under ROCR=2,0 selects survivor 1 (card 0)" \
+ "gfx90a|AMD Instinct MI210" \
+ "$(STUB_ROCR=2,0 STUB_AMDSMI_E="$WORK/smi_e_identity" summary "$WORK/empty" "$WORK/smi_three" 1)"
+assert_eq "amd-smi: CUDA_VISIBLE_DEVICES, HIP's alias, indexes the list" \
+ "gfx1201|AMD Radeon AI PRO R9700" \
+ "$(STUB_CUDA=2 STUB_AMDSMI_E="$WORK/smi_e_identity" summary "$WORK/empty" "$WORK/smi_three")"
+assert_eq "amd-smi: an empty HIP mask still leaves ROCR=1's survivor" \
+ "gfx1100|AMD Radeon RX 7900 XTX" \
+ "$(STUB_ROCR=1 STUB_HIP_EMPTY=1 STUB_AMDSMI_E="$WORK/smi_e_identity" summary "$WORK/empty" "$WORK/smi_three")"
+assert_eq "amd-smi: a repeated ROCr ordinal ends the survivors (ROCR=0,0,1 leaves card 0 only)" \
+ "gfx90a|AMD Instinct MI210" \
+ "$(STUB_ROCR=0,0,1 STUB_AMDSMI_E="$WORK/smi_e_identity" summary "$WORK/empty" "$WORK/smi_three" 2)"
+assert_eq "amd-smi: an out-of-range ROCr ordinal ends the survivors (ROCR=0,99,1)" \
+ "gfx90a|AMD Instinct MI210" \
+ "$(STUB_ROCR=0,99,1 STUB_AMDSMI_E="$WORK/smi_e_identity" summary "$WORK/empty" "$WORK/smi_three" 1)"
+assert_eq "amd-smi: a UUID in ROCR over unlike adapters declines instead of guessing" \
+ "|" \
+ "$(STUB_ROCR=GPU-4b2c1a9f8d3e6f7a,1 STUB_AMDSMI_E="$WORK/smi_e_identity" summary "$WORK/empty" "$WORK/smi_three")"
+
echo "=== detected, but no arm produced a record ==="
# Under `set -euo pipefail` an unassigned variable is not an empty string, it is a fatal
# error: `unsloth studio update` dies here instead of falling through to a source build.
assert_eq "the KFD-shaped path reaches the end instead of aborting on set -u" \
"|" "$(kfd_shape_summary)"
assert_eq "every variable the selection block reads is initialised up front" \
- "" "$(grep -oE '\$\{?_setup_(gfx|gfx_all|mkt|amd_records|amd_detected|nvidia_usable)\b' \
+ "" "$(grep -oE '\$\{?_setup_(gfx|gfx_all|mkt|amd_records|amd_detected|amd_probe|rocr_uuid_declined|nvidia_usable)\b' \
"$WORK/select.sh" | tr -d '${' | sort -u \
| while read -r _v; do grep -q "^$_v=" "$WORK/init.sh" || echo "$_v"; done | tr '\n' ' ' | sed 's/ $//')"
assert_eq "amd-smi answers list but not static --asic" \
diff --git a/tests/studio/install/test_amd_hybrid_igpu_routing_11143.py b/tests/studio/install/test_amd_hybrid_igpu_routing_11143.py
index bdd481b1444..57fca98f81b 100644
--- a/tests/studio/install/test_amd_hybrid_igpu_routing_11143.py
+++ b/tests/studio/install/test_amd_hybrid_igpu_routing_11143.py
@@ -84,6 +84,41 @@ def test_gfx906_is_never_the_repick(unmasked):
assert ILP._pick_rocm_gfx_target(_rocminfo("gfx1036", "gfx906")) == "gfx1036"
+def test_rocminfo_output_is_not_masked_by_rocr_twice(unmasked):
+ """rocminfo already applied ROCR_VISIBLE_DEVICES=1,0: its survivors are R9700, iGPU."""
+ out = _rocminfo("gfx1201", "gfx1036")
+ unmasked.setenv("ROCR_VISIBLE_DEVICES", "1,0")
+ assert ILP._pick_rocm_gfx_target(out, rocr_filtered = True) == "gfx1201"
+ unmasked.setenv("HIP_VISIBLE_DEVICES", "1")
+ assert ILP._pick_rocm_gfx_target(out, rocr_filtered = True) == "gfx1036"
+ unmasked.delenv("HIP_VISIBLE_DEVICES")
+ assert ILP._pick_rocm_gfx_target(out) == "gfx1036"
+
+
+def test_rocr_over_filtered_output_is_still_an_explicit_choice(unmasked):
+ """ROCR=0,1 leaves the iGPU first; it was chosen, so the discrete repick must not undo it."""
+ out = _rocminfo("gfx1036", "gfx1200")
+ unmasked.setenv("ROCR_VISIBLE_DEVICES", "0,1")
+ assert ILP._pick_rocm_gfx_target(out, rocr_filtered = True) == "gfx1036"
+ unmasked.setenv("ROCR_VISIBLE_DEVICES", "-1")
+ assert ILP._pick_rocm_gfx_target(out, rocr_filtered = True) is None
+
+
+def test_the_rocminfo_probe_says_its_output_is_rocr_filtered():
+ import ast
+
+ tree = ast.parse(Path(ILP.__file__).read_text(encoding = "utf-8"))
+ fn = next(
+ n for n in ast.walk(tree) if isinstance(n, ast.FunctionDef) and n.name == "detect_host"
+ )
+ calls = [
+ c
+ for c in ast.walk(fn)
+ if isinstance(c, ast.Call) and getattr(c.func, "id", "") == "_pick_rocm_gfx_target"
+ ]
+ assert any(k.arg == "rocr_filtered" for c in calls for k in c.keywords)
+
+
@pytest.mark.parametrize("env", _VISIBILITY_ENV)
def test_an_explicit_visibility_mask_still_wins(monkeypatch, env):
"""HIP_VISIBLE_DEVICES=0 on the hybrid host selects the iGPU on purpose -- it is the
diff --git a/tests/studio/install/test_rocm_support.py b/tests/studio/install/test_rocm_support.py
index 0811f6d5800..6d4322aea1f 100644
--- a/tests/studio/install/test_rocm_support.py
+++ b/tests/studio/install/test_rocm_support.py
@@ -2289,6 +2289,16 @@ class TestGfx1102Rocm64Floor:
)
assert self._run_install_sh_routing(preamble) == "rocm6.1"
+ @pytest.mark.parametrize("rocr, hip", (("0,0,1", "2"), ("0,99,1", "1")))
+ def test_install_sh_ends_rocr_survivors_at_a_repeated_or_missing_ordinal(self, rocr, hip):
+ """Only card 0 survives, so the HIP index falls back to it, never the gfx1200."""
+ preamble = (
+ "rocminfo() { return 1; }\n"
+ + self._amd_smi_stub("gfx1100", "gfx1200")
+ + f"export ROCR_VISIBLE_DEVICES={rocr}; export HIP_VISIBLE_DEVICES={hip}"
+ )
+ assert self._run_install_sh_routing(preamble) == "rocm6.1"
+
def test_install_sh_declines_a_rocr_uuid_mask_over_unlike_amd_smi_adapters(self):
"""A UUID names a device but no position, so no survivor is known to be it."""
preamble = (