Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Give a mixed NVIDIA plus AMD host a way to ask for the ROCm torch stack (#10474)

Daniel Hanb78f334a28e0
diff --git a/install.sh b/install.sh
index 783ffbfef97..13ee08d39a1 100755
--- a/install.sh
+++ b/install.sh
@@ -3800,15 +3800,31 @@ _ensure_rocm_probe_env() {
fi
}
+# Whether this run was TOLD to install ROCm torch, whatever else the host has: the automatic
+# profile stops probing AMD once CUDA is usable, leaving a mixed host no route (#10450). One
+# torch install serves one vendor, so this SWAPS the stack.
+# Mirrors install_python_stack._rocm_torch_explicitly_requested; keep the two in step.
+_rocm_torch_explicitly_requested() {
+ # Trimmed, because the Python twin reads the same variable through .strip(): untrimmed,
+ # " true " exports an authoritative CUDA backend the Python half then cannot override.
+ case "$(printf '%s' "${UNSLOTH_FORCE_ROCM_TORCH:-}" \
+ | sed 's/^[[:space:]]*//; s/[[:space:]]*$//' | tr '[:upper:]' '[:lower:]')" in
+ 1|true|yes|on) return 0 ;;
+ *) return 1 ;;
+ esac
+}
+
# Whether ROCm can see an AMD GPU. A usable NVIDIA card wins by default; a diagnosis that
-# has already established the run opens AMD nodes passes "ignore-nvidia" to skip that.
+# has already established the run opens AMD nodes passes "ignore-nvidia" to skip that, and an
+# explicit UNSLOTH_FORCE_ROCM_TORCH request relaxes it for every caller.
# The veto is inside this function rather than in a wrapper, because every tests/sh harness
# lifts probes one function at a time by name (sed -n '/^_name()/,/^}/p'), and a wrapper
# whose helper is not also lifted calls nothing: the ROCm branch would then fall silently
# through to the CPU wheel index.
_has_amd_rocm_gpu() {
_ensure_rocm_probe_env
- if [ "${1:-}" != "ignore-nvidia" ] && _has_usable_nvidia_gpu; then
+ if [ "${1:-}" != "ignore-nvidia" ] && _has_usable_nvidia_gpu && \
+ ! _rocm_torch_explicitly_requested; then
return 1
fi
if command -v rocminfo >/dev/null 2>&1 && \
@@ -3826,6 +3842,269 @@ _has_amd_rocm_gpu() {
return 1
}
+# AMD silicon this host can point at WITHOUT trusting a declared arch:
+# _infer_linux_amd_gfx_arch returns UNSLOTH_ROCM_GFX_ARCH before it looks at hardware, so it
+# cannot answer "is there a card" and a stale one would hand a working CUDA stack to no GPU.
+_amd_hardware_corroborated() {
+ _amd_gpu_present_via_pci && return 0
+ # WSL enumerates no PCI display device, and neither /dev/dxg (an NVIDIA passthrough creates
+ # it too) nor a leftover librocdxg names a vendor: beside an NVIDIA card the runtime must
+ # name an agent itself, hence "physical" mode.
+ if [ -e /dev/dxg ] || grep -qi microsoft /proc/version 2>/dev/null; then
+ for _ahc_d in /opt/rocm/lib /opt/rocm/lib64 /opt/rocm-*/lib /opt/rocm-*/lib64; do
+ { [ -e "$_ahc_d/librocdxg.so" ] || [ -e "$_ahc_d/librocdxg.so.1" ]; } || continue
+ _has_usable_nvidia_gpu || return 0
+ [ -n "$(_probe_amd_gfx_arch physical 2>/dev/null)" ] && return 0
+ return 1
+ done
+ fi
+ return 1
+}
+
+# Whether ANY index this installer can pick carries kernels for this arch. An arch in neither
+# (gfx1010, RDNA 1) must never depose a card that can.
+# Mirrors _gfx_has_a_wheel_route / _GENERIC_ROCM_WHEEL_GFX in install_python_stack.py.
+_amd_gfx_has_wheel_route() {
+ # A per-arch index carries its arch whatever the generic tag resolves to, so it answers first.
+ _amd_arch_index_family_for_gfx "$1" >/dev/null 2>&1 && return 0
+ case "$1" in
+ gfx900|gfx906|gfx908|gfx90a|gfx942|gfx950) : ;;
+ gfx1030|gfx1100|gfx1101|gfx1102|gfx1150|gfx1151|gfx1200|gfx1201) : ;;
+ *) return 1 ;;
+ esac
+ _amd_generic_tag_carries_gfx "$1"
+}
+
+# Whether the generic wheel THIS host resolves carries kernels for an arch whose only route is
+# that wheel. Tag comes from the installed ROCm version, so a stale /opt/rocm resolves a wheel
+# predating the card; unreadable answers NO. Mirrors _GENERIC_WHEEL_GFX_MIN_ROCM (python twin).
+_amd_generic_tag_carries_gfx() {
+ case "$1" in
+ gfx950|gfx1150|gfx1151) _agtc_min_major=7; _agtc_min_minor=0 ;;
+ gfx1200|gfx1201) _agtc_min_major=6; _agtc_min_minor=4 ;;
+ *) return 0 ;;
+ esac
+ _agtc_tag=$(_detect_rocm_version_tag 2>/dev/null) || _agtc_tag=""
+ _agtc_ver=${_agtc_tag#rocm}
+ _agtc_major=$(printf '%s' "$_agtc_ver" | cut -d. -f1)
+ _agtc_minor=$(printf '%s' "$_agtc_ver" | cut -d. -f2)
+ case "$_agtc_major$_agtc_minor" in
+ ''|*[!0-9]*) return 1 ;;
+ esac
+ [ "$_agtc_major" -gt "$_agtc_min_major" ] && return 0
+ [ "$_agtc_major" -eq "$_agtc_min_major" ] && [ "$_agtc_minor" -ge "$_agtc_min_minor" ] && return 0
+ return 1
+}
+
+# True when a set mask exposes NO GPU at either layer. ROCr filters BENEATH HIP, and
+# CUDA_VISIBLE_DEVICES is HIP's alias, read only when HIP is unset. ${VAR+x} not ${VAR:-}: a
+# SET-but-empty mask hides every device. Mirrors _visible_masks_select_no_gpu.
+_amd_visible_masks_select_no_gpu() {
+ if [ -n "${HIP_VISIBLE_DEVICES+x}" ]; then
+ _avm_hip=HIP_VISIBLE_DEVICES
+ else
+ _avm_hip=CUDA_VISIBLE_DEVICES
+ fi
+ for _avm_var in ROCR_VISIBLE_DEVICES "$_avm_hip"; do
+ eval "_avm_set=\${$_avm_var+x}"
+ eval "_avm_val=\${$_avm_var:-}"
+ [ -n "$_avm_set" ] || continue
+ case "$(printf '%s' "$_avm_val" | tr -d '[:space:]')" in
+ ''|-1) return 0 ;;
+ esac
+ done
+ return 1
+}
+
+# One mask layer applied to a per-DEVICE list, in mask order, never deduplicated. Survivors are
+# the PREFIX of resolvable ordinals, since CUDA and HIP stop at the first entry naming no device.
+# A third argument of "rocr" also ends the prefix at a REPEATED ordinal, which ROCr terminates on
+# (ROCR-Runtime, core/inc/amd_filter_device.h); clr documents no such rule for HIP.
+_amd_mask_survivors() {
+ # The mask arrives through the environment, not `awk -v`: an assignment operand is
+ # ESCAPE-PROCESSED, so HIP_VISIBLE_DEVICES='\061' became the ordinal 1 here while the Python
+ # twin rejects it at int().
+ printf '%s\n' "$1" | _ams_vis="$2" _ams_layer="${3:-}" awk '
+ BEGIN { vis = ENVIRON["_ams_vis"]; layer = ENVIRON["_ams_layer"] }
+ NF { vals[n++] = $0 }
+ END {
+ count = split(vis, want, ",")
+ for (i = 1; i <= count; i++) {
+ gsub(/^[ \t]+|[ \t]+$/, "", want[i])
+ if (want[i] !~ /^[0-9]+$/) break
+ idx = want[i] + 0
+ if (idx >= n) break
+ if (layer == "rocr" && (idx in seen)) break
+ seen[idx] = 1
+ print vals[idx]
+ }
+ }'
+}
+
+# The single arch the runtime will hand torch, empty when the masks resolve nothing. Resolved
+# here, not by a probe: rocminfo honours only ROCR_VISIBLE_DEVICES and amd-smi neither. The
+# layers COMPOSE, ROCr filtering and renumbering beneath HIP, and the FIRST survivor wins.
+_amd_runtime_gfx_target() {
+ _argt_list="$1"
+ if [ -n "${ROCR_VISIBLE_DEVICES:-}" ]; then
+ _argt_list=$(_amd_mask_survivors "$_argt_list" "$ROCR_VISIBLE_DEVICES" rocr)
+ [ -n "$_argt_list" ] || return 0
+ fi
+ # CUDA_VISIBLE_DEVICES is the HIP alias and clr reads it only when HIP itself is unset.
+ _argt_hip="${HIP_VISIBLE_DEVICES:-}"
+ if [ -z "${HIP_VISIBLE_DEVICES+x}" ]; then
+ _argt_hip="${CUDA_VISIBLE_DEVICES:-}"
+ fi
+ if [ -n "$_argt_hip" ]; then
+ _argt_list=$(_amd_mask_survivors "$_argt_list" "$_argt_hip")
+ [ -n "$_argt_list" ] || return 0
+ fi
+ printf '%s\n' "$_argt_list" | awk 'NF { print; exit }'
+}
+
+# The AMD integrated GPUs that shadow a discrete card by enumerating ahead of it. Mirror of
+# _SHADOWING_INTEGRATED_GFX, held to it by tests/studio/install/test_rocm_arch_table_parity.py:
+# one installer having this table and not the other is the shape of #7264 / #7277 / #7293.
+_amd_gfx_is_shadowing_integrated() {
+ case "$1" in
+ gfx90c|gfx1013|gfx1033|gfx1035|gfx1036|gfx1103|gfx1153) return 0 ;;
+ esac
+ return 1
+}
+
+# The card to install for when enumeration put an integrated GPU first (#7776): one arch picks
+# the wheel family, so letting the APU decide strands the discrete card. gfx906 is never a
+# candidate, since naming it on a mixed host strands BOTH cards.
+_amd_prefer_discrete_gfx() {
+ _apdg_devs="$1"
+ _apdg_sel="$2"
+ if ! _amd_gfx_is_shadowing_integrated "$_apdg_sel"; then
+ printf '%s' "$_apdg_sel"
+ return 0
+ fi
+ if [ -n "${HIP_VISIBLE_DEVICES+x}" ] || [ -n "${ROCR_VISIBLE_DEVICES+x}" ] || \
+ [ -n "${CUDA_VISIBLE_DEVICES+x}" ]; then
+ printf '%s' "$_apdg_sel"
+ return 0
+ fi
+ _apdg_others=$(printf '%s\n' "$_apdg_devs" | awk 'NF' | while IFS= read -r _apdg_g; do
+ _amd_gfx_is_shadowing_integrated "$_apdg_g" && continue
+ [ "$_apdg_g" = gfx906 ] && continue
+ printf '%s\n' "$_apdg_g"
+ done)
+ _apdg_pick=$(printf '%s\n' "$_apdg_others" | awk 'NF' | while IFS= read -r _apdg_g; do
+ _amd_gfx_has_wheel_route "$_apdg_g" && printf '%s\n' "$_apdg_g"
+ done | awk 'NF { print; exit }')
+ if [ -z "$_apdg_pick" ] && ! _amd_gfx_has_wheel_route "$_apdg_sel"; then
+ _apdg_pick=$(printf '%s\n' "$_apdg_others" | awk 'NF { print; exit }')
+ fi
+ [ -n "$_apdg_pick" ] || _apdg_pick="$_apdg_sel"
+ printf '%s' "$_apdg_pick"
+}
+
+# Whether the request has something to swap TO: a corroborated card whose arch an index can
+# serve. Presence is not that bar -- gfx1010 is present with no route, and gfx906 drops when a
+# second AMD arch is present.
+_amd_request_has_a_wheel_route() {
+ # The card this run hands torch, published for get_torch_index_url's miscomputing-arch gate,
+ # which has only the unmasked inventory to judge on. Cleared on ENTRY so a previous call can
+ # never answer for this one.
+ _AMD_REQUEST_TARGET_GFX=""
+ # Only a PROBE-resolved target may clear a safety gate, or a declared gfx1030 would exempt a
+ # physical gfx1033 from the Van Gogh gate. Declared still routes (the Strix workaround
+ # declares gfx1100 on a gfx1151), it just cannot vouch for the machine.
+ _AMD_REQUEST_TARGET_SOURCE=""
+ # A mask exposing no device is a deliberate no-GPU selection, not a detection miss.
+ _amd_visible_masks_select_no_gpu && return 1
+ # A DECLARED arch decides when there is one, since get_torch_index_url installs from it.
+ # Corroborated first, because the variable answers on a host with no AMD GPU at all.
+ _arwr_decl=$(printf '%s' "${UNSLOTH_ROCM_GFX_ARCH:-}" \
+ | tr '[:upper:]' '[:lower:]' | sed 's/:.*$//' | tr -d '[:space:]')
+ if [ -n "$_arwr_decl" ]; then
+ _amd_hardware_corroborated || _kfd_gfx_targets 2>/dev/null | grep -q . || return 1
+ # The arch names what to BUILD for, not whether the runtime exposes a device to build it
+ # for. Asked only when a mask is set AND the device list is knowable: an unknowable one
+ # leaves the declared arch alone rather than declining on no evidence.
+ if [ -n "${HIP_VISIBLE_DEVICES+x}" ] || [ -n "${ROCR_VISIBLE_DEVICES+x}" ] || \
+ [ -n "${CUDA_VISIBLE_DEVICES+x}" ]; then
+ _arwr_dd=$(_amd_ordered_gfx_devices 2>/dev/null | sed 's/:.*$//' \
+ | tr '[:upper:]' '[:lower:]' | awk 'NF')
+ [ -n "$_arwr_dd" ] || _arwr_dd=$(_kfd_gfx_targets 2>/dev/null | sed 's/:.*$//' \
+ | tr '[:upper:]' '[:lower:]' | awk 'NF')
+ if [ -n "$_arwr_dd" ] && [ -z "$(_amd_runtime_gfx_target "$_arwr_dd")" ]; then
+ return 1
+ fi
+ fi
+ if _amd_gfx_has_wheel_route "$_arwr_decl"; then
+ _AMD_REQUEST_TARGET_GFX="$_arwr_decl"
+ _AMD_REQUEST_TARGET_SOURCE=declared
+ return 0
+ fi
+ return 1
+ fi
+ _arwr_all=$(_probe_amd_gfx_arch physical 2>/dev/null || true)
+ [ -n "$_arwr_all" ] || _arwr_all=$(_kfd_gfx_targets 2>/dev/null || true)
+ if [ -z "$_arwr_all" ]; then
+ # Runtime-less but inferable, which a pure-AMD host is already served on. Requiring a working
+ # ROCm runtime would answer differently for the same silicon by whether an NVIDIA card sits
+ # beside it.
+ _amd_hardware_corroborated || return 1
+ _arwr_all=$(_infer_linux_amd_gfx_arch 2>/dev/null || true)
+ fi
+ _arwr_archs=$(printf '%s\n' "$_arwr_all" | sed 's/:.*$//' \
+ | tr '[:upper:]' '[:lower:]' | awk 'NF')
+ [ -n "$_arwr_archs" ] || return 1
+ # The PHYSICAL count, before any mask narrows the list: gfx906's rocm6.3 tag opens only when
+ # it is the sole arch, and the reroute granting it inspects the unmasked inventory.
+ _arwr_count=$(printf '%s\n' "$_arwr_archs" | sort -u | wc -l | tr -d ' ')
+ # The one card this run will hand torch; empty when the masks named something unresolvable.
+ # Fails CLOSED, since a wrong yes replaces a working CUDA stack with kernel-less wheels.
+ _arwr_devs=$(_amd_ordered_gfx_devices 2>/dev/null | sed 's/:.*$//' \
+ | tr '[:upper:]' '[:lower:]' | awk 'NF')
+ if [ -z "$_arwr_devs" ]; then
+ # rocminfo is the only per-device source the masks index. amd-smi enumerates in KFD DISCOVERY
+ # order while the kernel topology IS HIP's and ROCr's: take it when it describes this machine.
+ _arwr_kfd=$(_kfd_gfx_targets 2>/dev/null | sed 's/:.*$//' \
+ | tr '[:upper:]' '[:lower:]' | awk 'NF')
+ if [ -n "$_arwr_kfd" ] && \
+ [ "$(printf '%s\n' "$_arwr_kfd" | awk 'NF' | wc -l | tr -d ' ')" \
+ = "$(printf '%s\n' "$_arwr_archs" | awk 'NF' | wc -l | tr -d ' ')" ]; then
+ _arwr_devs="$_arwr_kfd"
+ fi
+ fi
+ if [ -n "$_arwr_devs" ]; then
+ _arwr_sel=$(_amd_runtime_gfx_target "$_arwr_devs")
+ elif [ "$_arwr_count" -eq 1 ]; then
+ # One arch on the whole host: every ordinal names it, so ordering cannot change the answer.
+ _arwr_sel=$(_amd_runtime_gfx_target "$_arwr_archs")
+ else
+ # Unlike adapters with no order the ordinals fit: which card the runtime hands torch is
+ # unanswerable here, so fail closed.
+ return 1
+ fi
+ [ -n "$_arwr_sel" ] || return 1
+ # Enumeration order is not a choice of card. The flat inventory only stands in where the
+ # count is 1 and no preference can apply anyway.
+ _arwr_pref="$_arwr_devs"
+ [ -n "$_arwr_pref" ] || _arwr_pref="$_arwr_archs"
+ _arwr_sel=$(_amd_prefer_discrete_gfx "$_arwr_pref" "$_arwr_sel")
+ [ "$_arwr_sel" = gfx906 ] && [ "$_arwr_count" -gt 1 ] && return 1
+ _amd_gfx_has_wheel_route "$_arwr_sel" || return 1
+ _AMD_REQUEST_TARGET_GFX="$_arwr_sel"
+ _AMD_REQUEST_TARGET_SOURCE=probe
+ return 0
+}
+
+# One place answers "does the NVIDIA card still win here", so index selection and the per-arch
+# reroutes cannot disagree: the reroutes probe afresh at top level, so get_torch_index_url
+# clearing its own _nvidia_detected never reached them (#10450).
+_nvidia_gpu_wins_over_amd() {
+ _has_usable_nvidia_gpu || return 1
+ if _rocm_torch_explicitly_requested && _amd_request_has_a_wheel_route; then
+ return 1
+ fi
+ return 0
+}
# Returns 0 if an AMD display GPU is on the PCI bus even when ROCm cannot use it (a Strix Halo iGPU with no /dev/kfd). Only sharpens the "no GPU detected" hint. vendor 0x1002 = AMD/ATI; class 0x03* = display controller.
_amd_gpu_present_via_pci() {
[ -d /sys/bus/pci/devices ] || return 1
@@ -4524,6 +4803,15 @@ _probe_amd_gfx_arch() {
printf '%s\n' "$_pg"
}
+# One gfx per GPU in ROCr enumeration order, the order a mask indexes. _probe_amd_gfx_arch
+# cannot serve: rocminfo names a target in BOTH "Name:" and "ISA Info", so a flat grep returns
+# two rows per card. Twin of install_python_stack._detect_amd_gfx_codes(dedup = False).
+_amd_ordered_gfx_devices() {
+ _ensure_rocm_probe_env
+ command -v rocminfo >/dev/null 2>&1 || return 0
+ (unset ROCR_VISIBLE_DEVICES HIP_VISIBLE_DEVICES HSA_OVERRIDE_GFX_VERSION; rocminfo 2>/dev/null) \
+ | _rocminfo_gpu_records | sed 's/|.*$//' | awk 'NF'
+}
# Classify the physical NVIDIA inventory for a cu126 fallback: "cu126" when it covers every GPU, "uncovered" for an incompatible mix, empty when no fallback is needed or the inventory is unreadable. CUDA_VISIBLE_DEVICES is ignored because the wheel must support the host. Shared decision with install.ps1 / setup.ps1 / install_python_stack.py.
_nvidia_cu126_verdict() {
# $2: capabilities already read from the driver library, one per line, when nvidia-smi cannot.
@@ -4700,6 +4988,11 @@ _detect_rocm_version_tag() {
# On CPU-only machines this returns the cpu index, avoiding the solver
# dead-end where --torch-backend=auto resolves to unsloth==2024.8.
get_torch_index_url() {
+ # Cleared on ENTRY, not only in the helper that sets them: the helper runs only when NVIDIA is
+ # detected AND the request is set, and the CUDA restore calls this a second time from a shell
+ # the reroutes have populated. A stale gfx1100/probe pair clears the Van Gogh gate on gfx1033.
+ _AMD_REQUEST_TARGET_GFX=""
+ _AMD_REQUEST_TARGET_SOURCE=""
_base="${UNSLOTH_PYTORCH_MIRROR:-https://download.pytorch.org/whl}"
_base="${_base%/}"
# An explicit override skips ALL GPU probing: the URL is verbatim, _FAMILY is its leaf.
@@ -4730,6 +5023,15 @@ get_torch_index_url() {
_smi="/usr/bin/nvidia-smi"
fi
fi
+ # Here rather than only in _has_amd_rocm_gpu, because this index is exported as
+ # UNSLOTH_TORCH_BACKEND, on which _ensure_rocm_torch returns immediately for "cuda":
+ # relaxing the AMD probe alone swapped nothing (#10450).
+ if [ "$_nvidia_detected" -eq 1 ] && _rocm_torch_explicitly_requested && \
+ _amd_request_has_a_wheel_route; then
+ echo "[INFO] UNSLOTH_FORCE_ROCM_TORCH is set and an AMD GPU is present -- selecting ROCm PyTorch over CUDA." >&2
+ echo "[INFO] One torch install serves one vendor: the NVIDIA card will not be available to training until this is unset and the installer re-run." >&2
+ _nvidia_detected=0
+ fi
if [ "$_nvidia_detected" -eq 0 ]; then
case "$(uname -m)" in
x86_64|amd64) : ;;
@@ -4780,13 +5082,29 @@ get_torch_index_url() {
[ -n "$_amd_gfx_gate_probe" ] || _amd_gfx_gate_probe="$_amd_gfx_probe"
_amd_gfx_tokens=" $(printf '%s\n' "$_amd_gfx_gate_probe" | sed 's/:.*$//' \
| tr '[:upper:]' '[:lower:]' | tr '\n' ' ')"
+ _amd_gfx_bad_arch=false
case "$_amd_gfx_tokens" in
- *" gfx1033 "*)
- echo "[WARN] AMD gfx1033 (Van Gogh) computes incorrect results under ROCm -- installing CPU-only PyTorch." >&2
- echo "[WARN] ROCm wheels install on it but training diverges to NaN and gradcheck fails; forward math is fine." >&2
- echo "[WARN] Details: studio/ROCM_RDNA2_APU.md. Override with UNSLOTH_TORCH_INDEX_URL if you want ROCm anyway." >&2
- echo "$_base/cpu"; return ;;
+ *" gfx1033 "*) _amd_gfx_bad_arch=true ;;
esac
+ # PRESENCE is the rule while the selected card is unknown. An honoured request has resolved
+ # the one card this run hands torch, so answering on a sibling it hid took the cpu index for
+ # a routable gfx1100. Probe-resolved only.
+ case "${_AMD_REQUEST_TARGET_GFX:-}" in
+ ""|gfx1033) : ;;
+ *)
+ # if/fi rather than `[ ... ] &&`: under set -e a failing test as the last
+ # command of the arm would abort the installer.
+ if [ "${_AMD_REQUEST_TARGET_SOURCE:-}" = probe ]; then
+ _amd_gfx_bad_arch=false
+ fi
+ ;;
+ esac
+ if [ "$_amd_gfx_bad_arch" = true ]; then
+ echo "[WARN] AMD gfx1033 (Van Gogh) computes incorrect results under ROCm -- installing CPU-only PyTorch." >&2
+ echo "[WARN] ROCm wheels install on it but training diverges to NaN and gradcheck fails; forward math is fine." >&2
+ echo "[WARN] Details: studio/ROCM_RDNA2_APU.md. Override with UNSLOTH_TORCH_INDEX_URL if you want ROCm anyway." >&2
+ echo "$_base/cpu"; return
+ fi
# end of the miscomputing-arch gate -- tests/sh/test_rocm_bad_arch_gate.sh lifts
# the block between the header comment above and this line, so keep both exact.
_rocm_tag=""
@@ -4899,6 +5217,16 @@ _torch_index_url_leaf() {
printf '%s' "${_tl_u##*/}" | tr '[:upper:]' '[:lower:]'
}
+# Did the resolution land on a ROCm index? The FINAL leaf only, so a mirror whose BASE path
+# contains rocm or gfx cannot read as one. Broader than _is_pip_rocm_family_leaf on purpose,
+# which declines the Radeon repo leaf rocm-rel-6.4.
+_torch_index_url_is_rocm() {
+ case "$(_torch_index_url_leaf "${1:-}")" in
+ rocm*|gfx*) return 0 ;;
+ *) return 1 ;;
+ esac
+}
+
# True for an EXACT ROCm family leaf; one that merely starts with rocm/gfx is a custom pin.
_is_pip_rocm_family_leaf() {
case "$1" in
@@ -5399,7 +5727,7 @@ case "$TORCH_INDEX_URL" in
*/cpu)
if [ "$_torch_index_pinned" = false ] && [ "$SKIP_TORCH" = false ] && \
[ -z "${UNSLOTH_ROCM_GFX_ARCH:-}" ] && \
- ! _has_usable_nvidia_gpu && _has_amd_rocm_gpu; then
+ ! _nvidia_gpu_wins_over_amd && _has_amd_rocm_gpu; then
_amd_probe_out=$(_probe_amd_gfx_arch)
# HSA_OVERRIDE_GFX_VERSION=11.0.0 is the standard Strix Halo workaround, and ROCr then reports the spoofed gfx1100. The llama.cpp path corrects that further down, which is too late for this branch: both the wheel family and the exported arch are derived from this probe, so an uncorrected gfx1151 would take gfx110X-all wheels and export gfx1100 to setup.sh.
_amd_spoof_inferred=$(_infer_linux_amd_gfx_arch 2>/dev/null || true)
@@ -5416,14 +5744,44 @@ case "$TORCH_INDEX_URL" in
|| _amd_probed_family=""
_amd_probed_gfx_first=$(_amd_sole_index_arch "$_amd_probe_out") \
|| _amd_probed_gfx_first=""
+ # Re-derived rather than read: _AMD_REQUEST_TARGET_GFX is assigned inside a command
+ # substitution above, and the predicate is idempotent.
+ _amd_reroute_target=""
+ if _rocm_torch_explicitly_requested && _amd_request_has_a_wheel_route; then
+ _amd_reroute_target="${_AMD_REQUEST_TARGET_GFX:-}"
+ fi
+ _amd_reroute_bad_arch=false
# A measured-bad arch anywhere in the inventory disqualifies the whole family, at the source not at each consumer: gfx1033 shares gfx103X-all with gfx1030-gfx1036, so the shared-family arm below would otherwise rewrite the cpu index the gate just chose back to ROCm wheels.
case " $(printf '%s\n' "$_amd_probe_out" | sed 's/:.*$//' \
| tr '[:upper:]' '[:lower:]' | tr '\n' ' ')" in
- *" gfx1033 "*)
- echo "[WARN] AMD gfx1033 (Van Gogh) is in the probed inventory -- not routing torch to the shared $_amd_probed_family index (studio/ROCM_RDNA2_APU.md)." >&2
- _amd_probed_family=""
- _amd_probed_gfx_first="" ;;
+ *" gfx1033 "*) _amd_reroute_bad_arch=true ;;
+ esac
+ # Probe-resolved only, exactly as the gate inside get_torch_index_url.
+ case "$_amd_reroute_target" in
+ ""|gfx1033) : ;;
+ *)
+ if [ "${_AMD_REQUEST_TARGET_SOURCE:-}" = probe ]; then
+ _amd_reroute_bad_arch=false
+ fi
+ ;;
esac
+ if [ "$_amd_reroute_bad_arch" = true ]; then
+ echo "[WARN] AMD gfx1033 (Van Gogh) is in the probed inventory -- not routing torch to the shared $_amd_probed_family index (studio/ROCM_RDNA2_APU.md)." >&2
+ _amd_probed_family=""
+ _amd_probed_gfx_first=""
+ fi
+ # Derived from the SELECTED card: _amd_agreed_index_family needs every physical GPU to
+ # share a family and answers empty for a cross-family pair, which left the cpu index
+ # unrewritten on exactly the host that resolved a routable target.
+ if [ "$_amd_reroute_bad_arch" = false ] && [ -n "$_amd_reroute_target" ] && \
+ [ "${_AMD_REQUEST_TARGET_SOURCE:-}" = probe ]; then
+ _amd_target_family=$(_amd_arch_index_family_for_gfx "$_amd_reroute_target") \
+ || _amd_target_family=""
+ if [ -n "$_amd_target_family" ]; then
+ _amd_probed_family="$_amd_target_family"
+ _amd_probed_gfx_first="$_amd_reroute_target"
+ fi
+ fi
if [ -n "${_amd_probed_family:-}" ] && \
[ -z "$(_detect_rocm_version_tag 2>/dev/null)" ]; then
_amd_no_rocm_version_reroute=true
@@ -5432,7 +5790,7 @@ case "$TORCH_INDEX_URL" in
;;
esac
if [ "$_torch_index_pinned" = false ] && [ "$SKIP_TORCH" = false ] && \
- ! _has_usable_nvidia_gpu && \
+ ! _nvidia_gpu_wins_over_amd && \
{ [ -n "${UNSLOTH_ROCM_GFX_ARCH:-}" ] || ! _has_amd_rocm_gpu || \
[ -z "$(_probe_amd_gfx_arch)" ] || \
[ "${_amd_no_rocm_version_reroute:-false}" = true ]; } && \
@@ -5524,6 +5882,21 @@ if [ "$_torch_index_pinned" = false ] && [ "$SKIP_TORCH" = false ] && \
esac
fi
+# The request buys a swap, never a downgrade. Asked of the RESOLVED index: the AMD branch
+# answers cpu for a ROCm too old, an arch no index covers, or a non-x86_64 host, and leaving
+# that would replace a working CUDA install with CPU torch.
+if [ "$_torch_index_pinned" = false ] && [ "$SKIP_TORCH" = false ] && \
+ _rocm_torch_explicitly_requested && _has_usable_nvidia_gpu; then
+ if ! _torch_index_url_is_rocm "$TORCH_INDEX_URL"; then
+ _cuda_fallback_index=$(UNSLOTH_FORCE_ROCM_TORCH=0 get_torch_index_url)
+ if [ -n "$_cuda_fallback_index" ] && \
+ [ "$_cuda_fallback_index" != "$TORCH_INDEX_URL" ]; then
+ echo "[WARN] UNSLOTH_FORCE_ROCM_TORCH is set, but no ROCm wheel index could be selected for this host." >&2
+ echo "[WARN] Keeping the CUDA build rather than installing CPU PyTorch on a machine with a working NVIDIA GPU." >&2
+ TORCH_INDEX_URL="$_cuda_fallback_index"
+ fi
+ fi
+fi
# Export the resolved torch backend ("cuda", "rocm" or "cpu") so setup.sh and install_python_stack.py know what was chosen here and can skip ROCm-specific repair steps. Classify on the FINAL path segment only: a custom UNSLOTH_PYTORCH_MIRROR whose base path happens to contain "rocm" or "gfx" must not mislabel a cu*/cpu index as ROCm (radeon repo URLs end in rocm-rel-X.Y/, Strix overrides in gfxNNNN/, so the trailing slash is stripped first). Lowercase the leaf so every gfx*/rocm*/cu* arm matches regardless of case (the canonical AMD RDNA4 leaf is gfx120X-all). CUDA is branded only on a real cu[0-9]* leaf, so a mirror leaf (/current) does NOT commit a CUDA backend; an unknown leaf leaves the var unset so the stack probes the GPU. Query and fragment are dropped first, then ALL trailing slashes, in lockstep with the shared _torch_index_url_leaf extractor.
_torch_index_leaf="${TORCH_INDEX_URL%%\?*}"
_torch_index_leaf="${_torch_index_leaf%%#*}"
@@ -5865,8 +6238,15 @@ fi
_TAURI_GPU_BRANCH=$(_tauri_gpu_branch "$_TAURI_TORCH_INDEX_FAMILY" "$_amd_gpu_radeon")
tauri_diag_marker "$_TAURI_GPU_BRANCH" "$_TAURI_TORCH_INDEX_FAMILY"
+
# ── GPU detection summary (mirrors install.ps1 step "gpu" block) ──
-if _has_usable_nvidia_gpu; then
+# Asked of the RESOLVED index, not the predicate that chose it: after the CUDA restore the
+# request is still set and an AMD card still present, so the predicate says "AMD wins" while
+# CUDA wheels are what gets installed. A PIN is exempt, because it names a wheel family rather
+# than a card: on an NVIDIA host pinned to a ROCm index there is no AMD card to describe, and
+# hiding the NVIDIA identity behind it reported hardware the machine does not have.
+if _has_usable_nvidia_gpu && \
+ { [ "$_torch_index_pinned" = true ] || ! _torch_index_url_is_rocm "$TORCH_INDEX_URL"; }; then
_nv_banner_fields
if [ -n "$_nv_name" ] && [ -n "$_nv_sm" ]; then
step "gpu" "$_nv_name ($_nv_sm)"
@@ -5878,7 +6258,7 @@ if _has_usable_nvidia_gpu; then
# An `if`, not `[ ... ] && substep ...`: the AND-list form leaves a non-zero status
# behind on the common path where there is no driver string to print.
if [ -n "$_nv_driver" ]; then substep "Driver: $_nv_driver"; fi
-elif case "$TORCH_INDEX_URL" in */rocm*|*/gfx*) true ;; *) false ;; esac; then
+elif _torch_index_url_is_rocm "$TORCH_INDEX_URL"; then
# Probe gfx arch for the display label, honouring HIP_VISIBLE_DEVICES
_ensure_rocm_probe_env
_gpu_disp_gfx_all=""
diff --git a/studio/backend/tests/test_amd_device_node_permissions.py b/studio/backend/tests/test_amd_device_node_permissions.py
index 74b885b5267..5d2f65e9156 100644
--- a/studio/backend/tests/test_amd_device_node_permissions.py
+++ b/studio/backend/tests/test_amd_device_node_permissions.py
@@ -4426,9 +4426,19 @@ def test_the_two_quoting_rules_are_the_same_rule(tmp_path):
assert out.stdout == shlex.quote(value), (value, out.stdout)
-# The two helpers every tests/sh ROCm harness lifts alongside _has_amd_rocm_gpu.
+# The helpers every tests/sh ROCm harness lifts alongside _has_amd_rocm_gpu. Adding a name
+# here is only honest once all five harnesses lift it too, which is what the test below the
+# invariant checks: each name must appear in every harness that lifts the probe.
_ROCM_PROBE_CALLEES_THE_SH_HARNESSES_LIFT = frozenset(
- {"_ensure_rocm_probe_env", "_has_usable_nvidia_gpu"}
+ {
+ "_ensure_rocm_probe_env",
+ "_has_usable_nvidia_gpu",
+ # Reads UNSLOTH_FORCE_ROCM_TORCH, the opt-in that lets a mixed NVIDIA+AMD host ask
+ # for the ROCm stack (#10450). Unlifted it is an undefined command, which exits
+ # non-zero, which reads as "not requested" -- the default answer, so a harness would
+ # keep passing while the branch under test was never the one the flag selects.
+ "_rocm_torch_explicitly_requested",
+ }
)
@@ -4457,6 +4467,27 @@ def test_the_rocm_probe_calls_nothing_the_shell_harnesses_do_not_lift():
assert called <= _ROCM_PROBE_CALLEES_THE_SH_HARNESSES_LIFT, called
+def test_the_harnesses_really_lift_every_name_the_allowlist_claims():
+ """The control on the allowlist above. That test only fails when the probe grows a call,
+ so widening the frozenset silences it whether or not the harnesses were updated -- which
+ is the same silent failure by a shorter route. Check the harnesses themselves: every
+ tests/sh file that lifts _has_amd_rocm_gpu must lift each allowlisted callee too."""
+ sh_dir = Path(__file__).resolve().parents[3] / "tests" / "sh"
+ harnesses = [
+ path
+ for path in sorted(sh_dir.glob("*.sh"))
+ if "_has_amd_rocm_gpu" in path.read_text(encoding = "utf-8")
+ ]
+ # Guard the guard: a glob that matched nothing would pass this vacuously.
+ assert len(harnesses) >= 5, [p.name for p in harnesses]
+ for path in harnesses:
+ source = path.read_text(encoding = "utf-8")
+ # A harness that only stubs the probe (bad_arch_gate does this for some cases) still
+ # lifts it elsewhere; asking for the name anywhere in the file is the right grain.
+ for callee in sorted(_ROCM_PROBE_CALLEES_THE_SH_HARNESSES_LIFT):
+ assert callee in source, f"{path.name} lifts _has_amd_rocm_gpu but not {callee}"
+
+
def test_that_check_sees_a_helper_the_harnesses_would_not_have():
"""The control. Without it the test above could be passing because the scan matches
nothing at all, which is what a name-based scan usually does when it is wrong."""
diff --git a/studio/backend/tests/test_rocm_torch_explicit_request.py b/studio/backend/tests/test_rocm_torch_explicit_request.py
new file mode 100644
index 00000000000..db413444d12
--- /dev/null
+++ b/studio/backend/tests/test_rocm_torch_explicit_request.py
@@ -0,0 +1,2684 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""A mixed NVIDIA+AMD host must have SOME route to its AMD card.
+
+Every installer stops probing AMD the moment nvidia-smi lists a GPU. That is the
+right default -- CUDA is the better-supported stack and a false AMD positive would
+swap a working install -- but it left a mixed host with no route at all: chat can be
+pointed at Vulkan in Settings, and training reads whatever torch was installed, so
+the AMD card is invisible to it. The only bypass was an index pin, which is
+undocumented and names a wheel family rather than a preference (#10450).
+
+``UNSLOTH_FORCE_ROCM_TORCH=1`` is the request, mirroring ``UNSLOTH_FORCE_VULKAN``
+for the llama.cpp bundle and the ``backend == "rocm"`` re-probe in
+``install_llama_prebuilt._route_to_vulkan_prebuilt``.
+
+One torch install serves one vendor, so the request SWAPS the stack: these tests
+assert it is honoured, and that it changes nothing at all unless it is set.
+"""
+
+from __future__ import annotations
+
+import importlib.util
+import os
+import subprocess
+import sys
+import types
+from pathlib import Path
+
+import pytest
+
+_STACK = Path(__file__).resolve().parents[2] / "install_python_stack.py"
+
+
[email protected](scope = "module")
+def stack():
+ """install_python_stack imported by path: it is a top-level installer script, not a package
+ module, so the backend's own import path does not reach it."""
+ spec = importlib.util.spec_from_file_location("_unsloth_install_python_stack", _STACK)
+ module = importlib.util.module_from_spec(spec)
+ sys.modules[spec.name] = module
+ spec.loader.exec_module(module)
+ return module
+
+
+class _StoppedAtProbe(BaseException):
+ """Raised by the stubbed ROCm version probe so nothing past it runs."""
+
+
[email protected](autouse = True)
+def _no_real_install(stack, monkeypatch):
+ """No test in this file may reach an installation."""
+
+ def _refuse(*args, **kwargs):
+ pytest.fail(f"the test reached a real install: pip_install{args[:2]}")
+
+ monkeypatch.setattr(stack, "pip_install", _refuse)
+ monkeypatch.setattr(stack, "pip_install_try", _refuse)
+
+
+def _reaches_the_version_probe(stack, monkeypatch) -> bool:
+ """Whether _ensure_rocm_torch gets as far as the ROCm version probe."""
+ reached = {"ran": False}
+
+ def _probe(*args, **kwargs):
+ reached["ran"] = True
+ raise _StoppedAtProbe
+
+ monkeypatch.setattr(stack, "_detect_rocm_version", _probe)
+ try:
+ stack._ensure_rocm_torch()
+ except _StoppedAtProbe:
+ pass
+ return reached["ran"]
+
+
[email protected]("value", ["1", "true", "TRUE", "yes", "on"])
+def test_the_request_is_recognised(stack, monkeypatch, value):
+ monkeypatch.setenv("UNSLOTH_FORCE_ROCM_TORCH", value)
+ assert stack._rocm_torch_explicitly_requested() is True
+
+
[email protected]("value", ["0", "false", "no", "", " "])
+def test_anything_else_is_not_a_request(stack, monkeypatch, value):
+ """Including the empty string: an exported-but-empty variable is how a shell passes "unset",
+ and reading it as a request would swap a CUDA host's stack."""
+ monkeypatch.setenv("UNSLOTH_FORCE_ROCM_TORCH", value)
+ assert stack._rocm_torch_explicitly_requested() is False
+
+
+def test_unset_is_not_a_request(stack, monkeypatch):
+ monkeypatch.delenv("UNSLOTH_FORCE_ROCM_TORCH", raising = False)
+ assert stack._rocm_torch_explicitly_requested() is False
+
+
+def test_an_nvidia_host_still_hides_the_amd_card_by_default(stack, monkeypatch):
+ """The control. Every assertion below is only meaningful if the default holds."""
+ monkeypatch.delenv("UNSLOTH_FORCE_ROCM_TORCH", raising = False)
+ monkeypatch.setattr(stack, "_has_usable_nvidia_gpu", lambda: True)
+ assert stack._has_rocm_gpu() is False
+
+
+def _rocminfo(monkeypatch, stack, output: str):
+ """A host where rocminfo is on PATH and prints ``output``; amd-smi absent."""
+ monkeypatch.setattr(
+ stack.shutil, "which", lambda c: "/usr/bin/rocminfo" if c == "rocminfo" else None
+ )
+ monkeypatch.setattr(
+ stack.subprocess,
+ "run",
+ lambda *a, **k: types.SimpleNamespace(returncode = 0, stdout = output),
+ )
+ # The sysfs fallback must not answer for the shelled probes above.
+ monkeypatch.setattr(stack.os.path, "isdir", lambda p: False)
+
+
+def test_the_request_lets_the_amd_probe_run_on_a_mixed_host(stack, monkeypatch):
+ """Fails before the fix: _has_rocm_gpu returned False on any NVIDIA host before consulting a
+ single AMD probe, so nothing downstream could see the AMD card."""
+ monkeypatch.setenv("UNSLOTH_FORCE_ROCM_TORCH", "1")
+ monkeypatch.setattr(stack, "_has_usable_nvidia_gpu", lambda: True)
+ _rocminfo(monkeypatch, stack, " Name: gfx1201")
+ assert stack._has_rocm_gpu() is True
+
+
+def test_the_same_host_without_the_request_still_answers_no(stack, monkeypatch):
+ """The pair to the test above, on an identical host: the ONLY difference is the variable, so a
+ probe that started answering for some other reason fails here."""
+ monkeypatch.delenv("UNSLOTH_FORCE_ROCM_TORCH", raising = False)
+ monkeypatch.setattr(stack, "_has_usable_nvidia_gpu", lambda: True)
+ _rocminfo(monkeypatch, stack, " Name: gfx1201")
+ assert stack._has_rocm_gpu() is False
+
+
+def test_the_request_does_not_invent_a_card(stack, monkeypatch):
+ """The request relaxes which vendor wins, not whether there is a card to serve."""
+ monkeypatch.setenv("UNSLOTH_FORCE_ROCM_TORCH", "1")
+ monkeypatch.setattr(stack, "_has_usable_nvidia_gpu", lambda: True)
+ _rocminfo(monkeypatch, stack, " Name: gfx000") # CPU agent only
+ assert stack._has_rocm_gpu() is False
+
+
+def _shell_function(name: str) -> str:
+ """The text of one function from install.sh, by brace matching."""
+ install_sh = Path(__file__).resolve().parents[3] / "install.sh"
+ lines = install_sh.read_text(encoding = "utf-8").splitlines()
+ start = next(i for i, line in enumerate(lines) if line.startswith(f"{name}() {{"))
+ depth = 0
+ for end in range(start, len(lines)):
+ depth += lines[end].count("{") - lines[end].count("}")
+ if depth == 0:
+ return "\n".join(lines[start : end + 1])
+ raise AssertionError(f"unterminated function {name}")
+
+
+def _clean_env(extra: "dict | None" = None) -> dict:
+ """The runner's environment minus the two things that decide these cases by themselves."""
+ env = {
+ k: v
+ for k, v in os.environ.items()
+ if not k.endswith("VISIBLE_DEVICES") and k != "UNSLOTH_ROCM_GFX_ARCH"
+ }
+ env.update(extra or {})
+ return env
+
+
+def _bash(script: str, *, env: "dict | None" = None) -> str:
+ """Run a lifted script under bash and return its stdout, failing with its own stderr."""
+ out = subprocess.run(
+ ["bash", "-c", script], capture_output = True, text = True, env = env or _clean_env()
+ )
+ assert out.returncode == 0, out.stderr
+ return out.stdout.strip()
+
+
+# `command -v rocminfo` must not reach a real one on a ROCm build host, or these answers
+# would depend on the machine running the suite. Each harness names its own inventory.
+_rocminfo_stub = "rocminfo() { return 1; }"
+
+
+def _wheel_route_defs(*, arch_family: bool = True, rocminfo: str = _rocminfo_stub) -> "list[str]":
+ """Everything _amd_request_has_a_wheel_route consults, lifted from install.sh."""
+ return [
+ _shell_function("_amd_hardware_corroborated"),
+ *([_shell_function("_amd_arch_index_family_for_gfx")] if arch_family else []),
+ _shell_function("_amd_gfx_has_wheel_route"),
+ _shell_function("_amd_visible_masks_select_no_gpu"),
+ _shell_function("_amd_generic_tag_carries_gfx"),
+ _shell_function("_amd_mask_survivors"),
+ _shell_function("_amd_runtime_gfx_target"),
+ _shell_function("_amd_gfx_is_shadowing_integrated"),
+ _shell_function("_amd_prefer_discrete_gfx"),
+ _shell_function("_rocminfo_gpu_records"),
+ _shell_function("_amd_ordered_gfx_devices"),
+ "_ensure_rocm_probe_env() { :; }",
+ rocminfo,
+ _shell_function("_amd_request_has_a_wheel_route"),
+ ]
+
+
+def _fake_rocminfo(devices: "list[str]") -> str:
+ """A rocminfo whose output has the shape the real one has."""
+ blocks = [
+ "Agent 1",
+ "*******",
+ " Name: AMD Ryzen 9",
+ " Marketing Name: AMD Ryzen 9",
+ " Device Type: CPU",
+ ]
+ for _i, _gfx in enumerate(devices):
+ blocks += [
+ "*******",
+ f"Agent {_i + 2}",
+ "*******",
+ f" Name: {_gfx}",
+ " Marketing Name: AMD Radeon Graphics",
+ " Device Type: GPU",
+ " ISA Info:",
+ " ISA 1",
+ f" Name: amdgcn-amd-amdhsa--{_gfx}",
+ ]
+ body = "\n".join(blocks).replace("'", "")
+ return "rocminfo() { cat <<'ROCMINFO_EOF'\n" + body + "\nROCMINFO_EOF\n}"
+
+
+def _index_url(env: str, stubs: str) -> str:
+ """get_torch_index_url() under a stubbed host, returning the wheel index it picks."""
+ script = "\n".join(
+ [
+ # Every case here describes a Linux x86_64 host, so say so rather than letting
+ # the RUNNER answer. get_torch_index_url returns the cpu index outright for any
+ # other `uname -m`, which on the ubuntu-24.04-arm leg turned these into "the
+ # request selected cpu": two of them failed for a reason that is about the
+ # runner, and the cpu-index CONTROLS below passed for the same wrong reason.
+ 'uname() { case "${1:-}" in -m) echo x86_64 ;; *) echo Linux ;; esac; }',
+ _shell_function("_rocm_torch_explicitly_requested"),
+ stubs,
+ # The stubs above name this host's arch family, so it is not lifted here.
+ *_wheel_route_defs(arch_family = False),
+ _shell_function("get_torch_index_url"),
+ f"{env} get_torch_index_url",
+ ]
+ )
+ return subprocess.run(
+ ["bash", "-c", script],
+ capture_output = True,
+ text = True,
+ env = _clean_env(),
+ ).stdout
+
+
+# A bash function definition ends at its closing brace, so these are newline
+# separated: two on one line is a syntax error, which the first version of this
+# harness produced and read as "the flag did nothing".
+_MIXED_HOST = "\n".join(
+ [
+ "_has_usable_nvidia_gpu() { return 0; }",
+ "_has_amd_rocm_gpu() { return 0; }",
+ "_probe_amd_gfx_arch() { echo gfx1201; }",
+ "_amd_arch_index_family_for_gfx() { echo gfx120X-all; }",
+ "_kfd_gfx_targets() { echo gfx1201; }",
+ "_infer_linux_amd_gfx_arch() { echo gfx1201; }",
+ "_amd_sole_index_arch() { echo gfx1201; }",
+ "_amd_gpu_present_via_pci() { return 0; }",
+ "_detect_rocm_version_tag() { echo rocm7.0; }",
+ "_amd_agreed_index_family() { echo gfx120X-all; }",
+ "_rocm_sdk_install_hint() { echo ''; }",
+ "nvidia-smi() { echo 'CUDA Version: 13.0'; }",
+ ]
+)
+
+
+def test_the_shell_installer_selects_a_rocm_index_under_the_request(stack):
+ """Fails before the fix: get_torch_index_url set _nvidia_detected=1 and returned a cu* family
+ whatever the AMD probes said, so the request could not swap anything."""
+ out = _index_url("UNSLOTH_FORCE_ROCM_TORCH=1", _MIXED_HOST)
+ assert "rocm" in out or "gfx" in out, out
+
+
+def test_the_same_host_without_the_request_still_selects_cuda(stack):
+ """The control, identical but for the variable."""
+ out = _index_url("UNSLOTH_FORCE_ROCM_TORCH=0", _MIXED_HOST)
+ assert "rocm" not in out and "gfx" not in out, out
+
+
+_NO_AMD_CARD = "\n".join(
+ [
+ _MIXED_HOST.replace(
+ "_has_amd_rocm_gpu() { return 0; }", "_has_amd_rocm_gpu() { return 1; }"
+ ),
+ # Every source the route test can read, silenced together. Silencing only the
+ # ROCm runtime leaves the arch probes answering, which is the host the
+ # runtime-less reroute exists for rather than a machine with no AMD card.
+ "_probe_amd_gfx_arch() { :; }",
+ "_kfd_gfx_targets() { :; }",
+ "_infer_linux_amd_gfx_arch() { :; }",
+ "_amd_gpu_present_via_pci() { return 1; }",
+ ]
+)
+
+
+def test_the_request_does_not_select_rocm_without_an_amd_card(stack):
+ """A typo on a pure NVIDIA box must not turn a working CUDA machine into a ROCm or CPU one: the
+ AMD route test still has to pass."""
+ out = _index_url("UNSLOTH_FORCE_ROCM_TORCH=1", _NO_AMD_CARD)
+ assert "rocm" not in out and "gfx" not in out, out
+
+
+ "rocm_gpu, archs",
+ [(True, ["gfx1100"]), (False, ["gfx1151"])],
+ ids = ["runtime-visible", "runtime-less-but-inferable"],
+)
+def test_the_cuda_repair_stands_down_under_the_request(stack, monkeypatch, rocm_gpu, archs):
+ """A standalone `studio update` leaves _TORCH_BACKEND empty, so the CUDA repair runs
+ first, sees an NVIDIA GPU beside the requested HIP build, reads it as poisoning and
+ reinstalls the CUDA trio. The second row is the control: a runtime-less host whose arch
+ is inferable is one _ensure_rocm_torch will serve, so repairing CUDA here only to have
+ ROCm forced back is a reinstall cycle on every update."""
+ _mixed_host(stack, monkeypatch, rocm_gpu = rocm_gpu, archs = archs, display = True)
+ probed = {"ran": False}
+ monkeypatch.setattr(
+ stack,
+ "_probe_torch_runtime",
+ lambda *a, **k: (
+ probed.__setitem__("ran", True),
+ (True, True, "2.11.0+rocm7.0", True, False),
+ )[1],
+ )
+ stack._ensure_cuda_torch()
+ assert probed["ran"] is False
+
+
+def test_the_cuda_repair_still_runs_without_the_request(stack, monkeypatch):
+ """The control: the same host, no request, and the repair must still classify."""
+ monkeypatch.delenv("UNSLOTH_FORCE_ROCM_TORCH", raising = False)
+ monkeypatch.setattr(stack, "_TORCH_BACKEND", "")
+ monkeypatch.setattr(stack, "IS_MACOS", False)
+ monkeypatch.setattr(stack, "IS_WINDOWS", False)
+ monkeypatch.setattr(stack, "NO_TORCH", False)
+ monkeypatch.setattr(stack, "_has_usable_nvidia_gpu", lambda: True)
+ monkeypatch.delenv("UNSLOTH_ROCM_TORCH_INSTALLED", raising = False)
+ monkeypatch.delenv("CUDA_VISIBLE_DEVICES", raising = False)
+ probed = {"ran": False}
+ monkeypatch.setattr(
+ stack,
+ "_probe_torch_runtime",
+ lambda *a, **k: (
+ probed.__setitem__("ran", True),
+ (True, True, "2.11.0+rocm7.0", True, False),
+ )[1],
+ )
+ monkeypatch.setattr(stack, "pip_install", lambda *a, **k: None)
+ stack._ensure_cuda_torch()
+ assert probed["ran"] is True
+
+
+def test_windows_is_not_swapped_by_the_request_alone(stack, monkeypatch):
+ """Windows picks the wheel in install.ps1 and republishes an expected tag that
+ _ensure_expected_torch_flavor restores, so honouring the request only in Python would
+ install ROCm and have it reverted."""
+ monkeypatch.setenv("UNSLOTH_FORCE_ROCM_TORCH", "1")
+ monkeypatch.setattr(stack, "IS_WINDOWS", True)
+ monkeypatch.setattr(stack, "IS_MACOS", False)
+ monkeypatch.setattr(stack, "_TORCH_BACKEND", "")
+ monkeypatch.setattr(stack, "_has_usable_nvidia_gpu", lambda: True)
+ monkeypatch.setattr(stack, "_explicit_rocm_torch_index_url", lambda: None)
+ monkeypatch.setattr(stack, "_explicit_unknown_family_torch_index_url", lambda: None)
+ monkeypatch.delenv("UNSLOTH_ROCM_TORCH_INSTALLED", raising = False)
+ detected = {"ran": False}
+ monkeypatch.setattr(
+ stack,
+ "_detect_windows_gfx_arch",
+ lambda *a, **k: (detected.__setitem__("ran", True), "gfx1151")[1],
+ )
+ stack._ensure_rocm_torch()
+ assert detected["ran"] is False
+
+
+def _nvidia_wins(env: str, stubs: str) -> bool:
+ """_nvidia_gpu_wins_over_amd() under a stubbed host."""
+ script = "\n".join(
+ [
+ _shell_function("_rocm_torch_explicitly_requested"),
+ stubs,
+ *_wheel_route_defs(),
+ _shell_function("_nvidia_gpu_wins_over_amd"),
+ f"{env} _nvidia_gpu_wins_over_amd && echo NVIDIA || echo AMD",
+ ]
+ )
+ out = subprocess.run(["bash", "-c", script], capture_output = True, text = True, env = _clean_env())
+ assert out.stdout.strip() in ("NVIDIA", "AMD"), out
+ return out.stdout.strip() == "NVIDIA"
+
+
+# The probes the route test reads, not its answer: stubbing _amd_request_has_a_wheel_route
+# itself would make every case below assert about the stub.
+_NO_AMD = "\n".join(
+ [
+ "_has_amd_rocm_gpu() { return 1; }",
+ "_probe_amd_gfx_arch() { :; }",
+ "_kfd_gfx_targets() { :; }",
+ "_infer_linux_amd_gfx_arch() { :; }",
+ "_amd_gpu_present_via_pci() { return 1; }",
+ ]
+)
+
+_ROUTABLE_AMD = "\n".join(
+ [
+ # Kept so a predicate asking the OLD presence question still RUNS: an undefined
+ # function would make the unroutable case below fail for the wrong reason.
+ "_has_amd_rocm_gpu() { return 0; }",
+ "_probe_amd_gfx_arch() { echo gfx1201; }",
+ "_kfd_gfx_targets() { echo gfx1201; }",
+ "_infer_linux_amd_gfx_arch() { echo gfx1201; }",
+ "_amd_gpu_present_via_pci() { return 0; }",
+ ]
+)
+
+_PURE_NVIDIA = "\n".join(["_has_usable_nvidia_gpu() { return 0; }", _NO_AMD])
+
+_PURE_AMD = "\n".join(["_has_usable_nvidia_gpu() { return 1; }", _ROUTABLE_AMD])
+
+_MIXED = "\n".join(["_has_usable_nvidia_gpu() { return 0; }", _ROUTABLE_AMD])
+
+
+def test_the_reroute_predicate_yields_to_the_request_on_a_mixed_host():
+ """The #10450 gap the selector fix alone left open: an unreadable ROCm version makes
+ get_torch_index_url return the cpu index deliberately, for the per-arch reroute to rewrite."""
+ assert not _nvidia_wins("UNSLOTH_FORCE_ROCM_TORCH=1", _MIXED)
+
+
+def test_the_same_mixed_host_without_the_request_keeps_cuda():
+ """The control, differing only in the environment."""
+ assert _nvidia_wins("", _MIXED)
+
+
+def test_the_request_does_not_yield_a_pure_nvidia_host():
+ """It relaxes which vendor wins, not whether there is a card to serve."""
+ assert _nvidia_wins("UNSLOTH_FORCE_ROCM_TORCH=1", _PURE_NVIDIA)
+
+
+def test_a_host_with_no_nvidia_card_is_unchanged_either_way():
+ """The automatic path is untouched: with no NVIDIA GPU the answer never depended on the
+ request, and must not start to."""
+ assert not _nvidia_wins("", _PURE_AMD)
+ assert not _nvidia_wins("UNSLOTH_FORCE_ROCM_TORCH=1", _PURE_AMD)
+
+
+def test_the_reroutes_ask_the_request_aware_predicate():
+ """The predicate is only worth testing if the reroutes actually consult it."""
+ install_sh = Path(__file__).resolve().parents[3] / "install.sh"
+ body = install_sh.read_text(encoding = "utf-8")
+ marker = "_amd_no_rocm_version_reroute=false"
+ tail = body[body.index(marker) :]
+ assert "! _nvidia_gpu_wins_over_amd" in tail
+ assert "! _has_usable_nvidia_gpu" not in tail, (
+ "a per-arch reroute still asks the bare NVIDIA probe, so the explicit "
+ "request cannot reach the AMD wheels it selected"
+ )
+
+
+def test_the_cuda_repair_still_runs_when_the_request_finds_no_amd_card(stack, monkeypatch):
+ """Standing down needs a card to stand down FOR, which is the shell selector's rule too: a
+ request on a box with no AMD GPU selects nothing and falls through to CUDA. Leaving the
+ variable set on a host whose AMD card has since been removed must not silence this repair,
+ because _ensure_rocm_torch then finds no target either and a stale HIP build would be left
+ on a working NVIDIA GPU with nothing to fix it."""
+ monkeypatch.setenv("UNSLOTH_FORCE_ROCM_TORCH", "1")
+ monkeypatch.setattr(stack, "_TORCH_BACKEND", "")
+ monkeypatch.setattr(stack, "IS_MACOS", False)
+ monkeypatch.setattr(stack, "IS_WINDOWS", False)
+ monkeypatch.setattr(stack, "NO_TORCH", False)
+ monkeypatch.setattr(stack, "_has_usable_nvidia_gpu", lambda: True)
+ monkeypatch.setattr(stack, "_has_rocm_gpu", lambda: False)
+ monkeypatch.delenv("UNSLOTH_ROCM_TORCH_INSTALLED", raising = False)
+ monkeypatch.delenv("CUDA_VISIBLE_DEVICES", raising = False)
+ probed = {"ran": False}
+ monkeypatch.setattr(
+ stack,
+ "_probe_torch_runtime",
+ lambda *a, **k: (
+ probed.__setitem__("ran", True),
+ (True, True, "2.11.0+rocm7.0", True, False),
+ )[1],
+ )
+ monkeypatch.setattr(stack, "pip_install", lambda *a, **k: None)
+ stack._ensure_cuda_torch()
+ assert probed["ran"] is True
+
+
+def _cuda_restore_block() -> str:
+ """install.sh's request-downgrade guard, lifted by text."""
+ install_sh = Path(__file__).resolve().parents[3] / "install.sh"
+ lines = install_sh.read_text(encoding = "utf-8").splitlines()
+ start = next(
+ (
+ i
+ for i, line in enumerate(lines)
+ if line.startswith('if [ "$_torch_index_pinned" = false ]')
+ and "_rocm_torch_explicitly_requested" in "".join(lines[i : i + 2])
+ ),
+ None,
+ )
+ if start is None:
+ return ""
+ end = next(i for i in range(start, len(lines)) if lines[i] == "fi")
+ return "\n".join(lines[start : end + 1])
+
+
+def _index_after_the_guard(env: str, resolved: str, cuda_answer: str) -> str:
+ """TORCH_INDEX_URL after the guard, given what the AMD branch resolved."""
+ script = "\n".join(
+ [
+ _shell_function("_rocm_torch_explicitly_requested"),
+ # Taken from install.sh rather than restated: a classifier restated
+ # here would agree with itself, and the mirror case below is exactly
+ # where it must not.
+ _shell_function("_torch_index_url_leaf"),
+ _shell_function("_torch_index_url_is_rocm"),
+ "_has_usable_nvidia_gpu() { return 0; }",
+ f"get_torch_index_url() {{ echo {cuda_answer!r}; }}",
+ "_torch_index_pinned=false",
+ "SKIP_TORCH=false",
+ f"TORCH_INDEX_URL={resolved!r}",
+ f"{env} true",
+ _cuda_restore_block(),
+ 'printf "%s\\n" "$TORCH_INDEX_URL"',
+ ]
+ )
+ return _bash(script, env = _clean_env(dict([env.split("=", 1)] if "=" in env else [])))
+
+
+_CUDA = "https://download.pytorch.org/whl/cu130"
+
+
+def test_a_dead_end_amd_route_keeps_cuda():
+ """ROCm 5.x, an arch no index covers, or a non-x86_64 host all end the AMD branch at the cpu
+ index."""
+ assert (
+ _index_after_the_guard(
+ "UNSLOTH_FORCE_ROCM_TORCH=1",
+ resolved = "https://download.pytorch.org/whl/cpu",
+ cuda_answer = _CUDA,
+ )
+ == _CUDA
+ )
+
+
+def test_a_resolved_rocm_index_is_left_alone():
+ """The control that makes the test above mean something: when the request DID reach ROCm
+ wheels, the guard must not undo it."""
+ rocm = "https://repo.amd.com/rocm/whl/gfx1151"
+ assert (
+ _index_after_the_guard(
+ "UNSLOTH_FORCE_ROCM_TORCH=1",
+ resolved = rocm,
+ cuda_answer = _CUDA,
+ )
+ == rocm
+ )
+
+
+def test_a_deliberate_cpu_install_without_the_request_is_untouched():
+ """No request, no guard."""
+ cpu = "https://download.pytorch.org/whl/cpu"
+ assert _index_after_the_guard("", resolved = cpu, cuda_answer = _CUDA) == cpu
+
+
+def test_a_mirror_whose_base_path_says_rocm_is_not_a_rocm_index():
+ """The whole-URL form this guard first used matched a mirror's BASE path, so
+ https://mirror.local/rocm-cache/cpu read as ROCm wheels and the guard stood down -- leaving
+ CPU torch on a working NVIDIA card, which is the one outcome it exists to prevent."""
+ assert (
+ _index_after_the_guard(
+ "UNSLOTH_FORCE_ROCM_TORCH=1",
+ resolved = "https://mirror.local/rocm-cache/cpu",
+ cuda_answer = _CUDA,
+ )
+ == _CUDA
+ )
+
+
+def test_a_mirrored_rocm_index_is_still_left_alone():
+ """The control for the case above, one path segment apart: the same mirror serving an actual
+ ROCm leaf must keep it."""
+ rocm = "https://mirror.local/rocm-cache/rocm7.2"
+ assert (
+ _index_after_the_guard(
+ "UNSLOTH_FORCE_ROCM_TORCH=1",
+ resolved = rocm,
+ cuda_answer = _CUDA,
+ )
+ == rocm
+ )
+
+
+def test_the_radeon_repo_leaf_counts_as_a_rocm_index():
+ """repo.radeon.com ends in rocm-rel-X.Y, which _is_pip_rocm_family_leaf declines -- correctly,
+ since it answers the narrower "is this a pip ROCm FAMILY index"."""
+ radeon = "https://repo.radeon.com/rocm/manylinux/rocm-rel-6.4/"
+ assert (
+ _index_after_the_guard(
+ "UNSLOTH_FORCE_ROCM_TORCH=1",
+ resolved = radeon,
+ cuda_answer = _CUDA,
+ )
+ == radeon
+ )
+
+
+def test_a_miscomputing_arch_host_keeps_the_cuda_repair(stack, monkeypatch):
+ """A card is necessary and not sufficient."""
+ monkeypatch.setenv("UNSLOTH_FORCE_ROCM_TORCH", "1")
+ monkeypatch.setattr(stack, "_TORCH_BACKEND", "")
+ monkeypatch.setattr(stack, "IS_MACOS", False)
+ monkeypatch.setattr(stack, "IS_WINDOWS", False)
+ monkeypatch.setattr(stack, "NO_TORCH", False)
+ monkeypatch.setattr(stack, "_has_usable_nvidia_gpu", lambda: True)
+ monkeypatch.setattr(stack, "_has_rocm_gpu", lambda: True)
+ monkeypatch.setattr(stack, "_miscomputing_arch_host", lambda: True)
+ monkeypatch.delenv("UNSLOTH_ROCM_TORCH_INSTALLED", raising = False)
+ monkeypatch.delenv("CUDA_VISIBLE_DEVICES", raising = False)
+ probed = {"ran": False}
+ monkeypatch.setattr(
+ stack,
+ "_probe_torch_runtime",
+ lambda *a, **k: (
+ probed.__setitem__("ran", True),
+ (True, True, "2.11.0+rocm7.0", True, False),
+ )[1],
+ )
+ monkeypatch.setattr(stack, "pip_install", lambda *a, **k: None)
+ stack._ensure_cuda_torch()
+ assert probed["ran"] is True
+
+
+def test_the_rocm_installer_bails_on_the_same_arch(stack, monkeypatch):
+ """The other half of the pair, and the reason the test above is not arbitrary: the helper the
+ repair now consults is the one _ensure_rocm_torch itself returns on, so the two sides of the
+ installer cannot disagree about whether a route exists."""
+ monkeypatch.setattr(stack, "IS_MACOS", False)
+ monkeypatch.setattr(stack, "IS_WINDOWS", False)
+ monkeypatch.setattr(stack, "_miscomputing_arch_host", lambda: True)
+ monkeypatch.setattr(stack, "_explicit_rocm_torch_index_url", lambda: None)
+ monkeypatch.setattr(stack, "_explicit_unknown_family_torch_index_url", lambda: None)
+ monkeypatch.setattr(stack, "_TORCH_BACKEND", "")
+ monkeypatch.setattr(stack, "_has_rocm_gpu", lambda: True)
+ monkeypatch.delenv("UNSLOTH_ROCM_TORCH_INSTALLED", raising = False)
+ installed = {"ran": False}
+ monkeypatch.setattr(
+ stack,
+ "pip_install",
+ lambda *a, **k: installed.__setitem__("ran", True),
+ )
+ stack._ensure_rocm_torch()
+ assert installed["ran"] is False
+
+
+def _gpu_summary_branch(
+ resolved: str,
+ request: str,
+ pinned: bool = False,
+) -> str:
+ """Which arm the GPU detection summary takes, given the index the resolution left."""
+ install_sh = Path(__file__).resolve().parents[3] / "install.sh"
+ lines = install_sh.read_text(encoding = "utf-8").splitlines()
+ head = next(
+ (l for l in lines if l.startswith("if _has_usable_nvidia_gpu && ")),
+ "if _nvidia_gpu_wins_over_amd; then",
+ )
+ # The condition spans a continuation; take the rest of it verbatim.
+ if head.rstrip().endswith("\\"):
+ _at = lines.index(head)
+ while lines[_at].rstrip().endswith("\\"):
+ _at += 1
+ head = head.rstrip().rstrip("\\") + " " + lines[_at].strip()
+ tail = next(
+ (l for l in lines if l.startswith("elif _torch_index_url_is_rocm")),
+ 'elif case "$TORCH_INDEX_URL" in */rocm*|*/gfx*) true ;; *) false ;; esac; then',
+ )
+ script = "\n".join(
+ [
+ _shell_function("_rocm_torch_explicitly_requested"),
+ _shell_function("_torch_index_url_leaf"),
+ _shell_function("_torch_index_url_is_rocm"),
+ _shell_function("_nvidia_gpu_wins_over_amd"),
+ "_has_usable_nvidia_gpu() { return 0; }",
+ "_has_amd_rocm_gpu() { return 0; }",
+ f"TORCH_INDEX_URL={resolved!r}",
+ f"_torch_index_pinned={'true' if pinned else 'false'}",
+ # export, not a VAR=VAL command prefix: a prefix applies to that one command and
+ # leaves the variable unset for everything after it, so the request would never
+ # be in effect and the first case below would pass without the fix.
+ f"export {request}" if request else ":",
+ head,
+ "echo nvidia",
+ tail,
+ "echo amd",
+ "else",
+ "echo amd-cpu-fallback",
+ "fi",
+ ]
+ )
+ return _bash(script)
+
+
+def test_a_pinned_rocm_index_still_reports_the_nvidia_card():
+ """A pin names a wheel family, not a card. An NVIDIA host pinned to a ROCm index has no AMD
+ card to describe, so reading the resolved index alone hid the NVIDIA identity behind an
+ "AMD ROCm" banner for hardware the machine does not have."""
+ assert (
+ _gpu_summary_branch("https://download.pytorch.org/whl/rocm7.2", "", pinned = True) == "nvidia"
+ )
+
+
+def test_an_unpinned_rocm_resolution_still_reports_amd():
+ """The control: the exemption is the pin, not the index. The same URL reached by resolution
+ still reports the card its wheels are for."""
+ assert _gpu_summary_branch("https://download.pytorch.org/whl/rocm7.2", "") == "amd"
+
+
+def test_the_summary_reports_cuda_after_the_cuda_restore():
+ """The mixed host whose request found no ROCm route: the guard put the CUDA index back, but the
+ request is still set and the AMD card is still there, so the reroute predicate still answers
+ "AMD wins"."""
+ assert _gpu_summary_branch(_CUDA, "UNSLOTH_FORCE_ROCM_TORCH=1") == "nvidia"
+
+
+def test_the_summary_still_reports_amd_when_the_request_won():
+ """The control: when the request DID reach ROCm wheels, the summary must report the card those
+ wheels are for, which is what the predicate was introduced to fix."""
+ assert (
+ _gpu_summary_branch(
+ "https://repo.amd.com/rocm/whl/gfx1151",
+ "UNSLOTH_FORCE_ROCM_TORCH=1",
+ )
+ == "amd"
+ )
+
+
+def test_the_summary_on_a_pure_cuda_install_is_unchanged():
+ """And the control for the control: no request, CUDA wheels, NVIDIA reported, which is what
+ every automatic install on an NVIDIA host must keep seeing."""
+ assert _gpu_summary_branch(_CUDA, "") == "nvidia"
+
+
+def _route_shell(probe: str, inferred: str, pci_ok: bool) -> bool:
+ """install.sh's request route test, on a stubbed host."""
+ script = "\n".join(
+ [
+ f'_probe_amd_gfx_arch() {{ printf "%s\\n" {probe!r}; }}',
+ "_kfd_gfx_targets() { :; }",
+ f'_infer_linux_amd_gfx_arch() {{ [ -n {inferred!r} ] && printf "%s\\n" {inferred!r}; }}',
+ f"_amd_gpu_present_via_pci() {{ return {0 if pci_ok else 1}; }}",
+ *_wheel_route_defs(),
+ "_detect_rocm_version_tag() { printf '%s\\n' rocm7.2; }",
+ "_amd_request_has_a_wheel_route && echo yes || echo no",
+ ]
+ )
+ return _bash(script, env = _clean_env()) == "yes"
+
+
+ "probe, inferred, pci_ok, routes",
+ [
+ # gfx1010 (RDNA 1) is in neither the generic wheel nor any per-arch index.
+ ("gfx1010", "", True, False),
+ ("gfx1100", "", True, True),
+ # A Strix box with no rocminfo, amd-smi or KFD node is served per-arch wheels on a
+ # pure-AMD host by the reroute this predicate gates, so it is a route here too.
+ ("", "gfx1151", True, True),
+ ("", "gfx1030", False, False),
+ # gfx906's only route is the rocm6.3 legacy tag, which opens solely when it is the
+ # sole arch on the machine.
+ ("gfx906", "", True, True),
+ ("gfx906\ngfx1010", "", True, False),
+ ],
+ ids = [
+ "no-index",
+ "generic-wheel",
+ "inferable-no-runtime",
+ "declared-without-a-card",
+ "gfx906-sole-arch",
+ "gfx906-mixed-amd",
+ ],
+)
+def test_the_request_needs_a_card_whose_arch_an_index_serves(probe, inferred, pci_ok, routes):
+ """Presence is not the bar: an arch no index can serve must never depose a card that can,
+ and each row is paired with its control."""
+ assert _route_shell(probe, inferred, pci_ok = pci_ok) is routes
+
+
+def _mixed_host(stack, monkeypatch, **over):
+ """The host this feature exists for, stated by difference."""
+ o = dict(
+ request = True,
+ nvidia = True,
+ rocm_gpu = True,
+ archs = ["gfx1100"],
+ devices = None,
+ masked = None,
+ kfd = [],
+ wsl = False,
+ display = None,
+ miscomputing = False,
+ machine = "x86_64",
+ windows = False,
+ macos = False,
+ backend = "",
+ inferred = None,
+ rocm = (6, 4),
+ torch = "2.13.0+cu130",
+ declared = None,
+ pin = None,
+ pin_url = None,
+ masks = {},
+ )
+ o.update(over)
+ if o["display"] is None:
+ o["display"] = o["rocm_gpu"]
+ devices = o["archs"] if o["devices"] is None else o["devices"]
+
+ monkeypatch.delenv("UNSLOTH_FORCE_ROCM_TORCH", raising = False)
+ if o["request"]:
+ monkeypatch.setenv("UNSLOTH_FORCE_ROCM_TORCH", "1")
+ for _var in (
+ "UNSLOTH_ROCM_GFX_ARCH",
+ "UNSLOTH_TORCH_INDEX_URL",
+ "UNSLOTH_TORCH_INDEX_FAMILY",
+ "UNSLOTH_ROCM_TORCH_INSTALLED",
+ "HIP_VISIBLE_DEVICES",
+ "ROCR_VISIBLE_DEVICES",
+ "CUDA_VISIBLE_DEVICES",
+ ):
+ monkeypatch.delenv(_var, raising = False)
+ for _var, _val in (
+ ("UNSLOTH_ROCM_GFX_ARCH", o["declared"]),
+ ("UNSLOTH_TORCH_INDEX_FAMILY", o["pin"]),
+ ("UNSLOTH_TORCH_INDEX_URL", o["pin_url"]),
+ ):
+ if _val is not None:
+ monkeypatch.setenv(_var, _val)
+ for _var, _val in o["masks"].items():
+ monkeypatch.setenv(_var, _val)
+
+ monkeypatch.setattr(stack.platform, "machine", lambda: o["machine"])
+ monkeypatch.setattr(stack, "IS_WINDOWS", o["windows"])
+ monkeypatch.setattr(stack, "IS_MACOS", o["macos"])
+ monkeypatch.setattr(stack, "NO_TORCH", False)
+ monkeypatch.setattr(stack, "_TORCH_BACKEND", o["backend"])
+ monkeypatch.setattr(stack, "_has_usable_nvidia_gpu", lambda: o["nvidia"])
+ monkeypatch.setattr(stack, "_has_rocm_gpu", lambda: o["rocm_gpu"])
+ monkeypatch.setattr(stack, "_kfd_gfx_targets", lambda *a, **k: list(o["kfd"]))
+ monkeypatch.setattr(stack, "_is_wsl", lambda: o["wsl"])
+ monkeypatch.setattr(stack, "_linux_amd_display_device_present", lambda: o["display"])
+ monkeypatch.setattr(stack, "_miscomputing_arch_host", lambda: o["miscomputing"])
+ monkeypatch.setattr(stack, "_infer_linux_amd_gfx_arch", lambda *a, **k: o["inferred"])
+ monkeypatch.setattr(stack, "_detect_rocm_version", lambda *a, **k: o["rocm"])
+ monkeypatch.setattr(
+ stack,
+ "_physical_amd_gfx_archs",
+ lambda: list(o["archs"]) or ([o["inferred"]] if o["inferred"] else []),
+ )
+ # The masked list is what a probe that HONOURS the mask returns; ignore_visible_masks asks
+ # for the machine before any mask, which is the distinction the ROCr layer rests on.
+ monkeypatch.setattr(
+ stack,
+ "_detect_amd_gfx_codes",
+ lambda **k: list(
+ devices if o["masked"] is None or k.get("ignore_visible_masks") else o["masked"]
+ ),
+ )
+ monkeypatch.setattr(
+ stack,
+ "_probe_torch_runtime",
+ lambda *a, **k: (True, True, o["torch"], "rocm" in o["torch"], ""),
+ )
+ # The pin readers are deliberately NOT stubbed: the environment above is already
+ # cleared, so they answer None on their own, and a case that sets a pin needs the real
+ # reader to see it.
+ return o
+
+
+def _viable(
+ stack,
+ monkeypatch,
+ *,
+ corroborated: bool,
+ archs: list,
+ miscomputing = False,
+ machine: str = "x86_64",
+):
+ _mixed_host(
+ stack,
+ monkeypatch,
+ request = False,
+ rocm_gpu = corroborated,
+ archs = archs,
+ miscomputing = miscomputing,
+ machine = machine,
+ )
+ return stack._forced_rocm_route_is_viable()
+
+
+def test_a_host_with_no_published_rocm_wheels_is_not_a_viable_route(stack, monkeypatch):
+ """_ensure_rocm_torch() returns on its Linux-x86_64 gate without installing anything, because
+ ROCm wheels are published nowhere else."""
+ assert (
+ _viable(stack, monkeypatch, corroborated = True, archs = ["gfx1100"], machine = "aarch64")
+ is False
+ )
+
+
+def test_the_same_host_on_x86_64_is_still_viable(stack, monkeypatch):
+ """The control: same card, same corroboration, an architecture the wheels exist for."""
+ assert (
+ _viable(stack, monkeypatch, corroborated = True, archs = ["gfx1100"], machine = "x86_64") is True
+ )
+
+
+def test_the_python_route_test_matches_the_shell_one(stack, monkeypatch):
+ """Both halves of the installer have to answer the same question the same way, or a standalone
+ `studio update` undoes what install.sh chose."""
+ assert _viable(stack, monkeypatch, corroborated = True, archs = ["gfx1010"]) is False
+ assert _viable(stack, monkeypatch, corroborated = True, archs = ["gfx1100"]) is True
+ assert _viable(stack, monkeypatch, corroborated = False, archs = ["gfx1030"]) is False
+ assert (
+ _viable(
+ stack,
+ monkeypatch,
+ corroborated = True,
+ archs = ["gfx1033"],
+ miscomputing = True,
+ )
+ is False
+ )
+
+
+def test_an_unroutable_card_keeps_the_cuda_repair(stack, monkeypatch):
+ """The repair had stood down for any host with a visible AMD GPU. With an unusable HIP build
+ and a gfx1010 beside a working NVIDIA card, nothing then classified the stale build,
+ _ensure_rocm_torch found no wheel tag, and the NVIDIA GPU was left with no working torch at
+ all."""
+ monkeypatch.setenv("UNSLOTH_FORCE_ROCM_TORCH", "1")
+ monkeypatch.setattr(stack, "_TORCH_BACKEND", "")
+ monkeypatch.setattr(stack, "IS_MACOS", False)
+ monkeypatch.setattr(stack, "IS_WINDOWS", False)
+ monkeypatch.setattr(stack, "NO_TORCH", False)
+ monkeypatch.setattr(stack, "_has_usable_nvidia_gpu", lambda: True)
+ monkeypatch.setattr(stack, "_has_rocm_gpu", lambda: True)
+ monkeypatch.setattr(stack, "_kfd_gfx_targets", lambda: [])
+ monkeypatch.setattr(stack, "_is_wsl", lambda: False)
+ monkeypatch.setattr(stack, "_linux_amd_display_device_present", lambda: True)
+ monkeypatch.setattr(stack, "_physical_amd_gfx_archs", lambda: ["gfx1010"])
+ monkeypatch.setattr(stack, "_miscomputing_arch_host", lambda: False)
+ monkeypatch.delenv("UNSLOTH_ROCM_TORCH_INSTALLED", raising = False)
+ monkeypatch.delenv("CUDA_VISIBLE_DEVICES", raising = False)
+ probed = {"ran": False}
+ monkeypatch.setattr(
+ stack,
+ "_probe_torch_runtime",
+ lambda *a, **k: (
+ probed.__setitem__("ran", True),
+ (True, True, "2.11.0+rocm7.0", True, False),
+ )[1],
+ )
+ monkeypatch.setattr(stack, "pip_install", lambda *a, **k: None)
+ stack._ensure_cuda_torch()
+ assert probed["ran"] is True
+
+
+def test_a_declared_arch_does_not_force_rocm_over_a_working_cuda_stack(stack, monkeypatch):
+ """The request skips the NVIDIA precedence return, leaving the presence gate as the only thing
+ between a stale UNSLOTH_ROCM_GFX_ARCH and per-arch AMD wheels installed over CUDA on a host
+ with no AMD card."""
+ monkeypatch.setenv("UNSLOTH_FORCE_ROCM_TORCH", "1")
+ monkeypatch.setenv("UNSLOTH_ROCM_GFX_ARCH", "gfx1030")
+ monkeypatch.setattr(stack, "IS_WINDOWS", False)
+ monkeypatch.setattr(stack, "IS_MACOS", False)
+ monkeypatch.setattr(stack, "_TORCH_BACKEND", "")
+ monkeypatch.setattr(stack, "_explicit_rocm_torch_index_url", lambda: None)
+ monkeypatch.setattr(stack, "_explicit_unknown_family_torch_index_url", lambda: None)
+ monkeypatch.setattr(stack, "_miscomputing_arch_host", lambda: False)
+ monkeypatch.setattr(stack, "_has_usable_nvidia_gpu", lambda: True)
+ monkeypatch.setattr(stack, "_has_rocm_gpu", lambda: False)
+ monkeypatch.setattr(stack, "_kfd_gfx_targets", lambda: [])
+ monkeypatch.setattr(stack, "_is_wsl", lambda: False)
+ monkeypatch.setattr(stack, "_linux_amd_display_device_present", lambda: False)
+ monkeypatch.delenv("UNSLOTH_ROCM_TORCH_INSTALLED", raising = False)
+ installed = {"ran": False}
+ monkeypatch.setattr(
+ stack,
+ "pip_install",
+ lambda *a, **k: installed.__setitem__("ran", True),
+ )
+ stack._ensure_rocm_torch()
+ assert installed["ran"] is False
+
+
+def test_a_real_card_with_a_declared_arch_is_still_served(stack, monkeypatch):
+ """The control: declaring an arch is the documented routing hint for a runtime-less host, so it
+ must keep working once the silicon is corroborated."""
+ monkeypatch.setenv("UNSLOTH_FORCE_ROCM_TORCH", "1")
+ monkeypatch.setenv("UNSLOTH_ROCM_GFX_ARCH", "gfx1151")
+ monkeypatch.setattr(stack, "IS_WINDOWS", False)
+ monkeypatch.setattr(stack, "IS_MACOS", False)
+ monkeypatch.setattr(stack.platform, "machine", lambda: "x86_64")
+ monkeypatch.setattr(stack, "_TORCH_BACKEND", "")
+ monkeypatch.setattr(stack, "_explicit_rocm_torch_index_url", lambda: None)
+ monkeypatch.setattr(stack, "_explicit_unknown_family_torch_index_url", lambda: None)
+ monkeypatch.setattr(stack, "_miscomputing_arch_host", lambda: False)
+ monkeypatch.setattr(stack, "_has_usable_nvidia_gpu", lambda: True)
+ monkeypatch.setattr(stack, "_has_rocm_gpu", lambda: False)
+ monkeypatch.setattr(stack, "_kfd_gfx_targets", lambda: [])
+ monkeypatch.setattr(stack, "_is_wsl", lambda: False)
+ monkeypatch.setattr(stack, "_linux_amd_display_device_present", lambda: True)
+ monkeypatch.delenv("UNSLOTH_ROCM_TORCH_INSTALLED", raising = False)
+ monkeypatch.setattr(
+ stack,
+ "_probe_torch_runtime",
+ lambda *a, **k: (True, True, "2.11.0", False, True),
+ )
+ # Not a no-op pip_install stub: this route installs through pip_install_try, which is
+ # a real subprocess too, so stopping at the probe is what keeps the test hermetic.
+ assert _reaches_the_version_probe(stack, monkeypatch) is True
+
+
+# A card ROCm sees perfectly well and no index can serve. The two questions disagree here,
+# which is the whole point: the old predicate yields the NVIDIA GPU, the new one does not.
+_UNROUTABLE_AMD = "\n".join(
+ [
+ "_has_usable_nvidia_gpu() { return 0; }",
+ "_has_amd_rocm_gpu() { return 0; }",
+ "_probe_amd_gfx_arch() { echo gfx1010; }",
+ "_kfd_gfx_targets() { echo gfx1010; }",
+ "_infer_linux_amd_gfx_arch() { echo gfx1010; }",
+ "_amd_gpu_present_via_pci() { return 0; }",
+ ]
+)
+
+
+def test_the_reroute_predicate_keeps_cuda_for_an_unroutable_card():
+ """The call site, not the helper: both per-arch reroutes consult this predicate, and on
+ presence alone a gfx1010 cleared the way for AMD wheels that carry no kernels for it."""
+ assert _nvidia_wins("UNSLOTH_FORCE_ROCM_TORCH=1", _UNROUTABLE_AMD)
+
+
+def test_the_reroute_predicate_still_yields_for_a_routable_one():
+ """The control: same host, routable arch, and the request must still win."""
+ assert not _nvidia_wins("UNSLOTH_FORCE_ROCM_TORCH=1", _MIXED)
+
+
+def _shell_request_flag(value: "str | None") -> bool:
+ """install.sh's own request test, run verbatim, for one value of the variable."""
+ # Through the environment rather than the script text: a tab quoted into a bash
+ # single-quoted string arrives as a literal backslash-t, so a harness that inlined the
+ # value would test a different string than the one named.
+ env = dict(os.environ)
+ env.pop("UNSLOTH_FORCE_ROCM_TORCH", None)
+ if value is not None:
+ env["UNSLOTH_FORCE_ROCM_TORCH"] = value
+ script = "\n".join(
+ [
+ _shell_function("_rocm_torch_explicitly_requested"),
+ "_rocm_torch_explicitly_requested && echo yes || echo no",
+ ]
+ )
+ return _bash(script, env = env) == "yes"
+
+
[email protected]("value", [" true ", "\ttrue", "1 ", " ON"])
+def test_the_shell_flag_is_trimmed_like_its_python_twin(value):
+ """install_python_stack.py reads this variable through .strip(); install.sh did not."""
+ assert _shell_request_flag(value) is True
+
+
[email protected]("value", [" ", "", "tru e", None])
+def test_trimming_does_not_widen_what_counts_as_a_request(value):
+ """The control."""
+ assert _shell_request_flag(value) is False
+
+
+def _route_shell_masked(
+ physical: "list[str]",
+ rocm_tag: str = "rocm7.2",
+ devices: "list[str] | None" = None,
+ kfd: "list[str] | None" = None,
+ **mask: str,
+) -> bool:
+ """The request route test on a masked host, with the mask resolution left live."""
+ return (
+ _bash(
+ _route_script(physical, rocm_tag, devices, kfd, tail = _ROUTE_YES_NO),
+ env = _route_env(mask),
+ )
+ == "yes"
+ )
+
+
+_ROUTE_YES_NO = "_amd_request_has_a_wheel_route && echo yes || echo no"
+# The published target, not the verdict: on a host whose integrated GPU enumerates first the
+# interesting failure is not whether ROCm wins but which card it wins FOR.
+_ROUTE_TARGET = (
+ '_amd_request_has_a_wheel_route >/dev/null 2>&1; printf "%s" "${_AMD_REQUEST_TARGET_GFX:-}"'
+)
+
+
+def _route_env(mask: dict) -> dict:
+ return _clean_env(mask)
+
+
+def _route_target_masked(
+ physical: "list[str]",
+ rocm_tag: str = "rocm7.2",
+ devices: "list[str] | None" = None,
+ kfd: "list[str] | None" = None,
+ **mask: str,
+) -> str:
+ """The gfx the request would install FOR, or "" when it declines."""
+ return _bash(
+ _route_script(physical, rocm_tag, devices, kfd, tail = _ROUTE_TARGET),
+ env = _route_env(mask),
+ )
+
+
+def _route_script(
+ physical: "list[str]",
+ rocm_tag: str,
+ devices: "list[str] | None",
+ kfd: "list[str] | None",
+ *,
+ tail: str,
+) -> str:
+ emit = 'printf "%s\\n" ' + " ".join(repr(a) for a in physical) if physical else ":"
+ # KFD node order is what the mask ordinals index, and once rocminfo says nothing it is
+ # the only ordered source left, so a test about ordering cannot leave it stubbed empty.
+ kfd_emit = 'printf "%s\\n" ' + " ".join(repr(a) for a in kfd) if kfd else ":"
+ script = "\n".join(
+ [
+ f"_probe_amd_gfx_arch() {{ {emit}; }}",
+ f"_kfd_gfx_targets() {{ {kfd_emit}; }}",
+ "_infer_linux_amd_gfx_arch() { :; }",
+ "_amd_gpu_present_via_pci() { return 0; }",
+ *_wheel_route_defs(rocminfo = _fake_rocminfo(physical if devices is None else devices)),
+ f"_detect_rocm_version_tag() {{ printf '%s\\n' {rocm_tag!r}; }}",
+ tail,
+ ]
+ )
+ return script
+
+
+ "physical, mask, routes",
+ [
+ # No probe can answer this: rocminfo is filtered only by ROCR_VISIBLE_DEVICES and
+ # amd-smi by neither, so a probe run under a HIP mask returns the whole machine and
+ # the route test answered yes on the card the mask had just hidden.
+ (["gfx1100", "gfx1010"], "1", False),
+ (["gfx1100", "gfx1010"], "0", True),
+ # Failing closed is about what cannot be resolved, not the whole variable: the
+ # runtime reads left to right and stops at the first entry naming no device, so the
+ # survivors are the PREFIX.
+ (["gfx1100", "gfx1010"], "0,GPU-abcdef", True),
+ (["gfx1100", "gfx1010"], "1,GPU-abcdef", False),
+ # gfx906's only route is the rocm6.3 legacy tag, and the reroute granting it inspects
+ # the UNMASKED inventory, refusing when a second AMD arch is present.
+ (["gfx906", "gfx1100"], "0", False),
+ # HIP remaps runtime ordinal 0 onto the head of the mask.
+ (["gfx1100", "gfx1010"], "1,0", False),
+ (["gfx1100", "gfx1010"], "0,1", True),
+ ],
+ ids = [
+ "selected-unroutable",
+ "selected-routable",
+ "prefix-resolves",
+ "prefix-unroutable",
+ "gfx906-not-sole-arch",
+ "mask-head-second-card",
+ "mask-head-first-card",
+ ],
+)
+def test_the_shell_route_judges_the_card_the_mask_selects(physical, mask, routes):
+ """install.sh's half of the rule above, on the same host shapes."""
+ assert _route_shell_masked(physical, HIP_VISIBLE_DEVICES = mask) is routes
+
+
+def test_the_flat_probes_duplicate_rows_do_not_shift_the_ordinals():
+ """rocminfo names each agent's target twice, so the flat probe returns four rows for a two-card
+ host."""
+ assert (
+ _route_shell_masked(
+ ["gfx1100", "gfx1100", "gfx1010", "gfx1010"],
+ devices = ["gfx1100", "gfx1010"],
+ HIP_VISIBLE_DEVICES = "1",
+ )
+ is False
+ )
+
+
+def test_the_same_duplicated_host_still_yields_for_the_routable_card():
+ """The control: same duplicated inventory, mask pointing at the card that does have a route."""
+ assert (
+ _route_shell_masked(
+ ["gfx1100", "gfx1100", "gfx1010", "gfx1010"],
+ devices = ["gfx1100", "gfx1010"],
+ HIP_VISIBLE_DEVICES = "0",
+ )
+ is True
+ )
+
+
+def test_a_mask_past_the_first_device_fails_closed_with_no_per_device_list():
+ """Only rocminfo enumerates in the order the masks index -- amd-smi orders by KFD discovery,
+ which is why _amd_smi_hip_order exists -- so a host without it cannot say which card an
+ ordinal names."""
+ assert _route_shell_masked(["gfx1010", "gfx1100"], devices = [], HIP_VISIBLE_DEVICES = "1") is False
+
+
+def test_one_arch_is_still_answerable_without_a_per_device_list():
+ """The control, and the reason this is not simply "no rocminfo, no route": with a single arch
+ on the whole host every ordinal names it, so no enumeration order can change the answer and
+ the flat inventory is enough."""
+ assert _route_shell_masked(["gfx1100", "gfx1100"], devices = [], HIP_VISIBLE_DEVICES = "0") is True
+
+
+def test_discovery_order_does_not_answer_for_runtime_device_zero():
+ """With no rocminfo the flat inventory came from amd-smi, which enumerates in KFD DISCOVERY
+ order -- the whole reason _amd_smi_hip_order exists -- so its first row can describe a
+ different GPU from the one HIP hands torch at ordinal 0. Reading it approved the swap off a
+ routable first row while the gfx1010 that actually runs has kernels in no wheel."""
+ assert _route_shell_masked(["gfx1100", "gfx1010"], devices = [], HIP_VISIBLE_DEVICES = "0") is False
+
+
+def test_the_kernel_topology_supplies_the_order_amd_smi_cannot():
+ """KFD node order IS the order HIP and ROCr index, so the repair is to read it rather than to
+ decline -- the same swap _runtime_gfx_target makes in install_python_stack.py."""
+ assert (
+ _route_shell_masked(
+ ["gfx1100", "gfx1010"],
+ devices = [],
+ kfd = ["gfx1010", "gfx1100"],
+ HIP_VISIBLE_DEVICES = "0",
+ )
+ is False
+ )
+
+
+def test_the_same_kernel_topology_still_yields_for_the_routable_card():
+ """The control: same host, same KFD order, the mask pointing at the card with a route."""
+ assert (
+ _route_shell_masked(
+ ["gfx1100", "gfx1010"],
+ devices = [],
+ kfd = ["gfx1010", "gfx1100"],
+ HIP_VISIBLE_DEVICES = "1",
+ )
+ is True
+ )
+
+
+ "mask, routes",
+ [("7", False), ("1", True)],
+ ids = ["mask-resolves-nothing", "mask-names-a-device"],
+)
+def test_a_declared_arch_does_not_answer_over_the_mask(mask, routes):
+ """UNSLOTH_ROCM_GFX_ARCH takes an early return above the mask resolution, so a declared
+ arch beside HIP_VISIBLE_DEVICES=7 approved the swap where HIP exposes no device at all.
+ The control keeps the escape hatch: the same declaration over a mask naming a device this
+ host has must still depose CUDA."""
+ assert (
+ _route_shell_masked(
+ ["gfx1100", "gfx1010"],
+ devices = ["gfx1100", "gfx1010"],
+ UNSLOTH_ROCM_GFX_ARCH = "gfx1100",
+ HIP_VISIBLE_DEVICES = mask,
+ )
+ is routes
+ )
+
+
+def test_the_rocr_layer_is_resolved_the_same_way():
+ """rocminfo does honour this one, but the route test must not depend on which probe answered:
+ amd-smi honours neither mask, so the resolution has to be its own."""
+ assert _route_shell_masked(["gfx1100", "gfx1010"], ROCR_VISIBLE_DEVICES = "1") is False
+
+
+ "mask",
+ [
+ {"HIP_VISIBLE_DEVICES": ""},
+ {"HIP_VISIBLE_DEVICES": "-1"},
+ {"ROCR_VISIBLE_DEVICES": "-1"},
+ {"ROCR_VISIBLE_DEVICES": "-1", "HIP_VISIBLE_DEVICES": "0"},
+ {"CUDA_VISIBLE_DEVICES": ""},
+ ],
+)
+def test_a_mask_that_exposes_no_device_leaves_cuda_alone(mask):
+ """A deliberate no-GPU selection, not a detection miss: there is nothing for the request to
+ swap TO, so falling back to the physical inventory here installed ROCm wheels on a host
+ whose runtime exposes no AMD GPU at all."""
+ assert _route_shell_masked(["gfx1100"], **mask) is False
+
+
+def test_an_unset_mask_is_not_a_no_gpu_selection():
+ """Its control, and the reason ${VAR+x} rather than ${VAR:-}: an unset mask hides nothing, and
+ reading it like a set-but-empty one would keep CUDA on every host."""
+ assert _route_shell_masked(["gfx1100"]) is True
+
+
[email protected]("value", ["GPU-abcdef", "9"])
+def test_a_mask_this_cannot_resolve_fails_closed(value):
+ """ROCr accepts UUIDs, and an ordinal can name no device at all."""
+ assert _route_shell_masked(["gfx1100", "gfx1010"], HIP_VISIBLE_DEVICES = value) is False
+
+
+def test_a_masked_gfx906_alone_is_still_routable():
+ """The control: on a host where gfx906 IS the sole arch, the legacy tag opens and a mask naming
+ it must not change that."""
+ assert _route_shell_masked(["gfx906"], HIP_VISIBLE_DEVICES = "0") is True
+
+
+def test_an_explicit_cuda_pin_outranks_the_request(stack, monkeypatch):
+ """_rocm_torch_explicitly_requested's docstring promises an index pin still outranks the
+ request, and _rocm_pin is the ROCm pin, so a CUDA pin leaves it None and the request skipped
+ the NVIDIA-precedence return."""
+ monkeypatch.setenv("UNSLOTH_FORCE_ROCM_TORCH", "1")
+ monkeypatch.setenv("UNSLOTH_TORCH_INDEX_FAMILY", "cu128")
+ monkeypatch.setattr(stack, "_TORCH_BACKEND", "")
+ monkeypatch.setattr(stack, "IS_WINDOWS", False)
+ monkeypatch.setattr(stack, "IS_MACOS", False)
+ monkeypatch.setattr(stack, "NO_TORCH", False)
+ monkeypatch.setattr(stack, "_has_usable_nvidia_gpu", lambda: True)
+ monkeypatch.setattr(stack, "_miscomputing_arch_host", lambda: False)
+ monkeypatch.delenv("UNSLOTH_ROCM_TORCH_INSTALLED", raising = False)
+ reached = {"ran": False}
+ monkeypatch.setattr(
+ stack,
+ "_has_rocm_gpu",
+ lambda: (reached.__setitem__("ran", True), True)[1],
+ )
+ stack._ensure_rocm_torch()
+ assert reached["ran"] is False
+
+
+def test_a_rocm_pin_is_still_honoured_over_an_nvidia_card(stack, monkeypatch):
+ """The control: the pin that names ROCm wheels must still win, or the fix would read as "any
+ pin keeps CUDA" and break the headless/CI case _rocm_pin exists for."""
+ monkeypatch.setenv("UNSLOTH_FORCE_ROCM_TORCH", "1")
+ monkeypatch.setenv("UNSLOTH_TORCH_INDEX_FAMILY", "rocm7.0")
+ monkeypatch.setattr(stack, "_TORCH_BACKEND", "")
+ monkeypatch.setattr(stack, "IS_WINDOWS", False)
+ monkeypatch.setattr(stack, "IS_MACOS", False)
+ monkeypatch.setattr(stack.platform, "machine", lambda: "x86_64")
+ monkeypatch.setattr(stack, "NO_TORCH", False)
+ monkeypatch.setattr(stack, "_has_usable_nvidia_gpu", lambda: True)
+ monkeypatch.delenv("UNSLOTH_ROCM_TORCH_INSTALLED", raising = False)
+ # A ROCm pin skips the whole vendor-precedence block, so reaching the version probe
+ # is what "the pin still won" looks like from outside.
+ assert _reaches_the_version_probe(stack, monkeypatch) is True
+
+
+def _viable_masked(
+ stack,
+ monkeypatch,
+ *,
+ devices: list,
+ masked: "list | None" = None,
+ inferred: "str | None" = None,
+ rocm: "tuple | None" = (6, 4),
+ declared: "str | None" = None,
+ **mask: str,
+) -> bool:
+ """_forced_rocm_route_is_viable on a masked host, with the resolution left live."""
+ _mixed_host(
+ stack,
+ monkeypatch,
+ request = False,
+ archs = devices,
+ devices = devices,
+ masked = masked,
+ inferred = inferred,
+ rocm = rocm,
+ declared = declared,
+ masks = mask,
+ )
+ return stack._forced_rocm_route_is_viable()
+
+
+ "devices, mask, viable",
+ [
+ # Which CARD the mask selects, not whether some arch on the bus has a route.
+ (["gfx1100", "gfx1010"], "1", False),
+ (["gfx1100", "gfx1010"], "0", True),
+ # _pick_visible_index folds an out-of-range ordinal and a UUID alike onto GPU 0,
+ # which is right for arch SELECTION and wrong for replacing a working stack: HIP
+ # exposes no device for either, so both fail closed. install.sh agrees on these.
+ (["gfx1100", "gfx1010"], "7", False),
+ (["gfx1100", "gfx1010"], "GPU-9d4f00a1", False),
+ (["gfx1010", "gfx1100"], "1", True),
+ # gfx1033 is refused outright (studio/ROCM_RDNA2_APU.md), keyed on the SELECTED
+ # card: a gfx1033 anywhere would be the host-wide reading this replaces.
+ (["gfx1033", "gfx1100"], "0", False),
+ (["gfx1033", "gfx1100"], "1", True),
+ ],
+ ids = [
+ "selected-unroutable",
+ "selected-routable",
+ "past-last-device",
+ "uuid",
+ "in-range-second",
+ "selected-miscomputing",
+ "healthy-sibling",
+ ],
+)
+def test_the_python_route_judges_the_card_the_mask_selects(
+ stack, monkeypatch, devices, mask, viable
+):
+ """Each row is paired with the same host masked the other way, so a fix that answered
+ "never viable" fails here."""
+ assert _viable_masked(stack, monkeypatch, devices = devices, HIP_VISIBLE_DEVICES = mask) is viable
+
+
+def test_a_mask_exposing_no_device_is_not_a_viable_route(stack, monkeypatch):
+ """The Python half of the same rule the shell now applies: a no-GPU mask is a deliberate
+ selection, so there is nothing to swap to."""
+ assert (
+ _viable_masked(stack, monkeypatch, devices = ["gfx1100"], HIP_VISIBLE_DEVICES = "-1") is False
+ )
+
+
+def test_an_unmasked_host_is_judged_exactly_as_before(stack, monkeypatch):
+ """And the control for all three: with no mask set the answer is the inventory's, which is what
+ this function did before and what the shell twin still does."""
+ assert _viable_masked(stack, monkeypatch, devices = ["gfx1100", "gfx1010"]) is True
+ assert _viable_masked(stack, monkeypatch, devices = ["gfx1010"]) is False
+
+
+def _rocm_repair_reached(
+ stack,
+ monkeypatch,
+ *,
+ archs: list,
+ viable: bool,
+ pin: "str | None" = None,
+ pin_url: "str | None" = None,
+) -> bool:
+ """Whether _ensure_rocm_torch gets past its vendor-precedence gates on a mixed host."""
+ _mixed_host(stack, monkeypatch, archs = archs, pin = pin, pin_url = pin_url)
+ monkeypatch.setattr(stack, "_forced_rocm_route_is_viable", lambda: viable)
+ return _reaches_the_version_probe(stack, monkeypatch)
+
+
+def test_the_request_needs_a_viable_route_to_bypass_nvidia(stack, monkeypatch):
+ """Presence was the bar, and presence is not the bar this feature states."""
+ assert _rocm_repair_reached(stack, monkeypatch, archs = ["gfx1010"], viable = False) is False
+
+
+def test_a_viable_route_still_lets_the_request_through(stack, monkeypatch):
+ """The control: the same mixed host with a card an index can serve."""
+ assert _rocm_repair_reached(stack, monkeypatch, archs = ["gfx1100"], viable = True) is True
+
+
[email protected]("family", ["cpu", "cu128"])
+def test_an_explicit_pin_of_another_family_outranks_the_request(stack, monkeypatch, family):
+ """A /cpu pin had the multi-GB ROCm stack installed here and then undone by _ensure_cpu_torch,
+ on every update."""
+ assert (
+ _rocm_repair_reached(stack, monkeypatch, archs = ["gfx1100"], viable = True, pin = family)
+ is False
+ )
+
+
+ "pin_url, reached",
+ [("https://download.pytorch.org/whl/rocm6.4", True), ("https://mirror.example/simple", False)],
+ ids = ["rocm-pin-agrees", "unknown-family-pin"],
+)
+def test_a_pin_of_another_family_is_the_only_pin_that_outranks_the_request(
+ stack, monkeypatch, pin_url, reached
+):
+ """A pin naming ROCm agrees with the request and must not be swept up by the CPU/CUDA
+ check. An unknown leaf was applied verbatim at install time, so _ensure_rocm_torch
+ declines to judge it and returns before any vendor gate -- pre-existing, not added here."""
+ assert (
+ _rocm_repair_reached(stack, monkeypatch, archs = ["gfx1100"], viable = True, pin_url = pin_url)
+ is reached
+ )
+
+
+def test_the_two_mask_layers_compose_in_the_order_the_runtime_reads_them():
+ """ROCr filters the physical list and renumbers the survivors; the HIP layer then indexes
+ those."""
+ assert (
+ _route_shell_masked(
+ ["gfx1100", "gfx1010"], ROCR_VISIBLE_DEVICES = "1", HIP_VISIBLE_DEVICES = "0"
+ )
+ is False
+ )
+
+
+def test_the_same_two_layers_pointing_at_the_routable_card_still_depose_cuda():
+ """The control: same host, same stacking, ROCr selecting the other card."""
+ assert (
+ _route_shell_masked(
+ ["gfx1100", "gfx1010"], ROCR_VISIBLE_DEVICES = "0", HIP_VISIBLE_DEVICES = "0"
+ )
+ is True
+ )
+
+
+def test_two_cards_of_one_arch_do_not_shift_the_ordinals():
+ """An ordinal names a DEVICE, not an architecture."""
+ assert _route_shell_masked(["gfx1010", "gfx1010", "gfx1100"], HIP_VISIBLE_DEVICES = "1") is False
+
+
+def test_the_same_host_selecting_past_the_pair_is_still_routable():
+ """The control for the cardinality rule: ordinal 2 on that host really is the gfx1100, and a
+ resolver that simply refused every repeated arch would fail this."""
+ assert _route_shell_masked(["gfx1010", "gfx1010", "gfx1100"], HIP_VISIBLE_DEVICES = "2") is True
+
+
+def test_a_declared_arch_decides_before_the_inventory():
+ """UNSLOTH_ROCM_GFX_ARCH is what _probe_amd_gfx_arch's default mode returns and what
+ get_torch_index_url installs from, so judging the physical inventory instead can pass on a
+ routable sibling while the wheels are chosen for the declared card."""
+ assert _route_shell_masked(["gfx1100"], UNSLOTH_ROCM_GFX_ARCH = "gfx1010") is False
+
+
+def test_a_declared_arch_that_is_routable_still_deposes_cuda():
+ """The control, and the #7301 host: the declaration exists to serve a runtime-less card, so a
+ routable one must still win even when the inventory disagrees with it."""
+ assert _route_shell_masked(["gfx1010"], UNSLOTH_ROCM_GFX_ARCH = "gfx1100") is True
+
+
+def test_a_generic_only_arch_is_judged_against_the_tag_this_host_resolves():
+ """gfx950 has no per-arch index, so its only route is the generic wheel -- and the tag comes
+ from the installed ROCm version, which a stale /opt/rocm beside a current amdgpu can leave
+ older than the card."""
+ assert _route_shell_masked(["gfx950"], rocm_tag = "rocm6.4") is False
+
+
+def test_the_same_arch_on_a_tag_that_carries_it_is_routable():
+ """The control: the identical host one tag later, where the wheel does carry gfx950."""
+ assert _route_shell_masked(["gfx950"], rocm_tag = "rocm7.0") is True
+
+
+def test_an_arch_with_its_own_index_is_not_held_to_the_generic_floor():
+ """The boundary: gfx1200 predates the rocm6.0 generic wheel too, but it has a per-arch AMD
+ index that carries it, and the reroute is what serves it."""
+ assert _route_shell_masked(["gfx1200"], rocm_tag = "rocm6.0") is True
+
+
+def _corroborated_on_wsl(
+ stack,
+ monkeypatch,
+ *,
+ nvidia: bool,
+ rocm_sees_a_gpu: bool = False,
+) -> bool:
+ """_amd_hardware_is_corroborated() on a WSL box, with the WSL evidence present."""
+ monkeypatch.setattr(stack, "IS_WINDOWS", False)
+ monkeypatch.setattr(stack, "IS_MACOS", False)
+ monkeypatch.setattr(stack, "_has_rocm_gpu", lambda: rocm_sees_a_gpu)
+ monkeypatch.setattr(stack, "_kfd_gfx_targets", lambda: [])
+ monkeypatch.setattr(stack, "_is_wsl", lambda: True)
+ monkeypatch.setattr(stack, "_wsl_rocm_runtime_present", lambda: True)
+ monkeypatch.setattr(stack, "_has_usable_nvidia_gpu", lambda: nvidia)
+ return stack._amd_hardware_is_corroborated()
+
+
+def test_leftover_wsl_rocm_files_are_not_a_card_beside_an_nvidia_gpu(stack, monkeypatch):
+ """/dev/dxg is the generic WSL GPU bridge an NVIDIA passthrough creates too, and librocdxg is a
+ file an uninstalled ROCm leaves behind, so neither names a vendor."""
+ assert _corroborated_on_wsl(stack, monkeypatch, nvidia = True) is False
+
+
+def test_the_same_wsl_box_with_no_nvidia_card_still_counts(stack, monkeypatch):
+ """The control, and why this is not simply "WSL never corroborates": with no NVIDIA GPU there
+ is no working stack to lose, so the leftover reading still beats declining and a WSL AMD
+ host whose runtime cannot answer keeps its route."""
+ assert _corroborated_on_wsl(stack, monkeypatch, nvidia = False) is True
+
+
+def test_a_wsl_box_whose_runtime_names_an_agent_counts_beside_nvidia(stack, monkeypatch):
+ """The other control: a real AMD adapter under WSL is still corroborated with an NVIDIA card
+ present, because the ROCm runtime enumerates it rather than a leftover file standing in for
+ it."""
+ assert _corroborated_on_wsl(stack, monkeypatch, nvidia = True, rocm_sees_a_gpu = True) is True
+
+
+def _needs_repair_passes_the_nvidia_gate(stack, monkeypatch, *, viable: bool) -> bool:
+ """Whether _amd_torch_needs_dependency_pass() gets past its NVIDIA fast path."""
+
+ class _Reached(Exception):
+ pass
+
+ monkeypatch.setattr(stack, "NO_TORCH", False)
+ monkeypatch.setattr(stack, "IS_LINUX", True)
+ monkeypatch.setattr(stack.platform, "machine", lambda: "x86_64")
+ monkeypatch.setattr(stack, "_TORCH_BACKEND", "rocm")
+ monkeypatch.setattr(stack, "_explicit_rocm_torch_index_url", lambda: None)
+ monkeypatch.setattr(stack, "_explicit_torch_index_url", lambda: None)
+ monkeypatch.setattr(stack, "_has_usable_nvidia_gpu", lambda: True)
+ monkeypatch.setattr(stack, "_rocm_torch_explicitly_requested", lambda: True)
+ monkeypatch.setattr(stack, "_forced_rocm_route_is_viable", lambda: viable)
+
+ def _sentinel():
+ raise _Reached
+
+ monkeypatch.setattr(stack, "_visible_masks_select_no_gpu", _sentinel)
+ try:
+ assert stack._amd_torch_needs_dependency_pass() is False
+ except _Reached:
+ return True
+ return False
+
+
+def test_an_unroutable_request_does_not_bypass_the_nvidia_fast_path(stack, monkeypatch):
+ """On a mixed host whose selected AMD card has no wheel route, _ensure_cuda_torch() and
+ _ensure_rocm_torch() both leave CUDA in place."""
+ assert _needs_repair_passes_the_nvidia_gate(stack, monkeypatch, viable = False) is False
+
+
+def test_a_routable_request_still_bypasses_it(stack, monkeypatch):
+ """The control: the request is the whole feature, so a viable one must still outrank the NVIDIA
+ card here or the repair it gates can never run."""
+ assert _needs_repair_passes_the_nvidia_gate(stack, monkeypatch, viable = True) is True
+
+
+def test_an_unresolvable_mask_does_not_answer_for_the_next_host(stack, monkeypatch):
+ """The flag is module state, so it is reset on entry rather than only where it is decided."""
+ assert (
+ _viable_masked(stack, monkeypatch, devices = ["gfx1100", "gfx1010"], HIP_VISIBLE_DEVICES = "7")
+ is False
+ )
+ monkeypatch.setenv("UNSLOTH_ROCM_GFX_ARCH", "gfx1100")
+ assert _viable_masked(stack, monkeypatch, devices = ["gfx1100", "gfx1010"]) is True
+
+
+def test_a_rocr_ordinal_past_the_last_device_is_not_a_viable_route(stack, monkeypatch):
+ """ROCr's own filter (ROCR-Runtime, core/inc/amd_filter_device.h) surfaces the tokens that are
+ "Legal and NOT Terminating", and an index terminates when it "lies outside the interval [0 -
+ (numGpuDevices - 1)]" -- so ROCR_VISIBLE_DEVICES=7 on a two-GPU box surfaces nothing and the
+ HIP layer above it indexes an empty list."""
+ assert (
+ _viable_masked(stack, monkeypatch, devices = ["gfx1100", "gfx1010"], ROCR_VISIBLE_DEVICES = "7")
+ is False
+ )
+
+
+def test_a_rocr_prefix_that_survives_is_still_a_viable_route(stack, monkeypatch):
+ """The control, and the reason this is a prefix rule rather than "any bad token declines":
+ ROCR_VISIBLE_DEVICES=0,7 terminates at the 7 and still surfaces device 0, which is routable."""
+ assert (
+ _viable_masked(
+ stack, monkeypatch, devices = ["gfx1100", "gfx1010"], ROCR_VISIBLE_DEVICES = "0,7"
+ )
+ is True
+ )
+
+
+def test_a_rocm_version_no_wheel_family_serves_is_not_a_viable_route(stack, monkeypatch):
+ """gfx908 is in the arch tables, so the route test said yes -- but on ROCm 5.7 no generic
+ rocmX.Y tag resolves, the missing-kernel reroute does not fire for an arch the generic wheel
+ does carry, and _ensure_rocm_torch prints "No PyTorch wheel for ROCm 5.7" and installs
+ nothing."""
+ assert _viable_masked(stack, monkeypatch, devices = ["gfx908"], rocm = (5, 7)) is False
+
+
+def test_the_same_arch_on_a_version_with_a_wheel_family_is_viable(stack, monkeypatch):
+ """The control: the same card on ROCm 6.4, where the tag resolves and the install proceeds."""
+ assert _viable_masked(stack, monkeypatch, devices = ["gfx908"], rocm = (6, 4)) is True
+
+
+def test_an_arch_the_generic_wheel_lacks_still_routes_on_an_old_version(stack, monkeypatch):
+ """The second control, and the one that keeps the version test from swallowing the repair it
+ exists beside: gfx1103 has no generic kernels at any tag, so _ensure_rocm_torch reroutes it
+ to AMD's per-arch index whatever the version reads."""
+ assert _viable_masked(stack, monkeypatch, devices = ["gfx1103"], rocm = (5, 7)) is True
+
+
+def test_the_route_test_does_not_read_the_hosts_rocm_version(stack, monkeypatch):
+ """The guard for the pin above, since its absence is not visible in a passing run."""
+ monkeypatch.setattr(stack, "_detect_rocm_version", lambda: None)
+ assert _viable_masked(stack, monkeypatch, devices = ["gfx1100"]) is True
+
+
+def test_a_declared_arch_does_not_answer_over_an_unresolvable_mask_in_python(stack, monkeypatch):
+ """The Python half of the shell rule above."""
+ monkeypatch.setenv("UNSLOTH_ROCM_GFX_ARCH", "gfx1100")
+ assert (
+ _viable_masked(stack, monkeypatch, devices = ["gfx1100", "gfx1010"], HIP_VISIBLE_DEVICES = "7")
+ is False
+ )
+
+
+def test_a_declared_arch_over_a_mask_that_resolves_is_still_viable_in_python(stack, monkeypatch):
+ """The control, and the reason the resolution is gated on a mask being set at all: the ordinary
+ declared-arch host has none, and must still route without paying for a probe."""
+ assert (
+ _viable_masked(
+ stack,
+ monkeypatch,
+ devices = ["gfx1100", "gfx1010"],
+ declared = "gfx1100",
+ HIP_VISIBLE_DEVICES = "1",
+ )
+ is True
+ )
+
+
+# A mixed AMD host: the arch the Deck gate refuses beside one it serves, with the kernel's
+# topology naming both in the order the masks index.
+_VAN_GOGH_PLUS_DGPU = "\n".join(
+ [
+ "_has_usable_nvidia_gpu() { return 0; }",
+ "_has_amd_rocm_gpu() { return 0; }",
+ "_probe_amd_gfx_arch() { printf '%s\\n' gfx1033 gfx1100; }",
+ # The real map, not a constant: without it gfx1033 has no per-arch family here and
+ # the route test declines for the wrong reason, so the control below would pass
+ # whatever the gate does.
+ '_amd_arch_index_family_for_gfx() { case "$1" in'
+ " gfx1100) echo gfx110X-all ;; gfx1033) echo gfx103X-all ;; *) return 1 ;; esac; }",
+ "_kfd_gfx_targets() { printf '%s\\n' gfx1033 gfx1100; }",
+ "_infer_linux_amd_gfx_arch() { :; }",
+ "_amd_gpu_present_via_pci() { return 0; }",
+ "_detect_rocm_version_tag() { echo rocm7.0; }",
+ "_amd_sole_index_arch() { :; }",
+ "_amd_agreed_index_family() { :; }",
+ "_rocm_sdk_install_hint() { echo ''; }",
+ "nvidia-smi() { echo 'CUDA Version: 13.0'; }",
+ ]
+)
+
+
+def test_the_bad_arch_gate_answers_for_the_selected_card_under_the_request(stack):
+ """The gate is a PRESENCE test because an unresolved host cannot say which card runs."""
+ out = _index_url("UNSLOTH_FORCE_ROCM_TORCH=1 HIP_VISIBLE_DEVICES=1", _VAN_GOGH_PLUS_DGPU)
+ assert out.strip().endswith("/rocm7.0"), out
+
+
+def test_the_same_request_selecting_the_deck_still_takes_the_cpu_index(stack):
+ """The control that keeps the narrowing honest: the request cannot buy ROCm wheels for the arch
+ measured to compute wrong answers."""
+ out = _index_url("UNSLOTH_FORCE_ROCM_TORCH=1 HIP_VISIBLE_DEVICES=0", _VAN_GOGH_PLUS_DGPU)
+ assert out.strip().endswith("/cpu"), out
+
+
+def test_the_presence_rule_is_unchanged_for_a_host_that_did_not_ask(stack):
+ """And the control for every other host: with no request nothing has resolved a target, so the
+ gate answers on presence exactly as before (#7776, studio/ROCM_RDNA2_APU.md)."""
+ out = _index_url(
+ "HIP_VISIBLE_DEVICES=1",
+ _VAN_GOGH_PLUS_DGPU.replace(
+ "_has_usable_nvidia_gpu() { return 0; }", "_has_usable_nvidia_gpu() { return 1; }"
+ ),
+ )
+ assert out.strip().endswith("/cpu"), out
+
+
+def test_a_repeated_rocr_ordinal_does_not_invent_a_device(stack, monkeypatch):
+ """ROCr terminates on an index that "maps to a device that has been previously selected" (ROCR-
+ Runtime, core/inc/amd_filter_device.h), so ROCR_VISIBLE_DEVICES=0,0 surfaces ONE device and
+ HIP_VISIBLE_DEVICES=1 above it then indexes nothing."""
+ assert (
+ _viable_masked(
+ stack,
+ monkeypatch,
+ devices = ["gfx1100", "gfx1010"],
+ ROCR_VISIBLE_DEVICES = "0,0",
+ HIP_VISIBLE_DEVICES = "1",
+ )
+ is False
+ )
+
+
+def test_two_distinct_rocr_ordinals_still_expose_both_devices(stack, monkeypatch):
+ """The control."""
+ assert (
+ _viable_masked(
+ stack,
+ monkeypatch,
+ devices = ["gfx1100", "gfx1010"],
+ ROCR_VISIBLE_DEVICES = "1,0",
+ HIP_VISIBLE_DEVICES = "1",
+ )
+ is True
+ )
+
+
+def test_the_rocr_subset_is_the_prefix_of_devices_the_mask_surfaces(stack, monkeypatch):
+ """The buckets behind the two cases above, since a route verdict cannot show WHICH list the HIP
+ layer was handed."""
+ monkeypatch.delenv("ROCR_VISIBLE_DEVICES", raising = False)
+ _devices = ["gfx1100", "gfx1010", "gfx1201"]
+ for _mask, _expected in (
+ ("0,0", ["gfx1100"]),
+ ("1,1,2", ["gfx1010"]),
+ ("0,7,1", ["gfx1100"]),
+ ("2,0", ["gfx1201", "gfx1100"]),
+ ("7", _devices),
+ ):
+ monkeypatch.setenv("ROCR_VISIBLE_DEVICES", _mask)
+ assert stack._rocr_visible_subset(list(_devices))[0] == _expected, _mask
+
+
+def test_a_declared_arch_resolves_its_mask_against_the_unmasked_list(stack, monkeypatch):
+ """With no KFD topology (WSL) the declared-arch path falls back to rocminfo, and rocminfo is
+ the one probe ROCR_VISIBLE_DEVICES renumbers."""
+ assert (
+ _viable_masked(
+ stack,
+ monkeypatch,
+ devices = ["gfx1100", "gfx1010"],
+ masked = ["gfx1010"],
+ declared = "gfx1100",
+ ROCR_VISIBLE_DEVICES = "1",
+ )
+ is True
+ )
+
+
+def test_the_same_declared_host_with_a_mask_past_its_last_device_still_declines(stack, monkeypatch):
+ """The control: the unmasked list is asked for so the ordinals can be judged, not so that every
+ ordinal passes."""
+ monkeypatch.setenv("UNSLOTH_ROCM_GFX_ARCH", "gfx1100")
+ assert (
+ _viable_masked(
+ stack,
+ monkeypatch,
+ devices = ["gfx1100", "gfx1010"],
+ masked = [],
+ ROCR_VISIBLE_DEVICES = "7",
+ )
+ is False
+ )
+
+
+def test_the_installer_stops_the_rocr_prefix_at_a_repeat_too():
+ """The shell half of the duplicate rule: _amd_mask_survivors printed the repeated device a
+ second time, so the HIP layer read a two-entry list and selected a phantom."""
+ assert (
+ _route_shell_masked(
+ ["gfx1100", "gfx1010"],
+ devices = ["gfx1100", "gfx1010"],
+ ROCR_VISIBLE_DEVICES = "0,0",
+ HIP_VISIBLE_DEVICES = "1",
+ )
+ is False
+ )
+
+
+def test_the_installer_still_exposes_both_devices_for_distinct_ordinals():
+ """The control, matching the Python one: ROCR_VISIBLE_DEVICES=1,0 renumbers rather than
+ truncates, so HIP ordinal 1 is the routable card and the request still deposes CUDA."""
+ assert (
+ _route_shell_masked(
+ ["gfx1100", "gfx1010"],
+ devices = ["gfx1100", "gfx1010"],
+ ROCR_VISIBLE_DEVICES = "1,0",
+ HIP_VISIBLE_DEVICES = "1",
+ )
+ is True
+ )
+
+
+def test_the_installer_does_not_apply_the_repeat_rule_to_the_hip_layer():
+ """The rule is ROCr's, cited from its own filter; clr documents no such termination for
+ HIP_VISIBLE_DEVICES, so the third argument scopes it to the layer it was read from."""
+ assert (
+ _route_shell_masked(
+ ["gfx1010", "gfx1100"],
+ devices = ["gfx1010", "gfx1100"],
+ HIP_VISIBLE_DEVICES = "1,1",
+ )
+ is True
+ )
+
+
+def test_an_unresolvable_mask_on_an_inferred_host_is_not_a_detection_miss(stack, monkeypatch):
+ """Nothing enumerates a device here, so the product name is the only arch on offer and
+ HIP_VISIBLE_DEVICES=1 indexes past it."""
+ assert (
+ _viable_masked(
+ stack,
+ monkeypatch,
+ devices = [],
+ inferred = "gfx1100",
+ HIP_VISIBLE_DEVICES = "1",
+ )
+ is False
+ )
+
+
+def test_the_same_inferred_host_without_a_mask_is_still_a_route(stack, monkeypatch):
+ """The control that keeps the feature: a runtime-less but inferable AMD card is deliberately
+ served per-arch wheels, so the rule must be about the mask and not about the host having no
+ runtime."""
+ assert _viable_masked(stack, monkeypatch, devices = [], inferred = "gfx1100") is True
+
+
+def test_the_same_inferred_host_selecting_its_only_card_is_still_a_route(stack, monkeypatch):
+ """The other control, and the one that pins WHICH ordinals decline: 0 names the single card the
+ name inferred, so it resolves and the route stands."""
+ assert (
+ _viable_masked(
+ stack,
+ monkeypatch,
+ devices = [],
+ inferred = "gfx1100",
+ HIP_VISIBLE_DEVICES = "0",
+ )
+ is True
+ )
+
+
+def test_the_installer_already_declines_that_mask():
+ """The installer twin of the case above, and the reason it is a Python-only fix: with one arch
+ and no per-device list, _amd_request_has_a_wheel_route resolves the mask against the single
+ row and takes `[ -n "$_arwr_sel" ] || return 1`, so it has always failed closed where the
+ Python half fell through to its inventory fallback."""
+ assert _route_shell_masked(["gfx1100"], devices = [], HIP_VISIBLE_DEVICES = "1") is False
+
+
+def test_the_installer_still_routes_that_host_unmasked():
+ """The control: the same one-arch host with no mask is exactly what the request exists to
+ serve, so the decline above must belong to the mask and not to the host shape."""
+ assert _route_shell_masked(["gfx1100"], devices = []) is True
+
+
+_UUID_MASK = "0,GPU-DEADBEEFDEADBEEF"
+
+
+ "devices, viable",
+ [(["gfx1010", "gfx1100"], False), (["gfx1100", "gfx1100"], True)],
+ ids = ["unlike-adapters", "one-architecture"],
+)
+def test_a_uuid_in_the_rocr_mask_is_a_rejection_only_where_it_is_ambiguous(
+ stack, monkeypatch, devices, viable
+):
+ """ROCR_VISIBLE_DEVICES may MIX ordinals and UUIDs, and a UUID names a device this
+ installer cannot place -- but where every ordinal gives the same arch there is nothing
+ ambiguous about it, which is why the branch asks about unlike adapters."""
+ assert (
+ _viable_masked(
+ stack,
+ monkeypatch,
+ devices = devices,
+ ROCR_VISIBLE_DEVICES = _UUID_MASK,
+ HIP_VISIBLE_DEVICES = "0",
+ )
+ is viable
+ )
+
+
+def test_the_named_arch_the_message_offers_resolves_the_ambiguity(stack, monkeypatch):
+ """The second control: the message names UNSLOTH_ROCM_GFX_ARCH as the way through, so setting
+ it must restore the route rather than leave the host declined for good."""
+ assert (
+ _viable_masked(
+ stack,
+ monkeypatch,
+ devices = ["gfx1010", "gfx1100"],
+ declared = "gfx1100",
+ ROCR_VISIBLE_DEVICES = _UUID_MASK,
+ HIP_VISIBLE_DEVICES = "0",
+ )
+ is True
+ )
+
+
+def test_the_ambiguity_flag_does_not_survive_the_next_host(stack, monkeypatch):
+ """It is a module global read by a separate function, so a stale True would decline a later,
+ unrelated host."""
+ monkeypatch.delenv("UNSLOTH_ROCM_GFX_ARCH", raising = False)
+ assert (
+ _viable_masked(
+ stack,
+ monkeypatch,
+ devices = ["gfx1010", "gfx1100"],
+ ROCR_VISIBLE_DEVICES = _UUID_MASK,
+ HIP_VISIBLE_DEVICES = "0",
+ )
+ is False
+ )
+ assert _viable_masked(stack, monkeypatch, devices = ["gfx1100"]) is True
+
+
+def test_an_integrated_gpu_enumerated_first_does_not_strand_the_discrete_card():
+ """A Ryzen APU enumerates ahead of the discrete Radeon beside it, and the wheel family is
+ picked for ONE arch."""
+ assert _route_shell_masked(["gfx90c", "gfx1200"]) is True
+ assert _route_target_masked(["gfx90c", "gfx1200"]) == "gfx1200"
+
+
+def test_a_routable_integrated_gpu_still_yields_to_the_discrete_one():
+ """The same divergence with a quieter symptom, and the reason the fix cannot just be "decline
+ when the first arch has no route": gfx1036 IS routable through gfx103X-all, so the shell
+ said yes and installed for the Raphael iGPU while a gfx1100 sat beside it."""
+ assert _route_target_masked(["gfx1036", "gfx1100"]) == "gfx1100"
+
+
+def test_a_discrete_card_enumerated_first_is_left_alone():
+ """The control: same two cards, opposite order."""
+ assert _route_target_masked(["gfx1200", "gfx90c"]) == "gfx1200"
+
+
+def test_an_integrated_gpu_alone_is_still_declined():
+ """The control that keeps the fix honest."""
+ assert _route_shell_masked(["gfx90c"]) is False
+ assert _route_target_masked(["gfx90c"]) == ""
+
+
+def test_an_unroutable_sibling_does_not_rescue_an_unroutable_integrated_pick():
+ """gfx1010 is in neither the generic wheel nor any per-arch index, so deposing gfx90c for it
+ buys nothing and the request still declines."""
+ assert _route_shell_masked(["gfx90c", "gfx1010"]) is False
+
+
+def test_gfx906_is_not_a_candidate_for_the_swap():
+ """gfx906's only route is the rocm6.3 legacy tag, which opens solely when it is the sole arch
+ on the host, so promoting it here would install a rocm7.x wheel whose BLAS has no gfx906
+ kernels and strand BOTH cards."""
+ assert _route_target_masked(["gfx1036", "gfx906"]) == "gfx1036"
+
+
+def test_a_pinned_host_keeps_the_device_the_user_named():
+ """A set mask is the user naming a device, and _visible_devices_pinned exempts the Python rule
+ for exactly that reason."""
+ assert _route_target_masked(["gfx90c", "gfx1200"], HIP_VISIBLE_DEVICES = "0") == ""
+ assert _route_shell_masked(["gfx90c", "gfx1200"], HIP_VISIBLE_DEVICES = "0") is False
+
+
+def _versionless_reroute_block() -> str:
+ """install.sh's versionless per-arch reroute, lifted by text."""
+ install_sh = Path(__file__).resolve().parents[3] / "install.sh"
+ lines = install_sh.read_text(encoding = "utf-8").splitlines()
+ start = lines.index("_amd_no_rocm_version_reroute=false")
+ end = next(i for i in range(start, len(lines)) if lines[i] == "esac")
+ return "\n".join(lines[start : end + 1])
+
+
+def _reroute_family(physical: "list[str]", **mask: str) -> str:
+ """The wheel family the versionless reroute leaves standing, or "" when it clears it."""
+ emit = 'printf "%s\\n" ' + " ".join(repr(a) for a in physical)
+ script = "\n".join(
+ [
+ "_torch_index_pinned=false",
+ "SKIP_TORCH=false",
+ 'TORCH_INDEX_URL="https://download.pytorch.org/whl/cpu"',
+ f"_probe_amd_gfx_arch() {{ {emit}; }}",
+ f"_kfd_gfx_targets() {{ {emit}; }}",
+ "_infer_linux_amd_gfx_arch() { :; }",
+ "_amd_gpu_present_via_pci() { return 0; }",
+ "_has_amd_rocm_gpu() { return 0; }",
+ # No NVIDIA card, so _nvidia_gpu_wins_over_amd answers no whatever the request
+ # did and the arms below differ only in the request and the mask.
+ "_has_usable_nvidia_gpu() { return 1; }",
+ _shell_function("_rocm_torch_explicitly_requested"),
+ *_wheel_route_defs(rocminfo = _fake_rocminfo(physical)),
+ _shell_function("_nvidia_gpu_wins_over_amd"),
+ _shell_function("_amd_probe_arches"),
+ _shell_function("_amd_agreed_index_family"),
+ _shell_function("_amd_sole_index_arch"),
+ _shell_function("_hsa_spoofed_physical_gfx"),
+ "_detect_rocm_version_tag() { :; }",
+ _versionless_reroute_block(),
+ 'printf "%s" "${_amd_probed_family:-}"',
+ ]
+ )
+ return _bash(script, env = _route_env(mask))
+
+
+# gfx1033 shares gfx103X-all with the card beside it, which is what makes the family
+# disqualification reach a routable sibling in the first place.
+_DECK_PLUS_RDNA2 = ["gfx1033", "gfx1030"]
+
+
+def test_the_versionless_reroute_keeps_the_family_the_request_selected():
+ """The reroute clears the family on PRESENCE of gfx1033, which is right only while the card
+ that will run is unknown."""
+ assert (
+ _reroute_family(_DECK_PLUS_RDNA2, UNSLOTH_FORCE_ROCM_TORCH = "1", HIP_VISIBLE_DEVICES = "1")
+ == "gfx103X-all"
+ )
+
+
+def test_the_same_request_selecting_the_deck_still_loses_the_family():
+ """The control that keeps the narrowing honest: the request cannot buy ROCm wheels for the arch
+ measured to compute wrong answers (studio/ROCM_RDNA2_APU.md)."""
+ assert (
+ _reroute_family(_DECK_PLUS_RDNA2, UNSLOTH_FORCE_ROCM_TORCH = "1", HIP_VISIBLE_DEVICES = "0")
+ == ""
+ )
+
+
+def test_the_presence_rule_still_holds_for_a_host_that_did_not_ask():
+ """And the control for every other host: with no request nothing resolved a target, so presence
+ disqualifies the shared family as before (#7776)."""
+ assert _reroute_family(_DECK_PLUS_RDNA2, HIP_VISIBLE_DEVICES = "1") == ""
+
+
+ "arch, declared, viable",
+ [
+ ("gfx1100", "gfx1100:sramecc-:xnack-", True),
+ ("gfx1100", "gfx1100", True),
+ ("gfx1010", "gfx1010:xnack-", False),
+ ],
+ ids = ["suffixed-routable", "bare-routable", "suffixed-unroutable"],
+)
+def test_a_feature_suffix_changes_the_spelling_and_not_the_answer(
+ stack, monkeypatch, arch, declared, viable
+):
+ """rocminfo prints gcnArchName with its feature flags, and that is the spelling users copy
+ into UNSLOTH_ROCM_GFX_ARCH. _amd_arch_index_url keys on the bare arch, so the suffixed
+ form answered None and declined a route the plain spelling gets on the same silicon. The
+ third row is the control: normalising the SPELLING must not normalise the ANSWER."""
+ assert (
+ _viable_masked(
+ stack,
+ monkeypatch,
+ devices = [arch],
+ inferred = declared,
+ rocm = (0, 0),
+ declared = declared,
+ )
+ is viable
+ )
+
+
+# ── The feature suffix rocminfo prints, which users copy verbatim ───────────────────────
+
+
+ "declared,family",
+ [
+ ("gfx1100", "gfx110X-all"),
+ ("gfx1100:sramecc-:xnack-", "gfx110X-all"),
+ ("gfx1151", "gfx1151"),
+ ("gfx1151:xnack-", "gfx1151"),
+ ],
+)
+def test_a_feature_suffix_routes_to_the_same_index_as_the_bare_arch(
+ stack, monkeypatch, declared, family
+):
+ """rocminfo prints gcnArchName as gfx1100:sramecc-:xnack- and that is what people paste into
+ UNSLOTH_ROCM_GFX_ARCH, but both arch tables are keyed on the bare arch."""
+ monkeypatch.setattr(stack, "IS_WINDOWS", False)
+ url = stack._amd_arch_index_url(declared)
+ assert url is not None and family in url
+
+
+def test_a_suffixed_miscomputing_arch_is_still_refused(stack, monkeypatch):
+ """The control that keeps the normalisation from opening a door."""
+ monkeypatch.setattr(stack, "IS_WINDOWS", False)
+ assert stack._amd_arch_index_url("gfx1033") is None
+ assert stack._amd_arch_index_url("gfx1033:xnack-") is None
+
+
+def test_an_unknown_arch_is_still_unrouted(stack, monkeypatch):
+ """The other control: normalising must not invent a route for an arch no index carries."""
+ monkeypatch.setattr(stack, "IS_WINDOWS", False)
+ assert stack._amd_arch_index_url("gfx1010") is None
+ assert stack._amd_arch_index_url("gfx1010:xnack-") is None
+
+
+# ── The Van Gogh gate, and what may exempt a host from it ──────────────────────────────
+
+
+def _bad_arch_verdict(
+ *,
+ declared: str,
+ probed: str,
+ request: str = "1",
+) -> str:
+ """Which index get_torch_index_url's miscomputing-arch gate leaves, for this host."""
+ lines = (
+ (Path(__file__).resolve().parents[3] / "install.sh")
+ .read_text(encoding = "utf-8")
+ .splitlines()
+ )
+ end = next(i for i, line in enumerate(lines) if "end of the miscomputing-arch gate" in line)
+ # The gate opens at its own header sentence; anchoring on the first
+ # `_amd_gfx_bad_arch=false` walking back would land inside the exemption branch.
+ start = next(
+ i
+ for i in range(end, 0, -1)
+ if "Archs measured to compute INCORRECTLY under ROCm" in lines[i]
+ )
+ # Wrapped in a function, because the lifted block ends in `return` and bash refuses
+ # that at top level -- unwrapped, execution ran on past the verdict.
+ gate = "_gate() {\n" + "\n".join(lines[start:end]) + "\n}"
+ script = "\n".join(
+ [
+ _shell_function("_rocm_torch_explicitly_requested"),
+ *_wheel_route_defs(arch_family = False),
+ "_amd_arch_index_family_for_gfx() { case $1 in gfx1030|gfx1033) echo gfx103X-all ;;"
+ " gfx1100) echo gfx110X-all ;; *) return 1 ;; esac; }",
+ "_has_usable_nvidia_gpu() { return 0; }",
+ "_amd_gpu_present_via_pci() { return 0; }",
+ f"_probe_amd_gfx_arch() {{ echo {probed}; }}",
+ f"_kfd_gfx_targets() {{ echo {probed}; }}",
+ f"_infer_linux_amd_gfx_arch() {{ echo {declared or probed}; }}",
+ "_detect_rocm_version_tag() { echo rocm7.1; }",
+ "_ensure_rocm_probe_env() { :; }",
+ '_base="https://download.pytorch.org/whl"',
+ f'_amd_gfx_probe="{probed}"',
+ "_amd_request_has_a_wheel_route || true",
+ gate,
+ # The gate either PRINTS an index and returns, or falls through printing nothing.
+ "_verdict=$(_gate || true)",
+ 'printf "%s\\n" "${_verdict:-kept}"',
+ ]
+ )
+ env = _clean_env({"UNSLOTH_FORCE_ROCM_TORCH": request})
+ if declared:
+ env["UNSLOTH_ROCM_GFX_ARCH"] = declared
+ out = subprocess.run(["bash", "-c", script], capture_output = True, text = True, env = env)
+ return (out.stdout.strip().splitlines() or [""])[-1]
+
+
+def test_a_declared_arch_cannot_exempt_a_physical_gfx1033():
+ """A Steam Deck beside an NVIDIA card, with a stale or copied UNSLOTH_ROCM_GFX_ARCH. The
+ declared gfx1030 is published as the request's target, and the gate used to read any non-
+ gfx1033 target as "the selected card is fine"."""
+ assert _bad_arch_verdict(declared = "gfx1030", probed = "gfx1033").endswith("/cpu")
+
+
+def test_a_probed_target_still_exempts_a_routable_sibling():
+ """The control, and the case the target was published for: the probe itself resolved a gfx1100,
+ so the gfx1033 in the inventory is a card the runtime did not select."""
+ assert _bad_arch_verdict(declared = "", probed = "gfx1100") == "kept"
+
+
+def test_the_gate_still_fires_with_no_request_at_all():
+ """The other control: presence remains the rule for every host that has not asked."""
+ assert _bad_arch_verdict(declared = "", probed = "gfx1033", request = "0").endswith("/cpu")
+
+
+def _reroute_family_for_target(
+ *,
+ inventory: list,
+ target: str,
+ source: str,
+ bad_arch: str = "false",
+) -> str:
+ """The wheel family the versionless reroute would use, for this host."""
+ lines = (
+ (Path(__file__).resolve().parents[3] / "install.sh")
+ .read_text(encoding = "utf-8")
+ .splitlines()
+ )
+ # From the bad-arch verdict, not from the target block alone: the gate CLEARS the family
+ # before the target may set it, and a lift that starts below that clearing tests a
+ # sequence install.sh never runs.
+ start = next(
+ i
+ for i, line in enumerate(lines)
+ if line.strip() == 'if [ "$_amd_reroute_bad_arch" = true ]; then'
+ )
+ # To the close of the OUTER if, by indentation: the block nests one, so the first
+ # bare "fi" is the inner one and cutting there left an unterminated script.
+ indent = len(lines[start]) - len(lines[start].lstrip())
+ # Two sibling blocks at this indent: the gate's clearing, then the target's assignment.
+ closes = [
+ i
+ for i in range(start + 1, len(lines))
+ if lines[i].strip() == "fi" and len(lines[i]) - len(lines[i].lstrip()) == indent
+ ]
+ block = "\n".join(lines[start : closes[1] + 1])
+ script = "\n".join(
+ [
+ _shell_function("_amd_arch_index_family_for_gfx"),
+ _shell_function("_amd_probe_arches"),
+ _shell_function("_amd_agreed_index_family"),
+ _shell_function("_amd_sole_index_arch"),
+ f'_amd_probe_out="{chr(10).join(inventory)}"',
+ '_amd_probed_family=$(_amd_agreed_index_family "$_amd_probe_out") || _amd_probed_family=""',
+ '_amd_probed_gfx_first=$(_amd_sole_index_arch "$_amd_probe_out") || _amd_probed_gfx_first=""',
+ f'_amd_reroute_target="{target}"',
+ f'_AMD_REQUEST_TARGET_SOURCE="{source}"',
+ f"_amd_reroute_bad_arch={bad_arch}",
+ block,
+ 'printf "%s\\n" "${_amd_probed_family:-none}"',
+ ]
+ )
+ return _bash(script)
+
+
+ "inventory, target, source, family",
+ [
+ (["gfx1033", "gfx1100"], "gfx1100", "probe", "gfx110X-all"),
+ (["gfx1033", "gfx1100"], "gfx1100", "declared", "none"),
+ (["gfx1100", "gfx1101"], "gfx1100", "probe", "gfx110X-all"),
+ ],
+ ids = ["cross-family-probe", "cross-family-declared", "single-family"],
+)
+def test_the_versionless_reroute_family_comes_from_the_selected_card(
+ inventory, target, source, family
+):
+ """_amd_agreed_index_family needs EVERY physical AMD GPU to share a family, so a
+ cross-family pair answered empty, the reroute never fired and the downgrade guard
+ restored CUDA -- the request ignored on exactly the host that resolved a routable
+ target. Probe-resolved only: a declared arch cannot name the card the runtime selected."""
+ assert _reroute_family_for_target(inventory = inventory, target = target, source = source) == family
+
+
+def test_a_disqualified_family_is_not_put_back_by_the_target():
+ """The control that matters most here."""
+ assert (
+ _reroute_family_for_target(
+ inventory = ["gfx1033"], target = "gfx1033", source = "probe", bad_arch = "true"
+ )
+ == "none"
+ )
+
+
+def _unreadable_version_host(stack, monkeypatch, gfx: str) -> None:
+ """A mixed NVIDIA + AMD host whose ROCm version cannot be read, carrying healthy CUDA torch:
+ the build the stand-down leaves in place when nothing replaces it."""
+ _mixed_host(stack, monkeypatch, archs = [gfx], kfd = [gfx], rocm = None)
+
+
[email protected]("gfx", ["gfx1102", "gfx1200", "gfx1201"])
+def test_an_unreadable_version_is_judged_as_the_installer_judges_it(stack, monkeypatch, gfx):
+ """The two halves must not read the same unreadable version differently."""
+ _unreadable_version_host(stack, monkeypatch, gfx)
+ assert stack._forced_rocm_route_is_viable() is False
+
+
[email protected]("gfx", ["gfx1151", "gfx1103"])
+def test_an_unreadable_version_still_serves_the_arches_that_do_install(stack, monkeypatch, gfx):
+ """The narrowing control."""
+ _unreadable_version_host(stack, monkeypatch, gfx)
+ assert stack._forced_rocm_route_is_viable() is True
+
+
[email protected]("gfx", ["gfx1102", "gfx1200", "gfx1201"])
+def test_a_readable_version_still_routes_the_same_three_arches(stack, monkeypatch, gfx):
+ """The other control: the change is scoped to an unreadable version."""
+ _unreadable_version_host(stack, monkeypatch, gfx)
+ monkeypatch.setattr(stack, "_detect_rocm_version", lambda: (6, 4))
+ assert stack._forced_rocm_route_is_viable() is True
+
+
+def _declared_on_physical(stack, monkeypatch, *, declared: str, physical: list) -> bool:
+ """_forced_rocm_route_is_viable for a DECLARED arch on a known physical inventory."""
+ monkeypatch.setenv("UNSLOTH_FORCE_ROCM_TORCH", "1")
+ monkeypatch.setenv("UNSLOTH_ROCM_GFX_ARCH", declared)
+ for _mask in ("HIP_VISIBLE_DEVICES", "ROCR_VISIBLE_DEVICES", "CUDA_VISIBLE_DEVICES"):
+ monkeypatch.delenv(_mask, raising = False)
+ monkeypatch.setattr(stack, "IS_WINDOWS", False)
+ monkeypatch.setattr(stack, "IS_MACOS", False)
+ monkeypatch.setattr(stack.platform, "machine", lambda: "x86_64")
+ monkeypatch.setattr(stack, "_has_usable_nvidia_gpu", lambda: True)
+ monkeypatch.setattr(stack, "_has_rocm_gpu", lambda: True)
+ monkeypatch.setattr(stack, "_is_wsl", lambda: False)
+ monkeypatch.setattr(stack, "_linux_amd_display_device_present", lambda: True)
+ monkeypatch.setattr(stack, "_physical_amd_gfx_archs", lambda: list(physical))
+ monkeypatch.setattr(stack, "_kfd_gfx_targets", lambda: list(physical))
+ monkeypatch.setattr(stack, "_detect_rocm_version", lambda: (6, 4))
+ monkeypatch.setattr(stack, "_explicit_rocm_torch_index_url", lambda: None)
+ return stack._forced_rocm_route_is_viable()
+
+
+def test_a_declared_arch_cannot_exempt_a_physical_gfx1033_in_python(stack, monkeypatch):
+ """A declaration is a build target the user typed, not a statement about the silicon."""
+ assert (
+ _declared_on_physical(
+ stack, monkeypatch, declared = "gfx1030", physical = ["gfx1033", "gfx1100"]
+ )
+ is False
+ )
+
+
+def test_a_probe_resolved_target_still_exempts_the_same_host(stack, monkeypatch):
+ """The control that keeps the narrowing honest, and the case install.sh spells out by name:
+ with NO declaration the probe resolves the target, the discrete-GPU preference selects the
+ gfx1100 over the integrated gfx1033, and that IS a statement about the silicon."""
+ monkeypatch.delenv("UNSLOTH_ROCM_GFX_ARCH", raising = False)
+ monkeypatch.setenv("UNSLOTH_FORCE_ROCM_TORCH", "1")
+ for _mask in ("HIP_VISIBLE_DEVICES", "ROCR_VISIBLE_DEVICES", "CUDA_VISIBLE_DEVICES"):
+ monkeypatch.delenv(_mask, raising = False)
+ monkeypatch.setattr(stack, "IS_WINDOWS", False)
+ monkeypatch.setattr(stack, "IS_MACOS", False)
+ monkeypatch.setattr(stack.platform, "machine", lambda: "x86_64")
+ monkeypatch.setattr(stack, "_has_usable_nvidia_gpu", lambda: True)
+ monkeypatch.setattr(stack, "_has_rocm_gpu", lambda: True)
+ monkeypatch.setattr(stack, "_is_wsl", lambda: False)
+ monkeypatch.setattr(stack, "_linux_amd_display_device_present", lambda: True)
+ monkeypatch.setattr(stack, "_infer_linux_amd_gfx_arch", lambda: None)
+ monkeypatch.setattr(stack, "_physical_amd_gfx_archs", lambda: ["gfx1033", "gfx1100"])
+ monkeypatch.setattr(stack, "_kfd_gfx_targets", lambda: ["gfx1033", "gfx1100"])
+ monkeypatch.setattr(
+ stack, "_detect_amd_gfx_codes", lambda dedup = True, **k: ["gfx1033", "gfx1100"]
+ )
+ monkeypatch.setattr(stack, "_detect_rocm_version", lambda: (6, 4))
+ monkeypatch.setattr(stack, "_explicit_rocm_torch_index_url", lambda: None)
+ # Proves the control is about the PROBE path: the target must be the discrete card, not
+ # the integrated one the guard above rejects a declaration for.
+ assert stack._runtime_gfx_target(None)[0] == "gfx1100"
+ assert stack._forced_rocm_route_is_viable() is True
+
+
+def test_a_declaration_still_routes_where_no_bad_arch_is_present(stack, monkeypatch):
+ """The second control: the guard keys on a MISCOMPUTING arch in the machine, not on the
+ declaration disagreeing with it."""
+ assert (
+ _declared_on_physical(
+ stack, monkeypatch, declared = "gfx1030", physical = ["gfx1030", "gfx1100"]
+ )
+ is True
+ )
+
+
+def test_the_documented_strix_declaration_still_routes(stack, monkeypatch):
+ """The third control, and the reason the guard cannot simply distrust declarations: the
+ documented Strix Halo workaround declares gfx1100 on a physical gfx1151, which install.sh
+ calls out by name."""
+ assert (
+ _declared_on_physical(stack, monkeypatch, declared = "gfx1100", physical = ["gfx1151"]) is True
+ )
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# The request must not authorise a swap it withdraws the moment the swap happens.
+
+
+def _viability_across_the_install(stack, monkeypatch, gfx, installed):
+ """_forced_rocm_route_is_viable() on one unchanging host, asked twice: once with the CUDA wheel
+ still installed, once with the ROCm wheel the request asked for in its place."""
+ monkeypatch.setenv("UNSLOTH_FORCE_ROCM_TORCH", "1")
+ monkeypatch.delenv("UNSLOTH_ROCM_GFX_ARCH", raising = False)
+ for _mask in ("HIP_VISIBLE_DEVICES", "ROCR_VISIBLE_DEVICES", "CUDA_VISIBLE_DEVICES"):
+ monkeypatch.delenv(_mask, raising = False)
+ monkeypatch.setattr(stack.platform, "machine", lambda: "x86_64")
+ monkeypatch.setattr(stack, "IS_WINDOWS", False)
+ monkeypatch.setattr(stack, "IS_MACOS", False)
+ monkeypatch.setattr(stack, "_has_usable_nvidia_gpu", lambda: True)
+ monkeypatch.setattr(stack, "_has_rocm_gpu", lambda: True)
+ monkeypatch.setattr(stack, "_kfd_gfx_targets", lambda *a, **k: [gfx])
+ monkeypatch.setattr(stack, "_detect_amd_gfx_codes", lambda *a, **k: [gfx])
+ monkeypatch.setattr(stack, "_physical_amd_gfx_archs", lambda *a, **k: [gfx])
+ monkeypatch.setattr(stack, "_infer_linux_amd_gfx_arch", lambda *a, **k: None)
+ monkeypatch.setattr(stack, "_detect_rocm_version", lambda *a, **k: None)
+ for _pin in (
+ "_explicit_rocm_torch_index_url",
+ "_explicit_cuda_torch_index_url",
+ "_explicit_cpu_torch_index_url",
+ "_explicit_torch_index_url",
+ ):
+ monkeypatch.setattr(stack, _pin, lambda: None)
+ monkeypatch.setattr(
+ stack, "_probe_torch_runtime", lambda *a, **k: (True, True, installed, "", "")
+ )
+ return stack._forced_rocm_route_is_viable()
+
+
+def test_the_route_is_still_viable_once_the_wheel_it_authorised_is_installed(stack, monkeypatch):
+ """Measured on gfx950 with an unreadable ROCm version, the one shape where
+ _rocm_torch_family_needs_repair is the only arm carrying the chain."""
+ before = _viability_across_the_install(stack, monkeypatch, "gfx950", "2.9.0+cu128")
+ after = _viability_across_the_install(stack, monkeypatch, "gfx950", "2.11.0+rocm7.1")
+ assert before is True, "the request never authorised the swap, so there is nothing to keep"
+ assert after is True, "the request withdrew the route its own install created"
+
+
+def test_an_installed_rocm_build_on_an_unroutable_card_is_still_not_a_route(stack, monkeypatch):
+ """The control on the rule above: "ROCm is already installed" must not become a blanket yes."""
+ assert _viability_across_the_install(stack, monkeypatch, "gfx1010", "2.11.0+rocm7.1") is False
+
+
+def test_the_cuda_repair_still_stands_down_after_the_swap(stack, monkeypatch):
+ """The consequence, at the caller."""
+ calls = []
+ monkeypatch.setattr(stack, "pip_install", lambda *a, **k: calls.append(a[:1]))
+ monkeypatch.setattr(stack, "pip_install_try", lambda *a, **k: calls.append(a[:1]))
+ monkeypatch.delenv("UNSLOTH_TORCH_BACKEND", raising = False)
+ monkeypatch.delenv("UNSLOTH_ROCM_TORCH_INSTALLED", raising = False)
+ _viability_across_the_install(stack, monkeypatch, "gfx950", "2.11.0+rocm7.1")
+ stack._ensure_cuda_torch()
+ assert not calls, f"the CUDA repair overwrote the requested ROCm build: {calls}"
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# A feature suffix must not change which pins the install gets.
+
+
+def _inferred_install_args(stack, monkeypatch, arch):
+ """The pip arguments _ensure_rocm_torch uses on its inferred-arch branch for ``arch``."""
+ calls = []
+ monkeypatch.setattr(stack, "pip_install", lambda *a, **k: calls.append(a))
+ monkeypatch.setattr(stack, "pip_install_try", lambda *a, **k: calls.append(a))
+ monkeypatch.setenv("UNSLOTH_ROCM_GFX_ARCH", arch)
+ monkeypatch.delenv("UNSLOTH_FORCE_ROCM_TORCH", raising = False)
+ # ROCm wheels are x86_64 only, so _ensure_rocm_torch returns on its arch gate anywhere
+ # else and this helper answers [] on the ubuntu-24.04-arm runner. The case is about the
+ # arch SPELLING, not the host's architecture, so the host is stated like the rest.
+ monkeypatch.setattr(stack.platform, "machine", lambda: "x86_64")
+ monkeypatch.setattr(stack, "IS_WINDOWS", False)
+ monkeypatch.setattr(stack, "IS_MACOS", False)
+ monkeypatch.setattr(stack, "_has_usable_nvidia_gpu", lambda: False)
+ monkeypatch.setattr(stack, "_has_rocm_gpu", lambda: False)
+ monkeypatch.setattr(
+ stack, "_probe_torch_runtime", lambda *a, **k: (True, True, "2.9.0", "", "")
+ )
+ monkeypatch.setattr(stack, "_detect_rocm_version", lambda *a, **k: None)
+ for _pin in ("_explicit_rocm_torch_index_url", "_explicit_torch_index_url"):
+ monkeypatch.setattr(stack, _pin, lambda: None)
+ stack._ensure_rocm_torch()
+ return [a for a in (calls[0] if calls else ()) if isinstance(a, str) and a.startswith("torch")]
+
+
+ "bare,suffixed",
+ [("gfx1151", "gfx1151:xnack-"), ("gfx1200", "gfx1200:sramecc+:xnack-")],
+)
+def test_a_suffixed_arch_installs_the_same_pins_as_the_bare_one(stack, monkeypatch, bare, suffixed):
+ """gcnArchName is what rocminfo prints and what users copy into UNSLOTH_ROCM_GFX_ARCH.
+ Stripping the suffix in _amd_arch_index_url is what opens the inferred-arch branch for that
+ spelling; the package table two lines down was still keyed on the raw string, so it missed
+ and installed unpinned torch/torchvision/torchaudio -- losing the ABI bound the table exists
+ to hold, on a host the bare spelling pins correctly."""
+ assert _inferred_install_args(stack, monkeypatch, suffixed) == _inferred_install_args(
+ stack, monkeypatch, bare
+ )
+
+
+def test_the_suffixed_arch_reaches_that_branch_at_all(stack, monkeypatch):
+ """The control: the assertion above is only worth anything if the suffixed spelling actually
+ installs something."""
+ assert _inferred_install_args(stack, monkeypatch, "gfx1151:xnack-")
+
+
[email protected]("arch", ["gfx1102:xnack-", "gfx1030:sramecc-"])
+def test_an_arch_outside_the_pin_table_is_still_bounded(stack, monkeypatch, arch):
+ """The suffix strip opens this branch for archs the pin table does not name, and the fallback
+ was three bare package names: those hosts reached an arch-index install with no companion
+ bound at all, where every other arch-index install carries one."""
+ assert _inferred_install_args(stack, monkeypatch, arch) == list(
+ stack._ROCM_ARCH_INDEX_TORCH_PKG_SPEC
+ )
+
+
+def test_an_arch_inside_the_pin_table_keeps_its_own_pins(stack, monkeypatch):
+ """The control: the fallback must not displace the table's tighter, per-arch bound."""
+ assert _inferred_install_args(stack, monkeypatch, "gfx1151") == list(
+ stack._WINDOWS_ROCM_TORCH_PKG_SPECS["gfx1151"]
+ )
+
+
+def _mask_probe_count(stack, monkeypatch, requested):
+ """How many device probes _runtime_gfx_target runs for a DECLARED arch under a set mask."""
+ probes = {"n": 0}
+
+ def _counted(*a, **k):
+ probes["n"] += 1
+ return []
+
+ monkeypatch.setenv("UNSLOTH_ROCM_GFX_ARCH", "gfx1100")
+ monkeypatch.setenv("HIP_VISIBLE_DEVICES", "0")
+ if requested:
+ monkeypatch.setenv("UNSLOTH_FORCE_ROCM_TORCH", "1")
+ else:
+ monkeypatch.delenv("UNSLOTH_FORCE_ROCM_TORCH", raising = False)
+ monkeypatch.setattr(stack, "_kfd_gfx_targets", _counted)
+ monkeypatch.setattr(stack, "_detect_amd_gfx_codes", _counted)
+ target = stack._runtime_gfx_target(None)[0]
+ assert target == "gfx1100", target
+ return probes["n"]
+
+
+def test_a_declared_arch_under_a_mask_costs_no_probe_unless_asked(stack, monkeypatch):
+ """Those probes only fill the two mask-provenance flags, which only the forced route reads,
+ and each one can spend its 15s timeout. Running them for every declared-arch host put that
+ cost into an ordinary `studio update` that had asked for nothing."""
+ assert _mask_probe_count(stack, monkeypatch, requested = False) == 0
+
+
+def test_the_host_that_did_ask_still_resolves_its_mask(stack, monkeypatch):
+ """The control: the flags are load-bearing for the route that reads them, so the request must
+ still pay for them."""
+ assert _mask_probe_count(stack, monkeypatch, requested = True) > 0
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# The two halves must read one mask string the same way.
+
+
+def test_an_escaped_mask_is_not_decoded_into_an_ordinal(stack):
+ """`awk -v vis=...` ESCAPE-PROCESSES its operand, so a literal four-character \061 arrived
+ inside awk as the ordinal 1 and the shell half selected the second card."""
+ script = "\n".join(
+ [_shell_function("_amd_mask_survivors"), 'printf "%s" "$(_amd_mask_survivors "$1" "$2")"']
+ )
+ out = subprocess.run(
+ ["bash", "-c", script, "_", "gfx1010\ngfx1100", "\\061"],
+ capture_output = True,
+ text = True,
+ env = _clean_env(),
+ )
+ assert out.returncode == 0, out.stderr
+ assert out.stdout == "", f"an escaped ordinal resolved a device: {out.stdout!r}"
+ # And the twin's answer on the same string, which is what it has to agree with.
+ os.environ["HIP_VISIBLE_DEVICES"] = "\\061"
+ try:
+ assert stack._hip_layer_mask_names_a_device(2) is False
+ finally:
+ os.environ.pop("HIP_VISIBLE_DEVICES", None)
+
+
+def test_a_plain_ordinal_still_selects_its_device(stack):
+ """The control: the rule above must reject the ESCAPE, not every mask."""
+ script = "\n".join(
+ [_shell_function("_amd_mask_survivors"), 'printf "%s" "$(_amd_mask_survivors "$1" "$2")"']
+ )
+ out = subprocess.run(
+ ["bash", "-c", script, "_", "gfx1010\ngfx1100", "1"],
+ capture_output = True,
+ text = True,
+ env = _clean_env(),
+ )
+ assert out.returncode == 0, out.stderr
+ assert out.stdout == "gfx1100", out.stdout
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# The miscomputing-arch gate must not inherit a target from an earlier call.
+
+
+def test_the_index_selector_does_not_inherit_a_previous_target(stack):
+ """_AMD_REQUEST_TARGET_GFX is cleared by the helper that sets it, and that helper runs only
+ when NVIDIA is detected AND the request is set."""
+ selector = _shell_function("get_torch_index_url")
+ assert '_AMD_REQUEST_TARGET_GFX=""' in selector.splitlines()[1] or any(
+ line.strip() == '_AMD_REQUEST_TARGET_GFX=""' for line in selector.splitlines()[:12]
+ ), "the selector must clear the published target on entry"
+ assert any(
+ line.strip() == '_AMD_REQUEST_TARGET_SOURCE=""' for line in selector.splitlines()[:12]
+ ), "the selector must clear the target SOURCE on entry too"
+
+
+ "miscomputing, viable",
+ [(True, False), (False, True)],
+ ids = ["every-arch-computes-wrong", "control-healthy-host"],
+)
+def test_a_host_gate_decides_when_no_probe_can_name_a_device(
+ stack, monkeypatch, miscomputing, viable
+):
+ """_miscomputing_arch_host is the only thing declining when no target resolves -- a Steam
+ Deck with no rocminfo, where the narrower target refusal never runs and the fallback would
+ otherwise approve the swap off the inventory."""
+ _mixed_host(
+ stack, monkeypatch, request = False, archs = ["gfx1033"], devices = [], miscomputing = miscomputing
+ )
+ assert stack._forced_rocm_route_is_viable() is viable
+
+
+ "request_set, sees_amd",
+ [("", False), ("UNSLOTH_FORCE_ROCM_TORCH=1", True)],
+ ids = ["default-nvidia-wins", "request-opens-the-probe"],
+)
+def test_the_shell_probe_hides_the_amd_card_until_it_is_asked(request_set, sees_amd):
+ """install.sh's half of the gate _has_rocm_gpu carries in Python: the AMD probe returns
+ early on a usable NVIDIA GPU unless this run asked for ROCm."""
+ script = "\n".join(
+ [
+ _shell_function("_rocm_torch_explicitly_requested"),
+ _shell_function("_ensure_rocm_probe_env"),
+ _shell_function("_has_amd_rocm_gpu"),
+ "_has_usable_nvidia_gpu() { return 0; }",
+ 'rocminfo() { echo " Name: gfx1100"; }',
+ f"{request_set} _has_amd_rocm_gpu && echo yes || echo no",
+ ]
+ )
+ assert _bash(script, env = _clean_env()) == ("yes" if sees_amd else "no")
diff --git a/studio/install_python_stack.py b/studio/install_python_stack.py
index 2120b84a6f9..76830041041 100644
--- a/studio/install_python_stack.py
+++ b/studio/install_python_stack.py
@@ -2204,6 +2204,9 @@ def _amd_arch_index_url(gfx_arch: str | None) -> str | None:
mirrored/air-gapped Linux repair reaches the index install.sh chose rather
than falling back to repo.amd.com. Both default to repo.amd.com when unset.
"""
+ # rocminfo prints gcnArchName with its feature suffix (gfx1100:sramecc-:xnack-), which is
+ # the spelling users copy into UNSLOTH_ROCM_GFX_ARCH; both tables key on the bare arch.
+ gfx_arch = (gfx_arch or "").strip().split(":")[0] or None
if IS_WINDOWS:
return _windows_rocm_index_url(gfx_arch)
# gfx1033 (Van Gogh) miscomputes under ROCm (studio/ROCM_RDNA2_APU.md). Without this,
@@ -2498,15 +2501,32 @@ def _persist_bnb_rocm_version(version: str) -> bool:
return True
+def _rocm_torch_explicitly_requested() -> bool:
+ """Whether UNSLOTH_FORCE_ROCM_TORCH asked for ROCm torch whatever else the host has.
+
+ Auto-detection stops probing AMD once CUDA is usable, leaving a mixed NVIDIA+AMD host no
+ route to its AMD card but an index pin, which names a wheel family rather than a
+ preference (#10450). Mirrors UNSLOTH_FORCE_VULKAN; a pin still outranks it. One torch
+ install serves one vendor, so this SWAPS the stack: NVIDIA is unavailable while it is set.
+ """
+ return (os.environ.get("UNSLOTH_FORCE_ROCM_TORCH") or "").strip().lower() in (
+ "1",
+ "true",
+ "yes",
+ "on",
+ )
+
+
def _has_rocm_gpu() -> bool:
"""Return True only if an actual AMD GPU is visible (not just ROCm tools installed).
- Always returns False when an NVIDIA GPU is present -- NVIDIA takes
- priority on mixed hosts and prevents every detection path below
- (rocminfo, amd-smi, KFD sysfs) from producing a false positive even
- if ROCm tools are installed alongside the NVIDIA driver.
+ Returns False when an NVIDIA GPU is present -- NVIDIA takes priority on mixed
+ hosts and prevents every detection path below (rocminfo, amd-smi, KFD sysfs) from
+ producing a false positive even if ROCm tools are installed alongside the NVIDIA
+ driver -- unless this run explicitly asked for ROCm, which is the one case where
+ the AMD card is the point.
"""
- if _has_usable_nvidia_gpu():
+ if _has_usable_nvidia_gpu() and not _rocm_torch_explicitly_requested():
return False
for cmd, check_fn in (
# rocminfo: real gfx GPU ids only (gfx000 = CPU agent, "gfx11-generic" = ISA line).
@@ -2628,6 +2648,16 @@ def _has_usable_nvidia_gpu() -> bool:
# keeps the plain indexing behaviour.
_LAST_AMD_GFX_PROBE: "str | None" = None
+# How the last _runtime_gfx_target call failed to resolve a target, which the target itself
+# cannot carry. Only the replace-the-stack callers read these; everything else keeps the guess.
+_LAST_HIP_MASK_RESOLVED = True
+# One layer down: _rocr_visible_subset keeps the whole list when the first ordinal names
+# nothing, so a mask exposing no device still reaches HIP as a full list.
+_LAST_ROCR_MASK_RESOLVED = True
+# Both masks resolved and the probes still could not say WHICH arch was selected, so "no
+# target" must not be read as a detection miss.
+_LAST_GFX_TARGET_AMBIGUOUS = False
+
def _detect_amd_gfx_codes(
dedup: bool = True,
@@ -2782,6 +2812,109 @@ def _gfx_route_on_host(gfx: "str | None", host_codes: "list[str] | None" = None)
)
+def _amd_hardware_is_corroborated() -> bool:
+ """AMD silicon this host can point at, with no declared arch anywhere in the chain.
+
+ _infer_linux_amd_gfx_arch() returns UNSLOTH_ROCM_GFX_ARCH before it looks at hardware, so
+ it cannot answer "is there a card". Under UNSLOTH_FORCE_ROCM_TORCH that matters: the
+ request skips the NVIDIA precedence return, so a stale arch would otherwise force AMD
+ wheels over a working CUDA stack on a host with no AMD GPU. Sources are the ones
+ _infer_linux_amd_gfx_arch uses on its own non-declared path.
+ """
+ if IS_WINDOWS or IS_MACOS:
+ return False
+ if _has_rocm_gpu() or _kfd_gfx_targets():
+ return True
+ if _is_wsl():
+ # WSL enumerates no PCI display device, and neither /dev/dxg (an NVIDIA passthrough creates
+ # it too) nor a leftover librocdxg names a vendor, so beside a usable NVIDIA card the
+ # runtime must name an agent itself.
+ return _wsl_rocm_runtime_present() and not _has_usable_nvidia_gpu()
+ return _linux_amd_display_device_present()
+
+
+def _forced_rocm_route_is_viable() -> bool:
+ """Whether the request has something to swap TO on this host.
+
+ The bar is _gfx_has_a_wheel_route's: an arch no index can serve must never depose a card
+ that can. Presence is not it (gfx1010 is present with no route). Runtime visibility is
+ not it either: a runtime-less but inferable AMD card is deliberately served per-arch
+ wheels on a pure-AMD host, so requiring rocminfo would answer differently for the same
+ silicon by whether an NVIDIA card sits beside it.
+ """
+ # ROCm wheels are x86_64 only, so _ensure_rocm_torch() installs nothing elsewhere and
+ # standing the CUDA repair down leaves the venv on whatever broken torch it had.
+ if platform.machine().lower() not in {"x86_64", "amd64"}:
+ return False
+ if not _amd_hardware_is_corroborated():
+ return False
+ if _miscomputing_arch_host():
+ return False
+ # The card the runtime will hand torch, not any sibling. _runtime_gfx_target returns the
+ # whole machine beside the target, which _gfx_route_on_host's gfx906 rule needs.
+ _inferred = (_infer_linux_amd_gfx_arch() or "").strip().lower().split(":")[0] or None
+ _target, _, _, _host_codes = _runtime_gfx_target(_inferred)
+ # A declared arch REPLACES the inventory, so a stale one hides a Van Gogh sibling
+ # (studio/ROCM_RDNA2_APU.md). Probe-resolved stays exempt, as install.sh already has it.
+ if (
+ (os.environ.get("UNSLOTH_ROCM_GFX_ARCH") or "").strip()
+ and _target is not None
+ and any(_gfx in _ROCM_MISCOMPUTING_GFX for _gfx in _physical_amd_gfx_archs())
+ ):
+ return False
+ # A mask HIP cannot resolve exposes no device, so there is nothing to swap to. Above the
+ # target test, or the fallback below approves the same host off its inventory.
+ if not _LAST_HIP_MASK_RESOLVED or not _LAST_ROCR_MASK_RESOLVED or _LAST_GFX_TARGET_AMBIGUOUS:
+ return False
+ if _target is not None:
+ # Keyed on the SELECTED target: _miscomputing_arch_host above requires EVERY arch to be bad,
+ # so a mask selecting the gfx1033 of a pair passed it and the install then declined.
+ if _target in _ROCM_MISCOMPUTING_GFX:
+ return False
+ # Below ROCm 6.0 no generic tag resolves, so _ensure_rocm_torch installs nothing while
+ # _ensure_cuda_torch has stood down. Ask its three version-independent arms verbatim.
+ _raw_ver = _detect_rocm_version()
+ _ver = _raw_ver or (0, 0)
+ _declared = (os.environ.get("UNSLOTH_ROCM_GFX_ARCH") or "").strip()
+ if _raw_ver is None:
+ # An unreadable version is not "ROCm 0.0" to _ensure_rocm_torch: its `ver is None` branch
+ # answers differently for gfx1102 / gfx1200 / gfx1201 than the (0, 0) spelling does.
+ _torch_ran, _torch_imp, _torch_ver, _, _ = _probe_torch_runtime()
+ _installed_ver = (_torch_ver or "").lower() if (_torch_ran and _torch_imp) else ""
+ # Already on a ROCm build: the arms below ask whether _ensure_rocm_torch would install
+ # SOMETHING, which after a successful swap is legitimately no, and reading that as "no
+ # route" let _ensure_cuda_torch overwrite it (gfx950, unreadable ROCm version).
+ if "rocm" in _installed_ver or "hip" in _installed_ver:
+ return _gfx_route_on_host(_target, _host_codes or [_target])
+ if (
+ _explicit_rocm_torch_index_url() is None
+ and not _inferred
+ and not _generic_rocm_wheel_lacks_kernels(_target)
+ and not _rocm_torch_family_needs_repair(_target, None, _host_codes or [_target])
+ and not _rocm_compat_reroute_pending(_target, (0, 0), _installed_ver)
+ ):
+ return False
+ return _gfx_route_on_host(_target, _host_codes or [_target])
+ if (
+ _explicit_rocm_torch_index_url() is None
+ and _generic_pytorch_rocm_tag(_ver) is None
+ and not _generic_rocm_wheel_lacks_kernels(_target, _ver)
+ and not (
+ _inferred
+ and (_declared or not _has_rocm_gpu())
+ and _amd_arch_index_url(_inferred) is not None
+ )
+ ):
+ return False
+ return _gfx_route_on_host(_target, _host_codes or [_target])
+ # No target resolved: a mask exposing no GPU is deliberate, anything else is a detection
+ # miss where the inventory is still the best evidence.
+ if _visible_masks_select_no_gpu():
+ return False
+ _archs = _physical_amd_gfx_archs()
+ return any(_gfx_route_on_host(_gfx, _archs) for _gfx in _archs)
+
+
def _gfx_has_a_wheel_route(gfx: "str | None") -> bool:
"""Whether ANY index this installer can pick carries kernels for ``gfx``.
@@ -2879,6 +3012,12 @@ def _runtime_gfx_target(
matter because a runtime-only ROCm install ships neither rocminfo nor amd-smi, and with
no target the callers keep a wheel with no kernels for this GPU.
"""
+ # Reset on entry, not only where decided, so a caller never reads a previous host shape's
+ # answer: every early return below leaves a mask that resolved or no list to index.
+ global _LAST_HIP_MASK_RESOLVED, _LAST_ROCR_MASK_RESOLVED, _LAST_GFX_TARGET_AMBIGUOUS
+ _LAST_HIP_MASK_RESOLVED = True
+ _LAST_ROCR_MASK_RESOLVED = True
+ _LAST_GFX_TARGET_AMBIGUOUS = False
# An empty (or "-1") mask selects NO GPU, deliberately, per _visible_devices_pinned.
# Decided before any probe runs, because no probe is filtered the way the reroutes need:
# only ROCR_VISIBLE_DEVICES reaches rocminfo, and amd-smi and KFD sysfs are filtered by
@@ -2901,6 +3040,21 @@ def _runtime_gfx_target(
# handing torch a gfx1100 agent the gfx1151 wheels have no code for). Only when the
# override names a DIFFERENT arch: naming the arch you spoofed TO is deliberate.
_spoofed = _explicit_gfx if _hsa_spoof_contradicts(_explicit_gfx) else None
+ # The arch names what to BUILD for; whether the runtime exposes a device to build it for is
+ # separate. Only for a host that ASKED, and only when a mask is set: the two flags below are
+ # read by the forced route alone, while the probes cost up to 15s each, so collecting them
+ # for every declared-arch host put a minute of timeouts into an ordinary `studio update`.
+ if _rocm_torch_explicitly_requested() and _visible_devices_pinned():
+ # KFD node order IS the order HIP and ROCr index, and answers on runtime-less hosts.
+ # ignore_visible_masks because the ordinals below index the list BEFORE the ROCr mask.
+ _mask_devices = _kfd_gfx_targets() or _detect_amd_gfx_codes(
+ dedup = False, ignore_visible_masks = True
+ )
+ if _mask_devices:
+ _LAST_ROCR_MASK_RESOLVED = _rocr_layer_mask_names_a_device(len(_mask_devices))
+ _LAST_HIP_MASK_RESOLVED = _hip_layer_mask_names_a_device(
+ len(_rocr_visible_subset(_mask_devices)[0])
+ )
return _explicit_gfx, [_explicit_gfx], _spoofed, [_explicit_gfx]
gfx_devices = _detect_amd_gfx_codes(dedup = False)
# Keyed to the userland probe: ROCr spoofs that reading and no other.
@@ -2935,6 +3089,9 @@ def _runtime_gfx_target(
f" cannot be read here, so the AMD per-gfx index is left alone.\n"
f" Set UNSLOTH_ROCM_GFX_ARCH to the arch you want wheels for.\n"
)
+ # A REJECTION, not a detection miss: without the flag a caller re-reads the arch just
+ # declined and stands a working CUDA install down for a swap that is then refused.
+ _LAST_HIP_MASK_RESOLVED = False
return None, [], None, []
gfx_devices = [inferred_linux_gfx]
# The machine as the probes saw it, before the ROCr layer reduces it to a lone survivor.
@@ -2982,6 +3139,8 @@ def _runtime_gfx_target(
gfx_devices = _kfd_ordered
_unlike_adapters = len(set(gfx_devices)) > 1
_discovery_ordered = False
+ # Against the list BEFORE the subset, which is what the ROCr ordinals index.
+ _LAST_ROCR_MASK_RESOLVED = _rocr_layer_mask_names_a_device(len(gfx_devices))
gfx_devices, _rocr_unresolved = _rocr_visible_subset(gfx_devices)
# A UUID names a device this cannot place. Judged against the list BEFORE the mask
# was applied: dropping the tokens that did resolve can leave one arch standing and
@@ -3015,7 +3174,12 @@ def _runtime_gfx_target(
f" selected cannot be read here, so the AMD per-gfx index is left alone.\n"
f" Set UNSLOTH_ROCM_GFX_ARCH to the arch you want wheels for.\n"
)
+ # A REJECTION, exactly as the mask branch above, and the message just said so.
+ _LAST_GFX_TARGET_AMBIGUOUS = True
return None, [], None, host_codes
+ # Beside the pick and against the same list, the one place both are known.
+ if gfx_devices:
+ _LAST_HIP_MASK_RESOLVED = _hip_layer_mask_names_a_device(len(gfx_devices))
runtime_gfx = (
gfx_devices[_pick_visible_index(len(gfx_devices), masks = _HIP_LAYER_MASKS)]
if gfx_devices
@@ -3263,6 +3427,65 @@ _HIP_LAYER_MASKS = ("HIP_VISIBLE_DEVICES", "CUDA_VISIBLE_DEVICES")
_INDEX_PROBE_LEN = 1 << 20
+def _hip_layer_mask_names_a_device(device_count: int) -> bool:
+ """Whether the HIP-layer mask, if set, names a device index this host has.
+
+ _pick_visible_index answers 0 for a mask it cannot resolve, because for arch SELECTION a
+ first-GPU guess beats no answer. Deciding whether to REPLACE a working CUDA stack is the
+ opposite question: HIP exposes no device, so the ROCm wheels would land where the runtime
+ hands torch nothing. install.sh fails closed on the same input.
+
+ First-set-wins between the two spellings, as _pick_visible_index documents, against the
+ list the HIP layer indexes (the ROCr survivors, which the caller has applied). True when
+ no HIP-layer mask is set, which is not the same as failing to resolve one.
+ """
+ for _env in _HIP_LAYER_MASKS:
+ _val = os.environ.get(_env)
+ if _val is None:
+ continue
+ _val = _val.strip()
+ # A no-GPU mask is deliberate, and _visible_masks_select_no_gpu already declines it.
+ if _val == "" or _val == "-1":
+ return False
+ _first = _val.split(",")[0].strip()
+ try:
+ return 0 <= int(_first) < device_count
+ except ValueError:
+ # A UUID or junk. AMD documents UUID forms for ROCR_VISIBLE_DEVICES, a different
+ # layer, resolved by _rocr_visible_subset.
+ return False
+ return True
+
+
+def _rocr_layer_mask_names_a_device(device_count: int) -> bool:
+ """Whether ROCR_VISIBLE_DEVICES, if set, leaves the runtime at least one device.
+
+ ROCr's filter (ROCR-Runtime, core/inc/amd_filter_device.h) keeps the tokens that are
+ "Legal and NOT Terminating", and an index terminates when it "lies outside the interval
+ [0 - (numGpuDevices - 1)]" or "maps to a device that has been previously selected". So
+ the survivors are a PREFIX: ROCR_VISIBLE_DEVICES=7 on a two-GPU box surfaces nothing.
+
+ _rocr_visible_subset keeps the whole list there instead, deliberately, so arch SELECTION
+ still guesses GPU 0. This is the other question -- whether to REPLACE a working CUDA
+ stack -- and it fails closed, as _hip_layer_mask_names_a_device does one layer up. A UUID
+ resolves to no position here. install.sh composes the same rule in _amd_mask_survivors.
+ """
+ _raw = (os.environ.get("ROCR_VISIBLE_DEVICES") or "").strip()
+ if not _raw:
+ return True
+ _selected: "set[int]" = set()
+ for _tok in _raw.split(","):
+ _tok = _tok.strip()
+ try:
+ _idx = int(_tok)
+ except ValueError:
+ break
+ if not (0 <= _idx < device_count) or _idx in _selected:
+ break
+ _selected.add(_idx)
+ return bool(_selected)
+
+
def _rocr_visible_subset(gfx_devices: "list[str]") -> "tuple[list[str], bool]":
"""Apply the ROCr layer to a device list no probe filtered.
@@ -3280,6 +3503,7 @@ def _rocr_visible_subset(gfx_devices: "list[str]") -> "tuple[list[str], bool]":
if not _raw or not gfx_devices:
return gfx_devices, False
_kept: "list[str]" = []
+ _seen: "set[int]" = set()
_unresolved = False
for _tok in _raw.split(","):
_tok = _tok.strip()
@@ -3288,12 +3512,14 @@ def _rocr_visible_subset(gfx_devices: "list[str]") -> "tuple[list[str], bool]":
except ValueError:
_unresolved = True # a UUID: this names a device, but not a position
continue
- if 0 <= _idx < len(gfx_devices):
- _kept.append(gfx_devices[_idx])
- # An out-of-range index keeps the whole list, deliberately: _pick_visible_index warns and
- # falls back to GPU 0 for that value (matching setup.ps1's Resolve-VisibleGpuIndex), and a
- # stricter rule here would split the two. ROCR_VISIBLE_DEVICES=1 on a one-GPU box is a
- # typo, and reading it as "no GPU" withdraws the repair from the hosts this exists for.
+ # The survivors are a PREFIX: an index terminates the mask when it falls outside the list OR
+ # repeats a selection, and appending past either invents a device the runtime never exposes.
+ if _idx in _seen or not (0 <= _idx < len(gfx_devices)):
+ break
+ _seen.add(_idx)
+ _kept.append(gfx_devices[_idx])
+ # A mask whose FIRST index resolves to nothing keeps the whole list, deliberately, matching
+ # _pick_visible_index. _LAST_ROCR_MASK_RESOLVED is the fail-closed half.
return (_kept or gfx_devices), _unresolved
@@ -3913,6 +4139,15 @@ def _ensure_cuda_torch(*, probe_only: bool = False) -> "bool | None":
# Never undo a deliberate ROCm install (setup.ps1 sets this marker).
if os.environ.get("UNSLOTH_ROCM_TORCH_INSTALLED") == "1":
return
+ # Nor one this run asked for: a standalone `studio update` leaves _TORCH_BACKEND empty, so this
+ # repair read the requested HIP build as poisoning and reinstalled CUDA for _ensure_rocm_torch
+ # to force ROCm back (#10450). Same wheel-ROUTE predicate, so the two cannot drift.
+ if (
+ _rocm_torch_explicitly_requested()
+ and _explicit_cuda_torch_index_url() is None
+ and _forced_rocm_route_is_viable()
+ ):
+ return
# An explicit CUDA pin commits to CUDA wheels and skips ALL GPU gates below.
_cuda_pinned = _explicit_cuda_torch_index_url() is not None
# CUDA_VISIBLE_DEVICES="" / "-1" hides the GPU; honour it unless a CUDA index is pinned.
@@ -5222,7 +5457,11 @@ def _amd_torch_needs_dependency_pass() -> bool:
if _explicit_rocm_torch_index_url() is None:
if _explicit_torch_index_url() is not None:
return False
- if _has_usable_nvidia_gpu():
+ # The request outranks NVIDIA only where it has somewhere to go, or this answers True
+ # forever and setup.sh reruns the dependency pass on every launch for an impossible swap.
+ if _has_usable_nvidia_gpu() and not (
+ _rocm_torch_explicitly_requested() and _forced_rocm_route_is_viable()
+ ):
return False
# A hidden layer either side leaves no target to classify. Same reading the routing
# guard uses, so the two can never drift.
@@ -5441,6 +5680,9 @@ def _ensure_rocm_torch() -> None:
if IS_WINDOWS:
# An explicit ROCm pin overrides the per-arch index: retry the PINNED one, not repo.amd.com.
_win_rocm_pin = _explicit_rocm_torch_index_url()
+ # UNSLOTH_FORCE_ROCM_TORCH is deliberately NOT read here: install.ps1 picks CUDA from the
+ # NVIDIA probe and setup.ps1 republishes a tag _ensure_expected_torch_flavor() restores, so
+ # honouring it only here is a multi-GB round trip. Windows needs both PowerShell installers.
if _win_rocm_pin is None and _has_usable_nvidia_gpu():
return
gfx_arch = _detect_windows_gfx_arch()
@@ -5528,13 +5770,28 @@ def _ensure_rocm_torch() -> None:
_infer_linux_amd_gfx_arch() if (_rocm_pin is None and not IS_WINDOWS) else None
)
if _rocm_pin is None:
- # NVIDIA takes precedence on mixed hosts (only if a GPU is usable).
- if _has_usable_nvidia_gpu():
+ # NVIDIA takes precedence unless this run asked for ROCm. The request relaxes which vendor
+ # wins, not whether there is a card, so the presence test below still has to pass; a pin of
+ # another known family outranks it.
+ if _has_usable_nvidia_gpu() and (
+ not _rocm_torch_explicitly_requested()
+ or _explicit_cuda_torch_index_url() is not None
+ or _explicit_cpu_torch_index_url() is not None
+ ):
return
# _has_rocm_gpu() (rocminfo / amd-smi rows) is the authoritative AMD-host signal;
# the old /opt/rocm-or-hipcc gate broke runtime-only ROCm installs.
if not _has_rocm_gpu() and not _inferred_linux_gfx:
return # no AMD GPU visible
+ # The request skipped the NVIDIA return, and _infer_linux_amd_gfx_arch() takes
+ # UNSLOTH_ROCM_GFX_ARCH before hardware, so a stale one satisfies the line above with no
+ # AMD card. Same viable ROUTE bar _ensure_cuda_torch stands down on.
+ if (
+ _rocm_torch_explicitly_requested()
+ and _has_usable_nvidia_gpu()
+ and not _forced_rocm_route_is_viable()
+ ):
+ return
ver = _detect_rocm_version()
if ver is None:
@@ -5605,8 +5862,17 @@ def _ensure_rocm_torch() -> None:
):
index_url = _amd_arch_index_url(_inferred_linux_gfx)
if index_url is not None:
+ # The bare arch _amd_arch_index_url resolved the URL from: stripping the suffix there is
+ # what OPENS this branch for "gfx1151:xnack-", and a table keyed on the raw string then
+ # misses and installs unpinned torch, losing this dict's ABI bound. _hsa_spoof_contradicts
+ # likewise reads a same-card HSA override as a spoof unless asked in bare form.
+ _bare_gfx = (_inferred_linux_gfx or "").strip().lower().split(":")[0]
+ # Falling back UNBOUNDED was the rest of that same defect: a suffixed arch reaching this
+ # branch for the first time skipped the bounds the reroute applies to every other
+ # arch-index install, so the newly opened route installed a companion set nothing
+ # constrained. Same tuple that path uses, so the two cannot drift.
_torch_pkg, _vision_pkg, _audio_pkg = _WINDOWS_ROCM_TORCH_PKG_SPECS.get(
- _inferred_linux_gfx, ("torch", "torchvision", "torchaudio")
+ _bare_gfx, _ROCM_ARCH_INDEX_TORCH_PKG_SPEC
)
_safe_print(
f" {_inferred_linux_gfx} inferred (ROCm runtime not visible) -- "
@@ -5631,8 +5897,8 @@ def _ensure_rocm_torch() -> None:
# _inferred_linux_gfx code objects alone, and a spoof naming another arch has ROCr
# hand them a device none of it matches (#7331). The install is already committed
# to that arch, so declining here only guarantees it is unusable.
- if _hsa_spoof_contradicts(_inferred_linux_gfx):
- _clear_confirmed_hsa_spoof(_inferred_linux_gfx)
+ if _hsa_spoof_contradicts(_bare_gfx):
+ _clear_confirmed_hsa_spoof(_bare_gfx)
# An explicit UNSLOTH_ROCM_GFX_ARCH=gfx906 pins the runtime target to the
# MI50 / Radeon VII path; it must win over the Strix probe-order detection
diff --git a/tests/sh/test_get_torch_index_url.sh b/tests/sh/test_get_torch_index_url.sh
index a011cdf2083..de4e234c1c6 100755
--- a/tests/sh/test_get_torch_index_url.sh
+++ b/tests/sh/test_get_torch_index_url.sh
@@ -33,6 +33,8 @@ _FAKE_ROCM_DIR=$(mktemp -d)
# references a helper that is not extracted here, the ROCm branch hits an
# undefined function, silently falls through to the CPU wheel index, and the
# ROCm assertions below fail.
+ sed -n '/^_rocm_torch_explicitly_requested()/,/^}/p' "$INSTALL_SH"
+ echo ""
sed -n '/^_ensure_rocm_probe_env()/,/^}/p' "$INSTALL_SH"
echo ""
sed -n '/^_probe_amd_gfx_arch()/,/^}/p' "$INSTALL_SH"
diff --git a/tests/sh/test_rocm_bad_arch_gate.sh b/tests/sh/test_rocm_bad_arch_gate.sh
index 3500a3c0bd4..82bb0919e3b 100755
--- a/tests/sh/test_rocm_bad_arch_gate.sh
+++ b/tests/sh/test_rocm_bad_arch_gate.sh
@@ -146,7 +146,8 @@ trap 'rm -rf "$_FN_FILE" "$_GATE_FILE" "$_REROUTE_FILE" "$_E2E_DIR" "$_FAKE_SMI_
# ROCm branch answer cpu and these pass for the wrong reason. The ROCm assertion below guards it.
{
for _fn in _run_bounded _cvd_hides_nvidia _has_amd_rocm_gpu _has_usable_nvidia_gpu \
- _ensure_rocm_probe_env _probe_amd_gfx_arch _amd_gfx_select_ordinals \
+ _ensure_rocm_probe_env _rocm_torch_explicitly_requested \
+ _probe_amd_gfx_arch _amd_gfx_select_ordinals \
_amd_gpu_present_via_pci \
_infer_amd_gfx_arch_from_gpu_name _infer_linux_amd_gfx_arch \
_amd_arch_index_family_for_gfx _trim_index_path_slashes \
diff --git a/tests/sh/test_rocm_no_version_arch_route.sh b/tests/sh/test_rocm_no_version_arch_route.sh
index e7fe41f3fa3..e1e87d94625 100644
--- a/tests/sh/test_rocm_no_version_arch_route.sh
+++ b/tests/sh/test_rocm_no_version_arch_route.sh
@@ -22,6 +22,8 @@ _FAKE_PROC_NV_DIR=$(mktemp -d)
echo ""
sed -n '/^_has_usable_nvidia_gpu()/,/^}/p' "$INSTALL_SH"
echo ""
+ sed -n '/^_rocm_torch_explicitly_requested()/,/^}/p' "$INSTALL_SH"
+ echo ""
sed -n '/^_ensure_rocm_probe_env()/,/^}/p' "$INSTALL_SH"
echo ""
sed -n '/^_probe_amd_gfx_arch()/,/^}/p' "$INSTALL_SH"
diff --git a/tests/sh/test_rocm_no_version_arch_route_e2e.sh b/tests/sh/test_rocm_no_version_arch_route_e2e.sh
index 7bf73c99b9b..72e62261cf7 100755
--- a/tests/sh/test_rocm_no_version_arch_route_e2e.sh
+++ b/tests/sh/test_rocm_no_version_arch_route_e2e.sh
@@ -85,7 +85,8 @@ for _fn in get_torch_index_url _has_amd_rocm_gpu _has_usable_nvidia_gpu \
_infer_amd_gfx_arch_from_gpu_name _amd_gpu_present_via_pci _run_bounded \
_rocm_tag_from_amd_smi _rocm_tag_from_hipconfig _rocm_tag_from_rpm \
_rocm_tag_from_dpkg _rocm_tag_from_version_file _highest_rocm_tag \
- _trim_index_path_slashes _cvd_hides_nvidia _ensure_rocm_probe_env; do
+ _trim_index_path_slashes _cvd_hides_nvidia _ensure_rocm_probe_env \
+ _rocm_torch_explicitly_requested; do
grep -q "^$_fn()" "$_FUNCS" || \
_fatal "install.sh no longer defines $_fn() at column 0 (splice would silently answer cpu)"
done
diff --git a/tests/sh/test_rocm_version_source_disagreement.sh b/tests/sh/test_rocm_version_source_disagreement.sh
index c3c500a98b5..0d04e394309 100755
--- a/tests/sh/test_rocm_version_source_disagreement.sh
+++ b/tests/sh/test_rocm_version_source_disagreement.sh
@@ -35,6 +35,8 @@ _FAKE_PROC_NV_DIR=$(mktemp -d)
echo ""
sed -n '/^_has_usable_nvidia_gpu()/,/^}/p' "$INSTALL_SH"
echo ""
+ sed -n '/^_rocm_torch_explicitly_requested()/,/^}/p' "$INSTALL_SH"
+ echo ""
sed -n '/^_ensure_rocm_probe_env()/,/^}/p' "$INSTALL_SH"
echo ""
sed -n '/^_probe_amd_gfx_arch()/,/^}/p' "$INSTALL_SH"
diff --git a/tests/studio/install/test_rocm_arch_table_parity.py b/tests/studio/install/test_rocm_arch_table_parity.py
index b595add0e5a..bb303cfbae0 100644
--- a/tests/studio/install/test_rocm_arch_table_parity.py
+++ b/tests/studio/install/test_rocm_arch_table_parity.py
@@ -712,10 +712,12 @@ class TestTorch211PinAllowlistParity:
class TestShadowingIntegratedGfxParity:
- """The shadowing-APU skip (#7776) exists twice: studio/setup.ps1 resolves the
- arch and builds $ROCmIndexUrl before it ever invokes the Python stack
- installer, so both copies of the list have to agree or one entry point keeps
- installing the iGPU's wheel family."""
+ """The shadowing-APU skip (#7776) exists four times now: studio/setup.ps1
+ resolves the arch and builds $ROCmIndexUrl before it ever invokes the Python
+ stack installer, install_llama_prebuilt.py honours setup's repick, and
+ install.sh answers the ROCm-request route on Linux before the Python stack
+ runs at all. Every copy has to agree or one entry point keeps installing the
+ iGPU's wheel family."""
_STRIX = {"gfx1150", "gfx1151", "gfx1152"}
@@ -734,6 +736,20 @@ class TestShadowingIntegratedGfxParity:
return set(ast.literal_eval(node.value.args[0]))
raise AssertionError("SHADOWING_INTEGRATED_GFX not found in install_llama_prebuilt.py")
+ def _install_sh_list(self):
+ body = _sh_function_body(
+ _INSTALL_SH.read_text(encoding = "utf-8"), "_amd_gfx_is_shadowing_integrated"
+ )
+ m = re.search(r"\n\s*(gfx[^)]*)\)\s*return 0", body)
+ assert m, "the shadowing-APU arm was not found in _amd_gfx_is_shadowing_integrated"
+ return {arch.strip() for arch in m.group(1).split("|")}
+
+ def test_install_sh_matches_install_python_stack(self):
+ # install.sh exports the family it chose and _ensure_rocm_torch returns on its first
+ # line for a non-ROCm one, so the shell picking the iGPU here is final and the Python
+ # preference never runs to correct it.
+ assert self._install_sh_list() == set(stack_mod._SHADOWING_INTEGRATED_GFX)
+
def test_setup_ps1_matches_install_python_stack(self):
assert self._setup_ps1_list() == set(stack_mod._SHADOWING_INTEGRATED_GFX)
@@ -748,6 +764,7 @@ class TestShadowingIntegratedGfxParity:
assert not (self._STRIX & set(stack_mod._SHADOWING_INTEGRATED_GFX))
assert not (self._STRIX & self._setup_ps1_list())
assert not (self._STRIX & self._prebuilt_list())
+ assert not (self._STRIX & self._install_sh_list())
if __name__ == "__main__":
diff --git a/tests/studio/install/test_rocm_support.py b/tests/studio/install/test_rocm_support.py
index e9a76b0d3f0..a1dab177fa9 100644
--- a/tests/studio/install/test_rocm_support.py
+++ b/tests/studio/install/test_rocm_support.py
@@ -3013,7 +3013,14 @@ class TestInstallShStructure:
)
def test_cuda_precedence(self):
- """ROCm detection runs only when NVIDIA is absent (check runtime ordering in get_torch_index_url)."""
+ """ROCm detection runs only when NVIDIA is absent, or when ROCm was ASKED for.
+
+ The automatic profile still gives CUDA precedence: that is the guarantee, and a
+ false AMD positive would swap a working install. The one exception is an
+ explicit UNSLOTH_FORCE_ROCM_TORCH request, which is the only route a mixed
+ NVIDIA+AMD host has to its AMD card (#10450), so any AMD probe reached before
+ the no-NVIDIA branch has to be guarded by that request and nothing else.
+ """
sh_path = PACKAGE_ROOT / "install.sh"
source = sh_path.read_text(encoding = "utf-8")
body = _extract_sh_function_body(source, "get_torch_index_url")
@@ -3023,16 +3030,43 @@ class TestInstallShStructure:
no_nvidia_branch = body.find('if [ "$_nvidia_detected" -eq 0 ]')
if no_nvidia_branch < 0:
no_nvidia_branch = body.find('if [ -z "$_smi" ]')
- rocm_call = body.find("_has_amd_rocm_gpu")
assert nvidia_call >= 0, "get_torch_index_url should call _has_usable_nvidia_gpu"
assert no_nvidia_branch >= 0, "get_torch_index_url should gate ROCm on no-nvidia branch"
- assert (
- rocm_call > no_nvidia_branch
- ), "ROCm detection should sit inside the 'no NVIDIA' branch"
assert (
nvidia_call < no_nvidia_branch
), "NVIDIA detection should run before the no-NVIDIA branch"
+ # The automatic path is unchanged: an AMD probe still sits inside the branch.
+ assert (
+ body.find("_has_amd_rocm_gpu", no_nvidia_branch) > no_nvidia_branch
+ ), "ROCm detection should sit inside the 'no NVIDIA' branch"
+
+ # Anything earlier has to be the explicit request, judged on the shell statement
+ # it belongs to rather than on the whole file: a bare probe before the branch
+ # would give AMD precedence over CUDA on every automatic install.
+ #
+ # Comment lines are dropped first, since the comment explaining the guard names
+ # the probe it guards, and continuation lines are then joined, since the guard
+ # and the probe sit either side of a backslash.
+ lines = [
+ line
+ for line in body[:no_nvidia_branch].splitlines()
+ if not line.lstrip().startswith("#")
+ ]
+ statement, guarded = [], []
+ for line in lines:
+ statement.append(line)
+ if not line.rstrip().endswith("\\"):
+ guarded.append(" ".join(statement))
+ statement = []
+ for stmt in guarded:
+ if "_has_amd_rocm_gpu" not in stmt:
+ continue
+ assert "_rocm_torch_explicitly_requested" in stmt, (
+ "an AMD probe before the no-NVIDIA branch must be guarded by "
+ f"_rocm_torch_explicitly_requested, got: {stmt.strip()!r}"
+ )
+
def test_bitsandbytes_amd_install(self):
"""install.sh should install bitsandbytes for AMD when ROCm detected."""
sh_path = PACKAGE_ROOT / "install.sh"